From 49385dd5812c8fc57ad35f8389bde35ba60da308 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:21:37 +0100
Subject: [PATCH 01/22] fix: scan linked worktree history and staged changes
---
justfile | 11 ++++++---
scripts/test_gitleaks.sh | 48 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 56 insertions(+), 3 deletions(-)
create mode 100644 scripts/test_gitleaks.sh
diff --git a/justfile b/justfile
index 02f8b29..d504f0e 100644
--- a/justfile
+++ b/justfile
@@ -133,7 +133,8 @@ lint:
docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/projects.yaml]}' .
@d=$(mktemp -d) && BOOTSTRAP_OUT_DIR="$d" ./bootstrap.sh dummy dummy >/dev/null && docker run --rm --network none -v "$d":/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: disable}}' .; rc=$?; rm -rf "$d"; exit $rc
docker run --rm --network none -v .:/repo:ro -w /repo {{actionlint}} -color
- docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh templates/run_scheduled.sh infra/generate-imports.sh
+ docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh scripts/test_gitleaks.sh templates/run_scheduled.sh infra/generate-imports.sh
+ sh scripts/test_gitleaks.sh
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo
# `just fmt` is the fix.
@@ -146,7 +147,7 @@ lint:
[ -z "$bad" ] || { echo "error: dashboards reference datasource uids that are not provisioned:" $bad >&2; exit 1; }
# Secrets in git history. Scans commits, not the working tree, so the
# gitignored .env never trips it. Needs full history (see ci.yml).
- docker run --rm --network none -v .:/repo:ro {{gitleaks}} git --redact --no-banner /repo
+ just _gitleaks
# The agent config every project host vendors, checked by the Alloy build
# the template actually pins.
docker run --rm --network none -v ./templates/alloy/config.alloy:/etc/alloy/config.alloy:ro -e COMPOSE_PROJECT_NAME=dummy -e ENVIRONMENT=dummy -e PROJECT=dummy -e OTEL_EXPORTER_OTLP_ENDPOINT=https://dummy -e OTLP_AUTH_TOKEN=dummy -e TELEMETRY_EDGE_KEY= $({{spoke_env}} just _image templates/compose.telemetry.yml alloy) validate /etc/alloy/config.alloy
@@ -184,7 +185,11 @@ infra-validate:
# gitleaks over the staged diff (the pre-commit hook; see .pre-commit-config.yaml).
_gitleaks-staged:
- @docker run --rm --network none -v .:/repo:ro {{gitleaks}} git --pre-commit --staged --redact --no-banner /repo
+ @just _gitleaks --pre-commit --staged
+
+_gitleaks *args:
+ # Linked worktrees point outside /repo at their shared Git metadata.
+ @gitdir=$(git rev-parse --path-format=absolute --git-common-dir) && docker run --rm --network none -v .:/repo:ro -v "$gitdir:$gitdir:ro" -e GIT_WORK_TREE=/repo --entrypoint sh {{gitleaks}} -c 'git -C /repo rev-parse --verify HEAD >/dev/null && exec gitleaks git --redact --no-banner "$@" /repo' -- {{args}}
# Format YAML in place (--user so the rewritten files stay yours).
fmt:
diff --git a/scripts/test_gitleaks.sh b/scripts/test_gitleaks.sh
new file mode 100644
index 0000000..9a5c91f
--- /dev/null
+++ b/scripts/test_gitleaks.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+# Git hooks export index and worktree paths; scratch repositories must not inherit them.
+for name in $(git rev-parse --local-env-vars); do unset "$name"; done
+root=$(pwd)
+d=$(mktemp -d "${TMPDIR:-/tmp}/monitoring-gitleaks.XXXXXX")
+trap 'rm -rf "$d"' EXIT HUP INT TERM
+
+git -c core.hooksPath=/dev/null init -q "$d/main"
+cd "$d/main"
+cat >.gitleaks.toml <<'EOF'
+[[rules]]
+id = "regression-fixture"
+description = "Synthetic regression credential"
+regex = '''fixture-secret-[0-9]{8}'''
+EOF
+git add .gitleaks.toml
+git -c core.hooksPath=/dev/null -c commit.gpgsign=false -c user.name=Test -c user.email=test@example.org commit -qm initial
+git -c core.hooksPath=/dev/null worktree add -q "$d/linked"
+cp "$root/justfile" "$d/linked/justfile"
+cd "$d/linked"
+mkdir dashboards
+printf '{}\n' >dashboards/fixture.json
+
+expect_leak() {
+ rc=0
+ just "$@" >"$d/output" 2>&1 || rc=$?
+ # just preserves the failing recipe status in its diagnostic, but exits 1 itself.
+ if [ "$rc" -ne 1 ] || ! grep -q 'leaks found: 1' "$d/output"; then
+ cat "$d/output" >&2
+ echo "error: $* did not detect the synthetic credential" >&2
+ exit 1
+ fi
+}
+
+just _gitleaks
+printf 'fixture-secret-%s\n' 12345678 >credential.txt
+git add credential.txt
+expect_leak _gitleaks-staged
+git -c core.hooksPath=/dev/null -c commit.gpgsign=false -c user.name=Test -c user.email=test@example.org commit -qm fixture
+expect_leak _gitleaks
+printf 'gitdir: /missing-git-dir\n' >.git
+if just _gitleaks >"$d/output" 2>&1; then
+ echo "error: a broken Git directory passed the scan" >&2
+ exit 1
+fi
+echo "gitleaks detects staged and committed credentials in linked worktrees and rejects broken Git directories"
From 84531d6426bb7884d74c2db8cb5e322ba911d5ad Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:22:52 +0100
Subject: [PATCH 02/22] ci: verify fresh demo setup and dependency licenses
---
.github/CONTRIBUTING.md | 5 +++
.github/ISSUE_TEMPLATE/bug.yml | 2 +-
.github/workflows/ci.yml | 3 ++
.github/workflows/cold-install.yml | 41 ++++++++++++++++++++
.github/workflows/infra.yml | 2 +
.github/workflows/licenses.yml | 42 +++++++++++++++++++++
.github/workflows/release.yml | 5 ++-
README.md | 2 +
compose.demo.yml | 6 +++
config/grafana/alerting/contact-points.yaml | 6 +--
10 files changed, 109 insertions(+), 5 deletions(-)
create mode 100644 .github/workflows/cold-install.yml
create mode 100644 .github/workflows/licenses.yml
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 8d7f139..0682184 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -29,6 +29,11 @@ validates every config, `just smoke` starts a throwaway copy of the stack and se
telemetry, and `just fmt` formats the YAML. CI runs `just lint`, `just validate` and
`just smoke`, plus `just infra-validate` when `infra/` changes. The
[Verification commands](../docs/RUNBOOK.md#verification-commands) table in the runbook lists every recipe.
+CI also checks the README demo on a fresh runner and checks the demo dependency licences.
+
+Add or update tests for each behaviour change, including bug fixes. Extend the assertions
+in `just lint` or `scripts/smoke.sh` for the affected path. Follow the coding standards
+checked by Ruff for Python, ShellCheck for shell scripts, and yamllint and yamlfmt for YAML.
## Write commits and pull requests
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
index e401e73..0a7e12f 100644
--- a/.github/ISSUE_TEMPLATE/bug.yml
+++ b/.github/ISSUE_TEMPLATE/bug.yml
@@ -31,4 +31,4 @@ body:
id: environment
attributes:
label: Environment
- description: Host operating system, and the Docker and Compose versions (`docker version`, `docker compose version`).
+ description: Operating system and Docker and Compose versions (`docker version`, `docker compose version`).
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ffe88bf..4c5fe41 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,6 +25,7 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# gitleaks scans git history; a shallow clone would pass vacuously.
with:
+ persist-credentials: false
fetch-depth: 0
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
# 600, or the exposure guards refuse it as world-readable.
@@ -36,6 +37,8 @@ jobs:
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
# 600, or the exposure guards refuse it as world-readable.
- run: install -m 600 .env.example .env
diff --git a/.github/workflows/cold-install.yml b/.github/workflows/cold-install.yml
new file mode 100644
index 0000000..651de7a
--- /dev/null
+++ b/.github/workflows/cold-install.yml
@@ -0,0 +1,41 @@
+name: cold install
+
+on:
+ pull_request:
+ schedule:
+ - cron: "47 5 * * 1"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ readme:
+ name: README demo, fresh runner
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
+ # Docker and Compose are supplied by the fresh runner; no build cache is restored.
+ - name: Run the README quick start
+ run: |
+ cp .env.example .env
+ just demo
+ - name: Check the demo API and Grafana
+ run: |
+ docker compose -p monitoring-demo -f compose.yml -f compose.demo.yml -f compose.sandbox.yml \
+ exec -T demo-load curl --fail --retry 20 --retry-all-errors --retry-delay 2 demo-api:8000/openapi.json
+ curl --fail --retry 20 --retry-all-errors --retry-delay 2 http://localhost:3002/api/health
+ - if: failure()
+ run: >-
+ docker compose -p monitoring-demo -f compose.yml -f compose.demo.yml -f compose.sandbox.yml
+ logs --no-color --tail=200
+ - if: always()
+ run: just demo-destroy
diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml
index 8f5c130..48465f3 100644
--- a/.github/workflows/infra.yml
+++ b/.github/workflows/infra.yml
@@ -20,6 +20,8 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
- run: install -m 600 .env.example .env # 600 or the exposure guards reject it
- run: just infra-validate
diff --git a/.github/workflows/licenses.yml b/.github/workflows/licenses.yml
new file mode 100644
index 0000000..cdf4ebc
--- /dev/null
+++ b/.github/workflows/licenses.yml
@@ -0,0 +1,42 @@
+name: demo dependency licenses
+
+on:
+ pull_request:
+ schedule:
+ - cron: "23 5 * * 1"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ licenses:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+ with:
+ version: "0.13.6"
+ enable-cache: false
+ - id: python
+ name: Read the demo runtime
+ run: |
+ version=$(sed -n 's/^FROM python:\([0-9]*\.[0-9]*\).*/\1/p' demo/Dockerfile)
+ test -n "$version"
+ echo "version=$version" >> "$GITHUB_OUTPUT"
+ # Resolve the same manifest as the demo image, including transitive packages.
+ - run: >-
+ uv pip compile --python-version "${{ steps.python.outputs.version }}"
+ demo/pyproject.toml -o "$RUNNER_TEMP/requirements.txt"
+ - name: List licenses and reject incompatibility with the repository's MIT license
+ run: >-
+ uv run --no-project --python "${{ steps.python.outputs.version }}"
+ --with-requirements "$RUNNER_TEMP/requirements.txt"
+ --with licensecheck==2026.0.8 licensecheck --license MIT -r "$RUNNER_TEMP/requirements.txt" --zero
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5e588cb..5ef3e0a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -11,16 +11,19 @@ on:
tags: ["v*"]
permissions:
- contents: write
+ contents: read
jobs:
release:
runs-on: ubuntu-latest
timeout-minutes: 15
+ permissions:
+ contents: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# gitleaks scans git history; a shallow clone would pass vacuously.
with:
+ persist-credentials: false
fetch-depth: 0
- uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
# The tag's section of CHANGELOG.md is the release body. No section, no
diff --git a/README.md b/README.md
index e1dc364..fd70d62 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,8 @@ This launches the core monitoring stack and a demo FastAPI service under simulat
- View **Dashboards → Service Health (RED)** for request rates, error rates, and latency traces.
- Stop and clean up: `just demo-destroy`.
+The demo displays sample telemetry but does not deliver alert notifications.
+

## Operating the stack
diff --git a/compose.demo.yml b/compose.demo.yml
index 704ac80..a2b284e 100644
--- a/compose.demo.yml
+++ b/compose.demo.yml
@@ -12,6 +12,12 @@ x-demo-logging: &demo-logging
max-file: "3"
services:
+ grafana:
+ # Demo notifications must not reach a production webhook or dead man's switch.
+ environment:
+ ALERT_WEBHOOK_URL: https://demo.invalid/alerts
+ HEARTBEAT_URL: https://demo.invalid/heartbeat
+
demo-api:
build: ./demo
restart: on-failure
diff --git a/config/grafana/alerting/contact-points.yaml b/config/grafana/alerting/contact-points.yaml
index 621bdb2..e084fa4 100644
--- a/config/grafana/alerting/contact-points.yaml
+++ b/config/grafana/alerting/contact-points.yaml
@@ -1,8 +1,8 @@
# Notification targets. Grafana expands $VAR from the environment.
#
-# An unset variable leaves the URL empty and every notification fails silently:
-# AlertDeliveryFailing routes to this same webhook, and the heartbeat stays
-# green. The exposure guard in `just up` is the only control.
+# Each receiver needs a URL or Grafana rejects contact point provisioning.
+# The demo and smoke stacks supply isolated placeholders. The exposure guard
+# in `just up` requires a production alert destination.
apiVersion: 1
From ca00a1aae1b080dd1a755131b5ac94d3bd75678c Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:21:06 +0100
Subject: [PATCH 03/22] chore(collector): use canonical processor and transform
names
---
config/otel-collector.yaml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml
index 467fff6..18a409a 100644
--- a/config/otel-collector.yaml
+++ b/config/otel-collector.yaml
@@ -90,7 +90,7 @@ processors:
# service.instance.id on each restart adds one every time. The default is
# unbounded. This cap is a runaway guard, not a budget: 28 streams cost ~51 MiB
# heap against the 400 MiB limiter (measured before `interval` was added).
- deltatocumulative:
+ delta_to_cumulative:
max_streams: 5000
# The count connector emits one datapoint per incoming batch, so 50 log batches
@@ -109,7 +109,7 @@ processors:
metric_statements:
- context: datapoint
statements:
- - set(time, Now())
+ - set(datapoint.time, Now())
# Sized to the container's mem_limit (512m in compose.yml); keep the two in step.
memory_limiter:
@@ -170,10 +170,10 @@ service:
processors: [memory_limiter, resource/department, batch]
exporters: [otlp_http/prometheus, count]
# The counters themselves. No batcher: it reordered timestamps ~1/min, which
- # deltatocumulative refused as out of order. `interval` does the collapsing.
+ # delta_to_cumulative refused as out of order. `interval` does the collapsing.
metrics/count:
receivers: [count]
- processors: [memory_limiter, deltatocumulative, interval, transform/count_ingest_time]
+ processors: [memory_limiter, delta_to_cumulative, interval, transform/count_ingest_time]
exporters: [otlp_http/prometheus_count]
telemetry:
metrics:
From 0e9fb2dbf874a2e941f836473cc30e1b2716fad8 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:25:33 +0100
Subject: [PATCH 04/22] fix(rum): report final LCP and session-window CLS
---
templates/web/README.md | 14 ++++-
templates/web/rum.html | 108 +++++++++++++++++++++++----------
templates/web/rum.test.mjs | 121 +++++++++++++++++++++++++++++++++++++
3 files changed, 208 insertions(+), 35 deletions(-)
create mode 100644 templates/web/rum.test.mjs
diff --git a/templates/web/README.md b/templates/web/README.md
index d0e61d6..fa9cae4 100644
--- a/templates/web/README.md
+++ b/templates/web/README.md
@@ -4,9 +4,17 @@ Lightweight, zero-dependency browser telemetry for static websites, documentatio
## What it captures
-1. **Core Web Vitals:** Largest Contentful Paint (LCP) and Cumulative Layout Shift (CLS).
-2. **Unhandled errors:** `window.onerror` exceptions and unhandled promise rejections.
-3. **Structured logs:** Shipped as standard OTLP/HTTP JSON (`/v1/logs`) to the central collector.
+1. **LCP:** The latest paint candidate, exported once at first interaction or page hide.
+2. **CLS:** The largest session of unexpected shifts, with gaps under one second and a span under five seconds. Exported on hide when the nonzero page maximum increases.
+3. **Unhandled errors:** `window.onerror` exceptions and unhandled promise rejections.
+4. **Structured logs:** Shipped as standard OTLP/HTTP JSON (`/v1/logs`) to the central collector.
+
+The script drains pending performance records before exporting. Showing the page
+again preserves its CLS maximum. It ignores LCP for pages loaded hidden and
+candidates painted after the first hide. These rules follow the
+[LCP](https://web.dev/articles/lcp) and [CLS](https://web.dev/articles/cls) definitions.
+It does not measure visits restored from the back/forward cache, prerendered
+page activation or content inside iframes.
## Integration
diff --git a/templates/web/rum.html b/templates/web/rum.html
index 4aa1ebb..ce02259 100644
--- a/templates/web/rum.html
+++ b/templates/web/rum.html
@@ -76,42 +76,86 @@
// --- Core Web Vitals ---
if ("PerformanceObserver" in window) {
- // LCP
- try {
- new PerformanceObserver(function(list) {
- const entries = list.getEntries();
- const lastEntry = entries[entries.length - 1];
- if (lastEntry) {
- sendLog("WebVital: LCP", "INFO", {
- vital_name: "LCP",
- duration_ms: Math.round(lastEntry.startTime),
- element: lastEntry.element ? (lastEntry.element.tagName || "element") : "unknown"
- });
+ let lcpObserver;
+ let latestLcp;
+ let lcpFinalized = document.visibilityState === "hidden";
+ let firstHiddenTime = lcpFinalized ? 0 : Infinity;
+ let clsObserver;
+ let sessionStart = 0;
+ let lastShiftTime = 0;
+ let sessionScore = 0;
+ let clsScore = 0;
+ let reportedCls = 0;
+
+ function updateLcp(entries) {
+ if (lcpFinalized) return;
+ for (const entry of entries) {
+ if (entry.startTime < firstHiddenTime) latestLcp = entry;
+ }
+ }
+
+ function finalizeLcp() {
+ if (lcpFinalized) return;
+ if (lcpObserver) {
+ updateLcp(lcpObserver.takeRecords());
+ lcpObserver.disconnect();
+ }
+ lcpFinalized = true;
+ if (latestLcp) {
+ sendLog("WebVital: LCP", "INFO", {
+ vital_name: "LCP",
+ duration_ms: Math.round(latestLcp.startTime),
+ element: latestLcp.element ? latestLcp.element.tagName || "element" : "unknown"
+ });
+ }
+ }
+
+ function updateCls(entries) {
+ for (const entry of entries) {
+ if (entry.hadRecentInput) continue;
+ // A session ends at a one-second gap or a five-second total span.
+ if (sessionScore > 0 && entry.startTime - lastShiftTime < 1000 && entry.startTime - sessionStart < 5000) {
+ sessionScore += entry.value;
+ } else {
+ sessionStart = entry.startTime;
+ sessionScore = entry.value;
}
- }).observe({ type: "largest-contentful-paint", buffered: true });
- } catch (e) {}
+ lastShiftTime = entry.startTime;
+ clsScore = Math.max(clsScore, sessionScore);
+ }
+ }
- // CLS
try {
- let clsValue = 0;
- new PerformanceObserver(function(list) {
- for (const entry of list.getEntries()) {
- if (!entry.hadRecentInput) {
- clsValue += entry.value;
- }
- }
- }).observe({ type: "layout-shift", buffered: true });
+ if (!lcpFinalized) {
+ lcpObserver = new PerformanceObserver((list) => updateLcp(list.getEntries()));
+ lcpObserver.observe({ type: "largest-contentful-paint", buffered: true });
+ }
+ } catch {
+ // Some browsers do not support this performance entry type.
+ }
+ try {
+ clsObserver = new PerformanceObserver((list) => updateCls(list.getEntries()));
+ clsObserver.observe({ type: "layout-shift", buffered: true });
+ } catch {
+ // Some browsers do not support this performance entry type.
+ }
- window.addEventListener("visibilitychange", function() {
- if (document.visibilityState === "hidden" && clsValue > 0) {
- sendLog("WebVital: CLS", "INFO", {
- vital_name: "CLS",
- cls_score: Math.round(clsValue * 1000) / 1000
- });
- clsValue = 0;
- }
- });
- } catch (e) {}
+ for (const event of ["pointerdown", "touchstart", "keydown", "scroll"]) {
+ window.addEventListener(event, finalizeLcp, { once: true, capture: true, passive: true });
+ }
+ document.addEventListener("visibilitychange", (event) => {
+ if (document.visibilityState !== "hidden") return;
+ firstHiddenTime = Math.min(firstHiddenTime, event.timeStamp);
+ finalizeLcp();
+ if (clsObserver) updateCls(clsObserver.takeRecords());
+ if (clsScore > reportedCls) {
+ sendLog("WebVital: CLS", "INFO", {
+ vital_name: "CLS",
+ cls_score: Math.round(clsScore * 1000) / 1000
+ });
+ reportedCls = clsScore;
+ }
+ });
}
// --- Uncaught Errors ---
diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs
new file mode 100644
index 0000000..8934ffc
--- /dev/null
+++ b/templates/web/rum.test.mjs
@@ -0,0 +1,121 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+import { runInNewContext } from 'node:vm';
+
+const source = readFileSync(new URL('./rum.html', import.meta.url), 'utf8').split('')[0];
+
+function browser(initialVisibility = 'visible') {
+ const logs = [];
+ const listeners = {};
+ const observers = {};
+ const on = (name, handler) => (listeners[name] ??= []).push(handler);
+ const document = { visibilityState: initialVisibility, addEventListener: on };
+ const window = {
+ __OTEL_ENDPOINT__: 'https://rum.example.org',
+ location: { origin: 'https://example.org', pathname: '/' },
+ addEventListener: on,
+ };
+ class PerformanceObserver {
+ pending = [];
+ disconnected = false;
+ constructor(callback) { this.callback = callback; }
+ observe({ type }) { observers[type] = this; }
+ takeRecords() { return this.pending.splice(0); }
+ disconnect() { this.disconnected = true; }
+ }
+ window.PerformanceObserver = PerformanceObserver;
+ runInNewContext(source, {
+ window, document, PerformanceObserver, navigator: { userAgent: 'test-browser' },
+ fetch: (_url, options) => {
+ logs.push(JSON.parse(options.body).resourceLogs[0].scopeLogs[0].logRecords[0]);
+ return Promise.resolve();
+ },
+ });
+ const emit = (name, timeStamp = 10000) => listeners[name]?.forEach((handler) => handler({ timeStamp }));
+ const deliver = (type, entries) => observers[type].callback({ getEntries: () => entries });
+ const hide = (time = 10000) => { document.visibilityState = 'hidden'; emit('visibilitychange', time); };
+ const values = (vital, key) => logs.filter((log) => log.body.stringValue === `WebVital: ${vital}`).map((log) => log.attributes.find((attr) => attr.key === key).value.doubleValue);
+ return { logs, document, observers, emit, deliver, hide, values };
+}
+const shift = (value, startTime, hadRecentInput = false) => ({ value, startTime, hadRecentInput });
+
+test('CLS reports the largest session instead of summing separate sessions', () => {
+ const page = browser();
+ page.deliver('layout-shift', [shift(0.12, 100), shift(0.13, 2100)]);
+ page.hide();
+ assert.deepEqual(page.values('CLS', 'cls_score'), [0.13]);
+});
+
+test('CLS sums shifts in the same session and excludes recent input', () => {
+ const page = browser();
+ page.deliver('layout-shift', [shift(0.12, 100), shift(0.13, 900), shift(0.8, 950, true)]);
+ page.hide();
+ assert.deepEqual(page.values('CLS', 'cls_score'), [0.25]);
+});
+
+test('CLS starts a new session at a one-second gap or five-second span', () => {
+ for (const times of [[100, 1100], [0, 900, 1800, 2700, 3600, 4500, 5000]]) {
+ const page = browser();
+ page.deliver('layout-shift', times.map((time) => shift(0.1, time)));
+ page.hide();
+ assert.deepEqual(page.values('CLS', 'cls_score'), [times.length === 2 ? 0.1 : 0.6]);
+ }
+});
+
+test('CLS drains queued shifts and retains the page maximum across hide/show cycles', () => {
+ const page = browser();
+ page.observers['layout-shift'].pending.push(shift(0.3, 100));
+ page.hide(200);
+ page.hide(200);
+ page.document.visibilityState = 'visible';
+ page.emit('visibilitychange', 300);
+ page.deliver('layout-shift', [shift(0.1, 2100)]);
+ page.hide(2200);
+ page.document.visibilityState = 'visible';
+ page.deliver('layout-shift', [shift(0.4, 4100)]);
+ page.hide(4200);
+ assert.deepEqual(page.values('CLS', 'cls_score'), [0.3, 0.4]);
+});
+
+test('LCP sends only the latest candidate at first interaction', () => {
+ const page = browser();
+ page.deliver('largest-contentful-paint', [{ startTime: 100 }]);
+ page.deliver('largest-contentful-paint', [{ startTime: 2000 }]);
+ assert.deepEqual(page.values('LCP', 'duration_ms'), []);
+ page.emit('pointerdown', 2100);
+ page.hide();
+ assert.deepEqual(page.values('LCP', 'duration_ms'), [2000]);
+ assert.equal(page.observers['largest-contentful-paint'].disconnected, true);
+});
+
+test('LCP drains queued candidates at hide and ignores later callbacks', () => {
+ const page = browser();
+ page.deliver('largest-contentful-paint', [{ startTime: 100 }]);
+ page.observers['largest-contentful-paint'].pending.push({ startTime: 2000 });
+ page.hide(2100);
+ page.deliver('largest-contentful-paint', [{ startTime: 3000 }]);
+ page.document.visibilityState = 'visible';
+ page.hide(4000);
+ assert.deepEqual(page.values('LCP', 'duration_ms'), [2000]);
+});
+
+test('LCP finalises on keyboard and scroll interactions', () => {
+ for (const event of ['keydown', 'scroll', 'pointerdown', 'touchstart']) {
+ const page = browser();
+ page.deliver('largest-contentful-paint', [{ startTime: 100 }]);
+ page.emit(event, 200);
+ assert.deepEqual(page.values('LCP', 'duration_ms'), [100]);
+ }
+});
+
+test('LCP excludes pages loaded hidden and candidates painted after first hide', () => {
+ const hiddenPage = browser('hidden');
+ hiddenPage.document.visibilityState = 'visible';
+ hiddenPage.emit('pointerdown');
+ assert.deepEqual(hiddenPage.values('LCP', 'duration_ms'), []);
+ const page = browser();
+ page.observers['largest-contentful-paint'].pending.push({ startTime: 3000 });
+ page.hide(2000);
+ assert.deepEqual(page.values('LCP', 'duration_ms'), []);
+});
From 12057ef766d66fa5f790a792917ea7587ca71b2d Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:27:40 +0100
Subject: [PATCH 05/22] fix(rum): distinguish zero CLS from unsupported
measurement
---
templates/web/README.md | 5 +++--
templates/web/rum.html | 5 +++--
templates/web/rum.test.mjs | 17 +++++++++++++++--
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/templates/web/README.md b/templates/web/README.md
index fa9cae4..86454ac 100644
--- a/templates/web/README.md
+++ b/templates/web/README.md
@@ -5,12 +5,13 @@ Lightweight, zero-dependency browser telemetry for static websites, documentatio
## What it captures
1. **LCP:** The latest paint candidate, exported once at first interaction or page hide.
-2. **CLS:** The largest session of unexpected shifts, with gaps under one second and a span under five seconds. Exported on hide when the nonzero page maximum increases.
+2. **CLS:** The largest session of unexpected shifts, with gaps under one second and a span under five seconds. Exported on the first hide, including zero, and on later hides when the page maximum increases.
3. **Unhandled errors:** `window.onerror` exceptions and unhandled promise rejections.
4. **Structured logs:** Shipped as standard OTLP/HTTP JSON (`/v1/logs`) to the central collector.
The script drains pending performance records before exporting. Showing the page
-again preserves its CLS maximum. It ignores LCP for pages loaded hidden and
+again preserves its CLS maximum. Later exports update the same page visit,
+not independent observations. Unsupported layout-shift observation produces no score. It ignores LCP for pages loaded hidden and
candidates painted after the first hide. These rules follow the
[LCP](https://web.dev/articles/lcp) and [CLS](https://web.dev/articles/cls) definitions.
It does not measure visits restored from the back/forward cache, prerendered
diff --git a/templates/web/rum.html b/templates/web/rum.html
index ce02259..f7e2aee 100644
--- a/templates/web/rum.html
+++ b/templates/web/rum.html
@@ -85,7 +85,7 @@
let lastShiftTime = 0;
let sessionScore = 0;
let clsScore = 0;
- let reportedCls = 0;
+ let reportedCls = -1;
function updateLcp(entries) {
if (lcpFinalized) return;
@@ -137,6 +137,7 @@
clsObserver = new PerformanceObserver((list) => updateCls(list.getEntries()));
clsObserver.observe({ type: "layout-shift", buffered: true });
} catch {
+ clsObserver = undefined;
// Some browsers do not support this performance entry type.
}
@@ -148,7 +149,7 @@
firstHiddenTime = Math.min(firstHiddenTime, event.timeStamp);
finalizeLcp();
if (clsObserver) updateCls(clsObserver.takeRecords());
- if (clsScore > reportedCls) {
+ if (clsObserver && clsScore > reportedCls) {
sendLog("WebVital: CLS", "INFO", {
vital_name: "CLS",
cls_score: Math.round(clsScore * 1000) / 1000
diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs
index 8934ffc..fbdfcaa 100644
--- a/templates/web/rum.test.mjs
+++ b/templates/web/rum.test.mjs
@@ -5,7 +5,7 @@ import { runInNewContext } from 'node:vm';
const source = readFileSync(new URL('./rum.html', import.meta.url), 'utf8').split('')[0];
-function browser(initialVisibility = 'visible') {
+function browser(initialVisibility = 'visible', unsupportedCls = false) {
const logs = [];
const listeners = {};
const observers = {};
@@ -20,7 +20,10 @@ function browser(initialVisibility = 'visible') {
pending = [];
disconnected = false;
constructor(callback) { this.callback = callback; }
- observe({ type }) { observers[type] = this; }
+ observe({ type }) {
+ if (unsupportedCls && type === 'layout-shift') throw new Error('unsupported');
+ observers[type] = this;
+ }
takeRecords() { return this.pending.splice(0); }
disconnect() { this.disconnected = true; }
}
@@ -119,3 +122,13 @@ test('LCP excludes pages loaded hidden and candidates painted after first hide',
page.hide(2000);
assert.deepEqual(page.values('LCP', 'duration_ms'), []);
});
+
+test('CLS reports a supported zero once and emits no score for unsupported observation', () => {
+ const supported = browser();
+ supported.hide();
+ supported.hide();
+ assert.deepEqual(supported.values('CLS', 'cls_score'), [0]);
+ const unsupported = browser('visible', true);
+ unsupported.hide();
+ assert.deepEqual(unsupported.values('CLS', 'cls_score'), []);
+});
From f3ec1dad51d44fad0f4001aa9e2eb18ec535a76d Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:28:24 +0100
Subject: [PATCH 06/22] test(rum): run vitals regressions during lint
---
justfile | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/justfile b/justfile
index d504f0e..cb7a16a 100644
--- a/justfile
+++ b/justfile
@@ -12,7 +12,8 @@ shellcheck := "koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f
ruff := "ghcr.io/astral-sh/ruff:0.14.2@sha256:636e27f3feb43800e44b0ad48c72811b500a2c6309d094b641a9bf2247f4dbff"
tofu := "ghcr.io/opentofu/opentofu:1.12.3@sha256:a0766d12f07b43e66f2ed40d7a8babe97d581d20339c68ad0ab561737af9a5b3"
gitleaks := "zricethezav/gitleaks:v8.30.1@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f"
-lint_images := jq + " " + yamllint + " " + yamlfmt + " " + actionlint + " " + shellcheck + " " + ruff + " " + tofu + " " + gitleaks
+node := "node:24-alpine@sha256:116b800c3754b440ee0aa3ed2f20f5bf3d99a554bfd9adeea09ea8017f6e85f5"
+lint_images := jq + " " + yamllint + " " + yamlfmt + " " + actionlint + " " + shellcheck + " " + ruff + " " + tofu + " " + gitleaks + " " + node
# `demo` and `smoke` each run a throwaway copy of the core stack under their own
# compose project and Grafana port (compose.sandbox.yml), so neither can touch a
@@ -137,6 +138,7 @@ lint:
sh scripts/test_gitleaks.sh
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo
+ docker run --rm --network none -v ./templates/web:/rum:ro {{node}} node --test --test-isolation=none /rum/rum.test.mjs
# `just fmt` is the fix.
docker run --rm --network none -v .:/code:ro -w /code {{yamlfmt}} -lint .
docker run --rm --network none -v ./infra:/infra:ro -w /infra {{tofu}} fmt -check
From 84a6a6271391c2df0f7f19df217caf602d43a092 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:29:52 +0100
Subject: [PATCH 07/22] fix(rum): check browser support before reporting CLS
---
templates/web/rum.html | 6 ++++--
templates/web/rum.test.mjs | 12 +++++++++++-
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/templates/web/rum.html b/templates/web/rum.html
index f7e2aee..338aabf 100644
--- a/templates/web/rum.html
+++ b/templates/web/rum.html
@@ -134,8 +134,10 @@
// Some browsers do not support this performance entry type.
}
try {
- clsObserver = new PerformanceObserver((list) => updateCls(list.getEntries()));
- clsObserver.observe({ type: "layout-shift", buffered: true });
+ if (PerformanceObserver.supportedEntryTypes?.includes("layout-shift")) {
+ clsObserver = new PerformanceObserver((list) => updateCls(list.getEntries()));
+ clsObserver.observe({ type: "layout-shift", buffered: true });
+ }
} catch {
clsObserver = undefined;
// Some browsers do not support this performance entry type.
diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs
index fbdfcaa..82552ed 100644
--- a/templates/web/rum.test.mjs
+++ b/templates/web/rum.test.mjs
@@ -17,11 +17,13 @@ function browser(initialVisibility = 'visible', unsupportedCls = false) {
addEventListener: on,
};
class PerformanceObserver {
+ static supportedEntryTypes = unsupportedCls === 'missing' ? undefined : unsupportedCls === 'silent' ? ['mark', 'measure'] : ['largest-contentful-paint', 'layout-shift'];
pending = [];
disconnected = false;
constructor(callback) { this.callback = callback; }
observe({ type }) {
- if (unsupportedCls && type === 'layout-shift') throw new Error('unsupported');
+ if (unsupportedCls === 'silent' && type === 'layout-shift') return;
+ if (unsupportedCls === true && type === 'layout-shift') throw new Error('unsupported');
observers[type] = this;
}
takeRecords() { return this.pending.splice(0); }
@@ -132,3 +134,11 @@ test('CLS reports a supported zero once and emits no score for unsupported obser
unsupported.hide();
assert.deepEqual(unsupported.values('CLS', 'cls_score'), []);
});
+
+test('CLS stays unknown when unsupported observe silently does nothing', () => {
+ for (const support of ['silent', 'missing']) {
+ const page = browser('visible', support);
+ page.hide();
+ assert.deepEqual(page.values('CLS', 'cls_score'), []);
+ }
+});
From 5850ad87e228e25049dc2868b15faacffe2f1ed0 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 15:36:04 +0100
Subject: [PATCH 08/22] fix: reject missing heartbeat provisioning URL
---
.env.example | 6 +++---
docs/RUNBOOK.md | 19 ++++++++++---------
justfile | 8 ++++----
3 files changed, 17 insertions(+), 16 deletions(-)
diff --git a/.env.example b/.env.example
index 7a668ef..f1b207f 100644
--- a/.env.example
+++ b/.env.example
@@ -25,9 +25,8 @@ OTLP_AUTH_TOKEN=local-dev-token
# Where Grafana delivers alert notifications (self-hosted ntfy, hosted ntfy, Slack, …).
# For self-hosted ntfy with basic auth publisher:
# ALERT_WEBHOOK_URL=http://grafana:password@ntfy:80/alerts?tpl=1&title={{or .commonAnnotations.summary .groupLabels.alertname}}&message={{or .commonAnnotations.description .commonAnnotations.summary "Alert firing"}}
-# Not optional: an empty value drops every alert silently while the heartbeat
-# below keeps reporting healthy. With the tunnel overlay active, `just up`
-# refuses to start without it.
+# Required: Grafana rejects contact point provisioning if this URL is empty.
+# With the tunnel overlay active, `just up` refuses to start without it.
ALERT_WEBHOOK_URL=
# Public URL Cloudflare Tunnel exposes ntfy on, e.g. https://ntfy.example.com
@@ -36,6 +35,7 @@ NTFY_BASE_URL=
# Dead man's switch ping target (e.g. https://hc-ping.com/). The Watchdog
# alert posts here every 5m; alert externally when pings stop.
+# Required: Grafana also rejects an empty heartbeat URL.
HEARTBEAT_URL=
# Optional. Lets `./bootstrap.sh` create a project's healthchecks.io checks.
diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md
index 4ebfba7..9ef24f8 100644
--- a/docs/RUNBOOK.md
+++ b/docs/RUNBOOK.md
@@ -180,10 +180,9 @@ them read-only. Edit the YAML.
- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five minutes.
Point it at a dead man's switch that raises the alarm when pings stop.
-Leaving either empty is not safe. Delivery fails silently while the heartbeat
-keeps pinging, so the switch reads healthy and every real alert is dropped.
-With the tunnel overlay active, `just up` refuses to start without
-`ALERT_WEBHOOK_URL`, and `AlertDeliveryFailing` fires on a failing notifier.
+Grafana rejects contact point provisioning if either URL is empty.
+With the tunnel overlay active, `just up` refuses to start without both URLs.
+`AlertDeliveryFailing` fires if a configured notifier fails to deliver alerts.
After changing either variable, run `docker compose up -d grafana`. A plain
`restart` keeps the old environment. Compose rebuilds a container's
@@ -314,17 +313,17 @@ When the tunnel overlay is active, `just up` validates critical security setting
- **`GRAFANA_ADMIN_PASSWORD`**: Refuses `change-me`. Set a strong password.
- **`GRAFANA_ROOT_URL`**: Must match the `https://` tunnel hostname for Grafana.
- **`GRAFANA_COOKIE_SECURE=true`**: Required when Grafana runs behind an HTTPS tunnel.
-- **`ALERT_WEBHOOK_URL`**: Cannot be empty; empty URLs drop all fired alerts.
+- **`ALERT_WEBHOOK_URL`**: Cannot be empty; Grafana requires an alert destination to provision its contact points.
- **`NTFY_BASE_URL`**: Must be an https:// URL when exposing ntfy via tunnel.
- **`GRAFANA_JWT_AUTH=true`**: Requires both `CF_ACCESS_TEAM_DOMAIN` and `CF_ACCESS_AUD` set.
- **File permissions**: Refuses to start if `.env`, `infra/terraform.tfvars`, or Terraform state files are readable by other users (mode must be `0600`).
-- **`HEARTBEAT_URL`**: Emits a warning if unset (the stack runs without a dead-man's switch).
+- **`HEARTBEAT_URL`**: Cannot be empty; Grafana also requires the heartbeat destination to provision its contact points.
### Verification commands
| Command | Action |
| --- | --- |
-| `just lint` | Runs static checks (compose, alert templates, YAML, workflows, shell, Python, JSON). |
+| `just lint` | Runs static checks (Compose, alert templates, YAML, workflows, shell, Python, JSON), plus secret-scan and browser RUM regression tests. |
| `just validate` | Validates each service config with its respective tool container. |
| `just check` | Runs `just lint` and `just validate`. |
| `just smoke` | Boots an isolated copy of the stack and asserts ingestion, storage, and query APIs end-to-end. |
@@ -391,8 +390,10 @@ one manual step: it bumps the constraint in `main.tf` but not the hashes in
account. Only a plan proves the provider still maps the config to the same
resources.
-Nothing watches the tool images pinned in the `justfile` (yamllint,
-actionlint, shellcheck, ruff, gitleaks, OpenTofu, jq). Bump those by hand.
+Nothing watches the tool images pinned in the `justfile` (yamllint, yamlfmt,
+actionlint, shellcheck, ruff, gitleaks, OpenTofu, jq, Node). Bump those by hand.
+Node runs the browser RUM regression tests inside a container; contributors
+do not need a local Node installation.
The backup recipes reuse the alpine image of `otel-queue-init` in
`compose.yml`.
diff --git a/justfile b/justfile
index cb7a16a..d7c3a9a 100644
--- a/justfile
+++ b/justfile
@@ -65,9 +65,9 @@ _expose-guards:
@case "${GRAFANA_ROOT_URL:-}" in https://*) ;; *) echo "error: GRAFANA_ROOT_URL must be the https:// tunnel hostname (got '${GRAFANA_ROOT_URL:-}'); every absolute URL Grafana generates comes from it" >&2; exit 1;; esac
@[ "${GRAFANA_COOKIE_SECURE:-false}" = "true" ] || { echo "error: GRAFANA_COOKIE_SECURE must be true when Grafana is served over HTTPS; set it in .env" >&2; exit 1; }
@[ "${GRAFANA_JWT_AUTH:-false}" != "true" ] || { [ -n "${CF_ACCESS_TEAM_DOMAIN:-}" ] && [ -n "${CF_ACCESS_AUD:-}" ]; } || { echo "error: GRAFANA_JWT_AUTH=true needs CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD in .env (cd infra && tofu output -raw grafana_access_team_domain grafana_access_aud)" >&2; exit 1; }
- @[ -n "${HEARTBEAT_URL:-}" ] || echo "WARNING: HEARTBEAT_URL is empty; the stack goes live without a dead-man's switch" >&2
+ @[ -n "${HEARTBEAT_URL:-}" ] || { echo "error: HEARTBEAT_URL is empty; Grafana rejects contact point provisioning. Set the dead man's switch URL in .env" >&2; exit 1; }
@for f in .env infra/terraform.tfvars infra/terraform.tfstate infra/terraform.tfstate.backup; do [ ! -e "$f" ] || case "$(stat -c %a "$f")" in *00) ;; *) echo "error: $f is readable by other users (mode $(stat -c %a "$f")); it holds live secrets, run: chmod 600 $f" >&2; exit 1;; esac; done
- @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; every alert would fire into an empty webhook URL and be dropped. The heartbeat keeps pinging either way, so this failure looks healthy from the outside. Set it, or comment out this guard" >&2; exit 1; }
+ @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; Grafana rejects contact point provisioning. Set the alert destination in .env" >&2; exit 1; }
@case "${NTFY_BASE_URL:-}" in ""|https://*) ;; *) echo "error: NTFY_BASE_URL must be an https:// URL when exposing via tunnel (got '${NTFY_BASE_URL:-}')" >&2; exit 1;; esac
# Core stack plus a demo telemetry source, isolated from any running stack (:3002).
@@ -90,7 +90,7 @@ restart service: _guard-if-exposed
# Validate every config in the repo.
check: lint validate
-# Static checks in tool images (~280 MB cold, seconds warm).
+# Static checks and regression tests in tool images.
lint:
# Digest-pinned, so anything already local is current; only fetch what is missing.
@printf '%s\n' {{lint_images}} | xargs -P 8 -I{} sh -c 'docker image inspect {} >/dev/null 2>&1 || docker pull -q {} >/dev/null'
@@ -109,7 +109,7 @@ lint:
# documented default is refused on its own.
@good="OTLP_AUTH_TOKEN=t GRAFANA_ADMIN_PASSWORD=p GRAFANA_ROOT_URL=https://g.example GRAFANA_COOKIE_SECURE=true GRAFANA_JWT_AUTH=true CF_ACCESS_TEAM_DOMAIN=d CF_ACCESS_AUD=a HEARTBEAT_URL=https://h ALERT_WEBHOOK_URL=https://w"; \
env $good just _expose-guards || { echo "error: exposure guards rejected a fully set environment" >&2; exit 1; }; \
- for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL= NTFY_BASE_URL=http://insecure; do \
+ for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= HEARTBEAT_URL= ALERT_WEBHOOK_URL= NTFY_BASE_URL=http://insecure; do \
! env $good $bad just _expose-guards 2>/dev/null || { echo "error: exposure guards accepted $bad" >&2; exit 1; }; \
done
# run_scheduled.sh, interrupted mid-job, must take the job down with it,
From d08ead41a7ffc2ee57376c5739498cde80f48738 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:09:56 +0100
Subject: [PATCH 09/22] fix(infra): reject failed optional DNS lookups
---
infra/generate-imports.sh | 5 ++++-
justfile | 3 ++-
scripts/test_imports.sh | 41 +++++++++++++++++++++++++++++++++++++++
3 files changed, 47 insertions(+), 2 deletions(-)
create mode 100644 scripts/test_imports.sh
diff --git a/infra/generate-imports.sh b/infra/generate-imports.sh
index f375aaa..c35582f 100755
--- a/infra/generate-imports.sh
+++ b/infra/generate-imports.sh
@@ -79,7 +79,10 @@ otel_record="$(lookup_dns_record "otel.$domain")" || {
die "no CNAME found for otel.$domain"
}
-ntfy_record="$(lookup_dns_record "ntfy.$domain" || true)"
+ntfy_record="$(lookup_dns_record "ntfy.$domain")" || {
+ [[ $? -ne 2 ]] || die "DNS lookup for ntfy.$domain failed (token lacks DNS:Read?)"
+ ntfy_record=""
+}
# The Access app may legitimately not exist; then the apply should create it.
access_apps="$(api "accounts/$account/access/apps?per_page=100")"
diff --git a/justfile b/justfile
index d7c3a9a..2515b04 100644
--- a/justfile
+++ b/justfile
@@ -134,8 +134,9 @@ lint:
docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/projects.yaml]}' .
@d=$(mktemp -d) && BOOTSTRAP_OUT_DIR="$d" ./bootstrap.sh dummy dummy >/dev/null && docker run --rm --network none -v "$d":/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: disable}}' .; rc=$?; rm -rf "$d"; exit $rc
docker run --rm --network none -v .:/repo:ro -w /repo {{actionlint}} -color
- docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh scripts/test_gitleaks.sh templates/run_scheduled.sh infra/generate-imports.sh
+ docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh scripts/test_gitleaks.sh scripts/test_imports.sh templates/run_scheduled.sh infra/generate-imports.sh
sh scripts/test_gitleaks.sh
+ bash scripts/test_imports.sh
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo
docker run --rm --network none -v ./templates/web:/rum:ro {{node}} node --test --test-isolation=none /rum/rum.test.mjs
diff --git a/scripts/test_imports.sh b/scripts/test_imports.sh
new file mode 100644
index 0000000..99404fb
--- /dev/null
+++ b/scripts/test_imports.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+scratch="$(mktemp -d "${TMPDIR:-/tmp}/monitoring-imports.XXXXXX")"
+trap 'rm -rf "$scratch"' EXIT
+mkdir "$scratch/bin" "$scratch/infra"
+cp "$root/infra/generate-imports.sh" "$scratch/infra/"
+cat >"$scratch/infra/terraform.tfvars" <<'EOF'
+account_id = "fixture-account"
+zone_id = "fixture-zone"
+domain = "fixture.example"
+EOF
+cat >"$scratch/bin/curl" <<'EOF'
+#!/usr/bin/env bash
+case "${*: -1}" in
+ *cfd_tunnel*) echo '{"result":[{"name":"cml-monitoring","id":"tunnel"}]}' ;;
+ *name=ntfy.*)
+ [[ "$NTFY_FIXTURE" != failure ]] || exit 22
+ if [[ "$NTFY_FIXTURE" == absent ]]; then echo '{"result":[]}'
+ else echo '{"result":[{"id":"ntfy-record"}]}'
+ fi ;;
+ *dns_records*) echo '{"result":[{"id":"dns-record"}]}' ;;
+ *) echo '{"result":[]}' ;;
+esac
+EOF
+chmod +x "$scratch/bin/curl"
+for fixture in present absent failure; do
+ rc=0
+ env -i PATH="$scratch/bin:$PATH" CLOUDFLARE_API_TOKEN=fixture NTFY_FIXTURE="$fixture" \
+ bash "$scratch/infra/generate-imports.sh" >"$scratch/out" 2>"$scratch/err" || rc=$?
+ case "$fixture" in
+ present) [[ "$rc" == 0 ]] && grep 'cloudflare_dns_record.ntfy' "$scratch/out" >/dev/null ;;
+ absent) [[ "$rc" == 0 ]] && ! grep 'cloudflare_dns_record.ntfy' "$scratch/out" >/dev/null ;;
+ failure) [[ "$rc" != 0 ]] && grep 'DNS lookup for ntfy.fixture.example failed' "$scratch/err" >/dev/null ;;
+ esac || {
+ echo "error: ntfy DNS $fixture check failed" >&2
+ exit 1
+ }
+done
+echo 'Import lookup checks passed'
From 8bcafa5bfb4dc6feb62dcf97dcb6bbf73012d4cf Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:10:32 +0100
Subject: [PATCH 10/22] docs: correct credential rotation and onboarding
procedures
---
docs/ONBOARDING.md | 13 +++++++++----
docs/RUNBOOK.md | 30 +++++++++++++++++++++++-------
templates/web/README.md | 10 +++++++---
3 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/docs/ONBOARDING.md b/docs/ONBOARDING.md
index 024632c..ab240b9 100644
--- a/docs/ONBOARDING.md
+++ b/docs/ONBOARDING.md
@@ -106,15 +106,19 @@ Prometheus reloads `config/targets/probes.yaml` automatically without a restart.
## Template 4: Browser Real User Monitoring (RUM)
-For client-side performance and error tracking on web applications or static portals, include the lightweight snippet from [`templates/web/rum.html`](../templates/web/rum.html):
+For browser performance and error tracking, first ask the hub administrator for
+a restricted public ingestion proxy. It must keep the collector bearer private,
+fix the project's identity, and limit payload size and request rate. Leave RUM
+disabled until that ingress and the collection agreement are ready.
+
+Then configure the snippet from [`templates/web/rum.html`](../templates/web/rum.html):
```html
```
@@ -125,7 +129,8 @@ What it captures:
- **Client Exceptions:** Unhandled `window.onerror` exceptions and unhandled promise rejections.
- **Log Streams:** Emits structured OTLP/HTTP JSON (`/v1/logs`) indexed into Loki.
-The collector permits cross-origin requests from `https://cmlplatform.github.io` and Leiden University domains (`*.leidenuniv.nl`). See [`templates/web/README.md`](../templates/web/README.md) for details.
+Never put the shared collector token in public HTML. CORS does not protect that
+credential. See [`templates/web/README.md`](../templates/web/README.md) for the ingress requirements.
---
diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md
index 9ef24f8..ee1d3d9 100644
--- a/docs/RUNBOOK.md
+++ b/docs/RUNBOOK.md
@@ -45,10 +45,12 @@ count, not time, is what grows the TSDB.
When one fires:
1. Check the Filesystem panel on Stack Health.
-2. Free space, or shorten a retention window and restart the service:
+2. Free space, or shorten a retention window:
`retention_period` in `config/loki.yaml`, `block_retention` in
`config/tempo.yaml`, or the `--storage.tsdb.retention.*` flags in
- `compose.yml`.
+ `compose.yml`. Restart Loki or Tempo after a config change. After changing
+ Prometheus command arguments, run `docker compose up -d prometheus` to
+ recreate its container with the new retention settings.
3. If disk pressure keeps returning, move Loki and Tempo to object storage.
The appendix of ADR 0001 describes the change.
@@ -106,8 +108,21 @@ on the host.
That builds a new tunnel and repoints all three CNAMEs (grafana, otel, ntfy) at it, and ingestion and
Grafana are unreachable for about a minute. Either way, put the new token
in `CLOUDFLARE_TUNNEL_TOKEN` in `.env`, then `just up`.
-- **Grafana admin password:** change `GRAFANA_ADMIN_PASSWORD` in `.env`,
- then `docker compose up -d grafana`.
+- **Grafana admin password:** changing `GRAFANA_ADMIN_PASSWORD` only initializes
+ a new database. To rotate an existing password, update its value in `.env`.
+ Enter the same value at the prompt below; it stays out of shell history and
+ process arguments:
+
+ ```bash
+ read -r -s -p 'New Grafana admin password: ' grafana_password
+ printf '\n'
+ printf '%s\n' "$grafana_password" | docker compose exec -T grafana grafana cli \
+ --homepath /usr/share/grafana admin reset-admin-password --password-from-stdin
+ unset grafana_password
+ docker compose up -d grafana
+ ```
+
+ Confirm that a fresh login accepts the new password and rejects the old one.
- **Alert webhook and heartbeat URLs:** the URL is the credential. Mint a new
topic or check at the provider, put it in `ALERT_WEBHOOK_URL` or
`HEARTBEAT_URL`, then `docker compose up -d grafana`. Grafana reads the
@@ -168,7 +183,7 @@ comparison stays true for the duration of `for`:
| `ContainerRestarting` | container-lifecycle | warning | 10m | Container restarts >3 times within 1 hour. |
| `ContainerOOMKilled` | container-lifecycle | warning | 0m | Container terminated by host kernel OOM killer. |
| `ProjectTelemetrySilent` | spoke | critical | 15m | Spoke service stopped shipping OTel telemetry. |
-| `ProjectsUncovered` | _coverage | warning | 5m | Telemetry received from unlisted project; register in projects.yaml. |
+| `ProjectsUncovered` | _coverage | warning | 15m | Telemetry received from unlisted project; register in projects.yaml. |
## Alert delivery
@@ -263,8 +278,9 @@ Zero Trust dashboard. The next apply reverts anything clicked in by hand.
```sh
just infra-validate # tofu init + validate, in a container
-cd infra && tofu plan # needs CLOUDFLARE_API_TOKEN exported
-cd infra && tofu apply
+cd infra # needs CLOUDFLARE_API_TOKEN exported
+tofu plan
+tofu apply
```
- **Granting or revoking Grafana access:** edit `grafana_allowed_emails` in
diff --git a/templates/web/README.md b/templates/web/README.md
index 86454ac..deed2e2 100644
--- a/templates/web/README.md
+++ b/templates/web/README.md
@@ -25,11 +25,10 @@ Add your endpoint and project details before loading the snippet:
```html
```
@@ -39,6 +38,11 @@ Paste the contents of [`rum.html`](rum.html) into your layout `` or before
## Ingestion and Security
+Error records contain only the event type and, when available, script origin/path,
+line and column. Messages, stacks and rejection values are excluded. Script URLs
+exclude credentials, queries and fragments; review paths for private values before
+enabling collection.
+
- **CORS origins:** The collector HTTP receiver permits cross-origin requests from `https://cmlplatform.github.io` and Leiden University domains (`https://*.leidenuniv.nl`).
-- **Credential safety:** Do not embed collector bearer secrets in public client HTML. For public apps, either omit `window.__OTEL_TOKEN__` and let an edge proxy (such as Cloudflare Workers or Nginx) inject `Authorization: Bearer ...`, or provision a scoped low-privilege token.
+- **Credential safety:** Never embed the collector bearer in public HTML. The collector supports one shared bearer, without per-project scopes. An administrator must provide a restricted proxy that injects that bearer, fixes the project identity, and limits payload size and request rate. Leave RUM disabled until the proxy and collection agreement are ready. CORS alone does not provide this boundary.
- **Log queries:** Logs are indexed in Loki by `service_name`, `project`, and `env`, viewable on the Grafana **Logs** dashboard.
From 523775664bc97f7cf922b561045ffddd50e86fa6 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:11:09 +0100
Subject: [PATCH 11/22] fix(rum): collect only technical error metadata
---
templates/web/rum.html | 24 ++++++++++++--------
templates/web/rum.test.mjs | 45 +++++++++++++++++++++++++++++++++-----
2 files changed, 54 insertions(+), 15 deletions(-)
diff --git a/templates/web/rum.html b/templates/web/rum.html
index 338aabf..1762a24 100644
--- a/templates/web/rum.html
+++ b/templates/web/rum.html
@@ -162,21 +162,27 @@
}
// --- Uncaught Errors ---
+ function scriptLocation(filename) {
+ try {
+ const url = new URL(filename, window.location.origin);
+ return url.protocol === "https:" || url.protocol === "http:" ? url.origin + url.pathname : "unknown";
+ } catch (_) {
+ return "unknown";
+ }
+ }
+
window.addEventListener("error", function(evt) {
- sendLog(evt.message || "Unhandled JavaScript Error", "ERROR", {
+ sendLog("Unhandled JavaScript error", "ERROR", {
error_type: "uncaught_exception",
- filename: evt.filename ? evt.filename.split("?")[0] : "unknown",
+ filename: evt.filename ? scriptLocation(evt.filename) : "unknown",
lineno: evt.lineno || 0,
- colno: evt.colno || 0,
- stack: evt.error ? evt.error.stack || "" : ""
+ colno: evt.colno || 0
});
});
- window.addEventListener("unhandledrejection", function(evt) {
- const reason = evt.reason;
- sendLog(reason && reason.message ? reason.message : String(reason), "ERROR", {
- error_type: "unhandled_promise_rejection",
- stack: reason && reason.stack ? reason.stack : ""
+ window.addEventListener("unhandledrejection", function() {
+ sendLog("Unhandled promise rejection", "ERROR", {
+ error_type: "unhandled_promise_rejection"
});
});
})();
diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs
index 82552ed..3b1737b 100644
--- a/templates/web/rum.test.mjs
+++ b/templates/web/rum.test.mjs
@@ -5,14 +5,15 @@ import { runInNewContext } from 'node:vm';
const source = readFileSync(new URL('./rum.html', import.meta.url), 'utf8').split('')[0];
-function browser(initialVisibility = 'visible', unsupportedCls = false) {
+function browser(initialVisibility = 'visible', unsupportedCls = false, endpoint = 'https://rum.example.org/') {
const logs = [];
+ const requests = [];
const listeners = {};
const observers = {};
const on = (name, handler) => (listeners[name] ??= []).push(handler);
const document = { visibilityState: initialVisibility, addEventListener: on };
const window = {
- __OTEL_ENDPOINT__: 'https://rum.example.org',
+ __OTEL_ENDPOINT__: endpoint,
location: { origin: 'https://example.org', pathname: '/' },
addEventListener: on,
};
@@ -31,20 +32,52 @@ function browser(initialVisibility = 'visible', unsupportedCls = false) {
}
window.PerformanceObserver = PerformanceObserver;
runInNewContext(source, {
- window, document, PerformanceObserver, navigator: { userAgent: 'test-browser' },
- fetch: (_url, options) => {
+ window, document, PerformanceObserver, URL, navigator: { userAgent: 'test-browser' },
+ fetch: (url, options) => {
+ requests.push({ url, options });
logs.push(JSON.parse(options.body).resourceLogs[0].scopeLogs[0].logRecords[0]);
return Promise.resolve();
},
});
- const emit = (name, timeStamp = 10000) => listeners[name]?.forEach((handler) => handler({ timeStamp }));
+ const emit = (name, timeStamp = 10000) => listeners[name]?.forEach((handler) => handler(typeof timeStamp === 'object' ? timeStamp : { timeStamp }));
const deliver = (type, entries) => observers[type].callback({ getEntries: () => entries });
const hide = (time = 10000) => { document.visibilityState = 'hidden'; emit('visibilitychange', time); };
const values = (vital, key) => logs.filter((log) => log.body.stringValue === `WebVital: ${vital}`).map((log) => log.attributes.find((attr) => attr.key === key).value.doubleValue);
- return { logs, document, observers, emit, deliver, hide, values };
+ return { logs, requests, document, observers, emit, deliver, hide, values };
}
const shift = (value, startTime, hadRecentInput = false) => ({ value, startTime, hadRecentInput });
+test('transport normalizes the endpoint and posts JSON with keepalive; no endpoint disables sending', () => {
+ const page = browser();
+ page.emit('error', {});
+ assert.equal(page.requests.length, 1);
+ const { url, options } = page.requests[0];
+ assert.equal(url, 'https://rum.example.org/v1/logs');
+ assert.equal(options.method, 'POST');
+ assert.equal(options.keepalive, true);
+ assert.deepEqual({ ...options.headers }, { 'Content-Type': 'application/json' });
+ assert.equal(JSON.parse(options.body).resourceLogs.length, 1);
+ const disabled = browser('visible', false, '');
+ disabled.emit('error', {});
+ assert.equal(disabled.requests.length, 0);
+});
+
+test('errors collect technical metadata without messages, stacks or URL credentials', () => {
+ const page = browser();
+ const privateValue = 'fixture-private-token';
+ page.emit('error', {
+ message: privateValue, error: { stack: privateValue },
+ filename: `https://user:${privateValue}@example.org/app.js?token=${privateValue}#${privateValue}`,
+ lineno: 4, colno: 2,
+ });
+ page.emit('unhandledrejection', { reason: { message: privateValue, stack: privateValue } });
+ page.emit('error', { filename: `data:text/javascript,${privateValue}` });
+ assert.equal(page.logs.length, 3);
+ assert.equal(JSON.stringify(page.requests).includes(privateValue), false);
+ assert.equal(page.logs[0].attributes.find(({ key }) => key === 'filename').value.stringValue, 'https://example.org/app.js');
+ assert.equal(page.logs[0].attributes.find(({ key }) => key === 'lineno').value.doubleValue, 4);
+});
+
test('CLS reports the largest session instead of summing separate sessions', () => {
const page = browser();
page.deliver('layout-shift', [shift(0.12, 100), shift(0.13, 2100)]);
From 0baa48bd983b1fb15927aa351e8493179b726274 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:11:43 +0100
Subject: [PATCH 12/22] fix(dashboard): show export failures for each signal
---
dashboards/stack-health.json | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json
index a195a18..50a6382 100644
--- a/dashboards/stack-health.json
+++ b/dashboards/stack-health.json
@@ -206,9 +206,19 @@
},
"targets": [
{
- "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))",
- "legendFormat": "{{exporter}}",
+ "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))",
+ "legendFormat": "{{exporter}} spans/s",
"refId": "A"
+ },
+ {
+ "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))",
+ "legendFormat": "{{exporter}} metric points/s",
+ "refId": "B"
+ },
+ {
+ "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))",
+ "legendFormat": "{{exporter}} log records/s",
+ "refId": "C"
}
],
"fieldConfig": {
From c4c8f99d9ba4dc0e5e8972fc2bee601226f0c7bc Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:11:10 +0100
Subject: [PATCH 13/22] fix: unpause interrupted backups and verify restored
volumes
---
justfile | 6 +++---
scripts/restore-check.sh | 37 ++++++++++++++++++++++++++++++++++
scripts/test_backup.sh | 43 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 83 insertions(+), 3 deletions(-)
create mode 100755 scripts/restore-check.sh
create mode 100644 scripts/test_backup.sh
diff --git a/justfile b/justfile
index 2515b04..70b4e75 100644
--- a/justfile
+++ b/justfile
@@ -214,7 +214,7 @@ _mounts project:
_backup project dir:
mkdir -p {{dir}}
@-docker compose -p {{project}} unpause {{stateful}} >/dev/null 2>&1
- @rc=0; img=$(just _image compose.yml otel-queue-init); m=$(just _mounts {{project}}); docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups "$img" sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz' || rc=$?; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack is still paused" >&2; rc=1; }; exit $rc
+ @trap 'rc=$?; trap - EXIT INT TERM; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack may still be paused" >&2; [ "$rc" -ne 0 ] || rc=1; }; exit "$rc"' EXIT; trap 'exit 130' INT; trap 'exit 143' TERM; img=$(just _image compose.yml otel-queue-init) && m=$(just _mounts {{project}}) && docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups "$img" sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz'
@just _newest-backup {{dir}}
# pre-restore-* snapshots share the dir and sort after monitoring-* by name.
@@ -239,8 +239,8 @@ _restore project file dir:
# Needs a booted smoke stack (`just smoke`). Run it after touching _backup/_restore.
# COMPOSE_FILE is pinned so the host's overlay list stays out, as for `smoke`.
# Round-trip backup and restore on the smoke stack.
-restore-check:
- @export COMPOSE_FILE=compose.yml:compose.sandbox.yml; d=$(mktemp -d) && just _backup {{smoke_project}} "$d" && f=$(ls "$d"/monitoring-*.tar.gz) && just _restore {{smoke_project}} "$f" "$d" && docker run --rm --network none -v {{smoke_project}}_grafana_data:/g:ro $(just _image compose.yml otel-queue-init) test -s /g/grafana.db && echo "Backup round-trip ok"; rc=$?; rm -rf "$d"; exit $rc
+restore-check project=smoke_project port=smoke_port:
+ @{{smoke_env}} SANDBOX_PORT={{port}} scripts/restore-check.sh {{project}} {{stateful}}
# `--wait` blocks on the healthchecks and fails if any container exits, so a
# crash-looping service is caught (after the full timeout). scripts/smoke.sh
diff --git a/scripts/restore-check.sh b/scripts/restore-check.sh
new file mode 100755
index 0000000..37f18d4
--- /dev/null
+++ b/scripts/restore-check.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# The caller supplies a booted disposable project; never use production volumes.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+export COMPOSE_FILE=compose.yml:compose.sandbox.yml
+project=$1
+shift
+img=$(just _image compose.yml otel-queue-init)
+d=$(mktemp -d "${TMPDIR:-/tmp}/monitoring-restore-check.XXXXXX")
+marker=${d##*/}
+mounts=()
+for service in "$@"; do
+ mounts+=(-v "${project}_${service}_data:/data/$service")
+done
+cleanup() {
+ rc=$?
+ trap - EXIT
+ docker run --rm --network none "${mounts[@]}" "$img" sh -c 'for volume in /data/*; do rm -f "$volume/$1" "$volume/$1-later"; done' _ "$marker" || { [ "$rc" -ne 0 ] || rc=1; }
+ rm -rf "$d"
+ exit "$rc"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+# Different contents in each volume catch missing or crossed mounts.
+docker run --rm --network none "${mounts[@]}" "$img" sh -ec 'for volume in /data/*; do printf "%s-before\n" "${volume##*/}" >"$volume/$1"; done' _ "$marker"
+just _backup "$project" "$d"
+f=$(find "$d" -name 'monitoring-*.tar.gz')
+docker run --rm --network none "${mounts[@]}" "$img" sh -ec 'for volume in /data/*; do printf "%s-after\n" "${volume##*/}" >"$volume/$1"; touch "$volume/$1-later"; done' _ "$marker"
+just _restore "$project" "$f" "$d"
+docker run --rm --network none "${mounts[@]}" "$img" sh -ec '
+for volume in /data/*; do
+ [ "$(cat "$volume/$1")" = "${volume##*/}-before" ] && [ ! -e "$volume/$1-later" ] || { echo "error: ${volume##*/} was not restored" >&2; exit 1; }
+done' _ "$marker"
+docker compose -p "$project" up -d --wait --wait-timeout 120
+docker compose -p "$project" exec -T grafana wget -qO- http://localhost:3000/api/health | grep -Eq '"database":[[:space:]]*"ok"'
+echo 'Backup round-trip ok: all five volumes restored; Grafana database healthy'
diff --git a/scripts/test_backup.sh b/scripts/test_backup.sh
new file mode 100644
index 0000000..5ddeedd
--- /dev/null
+++ b/scripts/test_backup.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+# Exercise the real recipe without pausing a Docker stack.
+set -eu
+cd "$(dirname "$0")/.."
+d=$(mktemp -d "${TMPDIR:-/tmp}/monitoring-backup-test.XXXXXX")
+trap 'rm -rf "$d"' EXIT
+cat >"$d/docker" <<'DOCKER'
+#!/bin/sh
+case "$*" in
+ *'config --format json'*) echo '{}' ;;
+ *' -er '*) echo archive-image ;;
+ *' pause '*) echo pause >>"$TEST_EVENTS"; [ "$TEST_CASE" != pause-failure ] || exit 6 ;;
+ *' unpause '*)
+ echo unpause >>"$TEST_EVENTS"
+ if { [ "$TEST_CASE" = unpause-failure ] || [ "$TEST_CASE" = both-fail ]; } && grep -qx pause "$TEST_EVENTS"; then exit 9; fi ;;
+ *'run --rm --network none'*)
+ touch "$(dirname "$TEST_EVENTS")/monitoring-test.tar.gz"
+ case "$TEST_CASE" in
+ TERM) kill -TERM "$PPID" ;;
+ INT) kill -INT "$PPID" ;;
+ archive-failure|both-fail) exit 7 ;;
+ esac ;;
+ *) echo "unexpected docker call" >&2; exit 1 ;;
+esac
+DOCKER
+chmod +x "$d/docker"
+for scenario in success TERM INT pause-failure archive-failure unpause-failure both-fail; do
+ : >"$d/events"
+ case "$scenario" in success) expected=0 ;; TERM) expected=143 ;; INT) expected=130 ;; pause-failure) expected=6 ;; archive-failure | both-fail) expected=7 ;; *) expected=1 ;; esac
+ rc=0
+ env -i PATH="$d:$PATH" HOME="$HOME" TEST_EVENTS="$d/events" TEST_CASE="$scenario" just _backup test-backup "$d" >"$d/output" 2>&1 || rc=$?
+ # just normalizes signal exits to 128+signal, and propagates recipe failures.
+ [ "$rc" -eq "$expected" ] || {
+ cat "$d/output" >&2
+ echo "wrong exit for $scenario: $rc" >&2
+ exit 1
+ }
+ [ "$(tail -1 "$d/events")" = unpause ] || {
+ echo "backup left services paused after $scenario" >&2
+ exit 1
+ }
+ echo "backup $scenario: cleanup attempted, exit $rc"
+done
From 6b9df4337274e535fa4b53e3d26479bdfa630349 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:14:50 +0100
Subject: [PATCH 14/22] test: run backup recovery regressions during lint
---
justfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/justfile b/justfile
index 70b4e75..54007b9 100644
--- a/justfile
+++ b/justfile
@@ -134,9 +134,10 @@ lint:
docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/projects.yaml]}' .
@d=$(mktemp -d) && BOOTSTRAP_OUT_DIR="$d" ./bootstrap.sh dummy dummy >/dev/null && docker run --rm --network none -v "$d":/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: disable}}' .; rc=$?; rm -rf "$d"; exit $rc
docker run --rm --network none -v .:/repo:ro -w /repo {{actionlint}} -color
- docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh scripts/test_gitleaks.sh scripts/test_imports.sh templates/run_scheduled.sh infra/generate-imports.sh
+ docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh scripts/restore-check.sh scripts/test_backup.sh scripts/test_gitleaks.sh scripts/test_imports.sh templates/run_scheduled.sh infra/generate-imports.sh
sh scripts/test_gitleaks.sh
bash scripts/test_imports.sh
+ sh scripts/test_backup.sh
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo
docker run --rm --network none -v ./templates/web:/rum:ro {{node}} node --test --test-isolation=none /rum/rum.test.mjs
From 45bc7a7c1ca8a98e6760baa96313c94971463d69 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:14:30 +0100
Subject: [PATCH 15/22] fix(alerts): keep notification credentials outside
grafana
---
.env.example | 7 +-
compose.demo.yml | 2 +-
compose.yml | 35 ++++-
config/grafana/alerting/contact-points.yaml | 10 +-
docs/RUNBOOK.md | 41 ++++--
notification-relay/relay.py | 141 ++++++++++++++++++
notification-relay/test_relay.py | 154 ++++++++++++++++++++
7 files changed, 361 insertions(+), 29 deletions(-)
create mode 100644 notification-relay/relay.py
create mode 100644 notification-relay/test_relay.py
diff --git a/.env.example b/.env.example
index f1b207f..b02463b 100644
--- a/.env.example
+++ b/.env.example
@@ -22,10 +22,11 @@ GRAFANA_COOKIE_SECURE=false
# openssl rand -hex 32
OTLP_AUTH_TOKEN=local-dev-token
-# Where Grafana delivers alert notifications (self-hosted ntfy, hosted ntfy, Slack, …).
+# Where the internal relay delivers Grafana alert notifications (self-hosted ntfy, hosted ntfy, Slack, …).
# For self-hosted ntfy with basic auth publisher:
# ALERT_WEBHOOK_URL=http://grafana:password@ntfy:80/alerts?tpl=1&title={{or .commonAnnotations.summary .groupLabels.alertname}}&message={{or .commonAnnotations.description .commonAnnotations.summary "Alert firing"}}
-# Required: Grafana rejects contact point provisioning if this URL is empty.
+# Only the relay receives this secret; Grafana stores an internal URL.
+# Empty disables delivery (the relay returns 503).
# With the tunnel overlay active, `just up` refuses to start without it.
ALERT_WEBHOOK_URL=
@@ -35,7 +36,7 @@ NTFY_BASE_URL=
# Dead man's switch ping target (e.g. https://hc-ping.com/). The Watchdog
# alert posts here every 5m; alert externally when pings stop.
-# Required: Grafana also rejects an empty heartbeat URL.
+# Only the relay receives this secret. Empty disables heartbeat delivery.
HEARTBEAT_URL=
# Optional. Lets `./bootstrap.sh` create a project's healthchecks.io checks.
diff --git a/compose.demo.yml b/compose.demo.yml
index 2b81bf5..6a03c70 100644
--- a/compose.demo.yml
+++ b/compose.demo.yml
@@ -12,7 +12,7 @@ x-demo-logging: &demo-logging
max-file: "3"
services:
- grafana:
+ notification-relay:
# Demo notifications must not reach a production webhook or dead man's switch.
environment:
ALERT_WEBHOOK_URL: https://demo.invalid/alerts
diff --git a/compose.yml b/compose.yml
index ab98a97..57b4512 100644
--- a/compose.yml
+++ b/compose.yml
@@ -163,10 +163,35 @@ services:
- /:/host:ro
networks: [backend]
+ notification-relay:
+ image: python:3.15.0-slim@sha256:f5b92f2a669a4c347044f29367f31983280b648250ca94dea66f98c53bc87e58
+ restart: unless-stopped
+ user: "65534:65534"
+ read_only: true
+ command: ["python", "-B", "/app/relay.py"]
+ volumes:
+ - ./notification-relay/relay.py:/app/relay.py:ro
+ environment:
+ ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-}
+ HEARTBEAT_URL: ${HEARTBEAT_URL:-}
+ healthcheck:
+ !!merge <<: *healthcheck
+ test:
+ - CMD
+ - python
+ - -c
+ - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=2).close()"
+ mem_limit: 64m
+ !!merge <<: *hardened
+ pids_limit: 32
+ networks: [notification-egress, notifications]
+
grafana:
image: grafana/grafana:13.3.0@sha256:488984b716c0862639f91a698adbe2862677a5a07bad42da5f03f3030883b205
restart: unless-stopped
depends_on:
+ notification-relay:
+ condition: service_healthy
loki:
condition: service_started
tempo:
@@ -183,9 +208,6 @@ services:
- grafana_data:/var/lib/grafana
environment:
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env}
- # Expanded by Grafana into the provisioned contact points.
- ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-}
- HEARTBEAT_URL: ${HEARTBEAT_URL:-}
GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000}
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/stack-health.json
# Secure cookies break plain-http localhost logins, so opt-in. With the
@@ -209,7 +231,7 @@ services:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
mem_limit: 1g
!!merge <<: *hardened
- networks: [default, backend]
+ networks: [default, backend, notifications]
blackbox-exporter:
image: prom/blackbox-exporter:v0.29.1@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13
@@ -241,13 +263,16 @@ services:
mem_limit: 128m
pids_limit: 128
!!merge <<: *hardened
- networks: [default, backend]
+ networks: [default, backend, notifications]
# Loki, Prometheus and Tempo have no authentication of their own, so only
# Grafana and the collector can reach them. cloudflared stays on `default`: its
# ingress list comes from Cloudflare at runtime, so an edited ingress cannot
# publish a backend that skips the collector's token gate.
networks:
+ notification-egress:
+ notifications:
+ internal: true
backend:
internal: true
diff --git a/config/grafana/alerting/contact-points.yaml b/config/grafana/alerting/contact-points.yaml
index e084fa4..5a5b14f 100644
--- a/config/grafana/alerting/contact-points.yaml
+++ b/config/grafana/alerting/contact-points.yaml
@@ -1,8 +1,4 @@
-# Notification targets. Grafana expands $VAR from the environment.
-#
-# Each receiver needs a URL or Grafana rejects contact point provisioning.
-# The demo and smoke stacks supply isolated placeholders. The exposure guard
-# in `just up` requires a production alert destination.
+# Destination credentials live only in notification-relay, outside Grafana.
apiVersion: 1
@@ -13,7 +9,7 @@ contactPoints:
- uid: cp-webhook-default
type: webhook
settings:
- url: $ALERT_WEBHOOK_URL
+ url: http://notification-relay:8080/alerts
- orgId: 1
name: heartbeat
@@ -21,6 +17,6 @@ contactPoints:
- uid: cp-heartbeat-dms
type: webhook
settings:
- url: $HEARTBEAT_URL
+ url: http://notification-relay:8080/heartbeat
# A resolved-notification would ping the switch and mask an outage.
disableResolveMessage: true
diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md
index ee1d3d9..e197803 100644
--- a/docs/RUNBOOK.md
+++ b/docs/RUNBOOK.md
@@ -125,8 +125,8 @@ on the host.
Confirm that a fresh login accepts the new password and rejects the old one.
- **Alert webhook and heartbeat URLs:** the URL is the credential. Mint a new
topic or check at the provider, put it in `ALERT_WEBHOOK_URL` or
- `HEARTBEAT_URL`, then `docker compose up -d grafana`. Grafana reads the
- contact points only at startup.
+ `HEARTBEAT_URL`, then `docker compose up -d notification-relay`. Revoke the
+ old capability URL at the provider after confirming delivery.
- **healthchecks.io API key:** regenerate it in the project's settings and
set `HEALTHCHECKS_API_KEY` in `.env`. Only `bootstrap.sh` reads it, so
nothing restarts.
@@ -137,9 +137,11 @@ on the host.
### Where every secret lives
Three gitignored files hold everything. `just backup` archives none of them,
-but the tarball contains Grafana's database, and Grafana stores the expanded
-contact points there. So the webhook and heartbeat URLs are inside every
-backup. Copy `.env` and `infra/terraform.tfstate` off-host together with the
+but the tarball contains Grafana's database. New contact points contain only
+internal relay routes; destination credentials live in the relay environment.
+Backups made before the relay migration contain the old destination URLs.
+Rotate those URLs when migrating and protect old backups until the URLs are
+revoked. Copy `.env` and `infra/terraform.tfstate` off-host together with the
backups and treat all three the same way.
| Secret | Lives in | Comes from |
@@ -195,12 +197,25 @@ them read-only. Edit the YAML.
- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five minutes.
Point it at a dead man's switch that raises the alarm when pings stop.
-Grafana rejects contact point provisioning if either URL is empty.
+The internal `notification-relay` owns both destination URLs. Grafana stores
+only `http://notification-relay:8080/alerts` and `/heartbeat`, so its Viewer
+contact point API cannot disclose destination credentials. The relay has no
+published port and its private network is separate from tunnel ingress. Only
+trusted stack services may call it; this does not isolate a compromised
+Grafana administrator or container. Keep the ntfy publisher account write-only.
+The relay forwards POST JSON with a 1 MiB body limit and a five-second socket
+timeout, refuses redirects, and reports delivery errors to Grafana without
+returning provider bodies or credentials. An empty destination returns 503.
With the tunnel overlay active, `just up` refuses to start without both URLs.
`AlertDeliveryFailing` fires if a configured notifier fails to deliver alerts.
-After changing either variable, run `docker compose up -d grafana`. A plain
-`restart` keeps the old environment. Compose rebuilds a container's
+When migrating from direct delivery, run
+`docker compose up -d notification-relay grafana` to replace Grafana's old
+environment and provision the internal routes. Rotate the old destination URLs
+after the new routes deliver successfully.
+
+After changing either variable, run
+`docker compose up -d notification-relay`. A plain `restart` keeps the old environment. Compose rebuilds a container's
environment only on `up`.
### Self-hosted alert delivery with ntfy
@@ -238,7 +253,7 @@ docker compose exec -e NTFY_PASSWORD="" ntfy ntfy user add grafa
docker compose exec ntfy ntfy access grafana alerts write-only
```
-#### 2. Configure Grafana contact point
+#### 2. Configure the notification relay
Point `ALERT_WEBHOOK_URL` in `.env` to the local ntfy instance using native
message templating (`?tpl=1`):
@@ -247,10 +262,10 @@ message templating (`?tpl=1`):
ALERT_WEBHOOK_URL=http://grafana:@ntfy:80/alerts?tpl=1&title={{or .commonAnnotations.summary .groupLabels.alertname}}&message={{or .commonAnnotations.description .commonAnnotations.summary "Alert firing"}}
```
-Apply the change to Grafana:
+Apply the change to the relay:
```sh
-docker compose up -d grafana
+docker compose up -d notification-relay
```
#### 3. Create subscriber users for operators
@@ -329,11 +344,11 @@ When the tunnel overlay is active, `just up` validates critical security setting
- **`GRAFANA_ADMIN_PASSWORD`**: Refuses `change-me`. Set a strong password.
- **`GRAFANA_ROOT_URL`**: Must match the `https://` tunnel hostname for Grafana.
- **`GRAFANA_COOKIE_SECURE=true`**: Required when Grafana runs behind an HTTPS tunnel.
-- **`ALERT_WEBHOOK_URL`**: Cannot be empty; Grafana requires an alert destination to provision its contact points.
+- **`ALERT_WEBHOOK_URL`**: Cannot be empty when exposing the stack; the relay needs an alert destination.
- **`NTFY_BASE_URL`**: Must be an https:// URL when exposing ntfy via tunnel.
- **`GRAFANA_JWT_AUTH=true`**: Requires both `CF_ACCESS_TEAM_DOMAIN` and `CF_ACCESS_AUD` set.
- **File permissions**: Refuses to start if `.env`, `infra/terraform.tfvars`, or Terraform state files are readable by other users (mode must be `0600`).
-- **`HEARTBEAT_URL`**: Cannot be empty; Grafana also requires the heartbeat destination to provision its contact points.
+- **`HEARTBEAT_URL`**: Cannot be empty when exposing the stack; the relay needs a heartbeat destination.
### Verification commands
diff --git a/notification-relay/relay.py b/notification-relay/relay.py
new file mode 100644
index 0000000..de019ac
--- /dev/null
+++ b/notification-relay/relay.py
@@ -0,0 +1,141 @@
+"""Forward Grafana notifications without exposing destination capabilities."""
+
+import base64
+import http.client
+import os
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib.parse import quote, unquote, urlsplit
+
+MAX_BODY = 1024 * 1024
+TIMEOUT = 5
+
+
+def validate_target(url):
+ """Reject invalid configured destinations without including secrets in errors."""
+ try:
+ parts = urlsplit(url)
+ if (
+ parts.scheme not in ("http", "https")
+ or not parts.hostname
+ or parts.fragment
+ ):
+ raise ValueError
+ _ = parts.port
+ if any(ord(char) < 32 or ord(char) == 127 for char in url):
+ raise ValueError
+ except ValueError:
+ raise ValueError("invalid notification destination") from None
+
+
+class RelayServer(HTTPServer):
+ """Keep destinations outside Grafana's environment and contact point store."""
+
+ def __init__(self, address, targets):
+ self.targets = targets
+ super().__init__(address, Handler)
+
+ def handle_error(self, request, client_address):
+ # Tracebacks from HTTP libraries may contain credential-bearing URLs.
+ print("notification request failed", flush=True)
+
+
+class Handler(BaseHTTPRequestHandler):
+ """Accept only bounded POST bodies on two fixed routes."""
+
+ def setup(self):
+ super().setup()
+ self.connection.settimeout(TIMEOUT)
+
+ def log_message(self, format, *args):
+ # Request paths and headers are untrusted and may contain credentials.
+ pass
+
+ def reply(self, status):
+ self.send_response(status)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+
+ def do_GET(self):
+ self.reply(204 if self.path == "/health" else 404)
+
+ def do_POST(self):
+ assert isinstance(self.server, RelayServer)
+ if self.path not in ("/alerts", "/heartbeat"):
+ self.reply(404)
+ return
+ url = self.server.targets.get(self.path, "")
+ if not url:
+ self.reply(503)
+ return
+ lengths = self.headers.get_all("Content-Length", [])
+ if (
+ self.headers.get("Transfer-Encoding")
+ or len(lengths) != 1
+ or not lengths[0].isdigit()
+ ):
+ self.reply(400)
+ return
+ try:
+ length = int(lengths[0])
+ except ValueError:
+ self.reply(400)
+ return
+ if length > MAX_BODY:
+ self.reply(413)
+ return
+ try:
+ body = self.rfile.read(length)
+ if len(body) != length:
+ self.reply(400)
+ return
+ parts = urlsplit(url)
+ connection_type = (
+ http.client.HTTPSConnection
+ if parts.scheme == "https"
+ else http.client.HTTPConnection
+ )
+ connection = connection_type(parts.hostname, parts.port, timeout=TIMEOUT)
+ try:
+ headers = {"Content-Type": "application/json"}
+ if parts.username is not None:
+ credentials = (
+ f"{unquote(parts.username)}:{unquote(parts.password or '')}"
+ )
+ headers["Authorization"] = (
+ "Basic " + base64.b64encode(credentials.encode()).decode()
+ )
+ path = parts.path or "/"
+ if parts.query:
+ path += "?" + parts.query
+ # Redirects must not move notification data or credentials elsewhere.
+ connection.request(
+ "POST", quote(path, safe="/%?&=:+,@!$'()*;-._~"), body, headers
+ )
+ status = connection.getresponse().status
+ finally:
+ connection.close()
+ self.reply(204 if 200 <= status < 300 else 502)
+ except (OSError, http.client.HTTPException, ValueError):
+ self.reply(502)
+
+
+def make_server(address, targets):
+ """Validate fixed destinations before accepting any requests."""
+ for target in targets.values():
+ if target:
+ validate_target(target)
+ return RelayServer(address, targets)
+
+
+if __name__ == "__main__":
+ try:
+ server = make_server(
+ ("0.0.0.0", 8080),
+ {
+ "/alerts": os.environ.get("ALERT_WEBHOOK_URL", ""),
+ "/heartbeat": os.environ.get("HEARTBEAT_URL", ""),
+ },
+ )
+ except ValueError:
+ raise SystemExit("invalid notification destination") from None
+ server.serve_forever()
diff --git a/notification-relay/test_relay.py b/notification-relay/test_relay.py
new file mode 100644
index 0000000..100b12c
--- /dev/null
+++ b/notification-relay/test_relay.py
@@ -0,0 +1,154 @@
+"""Exercise the fixed-route delivery boundary with local HTTP endpoints."""
+
+import base64
+import http.client
+import threading
+import time
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import ClassVar
+from unittest.mock import patch
+
+import relay
+
+
+class Destination(BaseHTTPRequestHandler):
+ status = 204
+ delay = 0.0
+ received: ClassVar[list] = []
+
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers["Content-Length"]))
+ self.received.append((self.path, body, self.headers.get("Authorization")))
+ time.sleep(self.delay)
+ self.send_response(self.status)
+ self.send_header("Location", "/must-not-follow")
+ self.end_headers()
+
+ def log_message(self, format, *args):
+ pass
+
+
+class RelayTest(unittest.TestCase):
+ def setUp(self):
+ Destination.received = []
+ Destination.status = 204
+ Destination.delay = 0
+ self.destination = HTTPServer(("127.0.0.1", 0), Destination)
+ self.addCleanup(self.destination.server_close)
+ threading.Thread(target=self.destination.serve_forever, daemon=True).start()
+ self.addCleanup(self.destination.shutdown)
+ port = self.destination.server_port
+ self.server = relay.make_server(
+ ("127.0.0.1", 0),
+ {
+ "/alerts": f"http://publisher:secret@127.0.0.1:{port}/topic?tpl=1",
+ "/heartbeat": f"http://127.0.0.1:{port}/ping",
+ },
+ )
+ self.addCleanup(self.server.server_close)
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
+ self.addCleanup(self.server.shutdown)
+
+ def request(self, path, body=b"{}", headers=None):
+ conn = http.client.HTTPConnection(
+ "127.0.0.1", self.server.server_port, timeout=3
+ )
+ self.addCleanup(conn.close)
+ conn.request("POST", path, body, headers or {})
+ response = conn.getresponse()
+ self.assertEqual(response.read(), b"")
+ return response.status
+
+ def test_delivers_only_fixed_routes_and_keeps_auth_out_of_response(self):
+ self.assertEqual(self.request("/alerts"), 204)
+ self.assertEqual(self.request("/heartbeat"), 204)
+ self.assertEqual(
+ Destination.received,
+ [
+ (
+ "/topic?tpl=1",
+ b"{}",
+ "Basic " + base64.b64encode(b"publisher:secret").decode(),
+ ),
+ ("/ping", b"{}", None),
+ ],
+ )
+ self.assertEqual(self.request("/alerts?url=http://elsewhere.invalid"), 404)
+ self.assertEqual(len(Destination.received), 2)
+
+ def test_encodes_native_ntfy_template_query(self):
+ self.server.targets["/alerts"] += (
+ '&message={{or .commonAnnotations.summary "Alert firing"}}'
+ )
+ self.assertEqual(self.request("/alerts"), 204)
+ self.assertIn(
+ "message=%7B%7Bor%20.commonAnnotations.summary%20%22Alert%20firing%22%7D%7D",
+ Destination.received[0][0],
+ )
+
+ def test_upstream_failures_and_redirects_fail_closed(self):
+ for status in (302, 400, 500):
+ Destination.status = status
+ self.assertEqual(self.request("/alerts"), 502)
+ self.assertEqual(len(Destination.received), 3)
+
+ def test_rejects_oversize_chunked_and_ambiguous_body(self):
+ self.assertEqual(
+ self.request(
+ "/alerts", headers={"Content-Length": str(relay.MAX_BODY + 1)}
+ ),
+ 413,
+ )
+ self.assertEqual(
+ self.request("/alerts", headers={"Transfer-Encoding": "chunked"}), 400
+ )
+ self.assertEqual(Destination.received, [])
+
+ def test_rejects_invalid_content_lengths(self):
+ for length in ("-1", "bad", "²", "9" * 5000):
+ self.assertEqual(
+ self.request("/alerts", headers={"Content-Length": length}), 400
+ )
+ self.assertEqual(Destination.received, [])
+
+ def test_upstream_timeout_is_generic_failure(self):
+ Destination.delay = 0.15
+ with patch.object(relay, "TIMEOUT", 0.05):
+ self.assertEqual(self.request("/alerts"), 502)
+
+ def test_rejects_duplicate_content_lengths(self):
+ conn = http.client.HTTPConnection(
+ "127.0.0.1", self.server.server_port, timeout=3
+ )
+ self.addCleanup(conn.close)
+ conn.putrequest("POST", "/alerts")
+ conn.putheader("Content-Length", "0")
+ conn.putheader("Content-Length", "1")
+ conn.endheaders()
+ self.assertEqual(conn.getresponse().status, 400)
+ self.assertEqual(Destination.received, [])
+
+ def test_unconfigured_route_reports_failure(self):
+ self.server.targets["/heartbeat"] = ""
+ self.assertEqual(self.request("/heartbeat"), 503)
+
+ def test_connection_failure_is_generic(self):
+ self.server.targets["/alerts"] = "http://127.0.0.1:1/secret"
+ self.assertEqual(self.request("/alerts"), 502)
+
+ def test_invalid_destinations_rejected_without_echoing_credentials(self):
+ for url in (
+ "ftp://secret@host/path",
+ "http://host:bad/path",
+ "http:///path",
+ "https://host/path#secret",
+ ):
+ with self.assertRaisesRegex(
+ ValueError, "^invalid notification destination$"
+ ):
+ relay.make_server(("127.0.0.1", 0), {"/alerts": url})
+
+
+if __name__ == "__main__":
+ unittest.main()
From f19b181e13d3c2ee7a73490db653afceebf3507a Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:24:35 +0100
Subject: [PATCH 16/22] test(smoke): verify fresh traces and target recovery
---
scripts/smoke.sh | 64 ++++++++++++++++++++++++++++++++++++++----------
1 file changed, 51 insertions(+), 13 deletions(-)
diff --git a/scripts/smoke.sh b/scripts/smoke.sh
index bf0f0f2..924a041 100755
--- a/scripts/smoke.sh
+++ b/scripts/smoke.sh
@@ -41,12 +41,11 @@ jq -e 'all(.isPaused | not)' <<<"$rules" >/dev/null \
|| die "paused alert rules never fire: $(jq -r '.[] | select(.isPaused) | .uid' <<<"$rules" | tr '\n' ' ')"
# --------------------------------------------------------------- contact points
-# The exact URLs the stack was given must come back: an empty value would
-# pass a "not literally $ALERT_WEBHOOK_URL" test and drop every alert.
+# External notification credentials belong to the relay, outside Grafana.
gf "$url/api/v1/provisioning/contact-points" \
- | jq -e --arg a "$ALERT_WEBHOOK_URL" --arg h "$HEARTBEAT_URL" \
+ | jq -e --arg a "http://notification-relay:8080/alerts" --arg h "http://notification-relay:8080/heartbeat" \
'[.[] | select(.uid | startswith("cp-")) | .settings.url] | sort == ([$a, $h] | sort)' >/dev/null \
- || die "contact points do not carry ALERT_WEBHOOK_URL and HEARTBEAT_URL; Grafana did not expand the provisioning file, or a receiver is missing"
+ || die "contact points do not target the internal notification relay"
# ------------------------------------------------------------------- JWT auth
# Grafana ignores an env key it no longer knows, so read the parsed setting
@@ -73,10 +72,23 @@ if [[ "${SMOKE_ALERTS:-}" == 1 ]]; then
fi
# ------------------------------------------------------------- data paths
-# One metric and one log through the collector's bearer auth, read back with
+# One metric, log and trace through the collector's bearer auth, read back with
# the identity labels the alert rules key on. Posted from inside the stack's
# network with Grafana's curl: the sandbox overlay publishes no ingestion ports.
-ts="$(date +%s)000000000"
+run_start="$(date +%s)"
+ts="${run_start}000000000"
+# A fresh value/body distinguishes this run without adding unbounded labels.
+smoke_value="$(od -An -N6 -tu8 /dev/urandom | tr -d ' ')"
+smoke_body="smoke $smoke_value"
+trace_id="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
+span_id="$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')"
+# Tempo returns OTLP JSON byte IDs as base64, unlike the hex lookup URL.
+# shellcheck disable=SC2001 # Convert hex byte pairs, rather than literal text.
+trace_b64="$(printf '%b' "$(sed 's/../\\x&/g' <<<"$trace_id")" | base64 | tr -d '\n')"
+# shellcheck disable=SC2001 # Convert hex byte pairs, rather than literal text.
+span_b64="$(printf '%b' "$(sed 's/../\\x&/g' <<<"$span_id")" | base64 | tr -d '\n')"
+counts_query='sum by (__name__) ({__name__=~"telemetry_(datapoints|logs|spans)_total",project="smoke",env="ci"})'
+counts_before="$(promq "$counts_query" | jq '[.data.result[] | {key: .metric.__name__, value: (.value[1] | tonumber)}] | from_entries')"
res='{"attributes":[{"key":"service.name","value":{"stringValue":"smoke"}},{"key":"project","value":{"stringValue":"smoke"}},{"key":"env","value":{"stringValue":"ci"}}]}'
# Token through curl's stdin config: argv is world-readable in ps, on the host
# and inside the container alike.
@@ -85,23 +97,40 @@ otlp() {
'curl -sf -K - -o /dev/null -H "Content-Type: application/json" --data-binary "$1" "http://otel-collector:4318/v1/$2"' \
_ "$2" "$1" <<<"header = \"Authorization: Bearer $(cfg_quote "$OTLP_AUTH_TOKEN")\""
}
-otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"1","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \
+otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"'"$smoke_value"'","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \
|| die "the collector refused an OTLP metric with the .env token"
-otlp logs '{"resourceLogs":[{"resource":'"$res"',"scopeLogs":[{"logRecords":[{"timeUnixNano":"'"$ts"'","body":{"stringValue":"smoke"}}]}]}]}' \
+otlp logs '{"resourceLogs":[{"resource":'"$res"',"scopeLogs":[{"logRecords":[{"timeUnixNano":"'"$ts"'","body":{"stringValue":"'"$smoke_body"'"}}]}]}]}' \
|| die "the collector refused an OTLP log with the .env token"
-smoke_metric() { promq "smoke_up{job=\"smoke\",project=\"smoke\",env=\"ci\",department=\"cml\"}" | jq -e '.data.result | length > 0' >/dev/null; }
+otlp traces '{"resourceSpans":[{"resource":'"$res"',"scopeSpans":[{"spans":[{"traceId":"'"$trace_id"'","spanId":"'"$span_id"'","name":"smoke","kind":1,"startTimeUnixNano":"'"$ts"'","endTimeUnixNano":"'"$((run_start + 1))000000000"'"}]}]}]}' \
+ || die "the collector refused an OTLP trace with the .env token"
+smoke_metric() { promq "timestamp(smoke_up{job=\"smoke\",project=\"smoke\",env=\"ci\",department=\"cml\"} == $smoke_value) >= $run_start" | jq -e '.data.result | length > 0' >/dev/null; }
smoke_log() {
gf -G --data-urlencode "query={project=\"smoke\",env=\"ci\",department=\"cml\"}" \
+ --data-urlencode "start=$ts" --data-urlencode "end=$(date +%s)000000000" \
"$url/api/datasources/proxy/uid/loki/loki/api/v1/query_range" \
- | jq -e 'any(.data.result[].values[][1]; . == "smoke")' >/dev/null
+ | jq -e --arg body "$smoke_body" 'any(.data.result[].values[][1]; . == $body)' >/dev/null
+}
+smoke_trace() {
+ gf -H 'Accept: application/json' "$url/api/datasources/proxy/uid/tempo/api/traces/$trace_id" \
+ | jq -e --arg trace "$trace_b64" --arg span "$span_b64" '
+ any(.batches[];
+ (.resource.attributes | map({key: .key, value: .value.stringValue}) | from_entries) as $a
+ | $a.project == "smoke" and $a.env == "ci" and $a.department == "cml"
+ and any(.scopeSpans[].spans[]; .traceId == $trace and .spanId == $span and .name == "smoke"))' >/dev/null
}
poll smoke_metric || die "the smoke metric never reached Prometheus with its project/env/department labels; see just smoke-logs"
poll smoke_log || die "the smoke log never reached Loki with its department label; see just smoke-logs"
+poll smoke_trace || die "the smoke trace never reached Tempo with its trace/span identity and project/env/department attributes; see just smoke-logs"
# The keystone alert (ProjectTelemetrySilent) and the coverage backstop key on the
# gateway's ingest counters, so a renamed metric or a dropped attribute would
# silence both without any other assertion noticing.
-counted() { promq "count by (project, env) ({__name__=~\"telemetry_.+_total\", project=\"smoke\", env=\"ci\"})" | jq -e '.data.result | length > 0' >/dev/null; }
-poll counted || die "the count connector never produced telemetry_*_total{project=smoke,env=ci}; the alert rules that key on it would never fire"
+counted() {
+ promq "$counts_query" | jq -e --argjson before "$counts_before" '
+ [.data.result[] | {key: .metric.__name__, value: (.value[1] | tonumber)}] | from_entries
+ | . as $after | ["telemetry_datapoints_total", "telemetry_logs_total", "telemetry_spans_total"]
+ | all(.[]; . as $name | ($after[$name] // 0) > ($before[$name] // 0))' >/dev/null
+}
+poll counted || die "the count connector never advanced all three telemetry signal counters for this smoke run; the alert rules that key on it would never fire"
# ------------------------------------------------------------- ntfy & blackbox
# Verify ntfy health endpoint answers over the docker network.
@@ -129,7 +158,16 @@ if [[ "${SMOKE_ALERTS:-}" == 1 ]]; then
}
POLL_MAX=60 POLL_SLEEP=5 poll target_down_firing || die "TargetDown did not fire within 5 minutes of stopping node-exporter"
docker compose -p "$project" start node-exporter >/dev/null
- echo "TargetDown fired for the stopped node-exporter"
+ node_recovered() {
+ promq 'up{job="node"}' | jq -e '.data.result | length > 0 and all(.[]; .value[1] == "1")' >/dev/null
+ }
+ target_down_normal() {
+ gf "$url/api/prometheus/grafana/api/v1/rules" \
+ | jq -e '.data.groups[].rules[] | select(.name == "TargetDown") | .state == "inactive"' >/dev/null
+ }
+ poll node_recovered || die "node-exporter did not resume scraping within a minute of restart"
+ POLL_MAX=60 POLL_SLEEP=5 poll target_down_normal || die "TargetDown did not return to normal within 5 minutes of restarting node-exporter"
+ echo "TargetDown fired, node-exporter resumed scraping, and TargetDown returned to normal"
fi
echo "Stack healthy"
From aa62c78900db3f4a09e3867b16569875fbd0c74d Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:23:58 +0100
Subject: [PATCH 17/22] fix(alerting): stop watchdog on notification failures
---
config/grafana/alerting/rules.yaml | 16 ++--
tests/watchdog.json | 128 +++++++++++++++++++++++++++++
2 files changed, 139 insertions(+), 5 deletions(-)
create mode 100644 tests/watchdog.json
diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml
index 52c3269..71b99ca 100644
--- a/config/grafana/alerting/rules.yaml
+++ b/config/grafana/alerting/rules.yaml
@@ -44,8 +44,8 @@ groups:
folder: Stack alerts
interval: 1m
rules:
- # Always firing. execErrState is Error so a broken query stops the
- # heartbeat and the switch alarms.
+ # Stop on failed deliveries, lost Grafana scrapes, or query errors. A new
+ # counter may start at 1; retain that evidence even if it disappears.
- uid: watchdog-heartbeat
title: Watchdog
!!merge <<: *rule_defaults
@@ -54,13 +54,19 @@ groups:
- !!merge <<: *query_node
model:
!!merge <<: *query_model
- expr: vector(1)
+ expr: >
+ (vector(1) and on() (min(up{job="grafana"}) == 1))
+ unless (sum(increase(grafana_alerting_notifications_failed_total{job="grafana"}[10m])) > 0
+ or sum(max_over_time(grafana_alerting_notifications_failed_total{job="grafana"}[10m])
+ unless grafana_alerting_notifications_failed_total{job="grafana"} offset 10m) > 0)
- *firing
labels:
severity: none
annotations:
- summary: "Alerting-pipeline heartbeat (always firing)"
- description: "Routed to a dead man's switch. Investigate if the pings stop."
+ summary: "Alerting-pipeline heartbeat"
+ description: >-
+ Routed to a dead man's switch. Pings stop when Grafana cannot be scraped
+ or recent notification failures appear in a 10-minute window. Investigate if the pings stop.
# Cannot page when the webhook itself is broken; the heartbeat covers that.
- uid: alert-delivery-failing
title: AlertDeliveryFailing
diff --git a/tests/watchdog.json b/tests/watchdog.json
new file mode 100644
index 0000000..346b3ff
--- /dev/null
+++ b/tests/watchdog.json
@@ -0,0 +1,128 @@
+{
+ "evaluation_interval": "1m",
+ "tests": [
+ {
+ "name": "healthy Grafana with no failures",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x20"},
+ {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"0+0x20"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ },
+ {
+ "name": "failure stops heartbeat and expires after ten minutes",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x30"},
+ {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"0+0x4 1+0x25"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"5m","exp_samples":[]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"13m","exp_samples":[]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"15m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ },
+ {
+ "name": "first counter sample already records a failed delivery",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x20"},
+ {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\",integration=\"webhook\"}","values":"_ _ 1+0x18"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"11m","exp_samples":[]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"12m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ },
+ {
+ "name": "counter reset followed by another failed delivery",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x30"},
+ {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"4+0x10 0 1+0x18"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"10m","exp_samples":[{"labels":"{}","value":1}]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"12m","exp_samples":[]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"22m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ },
+ {
+ "name": "no counters before the first notification",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x20"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ },
+ {
+ "name": "Grafana scrape fails",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1 0+0x20"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]}
+ ]
+ },
+ {
+ "name": "Grafana scrape disappears",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1 stale"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]}
+ ]
+ },
+ {
+ "name": "Grafana has never been scraped",
+ "interval": "1m",
+ "input_series": [
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]}
+ ]
+ },
+ {
+ "name": "failure counters from another job do not suppress Grafana",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x20"},
+ {"series":"grafana_alerting_notifications_failed_total{job=\"unrelated\"}","values":"1+1x20"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ },
+ {
+ "name": "one failed Grafana scrape suppresses heartbeat",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\",instance=\"healthy\"}","values":"1+0x20"},
+ {"series":"up{job=\"grafana\",instance=\"failed\"}","values":"0+0x20"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]}
+ ]
+ },
+ {
+ "name": "new failure counter disappears before its failure ages out",
+ "interval": "1m",
+ "input_series": [
+ {"series":"up{job=\"grafana\"}","values":"1+0x20"},
+ {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"_ _ 1 1 stale"}
+ ],
+ "promql_expr_test": [
+ {"expr":"WATCHDOG_EXPR","eval_time":"5m","exp_samples":[]},
+ {"expr":"WATCHDOG_EXPR","eval_time":"14m","exp_samples":[{"labels":"{}","value":1}]}
+ ]
+ }
+ ]
+}
From 293d7cc7afce3f61af8533b425fbbc11adffec09 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 16:29:23 +0100
Subject: [PATCH 18/22] test: verify notification and heartbeat safeguards
---
docs/RUNBOOK.md | 15 +++++++++++----
justfile | 9 +++++++--
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md
index e197803..110a6b7 100644
--- a/docs/RUNBOOK.md
+++ b/docs/RUNBOOK.md
@@ -168,7 +168,7 @@ comparison stays true for the duration of `for`:
| Rule | Group | Severity | For | Trigger & Intent |
| --- | --- | --- | --- | --- |
-| `Watchdog` | meta | none | 0m | Heartbeat pinging dead man's switch (`vector(1)`); always firing. |
+| `Watchdog` | meta | none | 0m | Heartbeat while Grafana scrapes succeed and recent notification deliveries have not failed. |
| `AlertDeliveryFailing` | meta | critical | 10m | Grafana failing to deliver webhook notifications. |
| `TargetDown` | stack-health | critical | 2m | Prometheus cannot scrape an internal job target (`up == 0`). |
| `OtelExportFailures` | stack-health | warning | 5m | OTel Collector failing to export spans, metrics, or logs. |
@@ -194,7 +194,7 @@ routing tree are provisioned from `config/grafana/alerting/`, so the UI shows
them read-only. Edit the YAML.
- `ALERT_WEBHOOK_URL` receives all alerts.
-- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five minutes.
+- `HEARTBEAT_URL` receives `Watchdog` every five minutes while delivery is healthy.
Point it at a dead man's switch that raises the alarm when pings stop.
The internal `notification-relay` owns both destination URLs. Grafana stores
@@ -208,6 +208,13 @@ timeout, refuses redirects, and reports delivery errors to Grafana without
returning provider bodies or credentials. An empty destination returns 503.
With the tunnel overlay active, `just up` refuses to start without both URLs.
`AlertDeliveryFailing` fires if a configured notifier fails to deliver alerts.
+Watchdog stops after a notification failure or a failed Grafana scrape. It resumes
+when scraping is healthy and notification failures leave the ten-minute query
+window. A failed channel can therefore trigger the external check even when
+Grafana itself is running. Resolved Watchdog notifications are disabled so they
+cannot send a misleading recovery ping. Check the failed destination before
+trusting resumed heartbeats; a quiet channel has no new delivery attempts to prove
+its recovery.
When migrating from direct delivery, run
`docker compose up -d notification-relay grafana` to replace Grafana's old
@@ -232,8 +239,8 @@ Cloudflare Tunnel with strict zero-trust authentication (`auth-default-access: "
+------------------------------------------------------------------+
| Monitoring Host |
| |
-| Grafana ------(http://ntfy:80/alerts?tpl=1...)------> ntfy |
-| (publisher) | |
+| Grafana --> notification-relay --> ntfy |
+| (publisher credentials) | |
| [auth.db] |
| ^ |
| | |
diff --git a/justfile b/justfile
index 54007b9..cf08fa4 100644
--- a/justfile
+++ b/justfile
@@ -65,9 +65,9 @@ _expose-guards:
@case "${GRAFANA_ROOT_URL:-}" in https://*) ;; *) echo "error: GRAFANA_ROOT_URL must be the https:// tunnel hostname (got '${GRAFANA_ROOT_URL:-}'); every absolute URL Grafana generates comes from it" >&2; exit 1;; esac
@[ "${GRAFANA_COOKIE_SECURE:-false}" = "true" ] || { echo "error: GRAFANA_COOKIE_SECURE must be true when Grafana is served over HTTPS; set it in .env" >&2; exit 1; }
@[ "${GRAFANA_JWT_AUTH:-false}" != "true" ] || { [ -n "${CF_ACCESS_TEAM_DOMAIN:-}" ] && [ -n "${CF_ACCESS_AUD:-}" ]; } || { echo "error: GRAFANA_JWT_AUTH=true needs CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD in .env (cd infra && tofu output -raw grafana_access_team_domain grafana_access_aud)" >&2; exit 1; }
- @[ -n "${HEARTBEAT_URL:-}" ] || { echo "error: HEARTBEAT_URL is empty; Grafana rejects contact point provisioning. Set the dead man's switch URL in .env" >&2; exit 1; }
+ @[ -n "${HEARTBEAT_URL:-}" ] || { echo "error: HEARTBEAT_URL is empty; the notification relay needs the dead man's switch URL in .env" >&2; exit 1; }
@for f in .env infra/terraform.tfvars infra/terraform.tfstate infra/terraform.tfstate.backup; do [ ! -e "$f" ] || case "$(stat -c %a "$f")" in *00) ;; *) echo "error: $f is readable by other users (mode $(stat -c %a "$f")); it holds live secrets, run: chmod 600 $f" >&2; exit 1;; esac; done
- @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; Grafana rejects contact point provisioning. Set the alert destination in .env" >&2; exit 1; }
+ @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; the notification relay needs the alert destination in .env" >&2; exit 1; }
@case "${NTFY_BASE_URL:-}" in ""|https://*) ;; *) echo "error: NTFY_BASE_URL must be an https:// URL when exposing via tunnel (got '${NTFY_BASE_URL:-}')" >&2; exit 1;; esac
# Core stack plus a demo telemetry source, isolated from any running stack (:3002).
@@ -138,8 +138,13 @@ lint:
sh scripts/test_gitleaks.sh
bash scripts/test_imports.sh
sh scripts/test_backup.sh
+ # Exercise the provisioned expression, including counters first seen after a failure.
+ @d=$(mktemp -d) && expr=$(awk '/uid: watchdog-heartbeat/{watchdog=1} watchdog && /expr: >/{expr=1;next} expr && /^ /{sub(/^ /, "");printf "%s ",$0;next} expr{exit}' config/grafana/alerting/rules.yaml) && test -n "$expr" && jq --arg expr "$expr" '.tests[].promql_expr_test[].expr = $expr' tests/watchdog.json >"$d/watchdog.json" && chmod 755 "$d" && chmod 644 "$d/watchdog.json" && docker run --rm --network none --entrypoint /bin/promtool -v "$d":/tests:ro $(just _image compose.yml prometheus) test rules /tests/watchdog.json; rc=$?; rm -rf "$d"; exit $rc
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo
docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo
+ docker run --rm --network none -v ./notification-relay:/relay:ro {{ruff}} check --no-cache /relay
+ docker run --rm --network none -v ./notification-relay:/relay:ro {{ruff}} format --check --no-cache /relay
+ docker run --rm --network none -v ./notification-relay:/relay:ro -w /relay $(just _image compose.yml notification-relay) python -B -m unittest discover
docker run --rm --network none -v ./templates/web:/rum:ro {{node}} node --test --test-isolation=none /rum/rum.test.mjs
# `just fmt` is the fix.
docker run --rm --network none -v .:/code:ro -w /code {{yamlfmt}} -lint .
From 646e27731124b7f78aade7116413abcbe48a9bad Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 17:10:18 +0100
Subject: [PATCH 19/22] test(rum): cover buffered performance history
---
templates/web/rum.test.mjs | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs
index 3b1737b..b883f5f 100644
--- a/templates/web/rum.test.mjs
+++ b/templates/web/rum.test.mjs
@@ -5,7 +5,7 @@ import { runInNewContext } from 'node:vm';
const source = readFileSync(new URL('./rum.html', import.meta.url), 'utf8').split('')[0];
-function browser(initialVisibility = 'visible', unsupportedCls = false, endpoint = 'https://rum.example.org/') {
+function browser(initialVisibility = 'visible', unsupportedCls = false, endpoint = 'https://rum.example.org/', initialEntries = {}) {
const logs = [];
const requests = [];
const listeners = {};
@@ -22,10 +22,11 @@ function browser(initialVisibility = 'visible', unsupportedCls = false, endpoint
pending = [];
disconnected = false;
constructor(callback) { this.callback = callback; }
- observe({ type }) {
+ observe({ type, buffered }) {
if (unsupportedCls === 'silent' && type === 'layout-shift') return;
if (unsupportedCls === true && type === 'layout-shift') throw new Error('unsupported');
observers[type] = this;
+ if (buffered) this.pending.push(...(initialEntries[type] ?? []));
}
takeRecords() { return this.pending.splice(0); }
disconnect() { this.disconnected = true; }
@@ -47,6 +48,16 @@ function browser(initialVisibility = 'visible', unsupportedCls = false, endpoint
}
const shift = (value, startTime, hadRecentInput = false) => ({ value, startTime, hadRecentInput });
+test('vitals include buffered entries recorded before initialization', () => {
+ const page = browser('visible', false, 'https://rum.example.org/', {
+ 'largest-contentful-paint': [{ startTime: 1200 }],
+ 'layout-shift': [shift(0.25, 400)],
+ });
+ page.hide(2000);
+ assert.deepEqual(page.values('LCP', 'duration_ms'), [1200]);
+ assert.deepEqual(page.values('CLS', 'cls_score'), [0.25]);
+});
+
test('transport normalizes the endpoint and posts JSON with keepalive; no endpoint disables sending', () => {
const page = browser();
page.emit('error', {});
From 6d3c4ae4197b0ee9c96039339bc582d0587ce9a7 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 17:12:21 +0100
Subject: [PATCH 20/22] fix(rum): omit browser credentials and referrers
---
templates/web/README.md | 1 +
templates/web/rum.html | 2 ++
templates/web/rum.test.mjs | 2 ++
3 files changed, 5 insertions(+)
diff --git a/templates/web/README.md b/templates/web/README.md
index deed2e2..b223cc1 100644
--- a/templates/web/README.md
+++ b/templates/web/README.md
@@ -42,6 +42,7 @@ Error records contain only the event type and, when available, script origin/pat
line and column. Messages, stacks and rejection values are excluded. Script URLs
exclude credentials, queries and fragments; review paths for private values before
enabling collection.
+Requests omit browser credentials and the Referer header.
- **CORS origins:** The collector HTTP receiver permits cross-origin requests from `https://cmlplatform.github.io` and Leiden University domains (`https://*.leidenuniv.nl`).
- **Credential safety:** Never embed the collector bearer in public HTML. The collector supports one shared bearer, without per-project scopes. An administrator must provide a restricted proxy that injects that bearer, fixes the project identity, and limits payload size and request rate. Leave RUM disabled until the proxy and collection agreement are ready. CORS alone does not provide this boundary.
diff --git a/templates/web/rum.html b/templates/web/rum.html
index 1762a24..0a5c809 100644
--- a/templates/web/rum.html
+++ b/templates/web/rum.html
@@ -65,6 +65,8 @@
headers: headers,
body: JSON.stringify(payload),
keepalive: true,
+ referrerPolicy: "no-referrer",
+ credentials: "omit",
mode: "cors"
}).catch(function() {});
} catch (e) {
diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs
index b883f5f..24b6cb6 100644
--- a/templates/web/rum.test.mjs
+++ b/templates/web/rum.test.mjs
@@ -66,6 +66,8 @@ test('transport normalizes the endpoint and posts JSON with keepalive; no endpoi
assert.equal(url, 'https://rum.example.org/v1/logs');
assert.equal(options.method, 'POST');
assert.equal(options.keepalive, true);
+ assert.equal(options.referrerPolicy, 'no-referrer');
+ assert.equal(options.credentials, 'omit');
assert.deepEqual({ ...options.headers }, { 'Content-Type': 'application/json' });
assert.equal(JSON.parse(options.body).resourceLogs.length, 1);
const disabled = browser('visible', false, '');
From 25c3b8a06261db1aa00206e30d76854366935002 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 17:12:38 +0100
Subject: [PATCH 21/22] docs(checks): declare jq prerequisite and surface
failures
---
.github/CONTRIBUTING.md | 3 ++-
scripts/test_imports.sh | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 0682184..b8873ae 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -22,7 +22,8 @@ request outside what the stack sets out to do is closed with a link to this sect
## Set up and check
-You need Docker with the Compose plugin and [`just`](https://github.com/casey/just).
+You need Docker with the Compose plugin, [`just`](https://github.com/casey/just),
+and `jq` for the local regression checks.
`just hooks` installs the git hooks (with [prek](https://prek.j178.dev), which sets up the
Python environment for the citation hooks itself). `just check` runs the static checks and
validates every config, `just smoke` starts a throwaway copy of the stack and sends it
diff --git a/scripts/test_imports.sh b/scripts/test_imports.sh
index 99404fb..45087ad 100644
--- a/scripts/test_imports.sh
+++ b/scripts/test_imports.sh
@@ -34,6 +34,7 @@ for fixture in present absent failure; do
absent) [[ "$rc" == 0 ]] && ! grep 'cloudflare_dns_record.ntfy' "$scratch/out" >/dev/null ;;
failure) [[ "$rc" != 0 ]] && grep 'DNS lookup for ntfy.fixture.example failed' "$scratch/err" >/dev/null ;;
esac || {
+ cat "$scratch/err" >&2
echo "error: ntfy DNS $fixture check failed" >&2
exit 1
}
From 56d15fcc7e877327306be4b7053b5b650c528ac5 Mon Sep 17 00:00:00 2001
From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
Date: Sat, 14 Nov 2026 17:43:02 +0100
Subject: [PATCH 22/22] fix(dashboard): distinguish export failure signals
---
dashboards/stack-health.json | 39 ++++++++++++++++++++++++++++++++----
1 file changed, 35 insertions(+), 4 deletions(-)
diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json
index 50a6382..9de697c 100644
--- a/dashboards/stack-health.json
+++ b/dashboards/stack-health.json
@@ -207,17 +207,17 @@
"targets": [
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))",
- "legendFormat": "{{exporter}} spans/s",
+ "legendFormat": "{{exporter}} spans/s (solid)",
"refId": "A"
},
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))",
- "legendFormat": "{{exporter}} metric points/s",
+ "legendFormat": "{{exporter}} metric points/s (dashed)",
"refId": "B"
},
{
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))",
- "legendFormat": "{{exporter}} log records/s",
+ "legendFormat": "{{exporter}} log records/s (dotted)",
"refId": "C"
}
],
@@ -234,7 +234,38 @@
"fillOpacity": 10
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "B"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dash",
+ "dash": [10, 10]
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "C"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "fill": "dot",
+ "dash": [0, 10]
+ }
+ }
+ ]
+ }
+ ]
}
},
{