diff --git a/.env.example b/.env.example index 7a668ef..b02463b 100644 --- a/.env.example +++ b/.env.example @@ -22,12 +22,12 @@ GRAFANA_COOKIE_SECURE=false # openssl rand -hex 32 OTLP_AUTH_TOKEN=local-dev-token -# Where Grafana delivers alert notifications (self-hosted ntfy, hosted ntfy, Slack, …). +# Where the internal relay delivers Grafana alert notifications (self-hosted ntfy, hosted ntfy, Slack, …). # For self-hosted ntfy with basic auth publisher: # ALERT_WEBHOOK_URL=http://grafana:password@ntfy:80/alerts?tpl=1&title={{or .commonAnnotations.summary .groupLabels.alertname}}&message={{or .commonAnnotations.description .commonAnnotations.summary "Alert firing"}} -# Not optional: an empty value drops every alert silently while the heartbeat -# below keeps reporting healthy. With the tunnel overlay active, `just up` -# refuses to start without it. +# Only the relay receives this secret; Grafana stores an internal URL. +# Empty disables delivery (the relay returns 503). +# With the tunnel overlay active, `just up` refuses to start without it. ALERT_WEBHOOK_URL= # Public URL Cloudflare Tunnel exposes ntfy on, e.g. https://ntfy.example.com @@ -36,6 +36,7 @@ NTFY_BASE_URL= # Dead man's switch ping target (e.g. https://hc-ping.com/). The Watchdog # alert posts here every 5m; alert externally when pings stop. +# Only the relay receives this secret. Empty disables heartbeat delivery. HEARTBEAT_URL= # Optional. Lets `./bootstrap.sh` create a project's healthchecks.io checks. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 8d7f139..b8873ae 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -22,13 +22,19 @@ request outside what the stack sets out to do is closed with a link to this sect ## Set up and check -You need Docker with the Compose plugin and [`just`](https://github.com/casey/just). +You need Docker with the Compose plugin, [`just`](https://github.com/casey/just), +and `jq` for the local regression checks. `just hooks` installs the git hooks (with [prek](https://prek.j178.dev), which sets up the Python environment for the citation hooks itself). `just check` runs the static checks and validates every config, `just smoke` starts a throwaway copy of the stack and sends it telemetry, and `just fmt` formats the YAML. CI runs `just lint`, `just validate` and `just smoke`, plus `just infra-validate` when `infra/` changes. The [Verification commands](../docs/RUNBOOK.md#verification-commands) table in the runbook lists every recipe. +CI also checks the README demo on a fresh runner and checks the demo dependency licences. + +Add or update tests for each behaviour change, including bug fixes. Extend the assertions +in `just lint` or `scripts/smoke.sh` for the affected path. Follow the coding standards +checked by Ruff for Python, ShellCheck for shell scripts, and yamllint and yamlfmt for YAML. ## Write commits and pull requests diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index e401e73..0a7e12f 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -31,4 +31,4 @@ body: id: environment attributes: label: Environment - description: Host operating system, and the Docker and Compose versions (`docker version`, `docker compose version`). + description: Operating system and Docker and Compose versions (`docker version`, `docker compose version`). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffe88bf..4c5fe41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # gitleaks scans git history; a shallow clone would pass vacuously. with: + persist-credentials: false fetch-depth: 0 - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 # 600, or the exposure guards refuse it as world-readable. @@ -36,6 +37,8 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 # 600, or the exposure guards refuse it as world-readable. - run: install -m 600 .env.example .env diff --git a/.github/workflows/cold-install.yml b/.github/workflows/cold-install.yml new file mode 100644 index 0000000..651de7a --- /dev/null +++ b/.github/workflows/cold-install.yml @@ -0,0 +1,41 @@ +name: cold install + +on: + pull_request: + schedule: + - cron: "47 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + readme: + name: README demo, fresh runner + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 + # Docker and Compose are supplied by the fresh runner; no build cache is restored. + - name: Run the README quick start + run: | + cp .env.example .env + just demo + - name: Check the demo API and Grafana + run: | + docker compose -p monitoring-demo -f compose.yml -f compose.demo.yml -f compose.sandbox.yml \ + exec -T demo-load curl --fail --retry 20 --retry-all-errors --retry-delay 2 demo-api:8000/openapi.json + curl --fail --retry 20 --retry-all-errors --retry-delay 2 http://localhost:3002/api/health + - if: failure() + run: >- + docker compose -p monitoring-demo -f compose.yml -f compose.demo.yml -f compose.sandbox.yml + logs --no-color --tail=200 + - if: always() + run: just demo-destroy diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml index 8f5c130..48465f3 100644 --- a/.github/workflows/infra.yml +++ b/.github/workflows/infra.yml @@ -20,6 +20,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 - run: install -m 600 .env.example .env # 600 or the exposure guards reject it - run: just infra-validate diff --git a/.github/workflows/licenses.yml b/.github/workflows/licenses.yml new file mode 100644 index 0000000..cdf4ebc --- /dev/null +++ b/.github/workflows/licenses.yml @@ -0,0 +1,42 @@ +name: demo dependency licenses + +on: + pull_request: + schedule: + - cron: "23 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + licenses: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.13.6" + enable-cache: false + - id: python + name: Read the demo runtime + run: | + version=$(sed -n 's/^FROM python:\([0-9]*\.[0-9]*\).*/\1/p' demo/Dockerfile) + test -n "$version" + echo "version=$version" >> "$GITHUB_OUTPUT" + # Resolve the same manifest as the demo image, including transitive packages. + - run: >- + uv pip compile --python-version "${{ steps.python.outputs.version }}" + demo/pyproject.toml -o "$RUNNER_TEMP/requirements.txt" + - name: List licenses and reject incompatibility with the repository's MIT license + run: >- + uv run --no-project --python "${{ steps.python.outputs.version }}" + --with-requirements "$RUNNER_TEMP/requirements.txt" + --with licensecheck==2026.0.8 licensecheck --license MIT -r "$RUNNER_TEMP/requirements.txt" --zero diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e588cb..5ef3e0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,16 +11,19 @@ on: tags: ["v*"] permissions: - contents: write + contents: read jobs: release: runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # gitleaks scans git history; a shallow clone would pass vacuously. with: + persist-credentials: false fetch-depth: 0 - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 # The tag's section of CHANGELOG.md is the release body. No section, no diff --git a/README.md b/README.md index e1dc364..fd70d62 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ This launches the core monitoring stack and a demo FastAPI service under simulat - View **Dashboards → Service Health (RED)** for request rates, error rates, and latency traces. - Stop and clean up: `just demo-destroy`. +The demo displays sample telemetry but does not deliver alert notifications. + ![Service Health dashboard](docs/img/service-health.png) ## Operating the stack diff --git a/compose.demo.yml b/compose.demo.yml index 0f2f6b7..6a03c70 100644 --- a/compose.demo.yml +++ b/compose.demo.yml @@ -12,6 +12,12 @@ x-demo-logging: &demo-logging max-file: "3" services: + notification-relay: + # Demo notifications must not reach a production webhook or dead man's switch. + environment: + ALERT_WEBHOOK_URL: https://demo.invalid/alerts + HEARTBEAT_URL: https://demo.invalid/heartbeat + demo-api: build: ./demo restart: on-failure diff --git a/compose.yml b/compose.yml index ab98a97..57b4512 100644 --- a/compose.yml +++ b/compose.yml @@ -163,10 +163,35 @@ services: - /:/host:ro networks: [backend] + notification-relay: + image: python:3.15.0-slim@sha256:f5b92f2a669a4c347044f29367f31983280b648250ca94dea66f98c53bc87e58 + restart: unless-stopped + user: "65534:65534" + read_only: true + command: ["python", "-B", "/app/relay.py"] + volumes: + - ./notification-relay/relay.py:/app/relay.py:ro + environment: + ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-} + HEARTBEAT_URL: ${HEARTBEAT_URL:-} + healthcheck: + !!merge <<: *healthcheck + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=2).close()" + mem_limit: 64m + !!merge <<: *hardened + pids_limit: 32 + networks: [notification-egress, notifications] + grafana: image: grafana/grafana:13.3.0@sha256:488984b716c0862639f91a698adbe2862677a5a07bad42da5f03f3030883b205 restart: unless-stopped depends_on: + notification-relay: + condition: service_healthy loki: condition: service_started tempo: @@ -183,9 +208,6 @@ services: - grafana_data:/var/lib/grafana environment: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} - # Expanded by Grafana into the provisioned contact points. - ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-} - HEARTBEAT_URL: ${HEARTBEAT_URL:-} GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000} GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/stack-health.json # Secure cookies break plain-http localhost logins, so opt-in. With the @@ -209,7 +231,7 @@ services: test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"] mem_limit: 1g !!merge <<: *hardened - networks: [default, backend] + networks: [default, backend, notifications] blackbox-exporter: image: prom/blackbox-exporter:v0.29.1@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13 @@ -241,13 +263,16 @@ services: mem_limit: 128m pids_limit: 128 !!merge <<: *hardened - networks: [default, backend] + networks: [default, backend, notifications] # Loki, Prometheus and Tempo have no authentication of their own, so only # Grafana and the collector can reach them. cloudflared stays on `default`: its # ingress list comes from Cloudflare at runtime, so an edited ingress cannot # publish a backend that skips the collector's token gate. networks: + notification-egress: + notifications: + internal: true backend: internal: true diff --git a/config/grafana/alerting/contact-points.yaml b/config/grafana/alerting/contact-points.yaml index 621bdb2..5a5b14f 100644 --- a/config/grafana/alerting/contact-points.yaml +++ b/config/grafana/alerting/contact-points.yaml @@ -1,8 +1,4 @@ -# Notification targets. Grafana expands $VAR from the environment. -# -# An unset variable leaves the URL empty and every notification fails silently: -# AlertDeliveryFailing routes to this same webhook, and the heartbeat stays -# green. The exposure guard in `just up` is the only control. +# Destination credentials live only in notification-relay, outside Grafana. apiVersion: 1 @@ -13,7 +9,7 @@ contactPoints: - uid: cp-webhook-default type: webhook settings: - url: $ALERT_WEBHOOK_URL + url: http://notification-relay:8080/alerts - orgId: 1 name: heartbeat @@ -21,6 +17,6 @@ contactPoints: - uid: cp-heartbeat-dms type: webhook settings: - url: $HEARTBEAT_URL + url: http://notification-relay:8080/heartbeat # A resolved-notification would ping the switch and mask an outage. disableResolveMessage: true diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml index 52c3269..71b99ca 100644 --- a/config/grafana/alerting/rules.yaml +++ b/config/grafana/alerting/rules.yaml @@ -44,8 +44,8 @@ groups: folder: Stack alerts interval: 1m rules: - # Always firing. execErrState is Error so a broken query stops the - # heartbeat and the switch alarms. + # Stop on failed deliveries, lost Grafana scrapes, or query errors. A new + # counter may start at 1; retain that evidence even if it disappears. - uid: watchdog-heartbeat title: Watchdog !!merge <<: *rule_defaults @@ -54,13 +54,19 @@ groups: - !!merge <<: *query_node model: !!merge <<: *query_model - expr: vector(1) + expr: > + (vector(1) and on() (min(up{job="grafana"}) == 1)) + unless (sum(increase(grafana_alerting_notifications_failed_total{job="grafana"}[10m])) > 0 + or sum(max_over_time(grafana_alerting_notifications_failed_total{job="grafana"}[10m]) + unless grafana_alerting_notifications_failed_total{job="grafana"} offset 10m) > 0) - *firing labels: severity: none annotations: - summary: "Alerting-pipeline heartbeat (always firing)" - description: "Routed to a dead man's switch. Investigate if the pings stop." + summary: "Alerting-pipeline heartbeat" + description: >- + Routed to a dead man's switch. Pings stop when Grafana cannot be scraped + or recent notification failures appear in a 10-minute window. Investigate if the pings stop. # Cannot page when the webhook itself is broken; the heartbeat covers that. - uid: alert-delivery-failing title: AlertDeliveryFailing diff --git a/config/otel-collector.yaml b/config/otel-collector.yaml index 467fff6..18a409a 100644 --- a/config/otel-collector.yaml +++ b/config/otel-collector.yaml @@ -90,7 +90,7 @@ processors: # service.instance.id on each restart adds one every time. The default is # unbounded. This cap is a runaway guard, not a budget: 28 streams cost ~51 MiB # heap against the 400 MiB limiter (measured before `interval` was added). - deltatocumulative: + delta_to_cumulative: max_streams: 5000 # The count connector emits one datapoint per incoming batch, so 50 log batches @@ -109,7 +109,7 @@ processors: metric_statements: - context: datapoint statements: - - set(time, Now()) + - set(datapoint.time, Now()) # Sized to the container's mem_limit (512m in compose.yml); keep the two in step. memory_limiter: @@ -170,10 +170,10 @@ service: processors: [memory_limiter, resource/department, batch] exporters: [otlp_http/prometheus, count] # The counters themselves. No batcher: it reordered timestamps ~1/min, which - # deltatocumulative refused as out of order. `interval` does the collapsing. + # delta_to_cumulative refused as out of order. `interval` does the collapsing. metrics/count: receivers: [count] - processors: [memory_limiter, deltatocumulative, interval, transform/count_ingest_time] + processors: [memory_limiter, delta_to_cumulative, interval, transform/count_ingest_time] exporters: [otlp_http/prometheus_count] telemetry: metrics: diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json index a195a18..9de697c 100644 --- a/dashboards/stack-health.json +++ b/dashboards/stack-health.json @@ -206,9 +206,19 @@ }, "targets": [ { - "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval])) or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", - "legendFormat": "{{exporter}}", + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[$__rate_interval]))", + "legendFormat": "{{exporter}} spans/s (solid)", "refId": "A" + }, + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points[$__rate_interval]))", + "legendFormat": "{{exporter}} metric points/s (dashed)", + "refId": "B" + }, + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))", + "legendFormat": "{{exporter}} log records/s (dotted)", + "refId": "C" } ], "fieldConfig": { @@ -224,7 +234,38 @@ "fillOpacity": 10 } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [10, 10] + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dot", + "dash": [0, 10] + } + } + ] + } + ] } }, { diff --git a/docs/ONBOARDING.md b/docs/ONBOARDING.md index 024632c..ab240b9 100644 --- a/docs/ONBOARDING.md +++ b/docs/ONBOARDING.md @@ -106,15 +106,19 @@ Prometheus reloads `config/targets/probes.yaml` automatically without a restart. ## Template 4: Browser Real User Monitoring (RUM) -For client-side performance and error tracking on web applications or static portals, include the lightweight snippet from [`templates/web/rum.html`](../templates/web/rum.html): +For browser performance and error tracking, first ask the hub administrator for +a restricted public ingestion proxy. It must keep the collector bearer private, +fix the project's identity, and limit payload size and request rate. Leave RUM +disabled until that ingress and the collection agreement are ready. + +Then configure the snippet from [`templates/web/rum.html`](../templates/web/rum.html): ```html ``` @@ -125,7 +129,8 @@ What it captures: - **Client Exceptions:** Unhandled `window.onerror` exceptions and unhandled promise rejections. - **Log Streams:** Emits structured OTLP/HTTP JSON (`/v1/logs`) indexed into Loki. -The collector permits cross-origin requests from `https://cmlplatform.github.io` and Leiden University domains (`*.leidenuniv.nl`). See [`templates/web/README.md`](../templates/web/README.md) for details. +Never put the shared collector token in public HTML. CORS does not protect that +credential. See [`templates/web/README.md`](../templates/web/README.md) for the ingress requirements. --- diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 4ebfba7..110a6b7 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -45,10 +45,12 @@ count, not time, is what grows the TSDB. When one fires: 1. Check the Filesystem panel on Stack Health. -2. Free space, or shorten a retention window and restart the service: +2. Free space, or shorten a retention window: `retention_period` in `config/loki.yaml`, `block_retention` in `config/tempo.yaml`, or the `--storage.tsdb.retention.*` flags in - `compose.yml`. + `compose.yml`. Restart Loki or Tempo after a config change. After changing + Prometheus command arguments, run `docker compose up -d prometheus` to + recreate its container with the new retention settings. 3. If disk pressure keeps returning, move Loki and Tempo to object storage. The appendix of ADR 0001 describes the change. @@ -106,12 +108,25 @@ on the host. That builds a new tunnel and repoints all three CNAMEs (grafana, otel, ntfy) at it, and ingestion and Grafana are unreachable for about a minute. Either way, put the new token in `CLOUDFLARE_TUNNEL_TOKEN` in `.env`, then `just up`. -- **Grafana admin password:** change `GRAFANA_ADMIN_PASSWORD` in `.env`, - then `docker compose up -d grafana`. +- **Grafana admin password:** changing `GRAFANA_ADMIN_PASSWORD` only initializes + a new database. To rotate an existing password, update its value in `.env`. + Enter the same value at the prompt below; it stays out of shell history and + process arguments: + + ```bash + read -r -s -p 'New Grafana admin password: ' grafana_password + printf '\n' + printf '%s\n' "$grafana_password" | docker compose exec -T grafana grafana cli \ + --homepath /usr/share/grafana admin reset-admin-password --password-from-stdin + unset grafana_password + docker compose up -d grafana + ``` + + Confirm that a fresh login accepts the new password and rejects the old one. - **Alert webhook and heartbeat URLs:** the URL is the credential. Mint a new topic or check at the provider, put it in `ALERT_WEBHOOK_URL` or - `HEARTBEAT_URL`, then `docker compose up -d grafana`. Grafana reads the - contact points only at startup. + `HEARTBEAT_URL`, then `docker compose up -d notification-relay`. Revoke the + old capability URL at the provider after confirming delivery. - **healthchecks.io API key:** regenerate it in the project's settings and set `HEALTHCHECKS_API_KEY` in `.env`. Only `bootstrap.sh` reads it, so nothing restarts. @@ -122,9 +137,11 @@ on the host. ### Where every secret lives Three gitignored files hold everything. `just backup` archives none of them, -but the tarball contains Grafana's database, and Grafana stores the expanded -contact points there. So the webhook and heartbeat URLs are inside every -backup. Copy `.env` and `infra/terraform.tfstate` off-host together with the +but the tarball contains Grafana's database. New contact points contain only +internal relay routes; destination credentials live in the relay environment. +Backups made before the relay migration contain the old destination URLs. +Rotate those URLs when migrating and protect old backups until the URLs are +revoked. Copy `.env` and `infra/terraform.tfstate` off-host together with the backups and treat all three the same way. | Secret | Lives in | Comes from | @@ -151,7 +168,7 @@ comparison stays true for the duration of `for`: | Rule | Group | Severity | For | Trigger & Intent | | --- | --- | --- | --- | --- | -| `Watchdog` | meta | none | 0m | Heartbeat pinging dead man's switch (`vector(1)`); always firing. | +| `Watchdog` | meta | none | 0m | Heartbeat while Grafana scrapes succeed and recent notification deliveries have not failed. | | `AlertDeliveryFailing` | meta | critical | 10m | Grafana failing to deliver webhook notifications. | | `TargetDown` | stack-health | critical | 2m | Prometheus cannot scrape an internal job target (`up == 0`). | | `OtelExportFailures` | stack-health | warning | 5m | OTel Collector failing to export spans, metrics, or logs. | @@ -168,7 +185,7 @@ comparison stays true for the duration of `for`: | `ContainerRestarting` | container-lifecycle | warning | 10m | Container restarts >3 times within 1 hour. | | `ContainerOOMKilled` | container-lifecycle | warning | 0m | Container terminated by host kernel OOM killer. | | `ProjectTelemetrySilent` | spoke | critical | 15m | Spoke service stopped shipping OTel telemetry. | -| `ProjectsUncovered` | _coverage | warning | 5m | Telemetry received from unlisted project; register in projects.yaml. | +| `ProjectsUncovered` | _coverage | warning | 15m | Telemetry received from unlisted project; register in projects.yaml. | ## Alert delivery @@ -177,16 +194,35 @@ routing tree are provisioned from `config/grafana/alerting/`, so the UI shows them read-only. Edit the YAML. - `ALERT_WEBHOOK_URL` receives all alerts. -- `HEARTBEAT_URL` receives the always-firing `Watchdog` every five minutes. +- `HEARTBEAT_URL` receives `Watchdog` every five minutes while delivery is healthy. Point it at a dead man's switch that raises the alarm when pings stop. -Leaving either empty is not safe. Delivery fails silently while the heartbeat -keeps pinging, so the switch reads healthy and every real alert is dropped. -With the tunnel overlay active, `just up` refuses to start without -`ALERT_WEBHOOK_URL`, and `AlertDeliveryFailing` fires on a failing notifier. - -After changing either variable, run `docker compose up -d grafana`. A plain -`restart` keeps the old environment. Compose rebuilds a container's +The internal `notification-relay` owns both destination URLs. Grafana stores +only `http://notification-relay:8080/alerts` and `/heartbeat`, so its Viewer +contact point API cannot disclose destination credentials. The relay has no +published port and its private network is separate from tunnel ingress. Only +trusted stack services may call it; this does not isolate a compromised +Grafana administrator or container. Keep the ntfy publisher account write-only. +The relay forwards POST JSON with a 1 MiB body limit and a five-second socket +timeout, refuses redirects, and reports delivery errors to Grafana without +returning provider bodies or credentials. An empty destination returns 503. +With the tunnel overlay active, `just up` refuses to start without both URLs. +`AlertDeliveryFailing` fires if a configured notifier fails to deliver alerts. +Watchdog stops after a notification failure or a failed Grafana scrape. It resumes +when scraping is healthy and notification failures leave the ten-minute query +window. A failed channel can therefore trigger the external check even when +Grafana itself is running. Resolved Watchdog notifications are disabled so they +cannot send a misleading recovery ping. Check the failed destination before +trusting resumed heartbeats; a quiet channel has no new delivery attempts to prove +its recovery. + +When migrating from direct delivery, run +`docker compose up -d notification-relay grafana` to replace Grafana's old +environment and provision the internal routes. Rotate the old destination URLs +after the new routes deliver successfully. + +After changing either variable, run +`docker compose up -d notification-relay`. A plain `restart` keeps the old environment. Compose rebuilds a container's environment only on `up`. ### Self-hosted alert delivery with ntfy @@ -203,8 +239,8 @@ Cloudflare Tunnel with strict zero-trust authentication (`auth-default-access: " +------------------------------------------------------------------+ | Monitoring Host | | | -| Grafana ------(http://ntfy:80/alerts?tpl=1...)------> ntfy | -| (publisher) | | +| Grafana --> notification-relay --> ntfy | +| (publisher credentials) | | | [auth.db] | | ^ | | | | @@ -224,7 +260,7 @@ docker compose exec -e NTFY_PASSWORD="" ntfy ntfy user add grafa docker compose exec ntfy ntfy access grafana alerts write-only ``` -#### 2. Configure Grafana contact point +#### 2. Configure the notification relay Point `ALERT_WEBHOOK_URL` in `.env` to the local ntfy instance using native message templating (`?tpl=1`): @@ -233,10 +269,10 @@ message templating (`?tpl=1`): ALERT_WEBHOOK_URL=http://grafana:@ntfy:80/alerts?tpl=1&title={{or .commonAnnotations.summary .groupLabels.alertname}}&message={{or .commonAnnotations.description .commonAnnotations.summary "Alert firing"}} ``` -Apply the change to Grafana: +Apply the change to the relay: ```sh -docker compose up -d grafana +docker compose up -d notification-relay ``` #### 3. Create subscriber users for operators @@ -264,8 +300,9 @@ Zero Trust dashboard. The next apply reverts anything clicked in by hand. ```sh just infra-validate # tofu init + validate, in a container -cd infra && tofu plan # needs CLOUDFLARE_API_TOKEN exported -cd infra && tofu apply +cd infra # needs CLOUDFLARE_API_TOKEN exported +tofu plan +tofu apply ``` - **Granting or revoking Grafana access:** edit `grafana_allowed_emails` in @@ -314,17 +351,17 @@ When the tunnel overlay is active, `just up` validates critical security setting - **`GRAFANA_ADMIN_PASSWORD`**: Refuses `change-me`. Set a strong password. - **`GRAFANA_ROOT_URL`**: Must match the `https://` tunnel hostname for Grafana. - **`GRAFANA_COOKIE_SECURE=true`**: Required when Grafana runs behind an HTTPS tunnel. -- **`ALERT_WEBHOOK_URL`**: Cannot be empty; empty URLs drop all fired alerts. +- **`ALERT_WEBHOOK_URL`**: Cannot be empty when exposing the stack; the relay needs an alert destination. - **`NTFY_BASE_URL`**: Must be an https:// URL when exposing ntfy via tunnel. - **`GRAFANA_JWT_AUTH=true`**: Requires both `CF_ACCESS_TEAM_DOMAIN` and `CF_ACCESS_AUD` set. - **File permissions**: Refuses to start if `.env`, `infra/terraform.tfvars`, or Terraform state files are readable by other users (mode must be `0600`). -- **`HEARTBEAT_URL`**: Emits a warning if unset (the stack runs without a dead-man's switch). +- **`HEARTBEAT_URL`**: Cannot be empty when exposing the stack; the relay needs a heartbeat destination. ### Verification commands | Command | Action | | --- | --- | -| `just lint` | Runs static checks (compose, alert templates, YAML, workflows, shell, Python, JSON). | +| `just lint` | Runs static checks (Compose, alert templates, YAML, workflows, shell, Python, JSON), plus secret-scan and browser RUM regression tests. | | `just validate` | Validates each service config with its respective tool container. | | `just check` | Runs `just lint` and `just validate`. | | `just smoke` | Boots an isolated copy of the stack and asserts ingestion, storage, and query APIs end-to-end. | @@ -391,8 +428,10 @@ one manual step: it bumps the constraint in `main.tf` but not the hashes in account. Only a plan proves the provider still maps the config to the same resources. -Nothing watches the tool images pinned in the `justfile` (yamllint, -actionlint, shellcheck, ruff, gitleaks, OpenTofu, jq). Bump those by hand. +Nothing watches the tool images pinned in the `justfile` (yamllint, yamlfmt, +actionlint, shellcheck, ruff, gitleaks, OpenTofu, jq, Node). Bump those by hand. +Node runs the browser RUM regression tests inside a container; contributors +do not need a local Node installation. The backup recipes reuse the alpine image of `otel-queue-init` in `compose.yml`. diff --git a/infra/generate-imports.sh b/infra/generate-imports.sh index f375aaa..c35582f 100755 --- a/infra/generate-imports.sh +++ b/infra/generate-imports.sh @@ -79,7 +79,10 @@ otel_record="$(lookup_dns_record "otel.$domain")" || { die "no CNAME found for otel.$domain" } -ntfy_record="$(lookup_dns_record "ntfy.$domain" || true)" +ntfy_record="$(lookup_dns_record "ntfy.$domain")" || { + [[ $? -ne 2 ]] || die "DNS lookup for ntfy.$domain failed (token lacks DNS:Read?)" + ntfy_record="" +} # The Access app may legitimately not exist; then the apply should create it. access_apps="$(api "accounts/$account/access/apps?per_page=100")" diff --git a/justfile b/justfile index 02f8b29..cf08fa4 100644 --- a/justfile +++ b/justfile @@ -12,7 +12,8 @@ shellcheck := "koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f ruff := "ghcr.io/astral-sh/ruff:0.14.2@sha256:636e27f3feb43800e44b0ad48c72811b500a2c6309d094b641a9bf2247f4dbff" tofu := "ghcr.io/opentofu/opentofu:1.12.3@sha256:a0766d12f07b43e66f2ed40d7a8babe97d581d20339c68ad0ab561737af9a5b3" gitleaks := "zricethezav/gitleaks:v8.30.1@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f" -lint_images := jq + " " + yamllint + " " + yamlfmt + " " + actionlint + " " + shellcheck + " " + ruff + " " + tofu + " " + gitleaks +node := "node:24-alpine@sha256:116b800c3754b440ee0aa3ed2f20f5bf3d99a554bfd9adeea09ea8017f6e85f5" +lint_images := jq + " " + yamllint + " " + yamlfmt + " " + actionlint + " " + shellcheck + " " + ruff + " " + tofu + " " + gitleaks + " " + node # `demo` and `smoke` each run a throwaway copy of the core stack under their own # compose project and Grafana port (compose.sandbox.yml), so neither can touch a @@ -64,9 +65,9 @@ _expose-guards: @case "${GRAFANA_ROOT_URL:-}" in https://*) ;; *) echo "error: GRAFANA_ROOT_URL must be the https:// tunnel hostname (got '${GRAFANA_ROOT_URL:-}'); every absolute URL Grafana generates comes from it" >&2; exit 1;; esac @[ "${GRAFANA_COOKIE_SECURE:-false}" = "true" ] || { echo "error: GRAFANA_COOKIE_SECURE must be true when Grafana is served over HTTPS; set it in .env" >&2; exit 1; } @[ "${GRAFANA_JWT_AUTH:-false}" != "true" ] || { [ -n "${CF_ACCESS_TEAM_DOMAIN:-}" ] && [ -n "${CF_ACCESS_AUD:-}" ]; } || { echo "error: GRAFANA_JWT_AUTH=true needs CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD in .env (cd infra && tofu output -raw grafana_access_team_domain grafana_access_aud)" >&2; exit 1; } - @[ -n "${HEARTBEAT_URL:-}" ] || echo "WARNING: HEARTBEAT_URL is empty; the stack goes live without a dead-man's switch" >&2 + @[ -n "${HEARTBEAT_URL:-}" ] || { echo "error: HEARTBEAT_URL is empty; the notification relay needs the dead man's switch URL in .env" >&2; exit 1; } @for f in .env infra/terraform.tfvars infra/terraform.tfstate infra/terraform.tfstate.backup; do [ ! -e "$f" ] || case "$(stat -c %a "$f")" in *00) ;; *) echo "error: $f is readable by other users (mode $(stat -c %a "$f")); it holds live secrets, run: chmod 600 $f" >&2; exit 1;; esac; done - @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; every alert would fire into an empty webhook URL and be dropped. The heartbeat keeps pinging either way, so this failure looks healthy from the outside. Set it, or comment out this guard" >&2; exit 1; } + @[ -n "${ALERT_WEBHOOK_URL:-}" ] || { echo "error: ALERT_WEBHOOK_URL is empty; the notification relay needs the alert destination in .env" >&2; exit 1; } @case "${NTFY_BASE_URL:-}" in ""|https://*) ;; *) echo "error: NTFY_BASE_URL must be an https:// URL when exposing via tunnel (got '${NTFY_BASE_URL:-}')" >&2; exit 1;; esac # Core stack plus a demo telemetry source, isolated from any running stack (:3002). @@ -89,7 +90,7 @@ restart service: _guard-if-exposed # Validate every config in the repo. check: lint validate -# Static checks in tool images (~280 MB cold, seconds warm). +# Static checks and regression tests in tool images. lint: # Digest-pinned, so anything already local is current; only fetch what is missing. @printf '%s\n' {{lint_images}} | xargs -P 8 -I{} sh -c 'docker image inspect {} >/dev/null 2>&1 || docker pull -q {} >/dev/null' @@ -108,7 +109,7 @@ lint: # documented default is refused on its own. @good="OTLP_AUTH_TOKEN=t GRAFANA_ADMIN_PASSWORD=p GRAFANA_ROOT_URL=https://g.example GRAFANA_COOKIE_SECURE=true GRAFANA_JWT_AUTH=true CF_ACCESS_TEAM_DOMAIN=d CF_ACCESS_AUD=a HEARTBEAT_URL=https://h ALERT_WEBHOOK_URL=https://w"; \ env $good just _expose-guards || { echo "error: exposure guards rejected a fully set environment" >&2; exit 1; }; \ - for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL= NTFY_BASE_URL=http://insecure; do \ + for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= HEARTBEAT_URL= ALERT_WEBHOOK_URL= NTFY_BASE_URL=http://insecure; do \ ! env $good $bad just _expose-guards 2>/dev/null || { echo "error: exposure guards accepted $bad" >&2; exit 1; }; \ done # run_scheduled.sh, interrupted mid-job, must take the job down with it, @@ -133,9 +134,18 @@ lint: docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/projects.yaml]}' . @d=$(mktemp -d) && BOOTSTRAP_OUT_DIR="$d" ./bootstrap.sh dummy dummy >/dev/null && docker run --rm --network none -v "$d":/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: disable}}' .; rc=$?; rm -rf "$d"; exit $rc docker run --rm --network none -v .:/repo:ro -w /repo {{actionlint}} -color - docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh templates/run_scheduled.sh infra/generate-imports.sh + docker run --rm --network none -v .:/mnt:ro {{shellcheck}} bootstrap.sh scripts/smoke.sh scripts/restore-check.sh scripts/test_backup.sh scripts/test_gitleaks.sh scripts/test_imports.sh templates/run_scheduled.sh infra/generate-imports.sh + sh scripts/test_gitleaks.sh + bash scripts/test_imports.sh + sh scripts/test_backup.sh + # Exercise the provisioned expression, including counters first seen after a failure. + @d=$(mktemp -d) && expr=$(awk '/uid: watchdog-heartbeat/{watchdog=1} watchdog && /expr: >/{expr=1;next} expr && /^ /{sub(/^ /, "");printf "%s ",$0;next} expr{exit}' config/grafana/alerting/rules.yaml) && test -n "$expr" && jq --arg expr "$expr" '.tests[].promql_expr_test[].expr = $expr' tests/watchdog.json >"$d/watchdog.json" && chmod 755 "$d" && chmod 644 "$d/watchdog.json" && docker run --rm --network none --entrypoint /bin/promtool -v "$d":/tests:ro $(just _image compose.yml prometheus) test rules /tests/watchdog.json; rc=$?; rm -rf "$d"; exit $rc docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo + docker run --rm --network none -v ./notification-relay:/relay:ro {{ruff}} check --no-cache /relay + docker run --rm --network none -v ./notification-relay:/relay:ro {{ruff}} format --check --no-cache /relay + docker run --rm --network none -v ./notification-relay:/relay:ro -w /relay $(just _image compose.yml notification-relay) python -B -m unittest discover + docker run --rm --network none -v ./templates/web:/rum:ro {{node}} node --test --test-isolation=none /rum/rum.test.mjs # `just fmt` is the fix. docker run --rm --network none -v .:/code:ro -w /code {{yamlfmt}} -lint . docker run --rm --network none -v ./infra:/infra:ro -w /infra {{tofu}} fmt -check @@ -146,7 +156,7 @@ lint: [ -z "$bad" ] || { echo "error: dashboards reference datasource uids that are not provisioned:" $bad >&2; exit 1; } # Secrets in git history. Scans commits, not the working tree, so the # gitignored .env never trips it. Needs full history (see ci.yml). - docker run --rm --network none -v .:/repo:ro {{gitleaks}} git --redact --no-banner /repo + just _gitleaks # The agent config every project host vendors, checked by the Alloy build # the template actually pins. docker run --rm --network none -v ./templates/alloy/config.alloy:/etc/alloy/config.alloy:ro -e COMPOSE_PROJECT_NAME=dummy -e ENVIRONMENT=dummy -e PROJECT=dummy -e OTEL_EXPORTER_OTLP_ENDPOINT=https://dummy -e OTLP_AUTH_TOKEN=dummy -e TELEMETRY_EDGE_KEY= $({{spoke_env}} just _image templates/compose.telemetry.yml alloy) validate /etc/alloy/config.alloy @@ -184,7 +194,11 @@ infra-validate: # gitleaks over the staged diff (the pre-commit hook; see .pre-commit-config.yaml). _gitleaks-staged: - @docker run --rm --network none -v .:/repo:ro {{gitleaks}} git --pre-commit --staged --redact --no-banner /repo + @just _gitleaks --pre-commit --staged + +_gitleaks *args: + # Linked worktrees point outside /repo at their shared Git metadata. + @gitdir=$(git rev-parse --path-format=absolute --git-common-dir) && docker run --rm --network none -v .:/repo:ro -v "$gitdir:$gitdir:ro" -e GIT_WORK_TREE=/repo --entrypoint sh {{gitleaks}} -c 'git -C /repo rev-parse --verify HEAD >/dev/null && exec gitleaks git --redact --no-banner "$@" /repo' -- {{args}} # Format YAML in place (--user so the rewritten files stay yours). fmt: @@ -206,7 +220,7 @@ _mounts project: _backup project dir: mkdir -p {{dir}} @-docker compose -p {{project}} unpause {{stateful}} >/dev/null 2>&1 - @rc=0; img=$(just _image compose.yml otel-queue-init); m=$(just _mounts {{project}}); docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups "$img" sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz' || rc=$?; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack is still paused" >&2; rc=1; }; exit $rc + @trap 'rc=$?; trap - EXIT INT TERM; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack may still be paused" >&2; [ "$rc" -ne 0 ] || rc=1; }; exit "$rc"' EXIT; trap 'exit 130' INT; trap 'exit 143' TERM; img=$(just _image compose.yml otel-queue-init) && m=$(just _mounts {{project}}) && docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups "$img" sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz' @just _newest-backup {{dir}} # pre-restore-* snapshots share the dir and sort after monitoring-* by name. @@ -231,8 +245,8 @@ _restore project file dir: # Needs a booted smoke stack (`just smoke`). Run it after touching _backup/_restore. # COMPOSE_FILE is pinned so the host's overlay list stays out, as for `smoke`. # Round-trip backup and restore on the smoke stack. -restore-check: - @export COMPOSE_FILE=compose.yml:compose.sandbox.yml; d=$(mktemp -d) && just _backup {{smoke_project}} "$d" && f=$(ls "$d"/monitoring-*.tar.gz) && just _restore {{smoke_project}} "$f" "$d" && docker run --rm --network none -v {{smoke_project}}_grafana_data:/g:ro $(just _image compose.yml otel-queue-init) test -s /g/grafana.db && echo "Backup round-trip ok"; rc=$?; rm -rf "$d"; exit $rc +restore-check project=smoke_project port=smoke_port: + @{{smoke_env}} SANDBOX_PORT={{port}} scripts/restore-check.sh {{project}} {{stateful}} # `--wait` blocks on the healthchecks and fails if any container exits, so a # crash-looping service is caught (after the full timeout). scripts/smoke.sh diff --git a/notification-relay/relay.py b/notification-relay/relay.py new file mode 100644 index 0000000..de019ac --- /dev/null +++ b/notification-relay/relay.py @@ -0,0 +1,141 @@ +"""Forward Grafana notifications without exposing destination capabilities.""" + +import base64 +import http.client +import os +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import quote, unquote, urlsplit + +MAX_BODY = 1024 * 1024 +TIMEOUT = 5 + + +def validate_target(url): + """Reject invalid configured destinations without including secrets in errors.""" + try: + parts = urlsplit(url) + if ( + parts.scheme not in ("http", "https") + or not parts.hostname + or parts.fragment + ): + raise ValueError + _ = parts.port + if any(ord(char) < 32 or ord(char) == 127 for char in url): + raise ValueError + except ValueError: + raise ValueError("invalid notification destination") from None + + +class RelayServer(HTTPServer): + """Keep destinations outside Grafana's environment and contact point store.""" + + def __init__(self, address, targets): + self.targets = targets + super().__init__(address, Handler) + + def handle_error(self, request, client_address): + # Tracebacks from HTTP libraries may contain credential-bearing URLs. + print("notification request failed", flush=True) + + +class Handler(BaseHTTPRequestHandler): + """Accept only bounded POST bodies on two fixed routes.""" + + def setup(self): + super().setup() + self.connection.settimeout(TIMEOUT) + + def log_message(self, format, *args): + # Request paths and headers are untrusted and may contain credentials. + pass + + def reply(self, status): + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self): + self.reply(204 if self.path == "/health" else 404) + + def do_POST(self): + assert isinstance(self.server, RelayServer) + if self.path not in ("/alerts", "/heartbeat"): + self.reply(404) + return + url = self.server.targets.get(self.path, "") + if not url: + self.reply(503) + return + lengths = self.headers.get_all("Content-Length", []) + if ( + self.headers.get("Transfer-Encoding") + or len(lengths) != 1 + or not lengths[0].isdigit() + ): + self.reply(400) + return + try: + length = int(lengths[0]) + except ValueError: + self.reply(400) + return + if length > MAX_BODY: + self.reply(413) + return + try: + body = self.rfile.read(length) + if len(body) != length: + self.reply(400) + return + parts = urlsplit(url) + connection_type = ( + http.client.HTTPSConnection + if parts.scheme == "https" + else http.client.HTTPConnection + ) + connection = connection_type(parts.hostname, parts.port, timeout=TIMEOUT) + try: + headers = {"Content-Type": "application/json"} + if parts.username is not None: + credentials = ( + f"{unquote(parts.username)}:{unquote(parts.password or '')}" + ) + headers["Authorization"] = ( + "Basic " + base64.b64encode(credentials.encode()).decode() + ) + path = parts.path or "/" + if parts.query: + path += "?" + parts.query + # Redirects must not move notification data or credentials elsewhere. + connection.request( + "POST", quote(path, safe="/%?&=:+,@!$'()*;-._~"), body, headers + ) + status = connection.getresponse().status + finally: + connection.close() + self.reply(204 if 200 <= status < 300 else 502) + except (OSError, http.client.HTTPException, ValueError): + self.reply(502) + + +def make_server(address, targets): + """Validate fixed destinations before accepting any requests.""" + for target in targets.values(): + if target: + validate_target(target) + return RelayServer(address, targets) + + +if __name__ == "__main__": + try: + server = make_server( + ("0.0.0.0", 8080), + { + "/alerts": os.environ.get("ALERT_WEBHOOK_URL", ""), + "/heartbeat": os.environ.get("HEARTBEAT_URL", ""), + }, + ) + except ValueError: + raise SystemExit("invalid notification destination") from None + server.serve_forever() diff --git a/notification-relay/test_relay.py b/notification-relay/test_relay.py new file mode 100644 index 0000000..100b12c --- /dev/null +++ b/notification-relay/test_relay.py @@ -0,0 +1,154 @@ +"""Exercise the fixed-route delivery boundary with local HTTP endpoints.""" + +import base64 +import http.client +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import ClassVar +from unittest.mock import patch + +import relay + + +class Destination(BaseHTTPRequestHandler): + status = 204 + delay = 0.0 + received: ClassVar[list] = [] + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + self.received.append((self.path, body, self.headers.get("Authorization"))) + time.sleep(self.delay) + self.send_response(self.status) + self.send_header("Location", "/must-not-follow") + self.end_headers() + + def log_message(self, format, *args): + pass + + +class RelayTest(unittest.TestCase): + def setUp(self): + Destination.received = [] + Destination.status = 204 + Destination.delay = 0 + self.destination = HTTPServer(("127.0.0.1", 0), Destination) + self.addCleanup(self.destination.server_close) + threading.Thread(target=self.destination.serve_forever, daemon=True).start() + self.addCleanup(self.destination.shutdown) + port = self.destination.server_port + self.server = relay.make_server( + ("127.0.0.1", 0), + { + "/alerts": f"http://publisher:secret@127.0.0.1:{port}/topic?tpl=1", + "/heartbeat": f"http://127.0.0.1:{port}/ping", + }, + ) + self.addCleanup(self.server.server_close) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.addCleanup(self.server.shutdown) + + def request(self, path, body=b"{}", headers=None): + conn = http.client.HTTPConnection( + "127.0.0.1", self.server.server_port, timeout=3 + ) + self.addCleanup(conn.close) + conn.request("POST", path, body, headers or {}) + response = conn.getresponse() + self.assertEqual(response.read(), b"") + return response.status + + def test_delivers_only_fixed_routes_and_keeps_auth_out_of_response(self): + self.assertEqual(self.request("/alerts"), 204) + self.assertEqual(self.request("/heartbeat"), 204) + self.assertEqual( + Destination.received, + [ + ( + "/topic?tpl=1", + b"{}", + "Basic " + base64.b64encode(b"publisher:secret").decode(), + ), + ("/ping", b"{}", None), + ], + ) + self.assertEqual(self.request("/alerts?url=http://elsewhere.invalid"), 404) + self.assertEqual(len(Destination.received), 2) + + def test_encodes_native_ntfy_template_query(self): + self.server.targets["/alerts"] += ( + '&message={{or .commonAnnotations.summary "Alert firing"}}' + ) + self.assertEqual(self.request("/alerts"), 204) + self.assertIn( + "message=%7B%7Bor%20.commonAnnotations.summary%20%22Alert%20firing%22%7D%7D", + Destination.received[0][0], + ) + + def test_upstream_failures_and_redirects_fail_closed(self): + for status in (302, 400, 500): + Destination.status = status + self.assertEqual(self.request("/alerts"), 502) + self.assertEqual(len(Destination.received), 3) + + def test_rejects_oversize_chunked_and_ambiguous_body(self): + self.assertEqual( + self.request( + "/alerts", headers={"Content-Length": str(relay.MAX_BODY + 1)} + ), + 413, + ) + self.assertEqual( + self.request("/alerts", headers={"Transfer-Encoding": "chunked"}), 400 + ) + self.assertEqual(Destination.received, []) + + def test_rejects_invalid_content_lengths(self): + for length in ("-1", "bad", "²", "9" * 5000): + self.assertEqual( + self.request("/alerts", headers={"Content-Length": length}), 400 + ) + self.assertEqual(Destination.received, []) + + def test_upstream_timeout_is_generic_failure(self): + Destination.delay = 0.15 + with patch.object(relay, "TIMEOUT", 0.05): + self.assertEqual(self.request("/alerts"), 502) + + def test_rejects_duplicate_content_lengths(self): + conn = http.client.HTTPConnection( + "127.0.0.1", self.server.server_port, timeout=3 + ) + self.addCleanup(conn.close) + conn.putrequest("POST", "/alerts") + conn.putheader("Content-Length", "0") + conn.putheader("Content-Length", "1") + conn.endheaders() + self.assertEqual(conn.getresponse().status, 400) + self.assertEqual(Destination.received, []) + + def test_unconfigured_route_reports_failure(self): + self.server.targets["/heartbeat"] = "" + self.assertEqual(self.request("/heartbeat"), 503) + + def test_connection_failure_is_generic(self): + self.server.targets["/alerts"] = "http://127.0.0.1:1/secret" + self.assertEqual(self.request("/alerts"), 502) + + def test_invalid_destinations_rejected_without_echoing_credentials(self): + for url in ( + "ftp://secret@host/path", + "http://host:bad/path", + "http:///path", + "https://host/path#secret", + ): + with self.assertRaisesRegex( + ValueError, "^invalid notification destination$" + ): + relay.make_server(("127.0.0.1", 0), {"/alerts": url}) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/restore-check.sh b/scripts/restore-check.sh new file mode 100755 index 0000000..37f18d4 --- /dev/null +++ b/scripts/restore-check.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# The caller supplies a booted disposable project; never use production volumes. +set -euo pipefail +cd "$(dirname "$0")/.." +export COMPOSE_FILE=compose.yml:compose.sandbox.yml +project=$1 +shift +img=$(just _image compose.yml otel-queue-init) +d=$(mktemp -d "${TMPDIR:-/tmp}/monitoring-restore-check.XXXXXX") +marker=${d##*/} +mounts=() +for service in "$@"; do + mounts+=(-v "${project}_${service}_data:/data/$service") +done +cleanup() { + rc=$? + trap - EXIT + docker run --rm --network none "${mounts[@]}" "$img" sh -c 'for volume in /data/*; do rm -f "$volume/$1" "$volume/$1-later"; done' _ "$marker" || { [ "$rc" -ne 0 ] || rc=1; } + rm -rf "$d" + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +# Different contents in each volume catch missing or crossed mounts. +docker run --rm --network none "${mounts[@]}" "$img" sh -ec 'for volume in /data/*; do printf "%s-before\n" "${volume##*/}" >"$volume/$1"; done' _ "$marker" +just _backup "$project" "$d" +f=$(find "$d" -name 'monitoring-*.tar.gz') +docker run --rm --network none "${mounts[@]}" "$img" sh -ec 'for volume in /data/*; do printf "%s-after\n" "${volume##*/}" >"$volume/$1"; touch "$volume/$1-later"; done' _ "$marker" +just _restore "$project" "$f" "$d" +docker run --rm --network none "${mounts[@]}" "$img" sh -ec ' +for volume in /data/*; do + [ "$(cat "$volume/$1")" = "${volume##*/}-before" ] && [ ! -e "$volume/$1-later" ] || { echo "error: ${volume##*/} was not restored" >&2; exit 1; } +done' _ "$marker" +docker compose -p "$project" up -d --wait --wait-timeout 120 +docker compose -p "$project" exec -T grafana wget -qO- http://localhost:3000/api/health | grep -Eq '"database":[[:space:]]*"ok"' +echo 'Backup round-trip ok: all five volumes restored; Grafana database healthy' diff --git a/scripts/smoke.sh b/scripts/smoke.sh index bf0f0f2..924a041 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -41,12 +41,11 @@ jq -e 'all(.isPaused | not)' <<<"$rules" >/dev/null \ || die "paused alert rules never fire: $(jq -r '.[] | select(.isPaused) | .uid' <<<"$rules" | tr '\n' ' ')" # --------------------------------------------------------------- contact points -# The exact URLs the stack was given must come back: an empty value would -# pass a "not literally $ALERT_WEBHOOK_URL" test and drop every alert. +# External notification credentials belong to the relay, outside Grafana. gf "$url/api/v1/provisioning/contact-points" \ - | jq -e --arg a "$ALERT_WEBHOOK_URL" --arg h "$HEARTBEAT_URL" \ + | jq -e --arg a "http://notification-relay:8080/alerts" --arg h "http://notification-relay:8080/heartbeat" \ '[.[] | select(.uid | startswith("cp-")) | .settings.url] | sort == ([$a, $h] | sort)' >/dev/null \ - || die "contact points do not carry ALERT_WEBHOOK_URL and HEARTBEAT_URL; Grafana did not expand the provisioning file, or a receiver is missing" + || die "contact points do not target the internal notification relay" # ------------------------------------------------------------------- JWT auth # Grafana ignores an env key it no longer knows, so read the parsed setting @@ -73,10 +72,23 @@ if [[ "${SMOKE_ALERTS:-}" == 1 ]]; then fi # ------------------------------------------------------------- data paths -# One metric and one log through the collector's bearer auth, read back with +# One metric, log and trace through the collector's bearer auth, read back with # the identity labels the alert rules key on. Posted from inside the stack's # network with Grafana's curl: the sandbox overlay publishes no ingestion ports. -ts="$(date +%s)000000000" +run_start="$(date +%s)" +ts="${run_start}000000000" +# A fresh value/body distinguishes this run without adding unbounded labels. +smoke_value="$(od -An -N6 -tu8 /dev/urandom | tr -d ' ')" +smoke_body="smoke $smoke_value" +trace_id="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" +span_id="$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" +# Tempo returns OTLP JSON byte IDs as base64, unlike the hex lookup URL. +# shellcheck disable=SC2001 # Convert hex byte pairs, rather than literal text. +trace_b64="$(printf '%b' "$(sed 's/../\\x&/g' <<<"$trace_id")" | base64 | tr -d '\n')" +# shellcheck disable=SC2001 # Convert hex byte pairs, rather than literal text. +span_b64="$(printf '%b' "$(sed 's/../\\x&/g' <<<"$span_id")" | base64 | tr -d '\n')" +counts_query='sum by (__name__) ({__name__=~"telemetry_(datapoints|logs|spans)_total",project="smoke",env="ci"})' +counts_before="$(promq "$counts_query" | jq '[.data.result[] | {key: .metric.__name__, value: (.value[1] | tonumber)}] | from_entries')" res='{"attributes":[{"key":"service.name","value":{"stringValue":"smoke"}},{"key":"project","value":{"stringValue":"smoke"}},{"key":"env","value":{"stringValue":"ci"}}]}' # Token through curl's stdin config: argv is world-readable in ps, on the host # and inside the container alike. @@ -85,23 +97,40 @@ otlp() { 'curl -sf -K - -o /dev/null -H "Content-Type: application/json" --data-binary "$1" "http://otel-collector:4318/v1/$2"' \ _ "$2" "$1" <<<"header = \"Authorization: Bearer $(cfg_quote "$OTLP_AUTH_TOKEN")\"" } -otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"1","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \ +otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"'"$smoke_value"'","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \ || die "the collector refused an OTLP metric with the .env token" -otlp logs '{"resourceLogs":[{"resource":'"$res"',"scopeLogs":[{"logRecords":[{"timeUnixNano":"'"$ts"'","body":{"stringValue":"smoke"}}]}]}]}' \ +otlp logs '{"resourceLogs":[{"resource":'"$res"',"scopeLogs":[{"logRecords":[{"timeUnixNano":"'"$ts"'","body":{"stringValue":"'"$smoke_body"'"}}]}]}]}' \ || die "the collector refused an OTLP log with the .env token" -smoke_metric() { promq "smoke_up{job=\"smoke\",project=\"smoke\",env=\"ci\",department=\"cml\"}" | jq -e '.data.result | length > 0' >/dev/null; } +otlp traces '{"resourceSpans":[{"resource":'"$res"',"scopeSpans":[{"spans":[{"traceId":"'"$trace_id"'","spanId":"'"$span_id"'","name":"smoke","kind":1,"startTimeUnixNano":"'"$ts"'","endTimeUnixNano":"'"$((run_start + 1))000000000"'"}]}]}]}' \ + || die "the collector refused an OTLP trace with the .env token" +smoke_metric() { promq "timestamp(smoke_up{job=\"smoke\",project=\"smoke\",env=\"ci\",department=\"cml\"} == $smoke_value) >= $run_start" | jq -e '.data.result | length > 0' >/dev/null; } smoke_log() { gf -G --data-urlencode "query={project=\"smoke\",env=\"ci\",department=\"cml\"}" \ + --data-urlencode "start=$ts" --data-urlencode "end=$(date +%s)000000000" \ "$url/api/datasources/proxy/uid/loki/loki/api/v1/query_range" \ - | jq -e 'any(.data.result[].values[][1]; . == "smoke")' >/dev/null + | jq -e --arg body "$smoke_body" 'any(.data.result[].values[][1]; . == $body)' >/dev/null +} +smoke_trace() { + gf -H 'Accept: application/json' "$url/api/datasources/proxy/uid/tempo/api/traces/$trace_id" \ + | jq -e --arg trace "$trace_b64" --arg span "$span_b64" ' + any(.batches[]; + (.resource.attributes | map({key: .key, value: .value.stringValue}) | from_entries) as $a + | $a.project == "smoke" and $a.env == "ci" and $a.department == "cml" + and any(.scopeSpans[].spans[]; .traceId == $trace and .spanId == $span and .name == "smoke"))' >/dev/null } poll smoke_metric || die "the smoke metric never reached Prometheus with its project/env/department labels; see just smoke-logs" poll smoke_log || die "the smoke log never reached Loki with its department label; see just smoke-logs" +poll smoke_trace || die "the smoke trace never reached Tempo with its trace/span identity and project/env/department attributes; see just smoke-logs" # The keystone alert (ProjectTelemetrySilent) and the coverage backstop key on the # gateway's ingest counters, so a renamed metric or a dropped attribute would # silence both without any other assertion noticing. -counted() { promq "count by (project, env) ({__name__=~\"telemetry_.+_total\", project=\"smoke\", env=\"ci\"})" | jq -e '.data.result | length > 0' >/dev/null; } -poll counted || die "the count connector never produced telemetry_*_total{project=smoke,env=ci}; the alert rules that key on it would never fire" +counted() { + promq "$counts_query" | jq -e --argjson before "$counts_before" ' + [.data.result[] | {key: .metric.__name__, value: (.value[1] | tonumber)}] | from_entries + | . as $after | ["telemetry_datapoints_total", "telemetry_logs_total", "telemetry_spans_total"] + | all(.[]; . as $name | ($after[$name] // 0) > ($before[$name] // 0))' >/dev/null +} +poll counted || die "the count connector never advanced all three telemetry signal counters for this smoke run; the alert rules that key on it would never fire" # ------------------------------------------------------------- ntfy & blackbox # Verify ntfy health endpoint answers over the docker network. @@ -129,7 +158,16 @@ if [[ "${SMOKE_ALERTS:-}" == 1 ]]; then } POLL_MAX=60 POLL_SLEEP=5 poll target_down_firing || die "TargetDown did not fire within 5 minutes of stopping node-exporter" docker compose -p "$project" start node-exporter >/dev/null - echo "TargetDown fired for the stopped node-exporter" + node_recovered() { + promq 'up{job="node"}' | jq -e '.data.result | length > 0 and all(.[]; .value[1] == "1")' >/dev/null + } + target_down_normal() { + gf "$url/api/prometheus/grafana/api/v1/rules" \ + | jq -e '.data.groups[].rules[] | select(.name == "TargetDown") | .state == "inactive"' >/dev/null + } + poll node_recovered || die "node-exporter did not resume scraping within a minute of restart" + POLL_MAX=60 POLL_SLEEP=5 poll target_down_normal || die "TargetDown did not return to normal within 5 minutes of restarting node-exporter" + echo "TargetDown fired, node-exporter resumed scraping, and TargetDown returned to normal" fi echo "Stack healthy" diff --git a/scripts/test_backup.sh b/scripts/test_backup.sh new file mode 100644 index 0000000..5ddeedd --- /dev/null +++ b/scripts/test_backup.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Exercise the real recipe without pausing a Docker stack. +set -eu +cd "$(dirname "$0")/.." +d=$(mktemp -d "${TMPDIR:-/tmp}/monitoring-backup-test.XXXXXX") +trap 'rm -rf "$d"' EXIT +cat >"$d/docker" <<'DOCKER' +#!/bin/sh +case "$*" in + *'config --format json'*) echo '{}' ;; + *' -er '*) echo archive-image ;; + *' pause '*) echo pause >>"$TEST_EVENTS"; [ "$TEST_CASE" != pause-failure ] || exit 6 ;; + *' unpause '*) + echo unpause >>"$TEST_EVENTS" + if { [ "$TEST_CASE" = unpause-failure ] || [ "$TEST_CASE" = both-fail ]; } && grep -qx pause "$TEST_EVENTS"; then exit 9; fi ;; + *'run --rm --network none'*) + touch "$(dirname "$TEST_EVENTS")/monitoring-test.tar.gz" + case "$TEST_CASE" in + TERM) kill -TERM "$PPID" ;; + INT) kill -INT "$PPID" ;; + archive-failure|both-fail) exit 7 ;; + esac ;; + *) echo "unexpected docker call" >&2; exit 1 ;; +esac +DOCKER +chmod +x "$d/docker" +for scenario in success TERM INT pause-failure archive-failure unpause-failure both-fail; do + : >"$d/events" + case "$scenario" in success) expected=0 ;; TERM) expected=143 ;; INT) expected=130 ;; pause-failure) expected=6 ;; archive-failure | both-fail) expected=7 ;; *) expected=1 ;; esac + rc=0 + env -i PATH="$d:$PATH" HOME="$HOME" TEST_EVENTS="$d/events" TEST_CASE="$scenario" just _backup test-backup "$d" >"$d/output" 2>&1 || rc=$? + # just normalizes signal exits to 128+signal, and propagates recipe failures. + [ "$rc" -eq "$expected" ] || { + cat "$d/output" >&2 + echo "wrong exit for $scenario: $rc" >&2 + exit 1 + } + [ "$(tail -1 "$d/events")" = unpause ] || { + echo "backup left services paused after $scenario" >&2 + exit 1 + } + echo "backup $scenario: cleanup attempted, exit $rc" +done diff --git a/scripts/test_gitleaks.sh b/scripts/test_gitleaks.sh new file mode 100644 index 0000000..9a5c91f --- /dev/null +++ b/scripts/test_gitleaks.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +# Git hooks export index and worktree paths; scratch repositories must not inherit them. +for name in $(git rev-parse --local-env-vars); do unset "$name"; done +root=$(pwd) +d=$(mktemp -d "${TMPDIR:-/tmp}/monitoring-gitleaks.XXXXXX") +trap 'rm -rf "$d"' EXIT HUP INT TERM + +git -c core.hooksPath=/dev/null init -q "$d/main" +cd "$d/main" +cat >.gitleaks.toml <<'EOF' +[[rules]] +id = "regression-fixture" +description = "Synthetic regression credential" +regex = '''fixture-secret-[0-9]{8}''' +EOF +git add .gitleaks.toml +git -c core.hooksPath=/dev/null -c commit.gpgsign=false -c user.name=Test -c user.email=test@example.org commit -qm initial +git -c core.hooksPath=/dev/null worktree add -q "$d/linked" +cp "$root/justfile" "$d/linked/justfile" +cd "$d/linked" +mkdir dashboards +printf '{}\n' >dashboards/fixture.json + +expect_leak() { + rc=0 + just "$@" >"$d/output" 2>&1 || rc=$? + # just preserves the failing recipe status in its diagnostic, but exits 1 itself. + if [ "$rc" -ne 1 ] || ! grep -q 'leaks found: 1' "$d/output"; then + cat "$d/output" >&2 + echo "error: $* did not detect the synthetic credential" >&2 + exit 1 + fi +} + +just _gitleaks +printf 'fixture-secret-%s\n' 12345678 >credential.txt +git add credential.txt +expect_leak _gitleaks-staged +git -c core.hooksPath=/dev/null -c commit.gpgsign=false -c user.name=Test -c user.email=test@example.org commit -qm fixture +expect_leak _gitleaks +printf 'gitdir: /missing-git-dir\n' >.git +if just _gitleaks >"$d/output" 2>&1; then + echo "error: a broken Git directory passed the scan" >&2 + exit 1 +fi +echo "gitleaks detects staged and committed credentials in linked worktrees and rejects broken Git directories" diff --git a/scripts/test_imports.sh b/scripts/test_imports.sh new file mode 100644 index 0000000..45087ad --- /dev/null +++ b/scripts/test_imports.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +scratch="$(mktemp -d "${TMPDIR:-/tmp}/monitoring-imports.XXXXXX")" +trap 'rm -rf "$scratch"' EXIT +mkdir "$scratch/bin" "$scratch/infra" +cp "$root/infra/generate-imports.sh" "$scratch/infra/" +cat >"$scratch/infra/terraform.tfvars" <<'EOF' +account_id = "fixture-account" +zone_id = "fixture-zone" +domain = "fixture.example" +EOF +cat >"$scratch/bin/curl" <<'EOF' +#!/usr/bin/env bash +case "${*: -1}" in + *cfd_tunnel*) echo '{"result":[{"name":"cml-monitoring","id":"tunnel"}]}' ;; + *name=ntfy.*) + [[ "$NTFY_FIXTURE" != failure ]] || exit 22 + if [[ "$NTFY_FIXTURE" == absent ]]; then echo '{"result":[]}' + else echo '{"result":[{"id":"ntfy-record"}]}' + fi ;; + *dns_records*) echo '{"result":[{"id":"dns-record"}]}' ;; + *) echo '{"result":[]}' ;; +esac +EOF +chmod +x "$scratch/bin/curl" +for fixture in present absent failure; do + rc=0 + env -i PATH="$scratch/bin:$PATH" CLOUDFLARE_API_TOKEN=fixture NTFY_FIXTURE="$fixture" \ + bash "$scratch/infra/generate-imports.sh" >"$scratch/out" 2>"$scratch/err" || rc=$? + case "$fixture" in + present) [[ "$rc" == 0 ]] && grep 'cloudflare_dns_record.ntfy' "$scratch/out" >/dev/null ;; + absent) [[ "$rc" == 0 ]] && ! grep 'cloudflare_dns_record.ntfy' "$scratch/out" >/dev/null ;; + failure) [[ "$rc" != 0 ]] && grep 'DNS lookup for ntfy.fixture.example failed' "$scratch/err" >/dev/null ;; + esac || { + cat "$scratch/err" >&2 + echo "error: ntfy DNS $fixture check failed" >&2 + exit 1 + } +done +echo 'Import lookup checks passed' diff --git a/templates/web/README.md b/templates/web/README.md index d0e61d6..b223cc1 100644 --- a/templates/web/README.md +++ b/templates/web/README.md @@ -4,9 +4,18 @@ Lightweight, zero-dependency browser telemetry for static websites, documentatio ## What it captures -1. **Core Web Vitals:** Largest Contentful Paint (LCP) and Cumulative Layout Shift (CLS). -2. **Unhandled errors:** `window.onerror` exceptions and unhandled promise rejections. -3. **Structured logs:** Shipped as standard OTLP/HTTP JSON (`/v1/logs`) to the central collector. +1. **LCP:** The latest paint candidate, exported once at first interaction or page hide. +2. **CLS:** The largest session of unexpected shifts, with gaps under one second and a span under five seconds. Exported on the first hide, including zero, and on later hides when the page maximum increases. +3. **Unhandled errors:** `window.onerror` exceptions and unhandled promise rejections. +4. **Structured logs:** Shipped as standard OTLP/HTTP JSON (`/v1/logs`) to the central collector. + +The script drains pending performance records before exporting. Showing the page +again preserves its CLS maximum. Later exports update the same page visit, +not independent observations. Unsupported layout-shift observation produces no score. It ignores LCP for pages loaded hidden and +candidates painted after the first hide. These rules follow the +[LCP](https://web.dev/articles/lcp) and [CLS](https://web.dev/articles/cls) definitions. +It does not measure visits restored from the back/forward cache, prerendered +page activation or content inside iframes. ## Integration @@ -16,11 +25,10 @@ Add your endpoint and project details before loading the snippet: ```html ``` @@ -30,6 +38,12 @@ Paste the contents of [`rum.html`](rum.html) into your layout `` or before ## Ingestion and Security +Error records contain only the event type and, when available, script origin/path, +line and column. Messages, stacks and rejection values are excluded. Script URLs +exclude credentials, queries and fragments; review paths for private values before +enabling collection. +Requests omit browser credentials and the Referer header. + - **CORS origins:** The collector HTTP receiver permits cross-origin requests from `https://cmlplatform.github.io` and Leiden University domains (`https://*.leidenuniv.nl`). -- **Credential safety:** Do not embed collector bearer secrets in public client HTML. For public apps, either omit `window.__OTEL_TOKEN__` and let an edge proxy (such as Cloudflare Workers or Nginx) inject `Authorization: Bearer ...`, or provision a scoped low-privilege token. +- **Credential safety:** Never embed the collector bearer in public HTML. The collector supports one shared bearer, without per-project scopes. An administrator must provide a restricted proxy that injects that bearer, fixes the project identity, and limits payload size and request rate. Leave RUM disabled until the proxy and collection agreement are ready. CORS alone does not provide this boundary. - **Log queries:** Logs are indexed in Loki by `service_name`, `project`, and `env`, viewable on the Grafana **Logs** dashboard. diff --git a/templates/web/rum.html b/templates/web/rum.html index 4aa1ebb..0a5c809 100644 --- a/templates/web/rum.html +++ b/templates/web/rum.html @@ -65,6 +65,8 @@ headers: headers, body: JSON.stringify(payload), keepalive: true, + referrerPolicy: "no-referrer", + credentials: "omit", mode: "cors" }).catch(function() {}); } catch (e) { @@ -76,60 +78,113 @@ // --- Core Web Vitals --- if ("PerformanceObserver" in window) { - // LCP - try { - new PerformanceObserver(function(list) { - const entries = list.getEntries(); - const lastEntry = entries[entries.length - 1]; - if (lastEntry) { - sendLog("WebVital: LCP", "INFO", { - vital_name: "LCP", - duration_ms: Math.round(lastEntry.startTime), - element: lastEntry.element ? (lastEntry.element.tagName || "element") : "unknown" - }); + let lcpObserver; + let latestLcp; + let lcpFinalized = document.visibilityState === "hidden"; + let firstHiddenTime = lcpFinalized ? 0 : Infinity; + let clsObserver; + let sessionStart = 0; + let lastShiftTime = 0; + let sessionScore = 0; + let clsScore = 0; + let reportedCls = -1; + + function updateLcp(entries) { + if (lcpFinalized) return; + for (const entry of entries) { + if (entry.startTime < firstHiddenTime) latestLcp = entry; + } + } + + function finalizeLcp() { + if (lcpFinalized) return; + if (lcpObserver) { + updateLcp(lcpObserver.takeRecords()); + lcpObserver.disconnect(); + } + lcpFinalized = true; + if (latestLcp) { + sendLog("WebVital: LCP", "INFO", { + vital_name: "LCP", + duration_ms: Math.round(latestLcp.startTime), + element: latestLcp.element ? latestLcp.element.tagName || "element" : "unknown" + }); + } + } + + function updateCls(entries) { + for (const entry of entries) { + if (entry.hadRecentInput) continue; + // A session ends at a one-second gap or a five-second total span. + if (sessionScore > 0 && entry.startTime - lastShiftTime < 1000 && entry.startTime - sessionStart < 5000) { + sessionScore += entry.value; + } else { + sessionStart = entry.startTime; + sessionScore = entry.value; } - }).observe({ type: "largest-contentful-paint", buffered: true }); - } catch (e) {} + lastShiftTime = entry.startTime; + clsScore = Math.max(clsScore, sessionScore); + } + } - // CLS try { - let clsValue = 0; - new PerformanceObserver(function(list) { - for (const entry of list.getEntries()) { - if (!entry.hadRecentInput) { - clsValue += entry.value; - } - } - }).observe({ type: "layout-shift", buffered: true }); + if (!lcpFinalized) { + lcpObserver = new PerformanceObserver((list) => updateLcp(list.getEntries())); + lcpObserver.observe({ type: "largest-contentful-paint", buffered: true }); + } + } catch { + // Some browsers do not support this performance entry type. + } + try { + if (PerformanceObserver.supportedEntryTypes?.includes("layout-shift")) { + clsObserver = new PerformanceObserver((list) => updateCls(list.getEntries())); + clsObserver.observe({ type: "layout-shift", buffered: true }); + } + } catch { + clsObserver = undefined; + // Some browsers do not support this performance entry type. + } - window.addEventListener("visibilitychange", function() { - if (document.visibilityState === "hidden" && clsValue > 0) { - sendLog("WebVital: CLS", "INFO", { - vital_name: "CLS", - cls_score: Math.round(clsValue * 1000) / 1000 - }); - clsValue = 0; - } - }); - } catch (e) {} + for (const event of ["pointerdown", "touchstart", "keydown", "scroll"]) { + window.addEventListener(event, finalizeLcp, { once: true, capture: true, passive: true }); + } + document.addEventListener("visibilitychange", (event) => { + if (document.visibilityState !== "hidden") return; + firstHiddenTime = Math.min(firstHiddenTime, event.timeStamp); + finalizeLcp(); + if (clsObserver) updateCls(clsObserver.takeRecords()); + if (clsObserver && clsScore > reportedCls) { + sendLog("WebVital: CLS", "INFO", { + vital_name: "CLS", + cls_score: Math.round(clsScore * 1000) / 1000 + }); + reportedCls = clsScore; + } + }); } // --- Uncaught Errors --- + function scriptLocation(filename) { + try { + const url = new URL(filename, window.location.origin); + return url.protocol === "https:" || url.protocol === "http:" ? url.origin + url.pathname : "unknown"; + } catch (_) { + return "unknown"; + } + } + window.addEventListener("error", function(evt) { - sendLog(evt.message || "Unhandled JavaScript Error", "ERROR", { + sendLog("Unhandled JavaScript error", "ERROR", { error_type: "uncaught_exception", - filename: evt.filename ? evt.filename.split("?")[0] : "unknown", + filename: evt.filename ? scriptLocation(evt.filename) : "unknown", lineno: evt.lineno || 0, - colno: evt.colno || 0, - stack: evt.error ? evt.error.stack || "" : "" + colno: evt.colno || 0 }); }); - window.addEventListener("unhandledrejection", function(evt) { - const reason = evt.reason; - sendLog(reason && reason.message ? reason.message : String(reason), "ERROR", { - error_type: "unhandled_promise_rejection", - stack: reason && reason.stack ? reason.stack : "" + window.addEventListener("unhandledrejection", function() { + sendLog("Unhandled promise rejection", "ERROR", { + error_type: "unhandled_promise_rejection" }); }); })(); diff --git a/templates/web/rum.test.mjs b/templates/web/rum.test.mjs new file mode 100644 index 0000000..24b6cb6 --- /dev/null +++ b/templates/web/rum.test.mjs @@ -0,0 +1,190 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import { runInNewContext } from 'node:vm'; + +const source = readFileSync(new URL('./rum.html', import.meta.url), 'utf8').split('')[0]; + +function browser(initialVisibility = 'visible', unsupportedCls = false, endpoint = 'https://rum.example.org/', initialEntries = {}) { + const logs = []; + const requests = []; + const listeners = {}; + const observers = {}; + const on = (name, handler) => (listeners[name] ??= []).push(handler); + const document = { visibilityState: initialVisibility, addEventListener: on }; + const window = { + __OTEL_ENDPOINT__: endpoint, + location: { origin: 'https://example.org', pathname: '/' }, + addEventListener: on, + }; + class PerformanceObserver { + static supportedEntryTypes = unsupportedCls === 'missing' ? undefined : unsupportedCls === 'silent' ? ['mark', 'measure'] : ['largest-contentful-paint', 'layout-shift']; + pending = []; + disconnected = false; + constructor(callback) { this.callback = callback; } + observe({ type, buffered }) { + if (unsupportedCls === 'silent' && type === 'layout-shift') return; + if (unsupportedCls === true && type === 'layout-shift') throw new Error('unsupported'); + observers[type] = this; + if (buffered) this.pending.push(...(initialEntries[type] ?? [])); + } + takeRecords() { return this.pending.splice(0); } + disconnect() { this.disconnected = true; } + } + window.PerformanceObserver = PerformanceObserver; + runInNewContext(source, { + window, document, PerformanceObserver, URL, navigator: { userAgent: 'test-browser' }, + fetch: (url, options) => { + requests.push({ url, options }); + logs.push(JSON.parse(options.body).resourceLogs[0].scopeLogs[0].logRecords[0]); + return Promise.resolve(); + }, + }); + const emit = (name, timeStamp = 10000) => listeners[name]?.forEach((handler) => handler(typeof timeStamp === 'object' ? timeStamp : { timeStamp })); + const deliver = (type, entries) => observers[type].callback({ getEntries: () => entries }); + const hide = (time = 10000) => { document.visibilityState = 'hidden'; emit('visibilitychange', time); }; + const values = (vital, key) => logs.filter((log) => log.body.stringValue === `WebVital: ${vital}`).map((log) => log.attributes.find((attr) => attr.key === key).value.doubleValue); + return { logs, requests, document, observers, emit, deliver, hide, values }; +} +const shift = (value, startTime, hadRecentInput = false) => ({ value, startTime, hadRecentInput }); + +test('vitals include buffered entries recorded before initialization', () => { + const page = browser('visible', false, 'https://rum.example.org/', { + 'largest-contentful-paint': [{ startTime: 1200 }], + 'layout-shift': [shift(0.25, 400)], + }); + page.hide(2000); + assert.deepEqual(page.values('LCP', 'duration_ms'), [1200]); + assert.deepEqual(page.values('CLS', 'cls_score'), [0.25]); +}); + +test('transport normalizes the endpoint and posts JSON with keepalive; no endpoint disables sending', () => { + const page = browser(); + page.emit('error', {}); + assert.equal(page.requests.length, 1); + const { url, options } = page.requests[0]; + assert.equal(url, 'https://rum.example.org/v1/logs'); + assert.equal(options.method, 'POST'); + assert.equal(options.keepalive, true); + assert.equal(options.referrerPolicy, 'no-referrer'); + assert.equal(options.credentials, 'omit'); + assert.deepEqual({ ...options.headers }, { 'Content-Type': 'application/json' }); + assert.equal(JSON.parse(options.body).resourceLogs.length, 1); + const disabled = browser('visible', false, ''); + disabled.emit('error', {}); + assert.equal(disabled.requests.length, 0); +}); + +test('errors collect technical metadata without messages, stacks or URL credentials', () => { + const page = browser(); + const privateValue = 'fixture-private-token'; + page.emit('error', { + message: privateValue, error: { stack: privateValue }, + filename: `https://user:${privateValue}@example.org/app.js?token=${privateValue}#${privateValue}`, + lineno: 4, colno: 2, + }); + page.emit('unhandledrejection', { reason: { message: privateValue, stack: privateValue } }); + page.emit('error', { filename: `data:text/javascript,${privateValue}` }); + assert.equal(page.logs.length, 3); + assert.equal(JSON.stringify(page.requests).includes(privateValue), false); + assert.equal(page.logs[0].attributes.find(({ key }) => key === 'filename').value.stringValue, 'https://example.org/app.js'); + assert.equal(page.logs[0].attributes.find(({ key }) => key === 'lineno').value.doubleValue, 4); +}); + +test('CLS reports the largest session instead of summing separate sessions', () => { + const page = browser(); + page.deliver('layout-shift', [shift(0.12, 100), shift(0.13, 2100)]); + page.hide(); + assert.deepEqual(page.values('CLS', 'cls_score'), [0.13]); +}); + +test('CLS sums shifts in the same session and excludes recent input', () => { + const page = browser(); + page.deliver('layout-shift', [shift(0.12, 100), shift(0.13, 900), shift(0.8, 950, true)]); + page.hide(); + assert.deepEqual(page.values('CLS', 'cls_score'), [0.25]); +}); + +test('CLS starts a new session at a one-second gap or five-second span', () => { + for (const times of [[100, 1100], [0, 900, 1800, 2700, 3600, 4500, 5000]]) { + const page = browser(); + page.deliver('layout-shift', times.map((time) => shift(0.1, time))); + page.hide(); + assert.deepEqual(page.values('CLS', 'cls_score'), [times.length === 2 ? 0.1 : 0.6]); + } +}); + +test('CLS drains queued shifts and retains the page maximum across hide/show cycles', () => { + const page = browser(); + page.observers['layout-shift'].pending.push(shift(0.3, 100)); + page.hide(200); + page.hide(200); + page.document.visibilityState = 'visible'; + page.emit('visibilitychange', 300); + page.deliver('layout-shift', [shift(0.1, 2100)]); + page.hide(2200); + page.document.visibilityState = 'visible'; + page.deliver('layout-shift', [shift(0.4, 4100)]); + page.hide(4200); + assert.deepEqual(page.values('CLS', 'cls_score'), [0.3, 0.4]); +}); + +test('LCP sends only the latest candidate at first interaction', () => { + const page = browser(); + page.deliver('largest-contentful-paint', [{ startTime: 100 }]); + page.deliver('largest-contentful-paint', [{ startTime: 2000 }]); + assert.deepEqual(page.values('LCP', 'duration_ms'), []); + page.emit('pointerdown', 2100); + page.hide(); + assert.deepEqual(page.values('LCP', 'duration_ms'), [2000]); + assert.equal(page.observers['largest-contentful-paint'].disconnected, true); +}); + +test('LCP drains queued candidates at hide and ignores later callbacks', () => { + const page = browser(); + page.deliver('largest-contentful-paint', [{ startTime: 100 }]); + page.observers['largest-contentful-paint'].pending.push({ startTime: 2000 }); + page.hide(2100); + page.deliver('largest-contentful-paint', [{ startTime: 3000 }]); + page.document.visibilityState = 'visible'; + page.hide(4000); + assert.deepEqual(page.values('LCP', 'duration_ms'), [2000]); +}); + +test('LCP finalises on keyboard and scroll interactions', () => { + for (const event of ['keydown', 'scroll', 'pointerdown', 'touchstart']) { + const page = browser(); + page.deliver('largest-contentful-paint', [{ startTime: 100 }]); + page.emit(event, 200); + assert.deepEqual(page.values('LCP', 'duration_ms'), [100]); + } +}); + +test('LCP excludes pages loaded hidden and candidates painted after first hide', () => { + const hiddenPage = browser('hidden'); + hiddenPage.document.visibilityState = 'visible'; + hiddenPage.emit('pointerdown'); + assert.deepEqual(hiddenPage.values('LCP', 'duration_ms'), []); + const page = browser(); + page.observers['largest-contentful-paint'].pending.push({ startTime: 3000 }); + page.hide(2000); + assert.deepEqual(page.values('LCP', 'duration_ms'), []); +}); + +test('CLS reports a supported zero once and emits no score for unsupported observation', () => { + const supported = browser(); + supported.hide(); + supported.hide(); + assert.deepEqual(supported.values('CLS', 'cls_score'), [0]); + const unsupported = browser('visible', true); + unsupported.hide(); + assert.deepEqual(unsupported.values('CLS', 'cls_score'), []); +}); + +test('CLS stays unknown when unsupported observe silently does nothing', () => { + for (const support of ['silent', 'missing']) { + const page = browser('visible', support); + page.hide(); + assert.deepEqual(page.values('CLS', 'cls_score'), []); + } +}); diff --git a/tests/watchdog.json b/tests/watchdog.json new file mode 100644 index 0000000..346b3ff --- /dev/null +++ b/tests/watchdog.json @@ -0,0 +1,128 @@ +{ + "evaluation_interval": "1m", + "tests": [ + { + "name": "healthy Grafana with no failures", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x20"}, + {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"0+0x20"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[{"labels":"{}","value":1}]} + ] + }, + { + "name": "failure stops heartbeat and expires after ten minutes", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x30"}, + {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"0+0x4 1+0x25"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"5m","exp_samples":[]}, + {"expr":"WATCHDOG_EXPR","eval_time":"13m","exp_samples":[]}, + {"expr":"WATCHDOG_EXPR","eval_time":"15m","exp_samples":[{"labels":"{}","value":1}]} + ] + }, + { + "name": "first counter sample already records a failed delivery", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x20"}, + {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\",integration=\"webhook\"}","values":"_ _ 1+0x18"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]}, + {"expr":"WATCHDOG_EXPR","eval_time":"11m","exp_samples":[]}, + {"expr":"WATCHDOG_EXPR","eval_time":"12m","exp_samples":[{"labels":"{}","value":1}]} + ] + }, + { + "name": "counter reset followed by another failed delivery", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x30"}, + {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"4+0x10 0 1+0x18"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"10m","exp_samples":[{"labels":"{}","value":1}]}, + {"expr":"WATCHDOG_EXPR","eval_time":"12m","exp_samples":[]}, + {"expr":"WATCHDOG_EXPR","eval_time":"22m","exp_samples":[{"labels":"{}","value":1}]} + ] + }, + { + "name": "no counters before the first notification", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x20"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[{"labels":"{}","value":1}]} + ] + }, + { + "name": "Grafana scrape fails", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1 0+0x20"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]} + ] + }, + { + "name": "Grafana scrape disappears", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1 stale"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]} + ] + }, + { + "name": "Grafana has never been scraped", + "interval": "1m", + "input_series": [ + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]} + ] + }, + { + "name": "failure counters from another job do not suppress Grafana", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x20"}, + {"series":"grafana_alerting_notifications_failed_total{job=\"unrelated\"}","values":"1+1x20"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[{"labels":"{}","value":1}]} + ] + }, + { + "name": "one failed Grafana scrape suppresses heartbeat", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\",instance=\"healthy\"}","values":"1+0x20"}, + {"series":"up{job=\"grafana\",instance=\"failed\"}","values":"0+0x20"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"2m","exp_samples":[]} + ] + }, + { + "name": "new failure counter disappears before its failure ages out", + "interval": "1m", + "input_series": [ + {"series":"up{job=\"grafana\"}","values":"1+0x20"}, + {"series":"grafana_alerting_notifications_failed_total{job=\"grafana\"}","values":"_ _ 1 1 stale"} + ], + "promql_expr_test": [ + {"expr":"WATCHDOG_EXPR","eval_time":"5m","exp_samples":[]}, + {"expr":"WATCHDOG_EXPR","eval_time":"14m","exp_samples":[{"labels":"{}","value":1}]} + ] + } + ] +}