diff --git a/config/grafana/alerting/projects.yaml b/config/grafana/alerting/projects.yaml index c952e06..a78255d 100644 --- a/config/grafana/alerting/projects.yaml +++ b/config/grafana/alerting/projects.yaml @@ -54,6 +54,7 @@ groups: labels: severity: critical annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "No telemetry from {{ $labels.project }}/{{ $labels.env }} in 15m" description: "Host down, Docker down, the agent down, tunnel down, token rotated wrong, or the collector is rejecting this project's data. Nothing on the project side can detect this on its own. Expect this ~20m after the last sample: absent() needs the series to go stale first." @@ -94,5 +95,6 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules" description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host. A literal `unknown` means the sender set no project or env resource attribute at all: fix the sender (docs/ONBOARDING.md), do not bootstrap a project by that name." diff --git a/config/grafana/alerting/rules.yaml b/config/grafana/alerting/rules.yaml index 71b99ca..cca3ead 100644 --- a/config/grafana/alerting/rules.yaml +++ b/config/grafana/alerting/rules.yaml @@ -63,6 +63,7 @@ groups: labels: severity: none annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-delivery summary: "Alerting-pipeline heartbeat" description: >- Routed to a dead man's switch. Pings stop when Grafana cannot be scraped @@ -81,6 +82,7 @@ groups: labels: severity: critical annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-delivery summary: "Grafana cannot deliver {{ $labels.integration }} notifications" description: >- Alerts are firing and going nowhere. Most likely an unset ALERT_WEBHOOK_URL, @@ -103,6 +105,7 @@ groups: labels: severity: critical annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Scrape target {{ $labels.job }} is down" description: "Prometheus cannot scrape {{ $labels.instance }} (job {{ $labels.job }}) for 2 minutes." - uid: otel-export-failures @@ -121,6 +124,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "OTel Collector failing to export via {{ $labels.exporter }}" description: >- The collector has been failing to deliver telemetry to a backend for @@ -148,6 +152,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog # Spoke series carry host_name; the central node-exporter only instance. summary: "Filesystem {{ $labels.mountpoint }} on {{ or $labels.host_name $labels.instance }} is over 80% full" description: >- @@ -170,6 +175,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: >- Filesystem {{ $labels.mountpoint }} on {{ or $labels.host_name $labels.instance }} fills up within 3 days at the current rate @@ -193,6 +199,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Memory on {{ or $labels.host_name $labels.instance }} is over 90% used" description: >- Sustained for 15 minutes. The OOM killer picks by RSS, so the next @@ -216,6 +223,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Prometheus holds {{ $values.QUERY }} active series, over the 30k ceiling" description: >- Find the label that exploded with @@ -241,6 +249,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Prometheus gained {{ $values.QUERY }} active series in 30 minutes" description: >- Far faster than onboarding a spoke explains, so a label has most @@ -267,6 +276,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Container {{ $labels.name }} is crash-looping" description: "{{ $labels.name }} on {{ $labels.host_name }} has restarted more than 3 times in the last hour." - uid: container-oom-killed @@ -282,6 +292,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Container {{ $labels.name }} was OOM-killed" description: "{{ $labels.name }} on {{ $labels.host_name }} hit an OOM kill in the last 5 minutes." - orgId: 1 @@ -307,6 +318,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} error rate above 5%" description: >- More than 5% of requests to {{ $labels.job }} @@ -329,6 +341,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} p99 latency above 2s" description: >- The slowest 1% of requests to {{ $labels.job }} @@ -347,6 +360,7 @@ groups: labels: severity: critical annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "Outside-in HTTP probe failed for {{ $labels.instance }}" description: "Blackbox prober failed to reach {{ $labels.instance }} for 2 minutes." # Google SRE multi-window burn rate alerts for a 99.5% availability SLO @@ -373,6 +387,7 @@ groups: labels: severity: critical annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} 14.4x error budget burn (fast burn)" description: >- Consuming 2% of the 30-day 99.5% error budget within 1 hour. @@ -400,6 +415,7 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "{{ $labels.project }}/{{ $labels.env }} {{ $labels.job }} 6x error budget burn (slow burn)" description: >- Consuming 5% of the 30-day 99.5% error budget within 6 hours. diff --git a/dashboards/logs.json b/dashboards/logs.json index 19f5105..61e67d2 100644 --- a/dashboards/logs.json +++ b/dashboards/logs.json @@ -7,6 +7,15 @@ "project" ], "links": [ + { + "title": "Service metrics", + "type": "link", + "icon": "dashboard", + "url": "/d/service-health/service-health", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, { "asDropdown": false, "icon": "dashboard", @@ -20,6 +29,13 @@ "tooltip": "Switch to another dashboard in the CML monitoring stack", "type": "dashboards", "url": "" + }, + { + "title": "Incident runbook", + "type": "link", + "icon": "book", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true } ], "time": { @@ -42,7 +58,7 @@ }, "definition": "label_values(project)", "query": "label_values(project)", - "refresh": 1, + "refresh": 2, "sort": 1, "current": {}, "options": [], @@ -58,7 +74,7 @@ }, "definition": "label_values({project=\"$project\"}, env)", "query": "label_values({project=\"$project\"}, env)", - "refresh": 1, + "refresh": 2, "sort": 1, "current": {}, "options": [], @@ -74,7 +90,7 @@ }, "definition": "label_values({project=\"$project\", env=\"$env\"}, service_name)", "query": "label_values({project=\"$project\", env=\"$env\"}, service_name)", - "refresh": 1, + "refresh": 2, "sort": 1, "current": { "text": "All", @@ -88,6 +104,21 @@ ] }, "panels": [ + { + "id": 100, + "type": "text", + "title": "Incident workflow", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "mode": "markdown", + "content": "Compare errors with All Logs. Follow trace IDs into Tempo. Confirm recovery with fresh Service metrics; missing logs do not prove health." + } + }, { "type": "timeseries", "title": "Log Rate by Service", @@ -96,7 +127,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "id": 1, "datasource": { @@ -105,7 +136,7 @@ }, "targets": [ { - "expr": "sum by (service_name) (rate({project=\"$project\", env=\"$env\", service_name=~\"$service\"}[$__auto]))", + "expr": "sum by (service_name) (rate({project=\"$project\", env=\"$env\", service_name=~\"${service:regex}\"}[$__auto]))", "legendFormat": "{{service_name}}", "refId": "A" } @@ -124,12 +155,12 @@ { "type": "timeseries", "title": "Log Rate by Level", - "description": "A rising error line is the cheapest early warning here.", + "description": "Rate by Loki-detected level. Missing levels or an empty result are not proof that errors stopped; compare All Logs and service metrics.", "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 4 }, "id": 2, "datasource": { @@ -138,7 +169,7 @@ }, "targets": [ { - "expr": "sum by (detected_level) (rate({project=\"$project\", env=\"$env\", service_name=~\"$service\"}[$__auto]))", + "expr": "sum by (detected_level) (rate({project=\"$project\", env=\"$env\", service_name=~\"${service:regex}\"}[$__auto]))", "legendFormat": "{{detected_level}}", "refId": "A" } @@ -162,7 +193,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "id": 3, "datasource": { @@ -171,7 +202,7 @@ }, "targets": [ { - "expr": "{project=\"$project\", env=\"$env\", service_name=~\"$service\"} | detected_level=~\"(?i)(error|critical|fatal|warn)\"", + "expr": "{project=\"$project\", env=\"$env\", service_name=~\"${service:regex}\"} | detected_level=~\"(?i)(error|critical|fatal|warn)\"", "refId": "A" } ], @@ -190,7 +221,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 22 }, "id": 4, "datasource": { @@ -199,7 +230,7 @@ }, "targets": [ { - "expr": "{project=\"$project\", env=\"$env\", service_name=~\"$service\"}", + "expr": "{project=\"$project\", env=\"$env\", service_name=~\"${service:regex}\"}", "refId": "A" } ], diff --git a/dashboards/service-health.json b/dashboards/service-health.json index 6ba2327..8f46f10 100644 --- a/dashboards/service-health.json +++ b/dashboards/service-health.json @@ -15,6 +15,15 @@ "otlp" ], "links": [ + { + "title": "Service logs", + "type": "link", + "icon": "dashboard", + "url": "/d/logs/logs", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, { "asDropdown": false, "icon": "dashboard", @@ -28,6 +37,13 @@ "tooltip": "Switch to another dashboard in the CML monitoring stack", "type": "dashboards", "url": "" + }, + { + "title": "Incident runbook", + "type": "link", + "icon": "book", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true } ], "templating": { @@ -46,7 +62,7 @@ "query": "label_values(http_server_request_duration_seconds_count, project)", "refId": "var" }, - "refresh": 1, + "refresh": 2, "sort": 1, "current": {}, "options": [], @@ -66,7 +82,7 @@ "query": "label_values(http_server_request_duration_seconds_count{project=\"$project\"}, env)", "refId": "var" }, - "refresh": 1, + "refresh": 2, "sort": 1, "current": {}, "options": [], @@ -86,15 +102,111 @@ "query": "label_values(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\"}, job)", "refId": "var" }, - "refresh": 1, + "refresh": 2, "sort": 1, - "current": {}, + "current": { + "text": "All", + "value": "$__all" + }, "options": [], - "hide": 0 + "hide": 0, + "includeAll": true, + "allValue": ".+" } ] }, "panels": [ + { + "id": 100, + "type": "text", + "title": "Incident workflow", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "mode": "markdown", + "content": "Check data age, errors and latency. Open Service logs; follow a trace ID or exemplar into Tempo. Confirm fresh traffic before recovery." + } + }, + { + "id": 13, + "type": "stat", + "title": "Newest Metric Age", + "description": "Age of the newest request-counter sample at the selected end time. More than two minutes indicates delayed data; Unknown means no current sample. A fresh sample does not prove requests succeeded.", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "time() - max(timestamp(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}))", + "instant": true, + "legendFormat": "Newest sample", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "noValue": "Unknown", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "orange", + "value": 120 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + } + }, { "type": "stat", "title": "Request Rate", @@ -103,7 +215,7 @@ "h": 4, "w": 4, "x": 0, - "y": 0 + "y": 8 }, "id": 7, "datasource": { @@ -112,7 +224,7 @@ }, "targets": [ { - "expr": "sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval]))", + "expr": "sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval]))", "instant": true, "legendFormat": "Requests", "refId": "A" @@ -132,7 +244,20 @@ "value": null } ] - } + }, + "noValue": "Unknown", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] }, "overrides": [] }, @@ -143,7 +268,7 @@ "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -154,12 +279,12 @@ { "type": "stat", "title": "Error Rate", - "description": "Percentage of requests resulting in 5xx HTTP server errors.", + "description": "HTTP 5xx responses as a percentage of observed requests. No traffic or missing telemetry is unknown, not 0% errors.", "gridPos": { "h": 4, "w": 4, "x": 4, - "y": 0 + "y": 8 }, "id": 8, "datasource": { @@ -168,7 +293,7 @@ }, "targets": [ { - "expr": "(100 * sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\", http_response_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval]))) or vector(0)", + "expr": "100 * ((sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\", http_response_status_code=~\"5..\"}[$__rate_interval])) or (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])) * 0)) / (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])) > 0))", "instant": true, "legendFormat": "Error Rate", "refId": "A" @@ -197,7 +322,20 @@ "value": 1.0 } ] - } + }, + "noValue": "Unknown", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] }, "overrides": [] }, @@ -208,7 +346,7 @@ "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -224,7 +362,7 @@ "h": 4, "w": 4, "x": 8, - "y": 0 + "y": 8 }, "id": 9, "datasource": { @@ -233,7 +371,7 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])))", "instant": true, "legendFormat": "p95", "refId": "A" @@ -261,7 +399,20 @@ "value": 1.5 } ] - } + }, + "noValue": "Unknown", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] }, "overrides": [] }, @@ -272,7 +423,7 @@ "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -288,7 +439,7 @@ "h": 4, "w": 4, "x": 12, - "y": 0 + "y": 8 }, "id": 10, "datasource": { @@ -297,7 +448,7 @@ }, "targets": [ { - "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])))", "instant": true, "legendFormat": "p50", "refId": "A" @@ -325,7 +476,20 @@ "value": 0.8 } ] - } + }, + "noValue": "Unknown", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] }, "overrides": [] }, @@ -336,7 +500,7 @@ "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -347,12 +511,12 @@ { "type": "stat", "title": "Availability (SLI)", - "description": "Success rate over the active time range. Target: >= 99.5%.", + "description": "Success rate over the active time range. Target: >= 99.5%. Request-based only; missing telemetry is not proof of uptime.", "gridPos": { "h": 4, "w": 4, "x": 16, - "y": 0 + "y": 8 }, "id": 11, "datasource": { @@ -361,7 +525,7 @@ }, "targets": [ { - "expr": "100 * (1 - ((sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\", http_response_status_code=~\"5..\"}[$__range])) or vector(0)) / (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__range])) > 0)))", + "expr": "100 * (1 - ((sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\", http_response_status_code=~\"5..\"}[$__range])) or vector(0)) / (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__range])) > 0)))", "instant": true, "legendFormat": "Availability", "refId": "A" @@ -391,7 +555,20 @@ "value": 99.5 } ] - } + }, + "noValue": "Unknown", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] }, "overrides": [] }, @@ -402,7 +579,7 @@ "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -413,12 +590,12 @@ { "type": "stat", "title": "Error Budget Remaining", - "description": "Remaining 99.5% SLO error budget in current window. Exhausted at 0%.", + "description": "Remaining 99.5% SLO error budget in current window. Exhausted at 0%. Request-based only; missing telemetry is not proof of uptime.", "gridPos": { "h": 4, "w": 4, "x": 20, - "y": 0 + "y": 8 }, "id": 12, "datasource": { @@ -427,7 +604,7 @@ }, "targets": [ { - "expr": "100 * (1 - (((sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\", http_response_status_code=~\"5..\"}[$__range])) or vector(0)) / (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__range])) > 0)) / 0.005))", + "expr": "100 * (1 - (((sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\", http_response_status_code=~\"5..\"}[$__range])) or vector(0)) / (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__range])) > 0)) / 0.005))", "instant": true, "legendFormat": "Budget Remaining", "refId": "A" @@ -455,7 +632,20 @@ "value": 50.0 } ] - } + }, + "noValue": "Unknown", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ] }, "overrides": [] }, @@ -466,7 +656,7 @@ "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "last" ], "fields": "", "values": false @@ -482,7 +672,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 12 }, "id": 1, "datasource": { @@ -491,7 +681,7 @@ }, "targets": [ { - "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval]))", + "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval]))", "legendFormat": "{{http_route}}", "refId": "A" } @@ -510,12 +700,12 @@ { "type": "timeseries", "title": "Error Rate over Time", - "description": "Share of server spans with STATUS_CODE_ERROR.", + "description": "HTTP 5xx responses as a percentage of observed requests. No traffic or missing telemetry is unknown, not 0% errors.", "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 12 }, "id": 2, "datasource": { @@ -524,7 +714,7 @@ }, "targets": [ { - "expr": "100 * sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\", http_response_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval]))", + "expr": "100 * ((sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\", http_response_status_code=~\"5..\"}[$__rate_interval])) or (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])) * 0)) / (sum(rate(http_server_request_duration_seconds_count{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])) > 0))", "legendFormat": "errors", "refId": "A" } @@ -557,12 +747,12 @@ { "type": "timeseries", "title": "Latency percentiles", - "description": "Quantiles over the applications' own request-duration histograms. Dots are exemplars — click one to open the exact trace in Tempo.", + "description": "Quantiles over the applications' own request-duration histograms. Dots are exemplars \u2014 click one to open the exact trace in Tempo.", "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 3, "datasource": { @@ -571,19 +761,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "exemplar": true }, { - "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B", "exemplar": true }, { - "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C", "exemplar": true @@ -603,12 +793,12 @@ { "type": "timeseries", "title": "Latency p95 by route", - "description": "From the service's own OTLP metrics (OTel SDK), i.e. the collector → Prometheus OTLP path.", + "description": "From the service's own OTLP metrics (OTel SDK), i.e. the collector \u2192 Prometheus OTLP path.", "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 4, "datasource": { @@ -617,7 +807,7 @@ }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=\"$service\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{project=\"$project\", env=\"$env\", job=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "{{http_route}}", "refId": "A" } @@ -641,7 +831,7 @@ "h": 8, "w": 24, "x": 0, - "y": 20 + "y": 28 }, "id": 6, "datasource": { @@ -650,7 +840,7 @@ }, "targets": [ { - "expr": "sum by (state) ({__name__=~\"db_client_connections_usage(_connections)?\", project=\"$project\", env=\"$env\", job=\"$service\"})", + "expr": "sum by (state) ({__name__=~\"db_client_connections_usage(_connections)?\", project=\"$project\", env=\"$env\", job=~\"${service:regex}\"})", "legendFormat": "{{state}}", "refId": "A" } @@ -678,7 +868,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 36 }, "id": 5, "datasource": { @@ -687,7 +877,7 @@ }, "targets": [ { - "expr": "{project=\"$project\", env=\"$env\", service_name=~\"$service\"}", + "expr": "{project=\"$project\", env=\"$env\", service_name=~\"${service:regex}\"}", "refId": "A" } ], diff --git a/dashboards/stack-health.json b/dashboards/stack-health.json index 9de697c..28c8a3f 100644 --- a/dashboards/stack-health.json +++ b/dashboards/stack-health.json @@ -27,9 +27,161 @@ "tooltip": "Switch to another dashboard in the CML monitoring stack", "type": "dashboards", "url": "" + }, + { + "title": "Incident runbook", + "type": "link", + "icon": "book", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true } ], "panels": [ + { + "id": 100, + "type": "text", + "title": "Incident workflow", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "mode": "markdown", + "content": "Check scrape status and alerts. Open a project below. Confirm fresh telemetry and resolved alerts; missing data means unknown." + } + }, + { + "id": 8, + "type": "table", + "title": "Scrape Status", + "description": "Current Prometheus scrape results, not application availability. A disappeared target has no value; consult Active Alerts and the runbook before declaring recovery.", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "up", + "instant": true, + "legendFormat": "{{job}} / {{instance}}", + "refId": "A", + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "Unknown", + "min": 0, + "max": 1, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Down", + "color": "red" + }, + "1": { + "text": "Up", + "color": "green" + } + } + }, + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "Unknown", + "color": "text" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 85 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Job" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Job", + "desc": false + } + ] + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "env": true, + "project": true, + "tier": true + }, + "indexByName": { + "Value": 0, + "job": 1, + "instance": 2 + }, + "renameByName": { + "job": "Job", + "instance": "Target", + "Value": "Status" + } + } + } + ] + }, { "type": "timeseries", "title": "Filesystem Used", @@ -38,7 +190,7 @@ "h": 8, "w": 8, "x": 0, - "y": 0 + "y": 12 }, "id": 1, "datasource": { @@ -76,7 +228,14 @@ } }, "overrides": [] - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] }, { "type": "timeseries", @@ -85,7 +244,7 @@ "h": 8, "w": 8, "x": 8, - "y": 0 + "y": 12 }, "id": 2, "datasource": { @@ -110,7 +269,14 @@ } }, "overrides": [] - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] }, { "type": "timeseries", @@ -119,7 +285,7 @@ "h": 8, "w": 8, "x": 16, - "y": 0 + "y": 12 }, "id": 3, "datasource": { @@ -144,7 +310,14 @@ } }, "overrides": [] - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] }, { "type": "timeseries", @@ -154,7 +327,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 20 }, "id": 4, "datasource": { @@ -187,7 +360,14 @@ } }, "overrides": [] - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] }, { "type": "timeseries", @@ -197,7 +377,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 20 }, "id": 5, "datasource": { @@ -245,7 +425,10 @@ "id": "custom.lineStyle", "value": { "fill": "dash", - "dash": [10, 10] + "dash": [ + 10, + 10 + ] } } ] @@ -260,23 +443,33 @@ "id": "custom.lineStyle", "value": { "fill": "dot", - "dash": [0, 10] + "dash": [ + 0, + 10 + ] } } ] } ] - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] }, { "type": "timeseries", "title": "Ingest by Project", - "description": "Telemetry accepted per project and environment, from the gateway's count connector. A project missing here is sending nothing, whichever signal it uses.", + "description": "Accepted telemetry by project and environment. Click a series to open its metrics or logs with the same time range. Missing or zero ingest does not prove a project is healthy.", "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 16 + "y": 28 }, "id": 7, "datasource": { @@ -306,21 +499,40 @@ "custom": { "lineWidth": 2, "fillOpacity": 10 - } + }, + "links": [ + { + "title": "Service metrics", + "url": "/d/service-health/service-health?var-project=${__field.labels.project:percentencode}&var-env=${__field.labels.env:percentencode}&${__url_time_range}", + "targetBlank": false + }, + { + "title": "Service logs", + "url": "/d/logs/logs?var-project=${__field.labels.project:percentencode}&var-env=${__field.labels.env:percentencode}&${__url_time_range}", + "targetBlank": false + } + ] }, "overrides": [] - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] }, { "type": "alertlist", "title": "Active Alerts", "id": 6, - "description": "Grafana-managed alerts. The old panel queried Prometheus's ALERTS series, which only exists while Prometheus evaluates rules \u2014 it stopped existing when alerting moved to Grafana (ADR 0002).", + "description": "Firing, pending, no-data and query-error states. An empty list alone does not prove the stack is healthy; check scrape status and data freshness.", "gridPos": { "h": 6, "w": 24, "x": 0, - "y": 24 + "y": 36 }, "options": { "viewMode": "list", @@ -331,11 +543,18 @@ "stateFilter": { "firing": true, "pending": true, - "noData": false, + "noData": true, "normal": false, "error": true } - } + }, + "links": [ + { + "title": "Runbook", + "url": "https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#incident-workflow", + "targetBlank": true + } + ] } ] } diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 110a6b7..2c9f8fe 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -9,6 +9,31 @@ All commands run from the repo root on the monitoring host. Start with In that capture, the red export-failure spike is a Tempo outage and the gap in the ingest panel is a backup/restore drill. +## Incident workflow + +1. Open **Stack Health** at the incident time. Check **Scrape Status** + and **Collector Export Failures** before trusting application charts. + A failed scrape or missing backend can hide the application's condition. +2. Find the affected project and environment in **Ingest by Project**. + Open **Service Health**, then select the service. Keep the incident time + range while comparing request rate, errors and latency. +3. Open **Logs** with the same project, environment and service. Expand an + error or warning. Follow its `trace_id` link into Tempo when one exists. + A log without a trace ID cannot open a correlated trace. +4. After a repair, check fresh telemetry and current scrape status. Confirm + the error rate falls with observed traffic and the alert resolves. + An old healthy chart or an empty error panel does not prove recovery. + +**Unknown** or **No data** means the query has no usable observations. +Check the selected filters, time range, telemetry age and backend status. +An idle service can have a zero request rate without a measurable error rate. +Availability measures successful requests among observed requests; it does +not measure wall-clock uptime. + +A panel error means its query failed. Open the panel's inspection menu for +the error, then check that backend's container logs. Widening the time range +can find historical data, but does not establish current health. + ## A service is down or misbehaving ```sh diff --git a/justfile b/justfile index cf08fa4..6b1f6e2 100644 --- a/justfile +++ b/justfile @@ -140,6 +140,10 @@ lint: sh scripts/test_backup.sh # Exercise the provisioned expression, including counters first seen after a failure. @d=$(mktemp -d) && expr=$(awk '/uid: watchdog-heartbeat/{watchdog=1} watchdog && /expr: >/{expr=1;next} expr && /^ /{sub(/^ /, "");printf "%s ",$0;next} expr{exit}' config/grafana/alerting/rules.yaml) && test -n "$expr" && jq --arg expr "$expr" '.tests[].promql_expr_test[].expr = $expr' tests/watchdog.json >"$d/watchdog.json" && chmod 755 "$d" && chmod 644 "$d/watchdog.json" && docker run --rm --network none --entrypoint /bin/promtool -v "$d":/tests:ro $(just _image compose.yml prometheus) test rules /tests/watchdog.json; rc=$?; rm -rf "$d"; exit $rc + # Generate query cases from the actual dashboard JSON so tests cannot drift. + @d=$(mktemp -d) && docker run --rm --network none -v .:/repo:ro -w /repo $(just _image compose.yml notification-relay) python -B scripts/test_dashboards.py >"$d/dashboard.yml" && chmod 755 "$d" && chmod 644 "$d/dashboard.yml" && docker run --rm --network none --entrypoint /bin/promtool -v "$d":/tests:ro $(just _image compose.yml prometheus) test rules /tests/dashboard.yml; rc=$?; rm -rf "$d"; exit $rc + docker run --rm --network none -v ./scripts/test_dashboards.py:/tests/test_dashboards.py:ro {{ruff}} check --no-cache /tests/test_dashboards.py + docker run --rm --network none -v ./scripts/test_dashboards.py:/tests/test_dashboards.py:ro {{ruff}} format --check --no-cache /tests/test_dashboards.py docker run --rm --network none -v ./demo:/demo:ro {{ruff}} check --no-cache /demo docker run --rm --network none -v ./demo:/demo:ro {{ruff}} format --check --no-cache /demo docker run --rm --network none -v ./notification-relay:/relay:ro {{ruff}} check --no-cache /relay diff --git a/scripts/test_dashboards.py b/scripts/test_dashboards.py new file mode 100644 index 0000000..dfe28e2 --- /dev/null +++ b/scripts/test_dashboards.py @@ -0,0 +1,93 @@ +"""Check incident links and render PromQL cases from the provisioned dashboards.""" + +import json +from pathlib import Path +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[1] +DASHBOARDS = { + name: json.loads((ROOT / "dashboards" / f"{name}.json").read_text()) + for name in ("stack-health", "service-health", "logs") +} + + +def panel(dashboard, title): + return next(p for p in DASHBOARDS[dashboard]["panels"] if p["title"] == title) + + +class IncidentContracts(unittest.TestCase): + def test_navigation_preserves_time_and_selection(self): + for name in ("service-health", "logs"): + destination = "logs" if name == "service-health" else "service-health" + link = next( + link + for link in DASHBOARDS[name]["links"] + if f"/d/{destination}/" in link.get("url", "") + ) + self.assertTrue(link["keepTime"]) + self.assertTrue(link["includeVars"]) + links = panel("stack-health", "Ingest by Project")["fieldConfig"]["defaults"][ + "links" + ] + for destination in ("service-health", "logs"): + self.assertTrue( + any(f"/d/{destination}/" in link.get("url", "") for link in links), + f"missing Ingest by Project link to {destination}", + ) + for link in links: + self.assertIn("${__field.labels.project:percentencode}", link["url"]) + self.assertIn("${__field.labels.env:percentencode}", link["url"]) + self.assertIn("${__url_time_range}", link["url"]) + + def test_unknown_values_and_staleness_remain_visible(self): + for p in DASHBOARDS["service-health"]["panels"]: + if p["type"] == "stat": + self.assertEqual(p["fieldConfig"]["defaults"]["noValue"], "Unknown") + self.assertIn( + "timestamp(", + panel("service-health", "Newest Metric Age")["targets"][0]["expr"], + ) + self.assertTrue( + panel("stack-health", "Active Alerts")["options"]["stateFilter"]["noData"] + ) + + def test_runbook_and_signal_styles(self): + for dashboard in DASHBOARDS.values(): + self.assertTrue( + any( + "RUNBOOK.md#incident-workflow" in x.get("url", "") + for x in dashboard["links"] + ) + ) + export = panel("stack-health", "Collector Export Failures") + self.assertIn("(solid)", export["targets"][0]["legendFormat"]) + self.assertIn("(dashed)", export["targets"][1]["legendFormat"]) + self.assertIn("(dotted)", export["targets"][2]["legendFormat"]) + + +def render_promtool_cases(): + template = (ROOT / "tests" / "dashboard-rules.test.yml").read_text() + for token, title in ( + ("ERROR_RATE", "Error Rate"), + ("ERROR_TIMESERIES", "Error Rate over Time"), + ("METRIC_AGE", "Newest Metric Age"), + ): + expr = panel("service-health", title)["targets"][0]["expr"] + for variable, value in ( + ("${service:regex}", "api"), + ("$project", "review"), + ("$env", "test"), + ("$service", "api"), + ("$__rate_interval", "2m"), + ): + expr = expr.replace(variable, value) + template = template.replace(f"expr: {token}", f"expr: {json.dumps(expr)}") + return template + + +if __name__ == "__main__": + result = unittest.main(exit=False).result + if not result.wasSuccessful(): + sys.exit(1) + sys.stdout.write(render_promtool_cases()) diff --git a/templates/alerting/projects.yaml.tmpl b/templates/alerting/projects.yaml.tmpl index f3a4636..43a74aa 100644 --- a/templates/alerting/projects.yaml.tmpl +++ b/templates/alerting/projects.yaml.tmpl @@ -54,6 +54,7 @@ groups: labels: severity: critical annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "No telemetry from {{ $labels.project }}/{{ $labels.env }} in 15m" description: "Host down, Docker down, the agent down, tunnel down, token rotated wrong, or the collector is rejecting this project's data. Nothing on the project side can detect this on its own. Expect this ~20m after the last sample: absent() needs the series to go stale first." @@ -94,5 +95,6 @@ groups: labels: severity: warning annotations: + runbook_url: https://github.com/CMLPlatform/monitoring/blob/main/docs/RUNBOOK.md#alert-catalog summary: "{{ $labels.project }}/{{ $labels.env }} is sending telemetry but has no alert rules" description: "Telemetry is arriving for a project/environment bootstrap.sh was never run for, so nothing would notice if it went silent. Run ./bootstrap.sh {{ $labels.project }} {{ $labels.env }} on the monitoring host. A literal `unknown` means the sender set no project or env resource attribute at all: fix the sender (docs/ONBOARDING.md), do not bootstrap a project by that name." diff --git a/tests/dashboard-rules.test.yml b/tests/dashboard-rules.test.yml new file mode 100644 index 0000000..83f460f --- /dev/null +++ b/tests/dashboard-rules.test.yml @@ -0,0 +1,92 @@ +fuzzy_compare: true +evaluation_interval: 30s +tests: + - name: traffic-without-5xx-is-zero-error + interval: 30s + input_series: + - series: 'http_server_request_duration_seconds_count{project="review",env="test",job="api",http_response_status_code="200"}' + values: '0+10x10' + promql_expr_test: + - expr: ERROR_RATE + eval_time: 5m + exp_samples: + - labels: '{}' + value: 0 + - expr: ERROR_TIMESERIES + eval_time: 5m + exp_samples: + - labels: '{}' + value: 0 + - expr: METRIC_AGE + eval_time: 5m + exp_samples: + - labels: '{}' + value: 0 + - name: failures-and-recovery + interval: 30s + input_series: + - series: 'http_server_request_duration_seconds_count{project="review",env="test",job="api",http_response_status_code="200"}' + values: '0+9x10 100+10x10' + - series: 'http_server_request_duration_seconds_count{project="review",env="test",job="api",http_response_status_code="500"}' + values: '0+1x10 10+0x10' + promql_expr_test: + - expr: ERROR_RATE + eval_time: 5m + exp_samples: + - labels: '{}' + value: 10 + - expr: ERROR_RATE + eval_time: 10m + exp_samples: + - labels: '{}' + value: 0 + - expr: ERROR_TIMESERIES + eval_time: 5m + exp_samples: + - labels: '{}' + value: 10 + - expr: ERROR_TIMESERIES + eval_time: 10m + exp_samples: + - labels: '{}' + value: 0 + - name: no-traffic-is-unknown + interval: 30s + input_series: + - series: 'http_server_request_duration_seconds_count{project="review",env="test",job="api",http_response_status_code="200"}' + values: '0+0x10' + promql_expr_test: + - expr: ERROR_RATE + eval_time: 5m + exp_samples: [] + - expr: ERROR_TIMESERIES + eval_time: 5m + exp_samples: [] + - name: missing-data-is-unknown + promql_expr_test: + - expr: ERROR_RATE + eval_time: 5m + exp_samples: [] + - expr: ERROR_TIMESERIES + eval_time: 5m + exp_samples: [] + - expr: METRIC_AGE + eval_time: 5m + exp_samples: [] + - name: delayed-then-stale-is-not-healthy-zero + interval: 30s + input_series: + - series: 'http_server_request_duration_seconds_count{project="review",env="test",job="api",http_response_status_code="200"}' + values: '0+10x4 _ _ _ _ _ _ _ _ _ _ _ _ _ _' + promql_expr_test: + - expr: METRIC_AGE + eval_time: 4m + exp_samples: + - labels: '{}' + value: 120 + - expr: METRIC_AGE + eval_time: 10m + exp_samples: [] + - expr: ERROR_RATE + eval_time: 10m + exp_samples: []