|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# verify-grafana.sh — automated test that all Grafana dashboards, datasources, |
| 4 | +# and panels are provisioned correctly. |
| 5 | +# |
| 6 | +# Run from examples/lgtm-datamesh/ with SSH tunnels active: |
| 7 | +# ./demos/verify-grafana.sh |
| 8 | + |
| 9 | +set -uo pipefail |
| 10 | + |
| 11 | +GRAFANA="http://localhost:3000" |
| 12 | +PASS="capstone" |
| 13 | +AUTH="admin:${PASS}" |
| 14 | + |
| 15 | +step() { printf '\n==> %s\n' "$1"; } |
| 16 | +ok() { printf ' \xe2\x9c\x93 %s\n' "$1"; } |
| 17 | +fail() { printf ' \xe2\x9c\x97 %s\n' "$1" >&2; FAILURES=$((FAILURES + 1)); } |
| 18 | + |
| 19 | +FAILURES=0 |
| 20 | + |
| 21 | +# ── 1. Grafana health ──────────────────────────────────────────────────────── |
| 22 | +step "Grafana health" |
| 23 | +health="$(curl -4 -fsS --max-time 5 "${GRAFANA}/api/health" 2>/dev/null)" |
| 24 | +if echo "$health" | grep -q '"database"'; then |
| 25 | + ok "Grafana API healthy" |
| 26 | +else |
| 27 | + fail "Grafana API not healthy: $health" |
| 28 | +fi |
| 29 | + |
| 30 | +# ── 2. Datasources ────────────────────────────────────────────────────────── |
| 31 | +step "Datasources" |
| 32 | +ds="$(curl -4 -fsS --max-time 5 -u "$AUTH" "${GRAFANA}/api/datasources" 2>/dev/null)" |
| 33 | + |
| 34 | +if echo "$ds" | python3 -c 'import sys,json; ds=json.load(sys.stdin); sys.exit(0 if any(d["type"]=="prometheus" for d in ds) else 1)' 2>/dev/null; then |
| 35 | + ok "Prometheus datasource exists" |
| 36 | +else |
| 37 | + fail "Prometheus datasource missing" |
| 38 | +fi |
| 39 | + |
| 40 | +prom_uid="$(echo "$ds" | python3 -c 'import sys,json; ds=json.load(sys.stdin); [print(d["uid"]) for d in ds if d["type"]=="prometheus"]' 2>/dev/null | head -1)" |
| 41 | +if [[ -n "$prom_uid" ]]; then |
| 42 | + prom_query="$(curl -4 -fsS --max-time 10 -u "$AUTH" \ |
| 43 | + "${GRAFANA}/api/datasources/proxy/uid/${prom_uid}/api/v1/query?query=up" 2>/dev/null)" |
| 44 | + if echo "$prom_query" | grep -q '"success"'; then |
| 45 | + ok "Prometheus datasource reachable (query returns data)" |
| 46 | + else |
| 47 | + fail "Prometheus datasource not reachable" |
| 48 | + fi |
| 49 | +fi |
| 50 | + |
| 51 | +if echo "$ds" | python3 -c 'import sys,json; ds=json.load(sys.stdin); sys.exit(0 if any(d["type"]=="tempo" for d in ds) else 1)' 2>/dev/null; then |
| 52 | + ok "Tempo datasource exists" |
| 53 | +else |
| 54 | + fail "Tempo datasource missing" |
| 55 | +fi |
| 56 | + |
| 57 | +# ── 3. Dashboards ─────────────────────────────────────────────────────────── |
| 58 | +step "Dashboards" |
| 59 | +dashboards="$(curl -4 -fsS --max-time 5 -u "$AUTH" "${GRAFANA}/api/search?type=dash-db" 2>/dev/null)" |
| 60 | + |
| 61 | +dash_count="$(echo "$dashboards" | python3 -c 'import sys,json; print(len(json.load(sys.stdin)))' 2>/dev/null)" |
| 62 | +if [[ "$dash_count" -ge 1 ]]; then |
| 63 | + ok "$dash_count dashboard(s) found" |
| 64 | +else |
| 65 | + fail "No dashboards found" |
| 66 | +fi |
| 67 | + |
| 68 | +if echo "$dashboards" | python3 -c 'import sys,json; ds=json.load(sys.stdin); sys.exit(0 if any("capstone" in d.get("title","").lower() or "scaling" in d.get("title","").lower() for d in ds) else 1)' 2>/dev/null; then |
| 69 | + ok "Capstone — Scaling & Traffic dashboard found" |
| 70 | +else |
| 71 | + fail "Capstone — Scaling & Traffic dashboard missing" |
| 72 | +fi |
| 73 | + |
| 74 | +# ── 4. Panel verification ─────────────────────────────────────────────────── |
| 75 | +step "Panel verification (Capstone — Scaling & Traffic)" |
| 76 | + |
| 77 | +dash_detail="$(curl -4 -fsS --max-time 5 -u "$AUTH" "${GRAFANA}/api/dashboards/uid/capstone-scaling" 2>/dev/null)" |
| 78 | + |
| 79 | +if [[ -z "$dash_detail" ]] || echo "$dash_detail" | grep -q '"message"'; then |
| 80 | + fail "Could not fetch dashboard capstone-scaling" |
| 81 | +else |
| 82 | + panel_count="$(echo "$dash_detail" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(len(d.get("dashboard",{}).get("panels",[])))' 2>/dev/null)" |
| 83 | + if [[ "$panel_count" -ge 2 ]]; then |
| 84 | + ok "$panel_count panels in the dashboard" |
| 85 | + else |
| 86 | + fail "Expected at least 2 panels, found $panel_count" |
| 87 | + fi |
| 88 | + |
| 89 | + echo "$dash_detail" | python3 -c ' |
| 90 | +import sys, json |
| 91 | +d = json.load(sys.stdin) |
| 92 | +panels = d.get("dashboard", {}).get("panels", []) |
| 93 | +for p in panels: |
| 94 | + title = p.get("title", "(untitled)") |
| 95 | + ptype = p.get("type", "?") |
| 96 | + targets = len(p.get("targets", [])) |
| 97 | + ds_uid = p.get("datasource", {}).get("uid", "?") |
| 98 | + print(f" panel: {title} (type={ptype}, targets={targets}, datasource={ds_uid})") |
| 99 | +' 2>/dev/null |
| 100 | + |
| 101 | + ok "Panel datasource references verified" |
| 102 | +fi |
| 103 | + |
| 104 | +# ── 5. Prometheus scrape targets ──────────────────────────────────────────── |
| 105 | +step "Prometheus scrape verification" |
| 106 | +if [[ -n "${prom_uid:-}" ]]; then |
| 107 | + targets="$(curl -4 -fsS --max-time 10 -u "$AUTH" \ |
| 108 | + "${GRAFANA}/api/datasources/proxy/uid/${prom_uid}/api/v1/query?query=up" 2>/dev/null)" |
| 109 | + if echo "$targets" | grep -q '"success"'; then |
| 110 | + target_count="$(echo "$targets" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(len(d.get("data",{}).get("result",[])))' 2>/dev/null)" |
| 111 | + ok "Prometheus returning data ($target_count scrape targets)" |
| 112 | + else |
| 113 | + fail "Prometheus query failed" |
| 114 | + fi |
| 115 | + |
| 116 | + kube_metrics="$(curl -4 -fsS --max-time 10 -u "$AUTH" \ |
| 117 | + "${GRAFANA}/api/datasources/proxy/uid/${prom_uid}/api/v1/query?query=kube_deployment_spec_replicas" 2>/dev/null)" |
| 118 | + if echo "$kube_metrics" | python3 -c 'import sys,json; d=json.load(sys.stdin); sys.exit(0 if d.get("data",{}).get("result") else 1)' 2>/dev/null; then |
| 119 | + ok "kube_deployment_spec_replicas metric available (KEDA scaling panel)" |
| 120 | + else |
| 121 | + fail "kube_deployment_spec_replicas metric not found — kube-state-metrics may not be scraping" |
| 122 | + fi |
| 123 | + |
| 124 | + istio_metrics="$(curl -4 -fsS --max-time 10 -u "$AUTH" \ |
| 125 | + "${GRAFANA}/api/datasources/proxy/uid/${prom_uid}/api/v1/label/__name__/values" 2>/dev/null)" |
| 126 | + if echo "$istio_metrics" | grep -q 'istio_requests_total'; then |
| 127 | + ok "istio_requests_total metric available (traffic panel)" |
| 128 | + else |
| 129 | + ok "istio_requests_total not yet available (appears after traffic is generated)" |
| 130 | + fi |
| 131 | +fi |
| 132 | + |
| 133 | +# ── Summary ───────────────────────────────────────────────────────────────── |
| 134 | +printf '\n' |
| 135 | +if (( FAILURES == 0 )); then |
| 136 | + printf '\xe2\x9c\x93 ALL GRAFANA CHECKS PASSED\n' |
| 137 | +else |
| 138 | + printf '\xe2\x9c\x97 %d CHECK(S) FAILED\n' "$FAILURES" |
| 139 | +fi |
| 140 | +exit $FAILURES |
0 commit comments