Repository navigation
Expand file tree
/
Copy pathcompose.yml
More file actions
286 lines (270 loc) · 10.4 KB
/
Copy pathcompose.yml
File metadata and controls
286 lines (270 loc) · 10.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
name: monitoring
# Central observability stack.
# OTel Collector → ingestion gateway (OTLP gRPC 4317 / HTTP 4318)
# Loki → logs
# Tempo → traces
# Prometheus → metrics
# Grafana → UI
#
# Do NOT publish :4317/:4318 to the public internet. The ports below bind to
# 127.0.0.1; expose them via Cloudflare Tunnel, Tailscale, WireGuard, or similar.
x-logging: &default-logging
driver: json-file
options:
max-size: "10m"
max-file: "3"
# Every service drops all capabilities and carries a pids limit. Each mem_limit
# is a ceiling sized from observed usage, so one runaway component cannot OOM a
# host that also runs production. Services that need a tighter pids_limit
# override it after the merge.
x-hardened: &hardened
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
pids_limit: 1024
logging: *default-logging
x-healthcheck: &healthcheck
interval: 15s
timeout: 3s
retries: 5
start_period: 30s
# Probe often while starting so `up --wait` returns on the first pass.
start_interval: 2s
services:
# A fresh named volume is root-owned and the collector image is distroless, so
# it cannot chown its own queue directory at startup. This one-shot service
# does it first. It lives here rather than in the justfile so that a plain
# `docker compose up` works, and so the volume stays Compose-managed: a
# `docker volume create` from outside leaves it unlabelled, and then
# `down --volumes` never removes it.
otel-queue-init:
image: alpine:3.24@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a
command: ["chown", "10001:10001", "/q"]
restart: "no"
network_mode: none
volumes:
- otel_queue:/q
cap_drop: [ALL]
cap_add: [CHOWN]
security_opt:
- no-new-privileges:true
pids_limit: 16
mem_limit: 32m
logging: *default-logging
otel-collector:
image: otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea
restart: unless-stopped
depends_on:
otel-queue-init:
condition: service_completed_successfully
loki:
condition: service_started
tempo:
condition: service_started
prometheus:
condition: service_healthy
ports:
- "127.0.0.1:4317:4317" # OTLP gRPC
- "127.0.0.1:4318:4318" # OTLP HTTP
volumes:
- ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro
# File-backed exporter queue. Must be owned by uid 10001; otel-queue-init
# chowns it before this service starts.
- otel_queue:/var/lib/otelcol/queue
command: ["--config=/etc/otelcol/config.yaml"]
environment:
OTLP_AUTH_TOKEN: ${OTLP_AUTH_TOKEN:?set OTLP_AUTH_TOKEN in .env}
# Keep in step with memory_limiter in config/otel-collector.yaml.
mem_limit: 512m
!!merge <<: *hardened
pids_limit: 512
networks: [default, backend]
loki:
image: grafana/loki:3.7.9@sha256:0db216b630ae1f348a1584c67f3cead8b4c1fc839169a39501db664744158e28
restart: unless-stopped
volumes:
- ./config/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
command: ["-config.file=/etc/loki/loki.yaml"]
# No healthcheck: the image is distroless, with no shell or wget to probe with.
mem_limit: 2g
!!merge <<: *hardened
networks: [backend]
tempo:
image: grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044
restart: unless-stopped
volumes:
- ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
command: ["-config.file=/etc/tempo/tempo.yaml"]
# Distroless too, so no healthcheck.
mem_limit: 2g
!!merge <<: *hardened
networks: [backend]
prometheus:
image: prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06
restart: unless-stopped
volumes:
- ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./config/targets:/etc/prometheus/targets:ro
- prometheus_data:/prometheus
command:
- --config.file=/etc/prometheus/prometheus.yaml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=30d
# Prometheus 3.x: OTLP ingestion is its own flag, not an --enable-feature entry.
- --web.enable-otlp-receiver
- --enable-feature=native-histograms,exemplar-storage
# Whichever limit hits first wins; the disk-space alert is the backstop.
- --storage.tsdb.retention.size=15GB
healthcheck:
!!merge <<: *healthcheck
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"]
mem_limit: 2g
!!merge <<: *hardened
networks: [backend]
node-exporter:
image: prom/node-exporter:v1.12.1@sha256:1b4e4438faca4dd7e001dd445d161a4a2091b0fededa84093b3a8dfeae1f1be0
restart: unless-stopped
# Same collector allowlist as the spokes' Alloy (templates/alloy/config.alloy).
# The defaults add ~1000 series on this host (cpufreq, thermal, cooling,
# per-collector scrape stats) that no dashboard, alert or runbook reads.
command:
- --path.rootfs=/host
- --collector.disable-defaults
- --collector.cpu
- --collector.diskstats
- --collector.filesystem
- --collector.hwmon
- --collector.loadavg
- --collector.meminfo
- --collector.netdev
- --collector.stat
- --collector.uname
pid: host
mem_limit: 128m
!!merge <<: *hardened
# Holds pid:host plus the whole host filesystem, hence the tightest limits.
read_only: true
pids_limit: 64
volumes:
# No rslave (unsupported on Docker Desktop) and no network_mode:host, so
# node_network_* describes the container veth, not host NICs.
- /:/host:ro
networks: [backend]
notification-relay:
image: python:3.15.0-slim@sha256:f5b92f2a669a4c347044f29367f31983280b648250ca94dea66f98c53bc87e58
restart: unless-stopped
user: "65534:65534"
read_only: true
command: ["python", "-B", "/app/relay.py"]
volumes:
- ./notification-relay/relay.py:/app/relay.py:ro
environment:
ALERT_WEBHOOK_URL: ${ALERT_WEBHOOK_URL:-}
HEARTBEAT_URL: ${HEARTBEAT_URL:-}
healthcheck:
!!merge <<: *healthcheck
test:
- CMD
- python
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=2).close()"
mem_limit: 64m
!!merge <<: *hardened
pids_limit: 32
networks: [notification-egress, notifications]
grafana:
image: grafana/grafana:13.3.0@sha256:488984b716c0862639f91a698adbe2862677a5a07bad42da5f03f3030883b205
restart: unless-stopped
depends_on:
notification-relay:
condition: service_healthy
loki:
condition: service_started
tempo:
condition: service_started
prometheus:
condition: service_healthy
ports:
- "127.0.0.1:3000:3000"
volumes:
- ./config/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro
- ./config/grafana/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro
- ./config/grafana/alerting:/etc/grafana/provisioning/alerting:ro
- ./dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
environment:
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env}
GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3000}
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/stack-health.json
# Secure cookies break plain-http localhost logins, so opt-in. With the
# tunnel overlay active, `just up` refuses to start without it.
GF_SECURITY_COOKIE_SECURE: ${GRAFANA_COOKIE_SECURE:-false}
GF_SECURITY_COOKIE_SAMESITE: strict
# Per-user identity from Cloudflare Access. Off until GRAFANA_JWT_AUTH=true
# plus CF_ACCESS_TEAM_DOMAIN and CF_ACCESS_AUD are set (the exposure guards
# enforce the pair). No fallback team domain: an unset name must fail
# closed, not fetch signing keys from a claimable cloudflareaccess.com
# subdomain. The aud pin is required because the JWK set is team-wide.
GF_AUTH_JWT_ENABLED: ${GRAFANA_JWT_AUTH:-false}
GF_AUTH_JWT_HEADER_NAME: Cf-Access-Jwt-Assertion
GF_AUTH_JWT_JWK_SET_URL: https://${CF_ACCESS_TEAM_DOMAIN:-}.cloudflareaccess.com/cdn-cgi/access/certs
GF_AUTH_JWT_EXPECT_CLAIMS: '{"aud":"${CF_ACCESS_AUD:-}"}'
GF_AUTH_JWT_EMAIL_CLAIM: email
GF_AUTH_JWT_USERNAME_CLAIM: email
GF_AUTH_JWT_AUTO_SIGN_UP: "true"
healthcheck:
!!merge <<: *healthcheck
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
mem_limit: 1g
!!merge <<: *hardened
networks: [default, backend, notifications]
blackbox-exporter:
image: prom/blackbox-exporter:v0.29.1@sha256:ea1c7aa3ef3f307cd03e85f34abb604478ccb90b567fc690da0cd9776da76b13
restart: unless-stopped
volumes:
- ./config/blackbox.yaml:/etc/blackbox_exporter/config.yml:ro
command: ["--config.file=/etc/blackbox_exporter/config.yml"]
healthcheck:
!!merge <<: *healthcheck
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9115/-/healthy"]
mem_limit: 64m
pids_limit: 64
read_only: true
!!merge <<: *hardened
networks: [default, backend]
ntfy:
image: binwiederhier/ntfy:v2.37.2@sha256:1875ddcf815c8a3c981d6d78c67a93452d0abba79b1ab9520b1f5f5238902bd7
restart: unless-stopped
volumes:
- ./config/ntfy/server.yml:/etc/ntfy/server.yml:ro
- ntfy_data:/var/lib/ntfy
command: ["serve", "--config=/etc/ntfy/server.yml"]
healthcheck:
!!merge <<: *healthcheck
test: ["CMD", "wget", "-q", "--spider", "http://localhost:80/v1/health"]
environment:
NTFY_BASE_URL: ${NTFY_BASE_URL:-http://localhost:80}
mem_limit: 128m
pids_limit: 128
!!merge <<: *hardened
networks: [default, backend, notifications]
# Loki, Prometheus and Tempo have no authentication of their own, so only
# Grafana and the collector can reach them. cloudflared stays on `default`: its
# ingress list comes from Cloudflare at runtime, so an edited ingress cannot
# publish a backend that skips the collector's token gate.
networks:
notification-egress:
notifications:
internal: true
backend:
internal: true
volumes:
loki_data:
tempo_data:
prometheus_data:
grafana_data:
ntfy_data:
# Not in `just backup`: seconds of in-flight telemetry, worthless by restore time.
otel_queue: