From 22fd651b6cb27c9b6d7caf3915aa436815e6706b Mon Sep 17 00:00:00 2001 From: Alexander Tarasov Date: Wed, 30 Sep 2026 10:56:05 +0200 Subject: [PATCH 1/3] ref(healthcheck): Share one python3 HTTP healthcheck for sentry web and snuba api Replace the web healthcheck's bash /dev/tcp probe, which needs a shell, with healthcheck/http.py, a generalized version of snuba/api_healthcheck.py that takes the URL as an argument. The healthcheck/ directory is mounted read-only at /healthcheck, like ./geoip, for sentry services and snuba-api. --- docker-compose.yml | 12 +++------ healthcheck/http.py | 55 +++++++++++++++++++++++++++++++++++++ snuba/api_healthcheck.py | 58 ---------------------------------------- 3 files changed, 59 insertions(+), 66 deletions(-) create mode 100755 healthcheck/http.py delete mode 100755 snuba/api_healthcheck.py diff --git a/docker-compose.yml b/docker-compose.yml index dbffd8c3051..632bb309953 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,7 @@ x-sentry-defaults: &sentry_defaults - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" + - "./healthcheck:/healthcheck:ro" x-snuba-defaults: &snuba_defaults <<: [*restart_policy, *pull_policy] depends_on: @@ -307,10 +308,10 @@ services: snuba-api: <<: *snuba_defaults volumes: - - "./snuba/api_healthcheck.py:/usr/src/snuba/api_healthcheck.py:ro" + - "./healthcheck:/healthcheck:ro" healthcheck: <<: *healthcheck_defaults - test: ["CMD", "python3", "/usr/src/snuba/api_healthcheck.py"] + test: ["CMD", "python3", "/healthcheck/http.py", "http://127.0.0.1:1218/health"] # Kafka consumer responsible for feeding events into Clickhouse snuba-errors-consumer: <<: *snuba_defaults @@ -465,12 +466,7 @@ services: healthcheck: <<: *healthcheck_defaults start_period: 5m - test: - - "CMD" - - "/bin/bash" - - "-c" - # Courtesy of https://unix.stackexchange.com/a/234089/108960 - - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' + test: ["CMD", "python3", "/healthcheck/http.py", "http://127.0.0.1:9000/_health/"] events-consumer: <<: *sentry_defaults command: run consumer ingest-events --consumer-group ingest-consumer --auto-offset-reset=earliest --no-strict-offset-reset --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} diff --git a/healthcheck/http.py b/healthcheck/http.py new file mode 100755 index 00000000000..b91b92d9b0b --- /dev/null +++ b/healthcheck/http.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +""" +HTTP healthcheck for self-hosted services. + +Usage: python3 /healthcheck/http.py + +GETs the URL and exits 0 if the response body contains "ok", else 1. On +failure, prints a one-line description to stderr rather than a full Python +traceback, so `docker inspect` output stays readable. + +Uses only the standard library, so it works in any image with python3 on +PATH, including distroless ones. docker-compose.yml mounts this directory +read-only at /healthcheck. +""" + +import sys +import urllib.error +import urllib.request + +TIMEOUT = 2 # seconds + + +def main(url: str) -> int: + try: + body = urllib.request.urlopen(url, timeout=TIMEOUT).read().decode() + except urllib.error.HTTPError as exc: + print(f"HTTP {exc.code} from {url}", file=sys.stderr) + return 1 + except urllib.error.URLError as exc: + # urlopen() wraps connection-phase failures (refused, DNS, etc.) here. + print(f"{url} unreachable: {exc.reason}", file=sys.stderr) + return 1 + except TimeoutError: + # A timeout firing during .read() (after urlopen returns) bubbles up + # as a bare TimeoutError from the underlying socket — not wrapped in + # URLError. Catch it explicitly so the message stays one-line. + print(f"timed out reading {url} after {TIMEOUT}s", file=sys.stderr) + return 1 + except OSError as exc: + # ConnectionResetError, etc. — anything else from the socket layer. + print(f"error against {url}: {exc}", file=sys.stderr) + return 1 + + if "ok" not in body: + print(f"response from {url} missing 'ok': {body!r}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("usage: http.py ", file=sys.stderr) + sys.exit(2) + sys.exit(main(sys.argv[1])) diff --git a/snuba/api_healthcheck.py b/snuba/api_healthcheck.py deleted file mode 100755 index dc5dd74340f..00000000000 --- a/snuba/api_healthcheck.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -""" -Healthcheck for the snuba api in self-hosted. - -GETs the snuba health endpoint and exits 0 if the response body -contains "ok", else 1. On failure, prints a one-line description to -stderr rather than a full Python traceback, so `docker inspect` -output stays readable. - -Mounted into the snuba-api container by docker-compose.yml. - -Optional overrides — set in the snuba-api container's environment -(e.g. via your own docker-compose.override.yml) if the defaults -don't fit: - - SNUBA_API_HEALTHCHECK_URL default http://127.0.0.1:1218/health - SNUBA_API_HEALTHCHECK_TIMEOUT default 2 (seconds) -""" - -import os -import sys -import urllib.error -import urllib.request - -URL = os.environ.get("SNUBA_API_HEALTHCHECK_URL") or "http://127.0.0.1:1218/health" -TIMEOUT = float(os.environ.get("SNUBA_API_HEALTHCHECK_TIMEOUT") or 2) - - -def main() -> int: - try: - body = urllib.request.urlopen(URL, timeout=TIMEOUT).read().decode() - except urllib.error.HTTPError as exc: - print(f"snuba api returned HTTP {exc.code} from {URL}", file=sys.stderr) - return 1 - except urllib.error.URLError as exc: - # urlopen() wraps connection-phase failures (refused, DNS, etc.) here. - print(f"snuba api unreachable at {URL}: {exc.reason}", file=sys.stderr) - return 1 - except TimeoutError: - # A timeout firing during .read() (after urlopen returns) bubbles up - # as a bare TimeoutError from the underlying socket — not wrapped in - # URLError. Catch it explicitly so the message stays one-line. - print(f"snuba api timed out reading {URL} after {TIMEOUT}s", file=sys.stderr) - return 1 - except OSError as exc: - # ConnectionResetError, etc. — anything else from the socket layer. - print(f"snuba api error against {URL}: {exc}", file=sys.stderr) - return 1 - - if "ok" not in body: - print(f"snuba api response missing 'ok' (from {URL}): {body!r}", file=sys.stderr) - return 1 - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From f34579306ac27a2edf949242b3f41aca3b993bc1 Mon Sep 17 00:00:00 2001 From: Alexander Tarasov Date: Wed, 30 Sep 2026 11:28:13 +0200 Subject: [PATCH 2/3] fix(healthcheck): Rename http.py so it doesn't shadow the stdlib http package Running python3 /healthcheck/http.py puts /healthcheck first on sys.path, so urllib's 'import http.client' picked up the script instead of the standard library and every healthcheck failed. --- docker-compose.yml | 4 ++-- healthcheck/{http.py => check_http.py} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename healthcheck/{http.py => check_http.py} (94%) diff --git a/docker-compose.yml b/docker-compose.yml index 632bb309953..7a87f94516d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -311,7 +311,7 @@ services: - "./healthcheck:/healthcheck:ro" healthcheck: <<: *healthcheck_defaults - test: ["CMD", "python3", "/healthcheck/http.py", "http://127.0.0.1:1218/health"] + test: ["CMD", "python3", "/healthcheck/check_http.py", "http://127.0.0.1:1218/health"] # Kafka consumer responsible for feeding events into Clickhouse snuba-errors-consumer: <<: *snuba_defaults @@ -466,7 +466,7 @@ services: healthcheck: <<: *healthcheck_defaults start_period: 5m - test: ["CMD", "python3", "/healthcheck/http.py", "http://127.0.0.1:9000/_health/"] + test: ["CMD", "python3", "/healthcheck/check_http.py", "http://127.0.0.1:9000/_health/"] events-consumer: <<: *sentry_defaults command: run consumer ingest-events --consumer-group ingest-consumer --auto-offset-reset=earliest --no-strict-offset-reset --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} diff --git a/healthcheck/http.py b/healthcheck/check_http.py similarity index 94% rename from healthcheck/http.py rename to healthcheck/check_http.py index b91b92d9b0b..2859c5e9d83 100755 --- a/healthcheck/http.py +++ b/healthcheck/check_http.py @@ -2,7 +2,7 @@ """ HTTP healthcheck for self-hosted services. -Usage: python3 /healthcheck/http.py +Usage: python3 /healthcheck/check_http.py GETs the URL and exits 0 if the response body contains "ok", else 1. On failure, prints a one-line description to stderr rather than a full Python @@ -50,6 +50,6 @@ def main(url: str) -> int: if __name__ == "__main__": if len(sys.argv) != 2: - print("usage: http.py ", file=sys.stderr) + print("usage: check_http.py ", file=sys.stderr) sys.exit(2) sys.exit(main(sys.argv[1])) From 8f96168a1707fd30fcf8fb08bf91ccd1c4aaca6a Mon Sep 17 00:00:00 2001 From: Alexander Tarasov Date: Wed, 30 Sep 2026 20:31:32 +0200 Subject: [PATCH 3/3] fix(healthcheck): Ignore proxy settings in check_http.py Docker can inject HTTP_PROXY/http_proxy into containers from the client config, and urllib would then send the local health request through the proxy. The target is always the container itself, so bypass proxies, like the clickhouse and seaweedfs healthchecks already do. --- healthcheck/check_http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/healthcheck/check_http.py b/healthcheck/check_http.py index 2859c5e9d83..79130ade68a 100755 --- a/healthcheck/check_http.py +++ b/healthcheck/check_http.py @@ -22,7 +22,10 @@ def main(url: str) -> int: try: - body = urllib.request.urlopen(url, timeout=TIMEOUT).read().decode() + # Ignore HTTP(S)_PROXY, which Docker may inject into containers: the + # target is always the container itself. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + body = opener.open(url, timeout=TIMEOUT).read().decode() except urllib.error.HTTPError as exc: print(f"HTTP {exc.code} from {url}", file=sys.stderr) return 1