From 95ead3a03a9d6656ff2c71baff32853d039aeb9f Mon Sep 17 00:00:00 2001 From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:23:30 +0100 Subject: [PATCH 1/2] feat: retain independent daily uptime evidence --- README.md | 3 + config/uptime-targets.json | 6 + docs/RUNBOOK.md | 4 + docs/UPTIME-EVIDENCE.md | 88 +++++++ justfile | 8 +- scripts/uptime_evidence.py | 445 ++++++++++++++++++++++++++++++++++ tests/test_uptime_evidence.py | 409 +++++++++++++++++++++++++++++++ 7 files changed, 962 insertions(+), 1 deletion(-) create mode 100644 config/uptime-targets.json create mode 100644 docs/UPTIME-EVIDENCE.md create mode 100755 scripts/uptime_evidence.py create mode 100644 tests/test_uptime_evidence.py diff --git a/README.md b/README.md index fd70d62..3f696c6 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ displays them side by side. The alert marker in `config/grafana/alerting/projects.yaml` guards active reporting. The `ProjectsUncovered` rule fires if an unlisted spoke begins transmitting. +For uptime evidence beyond Prometheus's 30-day retention, run the +[independent probe and daily JSON reports](docs/UPTIME-EVIDENCE.md) on a second host. + ## Connect your project Connecting a service takes about five minutes without application code changes. diff --git a/config/uptime-targets.json b/config/uptime-targets.json new file mode 100644 index 0000000..0b8047a --- /dev/null +++ b/config/uptime-targets.json @@ -0,0 +1,6 @@ +[ + {"id": "r9lab-app", "url": "https://app.r9lab.io", "project": "r9lab", "env": "prod"}, + {"id": "r9lab-docs", "url": "https://docs.r9lab.io", "project": "r9lab", "env": "prod"}, + {"id": "r9lab-api", "url": "https://api.r9lab.io/health", "project": "r9lab", "env": "prod"}, + {"id": "cml-gallery", "url": "https://cmlplatform.github.io/cml-gallery/", "project": "cml-gallery", "env": "prod"} +] diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 110a6b7..f97ea83 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -33,6 +33,10 @@ Retention is only partially size-bounded: | Loki chunks | 30d | none | | Tempo blocks | 7d | none | +For evidence beyond raw metric retention, use the +[independent uptime reports](UPTIME-EVIDENCE.md). Their evidence directory lives +on a second host and needs its own backup; `just backup` does not include it. + Loki and Tempo cannot cap their total size, so the disk alert at 80% is the backstop. Two warnings fire earlier. `HostDiskFilling` means a 6-hour linear fit says a filesystem is full within 3 days. `PrometheusCardinalityHigh` diff --git a/docs/UPTIME-EVIDENCE.md b/docs/UPTIME-EVIDENCE.md new file mode 100644 index 0000000..7d23b25 --- /dev/null +++ b/docs/UPTIME-EVIDENCE.md @@ -0,0 +1,88 @@ +# Retain independent uptime evidence + +Prometheus keeps at most 30 days of raw metrics. Use the independent probe to +retain daily JSON reports beyond that window. Run it on a second host outside +the monitoring hub's failure domain. The repository does not install a schedule +or choose a storage host. + +## Prepare the second host + +The commands below require Linux, `uv`, Python 3.15, GNU `timeout`, +a checkout at `/opt/monitoring`, and a writable evidence directory at +`/var/lib/monitoring-uptime`. +Provision Python 3.15 with the host's normal tooling before scheduling. +Run the manual commands first so cron never needs to provision the runtime. +Use a dedicated, unprivileged account. Keep this directory private and outside +the checkout. The probe needs outbound HTTP/S access, but no monitoring credentials. + +1. Review `config/uptime-targets.json`. This is the independent probe's inventory; + `just probe-add` only updates the hub's Blackbox inventory. + Use final URLs: every redirect counts as failure. A probe accepts 2xx response + headers without reading the body, ignores proxy environment variables, and + enforces a ten-second deadline per target. +2. Register the inventory with today's UTC date: + + ```sh + cd /opt/monitoring + uv run --no-project --python 3.15 scripts/uptime_evidence.py register --store /var/lib/monitoring-uptime --targets config/uptime-targets.json --vantage second-host --effective "$(date -u +%F)" + ``` + +3. Run one probe manually: + + ```sh + uv run --no-project --python 3.15 scripts/uptime_evidence.py probe --store /var/lib/monitoring-uptime + ``` + +4. Inspect the JSON output. Check that each target has an observation. Exit 1 + means a target failed, exit 2 means invalid input, and exit 3 means the command + could not complete. + +For inventory changes, register a new manifest effective on a future UTC day. +Historical membership comes from these manifests, rather than the current +configuration. The first registration day includes unknown slots before setup. + +## Schedule collection and daily reports + +Configure the second host's scheduler as the probe account. Use an absolute path +to its `uv` executable. The following crontab assumes `/usr/local/bin/uv` and +GNU `date`; adjust them before installation. `CRON_TZ=UTC` requires a cron +implementation that supports it. Otherwise configure equivalent UTC jobs in +the host's existing scheduler. + +```cron +CRON_TZ=UTC +*/5 * * * * cd /opt/monitoring && timeout 240 /usr/local/bin/uv run --no-project --python 3.15 scripts/uptime_evidence.py probe --store /var/lib/monitoring-uptime +17 0 * * * cd /opt/monitoring && /usr/local/bin/uv run --no-project --python 3.15 scripts/uptime_evidence.py summarize --store /var/lib/monitoring-uptime --date "$(date -u -d yesterday +\%F)" +``` + +The four-minute outer timeout bounds a stalled probe job, including DNS lookup. +An interrupted probe can leave targets unobserved; those slots remain unknown. +Monitor nonzero job exits with the host's existing job monitoring. A missing +schedule produces unknown intervals, not successful probes. A failed summary +job can be rerun for its original date. Reports reject conflicting replacements. + +Back up the **entire evidence directory** to a separate destination using the +host's existing backup system. Retain at least six months. Do not prune raw +observations or manifests until their reports and backup copies are verified. +The hub's `just backup` does not include this directory. + +## Read a historical period + +```sh +uv run --no-project --python 3.15 scripts/uptime_evidence.py report --store /var/lib/monitoring-uptime --start 2026-01-01 --end 2026-06-30 +``` + +Use dates covered by registered inventories. Reports count expected five-minute +slots separately as successful, failed or unknown. Compare success against +expected slots to avoid hiding gaps; success among observed slots describes +only measured requests. Keep coverage alongside any percentage. + +The result measures HTTP reachability from the named probe location. It does +not establish continuous availability between probes, application correctness, +or availability from every user's network. Keep it separate from hub probes +and request-based service indicators. An outage of the probe host creates +unknown evidence. It cannot retrospectively establish uptime before collection. + +To prove recovery after hub replacement, restore the evidence directory on a +clean second host and rerun the historical report. Compare its JSON with the +original output. No connection to Prometheus should be necessary. diff --git a/justfile b/justfile index cf08fa4..5346459 100644 --- a/justfile +++ b/justfile @@ -90,8 +90,14 @@ restart service: _guard-if-exposed # Validate every config in the repo. check: lint validate +# Independent uptime evidence: same pinned Python as the notification relay. +uptime-test: + docker run --rm --network none -v .:/repo:ro -w /repo {{ruff}} check --no-cache scripts/uptime_evidence.py tests/test_uptime_evidence.py + docker run --rm --network none -v .:/repo:ro -w /repo {{ruff}} format --check --no-cache scripts/uptime_evidence.py tests/test_uptime_evidence.py + docker run --rm --network none -v .:/repo:ro -w /repo $(just _image compose.yml notification-relay) python -B -m unittest discover -s tests -p test_uptime_evidence.py + # Static checks and regression tests in tool images. -lint: +lint: uptime-test # Digest-pinned, so anything already local is current; only fetch what is missing. @printf '%s\n' {{lint_images}} | xargs -P 8 -I{} sh -c 'docker image inspect {} >/dev/null 2>&1 || docker pull -q {} >/dev/null' # Explicit -f, not the host's COMPOSE_FILE, so lint means the same here as in CI. diff --git a/scripts/uptime_evidence.py b/scripts/uptime_evidence.py new file mode 100755 index 0000000..aed9274 --- /dev/null +++ b/scripts/uptime_evidence.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.15" +# /// +"""Retain independent HTTP probe observations and complete UTC daily reports.""" + +import argparse +import http.client +import io +import json +import os +import re +import signal +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from datetime import UTC, date, datetime, timedelta +from pathlib import Path + +SLOT_SECONDS = 300 +SLOTS_PER_DAY = 86400 // SLOT_SECONDS +TIMEOUT_SECONDS = 10 + + +class EvidenceError(ValueError): + """Stored evidence is invalid or conflicts with a new observation.""" + + +def utc_now(): + return datetime.now(UTC) + + +def parse_date(value): + try: + result = date.fromisoformat(value) + if result.isoformat() != value: + raise ValueError + return result + except ValueError: + raise argparse.ArgumentTypeError("use YYYY-MM-DD") from None + + +def read_json(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def write_once(path, record): + """Publish a complete file without replacing any existing evidence.""" + data = ( + json.dumps(record, sort_keys=True, indent=2, allow_nan=False) + "\n" + ).encode() + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(dir=path.parent, prefix=".pending-") + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + except FileExistsError: + if path.read_bytes() != data: + raise EvidenceError( + "conflicting evidence; existing file retained" + ) from None + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + os.unlink(temporary) + + +def validate_targets(targets): + if not isinstance(targets, list) or not targets: + raise ValueError("targets must be a nonempty JSON array") + identifiers = set() + for target in targets: + if not isinstance(target, dict) or set(target) != { + "id", + "url", + "project", + "env", + }: + raise ValueError("each target needs exactly id, url, project and env") + for field in ("id", "project", "env"): + if not isinstance(target[field], str) or not re.fullmatch( + r"[a-z0-9][a-z0-9_-]{0,63}", target[field] + ): + raise ValueError( + f"target {field} must use 1–64 lowercase letters, digits, _ or -" + ) + if target["id"] in identifiers: + raise ValueError("target ids must be unique") + identifiers.add(target["id"]) + url = target["url"] + if not isinstance(url, str) or any( + not char.isprintable() or char.isspace() for char in url + ): + raise ValueError( + "target URL must contain no whitespace or control characters" + ) + parsed = urllib.parse.urlsplit(url) + if ( + parsed.scheme not in ("http", "https") + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise ValueError( + "target URL must be HTTP(S) without credentials, query or fragment" + ) + if parsed.port is not None and not 1 <= parsed.port <= 65535: + raise ValueError("target URL port must be between 1 and 65535") + return targets + + +def inventory(store, day): + choices = sorted((store / "inventories").glob("*.json")) + eligible = [path for path in choices if parse_date(path.stem) <= day] + if not eligible: + raise ValueError( + "no inventory covers this date; register before collecting evidence" + ) + record = read_json(eligible[-1]) + if ( + record.get("schema_version") != 1 + or record.get("effective") != eligible[-1].stem + ): + raise EvidenceError("invalid inventory metadata") + if not isinstance(record.get("vantage"), str) or not re.fullmatch( + r"[a-z0-9][a-z0-9_-]{0,63}", record["vantage"] + ): + raise EvidenceError("invalid inventory vantage") + validate_targets(record.get("targets")) + return record + + +def register(store, targets, vantage, effective, now): + if effective < now.date(): + raise ValueError("inventory cannot be registered retrospectively") + if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", vantage): + raise ValueError("vantage must use 1–64 lowercase letters, digits, _ or -") + targets = validate_targets(targets) + previous = sorted((store / "inventories").glob("*.json")) + destination = store / "inventories" / f"{effective}.json" + if previous and effective == now.date() and not destination.exists(): + raise ValueError("inventory changes must start on a future UTC date") + if previous and read_json(previous[0])["vantage"] != vantage: + raise ValueError("one evidence directory must keep a single vantage") + record = { + "schema_version": 1, + "effective": str(effective), + "vantage": vantage, + "targets": targets, + } + write_once(store / "inventories" / f"{effective}.json", record) + return record + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + # Follow-up requests could leave the configured target or collect a login page. + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def request_status(url): + def deadline(_signum, _frame): + raise TimeoutError + + previous = signal.signal(signal.SIGALRM, deadline) + signal.setitimer(signal.ITIMER_REAL, TIMEOUT_SECONDS) + try: + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), NoRedirect() + ) + request = urllib.request.Request( + url, headers={"User-Agent": "uptime-evidence/1"} + ) + try: + with opener.open(request, timeout=TIMEOUT_SECONDS) as response: + return response.status + except urllib.error.HTTPError as error: + status = error.code + error.close() + return status + except (OSError, urllib.error.URLError, http.client.HTTPException, ValueError): + return None + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def observation_path(store, day, target, slot): + return store / "observations" / str(day) / target / f"{slot:03d}.json" + + +def probe(store): + now = utc_now() + manifest = inventory(store, now.date()) + results = [] + for target in manifest["targets"]: + started = utc_now() + if started.date() != now.date(): + raise ValueError( + "UTC day changed during probe; rerun with the current inventory" + ) + slot = ( + started.hour * 3600 + started.minute * 60 + started.second + ) // SLOT_SECONDS + path = observation_path(store, started.date(), target["id"], slot) + if path.exists(): + record = read_json(path) + validate_observation( + record, target, started.date(), slot, manifest["vantage"] + ) + else: + status = request_status(target["url"]) + record = { + "schema_version": 1, + "target": target, + "vantage": manifest["vantage"], + "started_at": started.isoformat(), + "slot": slot, + "status": status, + "outcome": "success" + if status is not None and 200 <= status < 300 + else "failure", + } + write_once(path, record) + results.append(record) + return {"schema_version": 1, "observations": results} + + +def validate_observation(record, target, day, slot, vantage): + started = datetime.fromisoformat(record["started_at"]) + status = record.get("status") + expected_outcome = ( + "success" if type(status) is int and 200 <= status < 300 else "failure" + ) + if ( + record.get("schema_version") != 1 + or record.get("target") != target + or record.get("vantage") != vantage + or started.utcoffset() != timedelta(0) + or started.date() != day + or (started.hour * 3600 + started.minute * 60 + started.second) // SLOT_SECONDS + != slot + or record.get("slot") != slot + or record.get("outcome") != expected_outcome + or ( + status is not None and (type(status) is not int or not 100 <= status <= 599) + ) + ): + raise EvidenceError("invalid observation; evidence was not changed") + return expected_outcome + + +def daily_report(store, day, now): + if day >= now.date(): + raise ValueError("reports require a complete UTC day before today") + manifest = inventory(store, day) + targets = [] + for target in manifest["targets"]: + counts = {"success": 0, "failure": 0, "unknown": 0} + slots = [] + for slot in range(SLOTS_PER_DAY): + path = observation_path(store, day, target["id"], slot) + outcome = ( + validate_observation( + read_json(path), target, day, slot, manifest["vantage"] + ) + if path.exists() + else "unknown" + ) + counts[outcome] += 1 + slots.append({"success": "S", "failure": "F", "unknown": "U"}[outcome]) + observed = counts["success"] + counts["failure"] + targets.append( + { + **target, + **counts, + "slot_outcomes": "".join(slots), + "expected_slots": SLOTS_PER_DAY, + "coverage": observed / SLOTS_PER_DAY, + "success_fraction_of_expected": counts["success"] / SLOTS_PER_DAY, + "success_fraction_of_observed": counts["success"] / observed + if observed + else None, + } + ) + return { + "schema_version": 1, + "date": str(day), + "window_start": f"{day}T00:00:00+00:00", + "window_end": f"{day + timedelta(days=1)}T00:00:00+00:00", + "slot_seconds": SLOT_SECONDS, + "measurement": "first HTTP GET per slot; 2xx success; redirects are failures; no body read", + "inventory_effective": manifest["effective"], + "vantage": manifest["vantage"], + "targets": targets, + } + + +def validate_daily(record, day): + if ( + record.get("schema_version") != 1 + or record.get("date") != str(day) + or record.get("window_start") != f"{day}T00:00:00+00:00" + or record.get("window_end") != f"{day + timedelta(days=1)}T00:00:00+00:00" + or record.get("slot_seconds") != SLOT_SECONDS + or not isinstance(record.get("vantage"), str) + or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", record["vantage"]) + or parse_date(record["inventory_effective"]) > day + ): + raise EvidenceError("invalid daily report metadata") + targets = record["targets"] + validate_targets( + [ + {key: item[key] for key in ("id", "url", "project", "env")} + for item in targets + ] + ) + for item in targets: + slots = item["slot_outcomes"] + if ( + not isinstance(slots, str) + or len(slots) != SLOTS_PER_DAY + or set(slots) - set("SFU") + ): + raise EvidenceError("invalid daily report intervals") + success, failure, unknown = (slots.count(char) for char in "SFU") + observed = success + failure + expected = { + "success": success, + "failure": failure, + "unknown": unknown, + "expected_slots": SLOTS_PER_DAY, + "coverage": observed / SLOTS_PER_DAY, + "success_fraction_of_expected": success / SLOTS_PER_DAY, + "success_fraction_of_observed": success / observed if observed else None, + } + if any(item.get(key) != value for key, value in expected.items()): + raise EvidenceError("daily report counts do not match intervals") + + +def main(): + parser = argparse.ArgumentParser( + description="Retain independent probe evidence. stdout is JSON; no response bodies are retained.", + epilog="Exit: 0 results, 1 probe failure, 2 invalid input, 3 cannot run, 130 interrupted.", + ) + parser.add_argument("--version", action="version", version="uptime-evidence 1.0.0") + commands = parser.add_subparsers(dest="command", required=True) + for name in ("register", "probe", "summarize", "report"): + command = commands.add_parser(name) + command.add_argument( + "--store", type=Path, required=True, help="off-hub evidence directory" + ) + if name == "register": + command.add_argument("--targets", type=Path, required=True) + command.add_argument("--vantage", required=True) + command.add_argument("--effective", type=parse_date, required=True) + elif name == "summarize": + command.add_argument("--date", type=parse_date, required=True) + elif name == "report": + command.add_argument("--start", type=parse_date, required=True) + command.add_argument("--end", type=parse_date, required=True) + args = parser.parse_args() + try: + now = utc_now() + result: dict + if args.command == "register": + result = register( + args.store, read_json(args.targets), args.vantage, args.effective, now + ) + elif args.command == "probe": + result = probe(args.store) + elif args.command == "summarize": + path = args.store / "daily" / f"{args.date}.json" + result = ( + read_json(path) + if path.exists() + else daily_report(args.store, args.date, now) + ) + validate_daily(result, args.date) + write_once(path, result) + else: + if ( + args.end < args.start + or args.end >= now.date() + or (args.end - args.start).days > 366 + ): + raise ValueError( + "report needs 1–367 complete UTC days in ascending order" + ) + days = [] + day = args.start + while day <= args.end: + path = args.store / "daily" / f"{day}.json" + record = ( + read_json(path) + if path.exists() + else daily_report(args.store, day, now) + ) + validate_daily(record, day) + days.append(record) + day += timedelta(days=1) + result = {"schema_version": 1, "days": days} + print(json.dumps(result, sort_keys=True, allow_nan=False)) + return ( + 1 + if args.command == "probe" + and any(item["outcome"] == "failure" for item in result["observations"]) + else 0 + ) + except EvidenceError: + print( + "error: invalid or conflicting stored evidence; existing files retained", + file=sys.stderr, + ) + return 3 + except ValueError as error: + parser.error(str(error)) + except Exception: # noqa: BLE001 - the CLI maps unexpected failures to exit 3 + print( + "error: could not read, probe or write evidence; check files and permissions", + file=sys.stderr, + ) + return 3 + except KeyboardInterrupt: + print("error: interrupted", file=sys.stderr) + signal.signal(signal.SIGINT, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGINT) + + +if __name__ == "__main__": + for stream in (sys.stdout, sys.stderr): + if isinstance(stream, io.TextIOWrapper): + stream.reconfigure(encoding="utf-8", newline="\n") + sys.exit(main()) diff --git a/tests/test_uptime_evidence.py b/tests/test_uptime_evidence.py new file mode 100644 index 0000000..6cc4030 --- /dev/null +++ b/tests/test_uptime_evidence.py @@ -0,0 +1,409 @@ +import contextlib +import http.client +import importlib.util +import io +import json +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from datetime import UTC, date, datetime, timedelta +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from unittest.mock import patch + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts/uptime_evidence.py" +SPEC = importlib.util.spec_from_file_location("uptime_evidence", SCRIPT_PATH) +assert SPEC is not None and SPEC.loader is not None and SPEC.origin is not None +evidence = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(evidence) +DAY = date(2026, 1, 1) +START = datetime(2026, 1, 1, tzinfo=UTC) +TARGET = { + "id": "demo", + "url": "https://example.test/health", + "project": "demo", + "env": "test", +} + + +class HTTPHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/slow": + time.sleep(0.3) + self.send_response( + 302 if self.path == "/redirect" else 503 if self.path == "/fail" else 204 + ) + if self.path == "/redirect": + self.send_header("Location", "/") + self.end_headers() + + def log_message(self, format, *args): + pass + + +class EvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.store = Path(self.temp.name) + evidence.register(self.store, [TARGET], "outside", DAY, START) + + def observe(self, slot=0, status=204): + stamp = START + timedelta(seconds=slot * 300) + record = { + "schema_version": 1, + "target": TARGET, + "vantage": "outside", + "started_at": stamp.isoformat(), + "slot": slot, + "status": status, + "outcome": "success" + if status is not None and 200 <= status < 300 + else "failure", + } + evidence.write_once( + evidence.observation_path(self.store, DAY, "demo", slot), record + ) + return record + + def test_missing_intervals_and_denominator(self): + self.observe(0) + self.observe(1, 503) + report = evidence.daily_report(self.store, DAY, START + timedelta(days=1)) + item = report["targets"][0] + self.assertEqual( + (item["success"], item["failure"], item["unknown"]), (1, 1, 286) + ) + self.assertEqual(item["slot_outcomes"], "SF" + "U" * 286) + self.assertAlmostEqual(item["coverage"], 2 / 288, delta=1e-12) + self.assertAlmostEqual(item["success_fraction_of_observed"], 0.5, delta=1e-12) + self.assertAlmostEqual( + item["success_fraction_of_expected"], 1 / 288, delta=1e-12 + ) + evidence.validate_daily(report, DAY) + + def test_wholly_missing_day_is_unknown(self): + record = evidence.daily_report( + self.store, DAY + timedelta(days=40), START + timedelta(days=41) + ) + self.assertEqual(record["targets"][0]["unknown"], 288) + self.assertIsNone(record["targets"][0]["success_fraction_of_observed"]) + + def test_inventory_history_not_current_targets(self): + newer = {**TARGET, "id": "replacement"} + tomorrow = DAY + timedelta(days=1) + evidence.register(self.store, [newer], "outside", tomorrow, START) + old = evidence.daily_report(self.store, DAY, START + timedelta(days=200)) + new = evidence.daily_report(self.store, tomorrow, START + timedelta(days=200)) + self.assertEqual(old["targets"][0]["id"], "demo") + self.assertEqual(new["targets"][0]["id"], "replacement") + with self.assertRaises(ValueError): + evidence.register( + self.store, [newer], "outside", DAY, START + timedelta(days=2) + ) + with self.assertRaises(ValueError): + evidence.register(self.store, [newer], "elsewhere", tomorrow, START) + + def test_same_day_inventory_change_cannot_remove_failure(self): + tomorrow = START + timedelta(days=1) + replacement = {**TARGET, "id": "replacement"} + with self.assertRaises(ValueError): + evidence.register( + self.store, [replacement], "outside", tomorrow.date(), tomorrow + ) + + def test_atomic_idempotence_and_conflict_retains_evidence(self): + path = self.store / "safe.json" + evidence.write_once(path, {"success": 1}) + evidence.write_once(path, {"success": 1}) + with self.assertRaises(ValueError): + evidence.write_once(path, {"success": 0}) + self.assertEqual(evidence.read_json(path), {"success": 1}) + self.assertEqual(list(self.store.glob(".pending-*")), []) + + def test_concurrent_writers_never_replace_evidence(self): + path = self.store / "concurrent.json" + errors = [] + + def write(value): + try: + evidence.write_once(path, value) + except ValueError: + errors.append(value) + + threads = [ + threading.Thread(target=write, args=({"value": value},)) for value in (0, 1) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(len(errors), 1) + self.assertNotEqual(evidence.read_json(path), errors[0]) + + def test_probe_rerun_does_not_repeat_or_hide_failure(self): + with ( + patch.object(evidence, "utc_now", return_value=START), + patch.object(evidence, "request_status", return_value=503) as request, + ): + first = evidence.probe(self.store) + again = evidence.probe(self.store) + self.assertEqual(request.call_count, 1) + self.assertEqual(first, again) + self.assertEqual(first["observations"][0]["outcome"], "failure") + + def test_actual_start_slot_and_utc_boundary(self): + stamp = START + timedelta(hours=23, minutes=59, seconds=59) + with ( + patch.object(evidence, "utc_now", return_value=stamp), + patch.object(evidence, "request_status", return_value=None), + ): + result = evidence.probe(self.store) + self.assertEqual(result["observations"][0]["slot"], 287) + with self.assertRaises(ValueError): + evidence.daily_report(self.store, DAY, stamp) + with self.assertRaises(ValueError): + evidence.daily_report(self.store, DAY + timedelta(days=1), stamp) + + def test_invalid_urls_fail_before_collection(self): + for url in ( + "ftp://example.test", + "https://user:pass@example.test", + "https://example.test/?token=x", + "https://example.test/#secret", + "https://example.test/\nheader", + "https://example.test:99999", + ): + with self.subTest(url=url), self.assertRaises(ValueError): + evidence.validate_targets([{**TARGET, "url": url}]) + with self.assertRaises(ValueError): + evidence.validate_targets([TARGET, TARGET]) + with self.assertRaises(ValueError): + evidence.validate_targets([{**TARGET, "id": "../elsewhere"}]) + + def test_corrupt_observation_and_report_rejected(self): + record = self.observe() + for mutation in ( + {"slot": 1}, + {"target": {**TARGET, "url": "https://elsewhere.test"}}, + {"started_at": "2026-01-02T00:00:00+00:00"}, + {"outcome": "failure"}, + {"status": True}, + ): + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + evidence.validate_observation( + {**record, **mutation}, TARGET, DAY, 0, "outside" + ) + report = evidence.daily_report(self.store, DAY, START + timedelta(days=1)) + report["targets"][0]["success"] = 288 + with self.assertRaises(ValueError): + evidence.validate_daily(report, DAY) + + def test_archive_reconstructs_after_raw_loss_and_hub_replacement(self): + self.observe() + report = evidence.daily_report(self.store, DAY, START + timedelta(days=1)) + evidence.write_once(self.store / "daily" / f"{DAY}.json", report) + shutil.rmtree(self.store / "observations") + shutil.rmtree(self.store / "inventories") + with ( + patch.object( + sys, + "argv", + [ + "uptime", + "report", + "--store", + str(self.store), + "--start", + str(DAY), + "--end", + str(DAY), + ], + ), + patch.object(evidence, "utc_now", return_value=START + timedelta(days=200)), + contextlib.redirect_stdout(io.StringIO()) as output, + ): + self.assertEqual(evidence.main(), 0) + self.assertEqual(json.loads(output.getvalue())["days"], [report]) + + def test_http_status_redirect_and_total_deadline(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), HTTPHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base = f"http://127.0.0.1:{server.server_port}" + self.assertEqual(evidence.request_status(base), 204) + self.assertEqual(evidence.request_status(base + "/fail"), 503) + self.assertEqual(evidence.request_status(base + "/redirect"), 302) + with patch.object(evidence, "TIMEOUT_SECONDS", 0.05): + self.assertIsNone(evidence.request_status(base + "/slow")) + + def test_malformed_http_is_failed_observation_and_collection_continues(self): + class BrokenOpener: + def open(self, *args, **kwargs): + raise http.client.BadStatusLine("BROKEN UPSTREAM") + + with patch.object( + evidence.urllib.request, "build_opener", return_value=BrokenOpener() + ): + self.assertIsNone(evidence.request_status(TARGET["url"])) + tomorrow = START + timedelta(days=1) + second = {**TARGET, "id": "second"} + evidence.register( + self.store, [TARGET, second], "outside", tomorrow.date(), START + ) + with ( + patch.object(evidence, "utc_now", return_value=tomorrow), + patch.object( + evidence.urllib.request, "build_opener", return_value=BrokenOpener() + ), + ): + result = evidence.probe(self.store) + self.assertEqual(len(result["observations"]), 2) + self.assertTrue( + all(item["outcome"] == "failure" for item in result["observations"]) + ) + + def test_probe_exit_status_reports_observed_failure(self): + for outcome, expected in (("success", 0), ("failure", 1)): + with ( + self.subTest(outcome=outcome), + patch.object( + sys, "argv", ["uptime", "probe", "--store", str(self.store)] + ), + patch.object( + evidence, + "probe", + return_value={ + "schema_version": 1, + "observations": [{"outcome": outcome}], + }, + ), + contextlib.redirect_stdout(io.StringIO()) as output, + ): + self.assertEqual(evidence.main(), expected) + self.assertEqual( + json.loads(output.getvalue())["observations"][0]["outcome"], outcome + ) + + def test_total_deadline_interrupts_blocking_dns_or_headers(self): + class BlockingOpener: + def open(self, *args, **kwargs): + time.sleep(1) + raise OSError + + started = time.monotonic() + with ( + patch.object(evidence, "TIMEOUT_SECONDS", 0.03), + patch.object( + evidence.urllib.request, "build_opener", return_value=BlockingOpener() + ), + ): + self.assertIsNone(evidence.request_status(TARGET["url"])) + self.assertLess(time.monotonic() - started, 0.5) + + def test_archived_summary_rerun_survives_raw_pruning(self): + self.observe() + record = evidence.daily_report(self.store, DAY, START + timedelta(days=1)) + path = self.store / "daily" / f"{DAY}.json" + evidence.write_once(path, record) + shutil.rmtree(self.store / "observations") + before = path.read_bytes() + with ( + patch.object( + sys, + "argv", + ["uptime", "summarize", "--store", str(self.store), "--date", str(DAY)], + ), + patch.object(evidence, "utc_now", return_value=START + timedelta(days=200)), + contextlib.redirect_stdout(io.StringIO()) as output, + ): + self.assertEqual(evidence.main(), 0) + self.assertEqual(json.loads(output.getvalue()), record) + self.assertEqual(path.read_bytes(), before) + + def test_cli_conflicting_inventory_returns_runtime_error_without_replacing(self): + original = self.store / "inventories" / f"{DAY}.json" + before = original.read_bytes() + targets = self.store / "targets.json" + targets.write_text( + json.dumps([{**TARGET, "url": "https://changed.example.test"}]), + encoding="utf-8", + ) + with ( + patch.object( + sys, + "argv", + [ + "uptime", + "register", + "--store", + str(self.store), + "--targets", + str(targets), + "--vantage", + "outside", + "--effective", + str(DAY), + ], + ), + patch.object(evidence, "utc_now", return_value=START), + contextlib.redirect_stdout(io.StringIO()) as output, + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(evidence.main(), 3) + self.assertEqual(output.getvalue(), "") + self.assertEqual(original.read_bytes(), before) + + def test_cli_help_usage_error_and_missing_file(self): + command = [sys.executable, "-B", str(SCRIPT_PATH)] + help_result = subprocess.run( + [*command, "--help"], check=False, capture_output=True, text=True + ) + self.assertEqual(help_result.returncode, 0) + self.assertIn("Exit: 0 results", help_result.stdout) + invalid = subprocess.run( + [ + *command, + "report", + "--store", + str(self.store), + "--start", + "bad", + "--end", + "2026-01-02", + ], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(invalid.returncode, 2) + self.assertEqual(invalid.stdout, "") + missing = subprocess.run( + [ + *command, + "register", + "--store", + str(self.store), + "--targets", + str(self.store / "absent"), + "--vantage", + "outside", + "--effective", + str(datetime.now(UTC).date()), + ], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(missing.returncode, 3) + self.assertNotIn("Traceback", missing.stderr) + + +if __name__ == "__main__": + unittest.main() From 079805728e2f2595703bc29752bbe416b109da6d Mon Sep 17 00:00:00 2001 From: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:53:53 +0100 Subject: [PATCH 2/2] fix: retain failures from unsupported http statuses --- scripts/uptime_evidence.py | 6 +++--- tests/test_uptime_evidence.py | 36 ++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/scripts/uptime_evidence.py b/scripts/uptime_evidence.py index aed9274..07b0923 100755 --- a/scripts/uptime_evidence.py +++ b/scripts/uptime_evidence.py @@ -184,13 +184,13 @@ def deadline(_signum, _frame): ) try: with opener.open(request, timeout=TIMEOUT_SECONDS) as response: - return response.status + status = response.status except urllib.error.HTTPError as error: status = error.code error.close() - return status - except (OSError, urllib.error.URLError, http.client.HTTPException, ValueError): + except OSError, urllib.error.URLError, http.client.HTTPException, ValueError: return None + return status if type(status) is int and 100 <= status <= 599 else None finally: signal.setitimer(signal.ITIMER_REAL, 0) signal.signal(signal.SIGALRM, previous) diff --git a/tests/test_uptime_evidence.py b/tests/test_uptime_evidence.py index 6cc4030..308297a 100644 --- a/tests/test_uptime_evidence.py +++ b/tests/test_uptime_evidence.py @@ -34,9 +34,8 @@ class HTTPHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == "/slow": time.sleep(0.3) - self.send_response( - 302 if self.path == "/redirect" else 503 if self.path == "/fail" else 204 - ) + statuses = {"/redirect": 302, "/fail": 503, "/unsupported": 600} + self.send_response(statuses.get(self.path, 204)) if self.path == "/redirect": self.send_header("Location", "/") self.end_headers() @@ -243,6 +242,37 @@ def test_http_status_redirect_and_total_deadline(self): with patch.object(evidence, "TIMEOUT_SECONDS", 0.05): self.assertIsNone(evidence.request_status(base + "/slow")) + def test_unsupported_http_status_remains_readable_failed_evidence(self): + server = ThreadingHTTPServer(("127.0.0.1", 0), HTTPHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + tomorrow = START + timedelta(days=1) + target = { + **TARGET, + "url": f"http://127.0.0.1:{server.server_port}/unsupported", + } + evidence.register(self.store, [target], "outside", tomorrow.date(), START) + with patch.object(evidence, "utc_now", return_value=tomorrow): + first = evidence.probe(self.store) + again = evidence.probe(self.store) + self.assertEqual(first, again) + self.assertIsNone(first["observations"][0]["status"]) + self.assertEqual(first["observations"][0]["outcome"], "failure") + report = evidence.daily_report( + self.store, tomorrow.date(), tomorrow + timedelta(days=1) + ) + item = report["targets"][0] + self.assertEqual( + (item["success"], item["failure"], item["unknown"]), (0, 1, 287) + ) + evidence.validate_daily(report, tomorrow.date()) + + def test_unsupported_normal_response_status_is_not_persistable(self): + with patch.object(evidence.urllib.request, "build_opener") as opener: + opener.return_value.open.return_value.__enter__.return_value.status = 600 + self.assertIsNone(evidence.request_status(TARGET["url"])) + def test_malformed_http_is_failed_observation_and_collection_continues(self): class BrokenOpener: def open(self, *args, **kwargs):