From 94619cee740f18f02db1b8f0807da4f60c6d8978 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 01/15] ci: add fail-closed verification evidence and managed host lanes --- tests/test_verification_evidence.py | 278 ++++++++++++++++++++++++++++ tools/__init__.py | 0 tools/verification.py | 241 ++++++++++++++++++++++++ tools/verification_environment.py | 55 ++++++ tools/verification_evidence.py | 217 ++++++++++++++++++++++ tools/verification_host.py | 152 +++++++++++++++ tools/verification_process.py | 81 ++++++++ tools/verification_wheel.py | 60 ++++++ 8 files changed, 1084 insertions(+) create mode 100644 tests/test_verification_evidence.py create mode 100644 tools/__init__.py create mode 100644 tools/verification.py create mode 100644 tools/verification_environment.py create mode 100644 tools/verification_evidence.py create mode 100644 tools/verification_host.py create mode 100644 tools/verification_process.py create mode 100644 tools/verification_wheel.py diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py new file mode 100644 index 0000000..3fda6de --- /dev/null +++ b/tests/test_verification_evidence.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import json +from pathlib import Path +import sys + +import pytest + +from tools.verification import execute, integration, pytest_command, verify +from tools.verification_environment import clean, doctor +from tools.verification_evidence import ( + APP_INSIGHTS_TEST, + VERSION_TEST, + Summary, + collect, + publish, + redact, + report, + save, +) +from tools.verification_process import child, run +from tools.verification_wheel import main as wheel_smoke + + +@pytest.mark.parametrize( + ("cases", "code", "status"), + [ + ("", 0, "INCOMPLETE"), + ("", 5, "INCOMPLETE"), + ("", 0, "INCOMPLETE"), + ("", 0, "INCOMPLETE"), + ("", 1, "FAIL"), + ("", 0, "PASS"), + ("", 124, "FAIL"), + ], +) +def test_required_lane_classification(tmp_path: Path, cases: str, code: int, status: str) -> None: + (tmp_path / "junit.xml").write_text(f"{cases}") + summary = collect(tmp_path, ["pytest"], code) + assert summary.status == status + assert (summary.exit_code == 0) == (status == "PASS") + + +@pytest.mark.parametrize( + ("lane", "status", "cloud_status"), + [ + ("azure-e2e", "INCOMPLETE", "INCOMPLETE"), + ("local-e2e", "PASS", "NOT_APPLICABLE"), + ], +) +def test_skipped_app_insights_never_certifies_cloud( + tmp_path: Path, lane: str, status: str, cloud_status: str +) -> None: + directory = tmp_path / lane + directory.mkdir() + (directory / "junit.xml").write_text( + f'' + f'' + ) + summary = collect(directory, ["pytest"], 0) + assert summary.status == status + assert summary.cloud_monitor == cloud_status + assert summary.deployed_version_match == "PASS" + assert summary.cloud_tests_not_run == (APP_INSIGHTS_TEST,) + + +def test_missing_required_cloud_assertion_is_incomplete(tmp_path: Path) -> None: + directory = tmp_path / "azure-e2e" + directory.mkdir() + (directory / "junit.xml").write_text( + f'' + ) + assert collect(directory, ["pytest"], 0).status == "INCOMPLETE" + + +def test_cloud_certificate_requires_both_observed_assertions(tmp_path: Path) -> None: + directory = tmp_path / "azure-e2e" + directory.mkdir() + (directory / "junit.xml").write_text( + f'' + f'' + ) + summary = collect(directory, ["pytest"], 0) + assert (summary.status, summary.cloud_monitor, summary.deployed_version_match) == ( + "PASS", + "PASS", + "PASS", + ) + assert summary.cloud_tests_not_run == () + + +@pytest.mark.parametrize( + "secret", + [ + "AccountKey=VERYSECRET;", + "SharedAccessSignature=VERYSECRET;", + "code=VERYSECRET&x=1", + "x-functions-key: VERYSECRET", + "Authorization: Bearer VERYSECRET", + "password=VERYSECRET", + "token=VERYSECRET", + "AzureWebJobsStorage=VERYSECRET", + "sas=VERYSECRET", + "api_key=VERYSECRET", + "DefaultEndpointsProtocol=https;AccountName=PRIVATE;AccountKey=VERYSECRET;", + "InstrumentationKey=VERYSECRET;IngestionEndpoint=https://VERYSECRET/;", + '"connection_string": "VERYSECRET"', + ], +) +def test_canaries_removed_from_junit_logs_console_and_summary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], secret: str +) -> None: + (tmp_path / "junit.xml").write_text( + "" + + secret.replace("&", "&") + + "" + ) + (tmp_path / "runner.log").write_text(secret) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary")) + summary = save(tmp_path, collect(tmp_path, ["pytest", secret], 0)) + publish(summary) + assert "VERYSECRET" not in capsys.readouterr().out + for path in tmp_path.rglob("*"): + if path.is_file(): + assert "VERYSECRET" not in path.read_text() + json.loads((tmp_path / "summary.json").read_text()) + + +def test_environment_secret_removed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CUSTOM_ACCESS_TOKEN", "opaque-sensitive-value") + assert redact("diagnostic opaque-sensitive-value") == "diagnostic [REDACTED]" + + +def test_saved_report_does_not_execute_command( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + directory = tmp_path / "test" + directory.mkdir() + (directory / "junit.xml").write_text("") + (directory / "coverage.xml").write_text('') + save(directory, collect(directory, ["not-an-executable"], 0)) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary")) + assert report(tmp_path) == 0 + data = json.loads((directory / "summary.json").read_text()) + assert data["schema_version"] == 1 + assert (data["selected"], data["passed"], data["failed"], data["skipped"]) == (1, 1, 0, 0) + assert data["coverage"] == 0.99 + assert str(directory / "junit.xml") in data["supporting_files"] + assert "passed=1" in (tmp_path / "job-summary").read_text() + + +def test_empty_report_is_read_only(tmp_path: Path) -> None: + assert report(tmp_path) == 1 + assert list(tmp_path.iterdir()) == [] + + +def test_missing_tools_fail_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("tools.verification_environment.shutil.which", lambda _: None) + assert doctor() != 0 + assert execute("host-smoke", []) != 0 + assert ( + json.loads(Path("test-results/host-smoke/summary.json").read_text())["status"] + == "INCOMPLETE" + ) + + +def test_host_prerequisites_do_not_require_mermaid(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "tools.verification_environment.shutil.which", + lambda tool: None if tool == "mmdc" else "/tools/" + tool, + ) + assert doctor(quality=False) == 0 + assert doctor() != 0 + + +def test_offline_selection_keeps_all_privacy_tests() -> None: + command = pytest_command("test", []) + assert "tests/" in command + assert "not e2e" in command + assert [arg for arg in command if arg.startswith("--ignore")] == [ + "--ignore=tests/e2e", + "--ignore=tests/host_smoke", + ] + assert "tests/test_otel_contract.py" in pytest_command("otel", []) + + +def test_clean_preserves_environments_storage_and_source(tmp_path: Path) -> None: + for name in (".venv", "src", "shared-azurite", "test-results", "dist", "build"): + (tmp_path / name).mkdir() + (tmp_path / name / "keep").touch() + clean(tmp_path) + assert sorted(p.name for p in tmp_path.iterdir()) == [".venv", "shared-azurite", "src"] + + +def test_clean_does_not_follow_artifact_symlink(tmp_path: Path) -> None: + (tmp_path / "shared").mkdir() + (tmp_path / "shared/keep").touch() + (tmp_path / "test-results").symlink_to(tmp_path / "shared", target_is_directory=True) + clean(tmp_path) + assert (tmp_path / "shared/keep").exists() + + +def test_process_failure_preserved(tmp_path: Path) -> None: + assert run([sys.executable, "-c", "raise SystemExit(7)"], tmp_path / "log") == 7 + assert run(["/nonexistent/verification-command"], tmp_path / "missing") == 127 + + +def test_interrupt_reaps_owned_child(tmp_path: Path) -> None: + with ( + pytest.raises(KeyboardInterrupt), + child([sys.executable, "-c", "import time; time.sleep(30)"], tmp_path / "log") as process, + ): + raise KeyboardInterrupt + assert process.poll() is not None + + +def test_timeout_reaps_and_redacts(tmp_path: Path) -> None: + code = run( + [sys.executable, "-u", "-c", "import time; print('password=VERYSECRET'); time.sleep(30)"], + tmp_path / "log", + 1, + ) + assert code == 124 + assert "VERYSECRET" not in (tmp_path / "log").read_text() + + +def test_wheel_smoke_rejects_stale_dist(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "dist").mkdir() + (tmp_path / "dist/stale.whl").write_bytes(b"stale") + + def empty_build(command: list[str], log: Path, timeout: float = 600) -> int: + assert "--outdir" in command + assert str(tmp_path / "dist") not in command + return 0 + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("tools.verification_wheel.run", empty_build) + assert wheel_smoke() != 0 + + +def test_integration_retains_separate_required_lanes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[str] = [] + + def fake_execute(lane: str, extra: list[str]) -> int: + calls.append(lane) + save(Path("test-results") / lane, Summary(1, lane, ("test",), 0, 0, 0, 0, "INCOMPLETE", 1)) + return 1 + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("tools.verification.execute", fake_execute) + assert integration() != 0 + assert calls == ["otel", "host-smoke", "host-matrix"] + assert ( + json.loads(Path("test-results/integration/summary.json").read_text())["status"] + == "INCOMPLETE" + ) + + +def test_verify_continues_after_failure_and_rejects_stale_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[str] = [] + + def fake_run(command: list[str], log: Path, timeout: float = 600) -> int: + calls.append(command[1]) + return 1 + + monkeypatch.chdir(tmp_path) + save(Path("test-results/test"), Summary(1, "test", ("old",), 1, 1, 0, 0, "PASS", 0)) + monkeypatch.setattr("tools.verification.run", fake_run) + assert verify() != 0 + assert calls == ["check", "test", "integration", "wheel-smoke", "docs"] + assert ( + json.loads(Path("test-results/verify/lanes.json").read_text())[1]["status"] == "INCOMPLETE" + ) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/verification.py b/tools/verification.py new file mode 100644 index 0000000..19cdbce --- /dev/null +++ b/tools/verification.py @@ -0,0 +1,241 @@ +"""Canonical verification CLI, shared by Make and CI's selected interpreter.""" + +from __future__ import annotations + +import argparse +from dataclasses import asdict, replace +import json +import os +from pathlib import Path +import shutil +import sys +import xml.etree.ElementTree as ET + +from tools.verification_environment import clean, doctor +from tools.verification_evidence import Status, Summary, collect, publish, report, save +from tools.verification_host import host_smoke +from tools.verification_process import run + +OTEL_TESTS = ( + "tests/test_otel.py", + "tests/test_otel_contract.py", + "tests/test_otel_filters.py", + "tests/test_host_config_otel.py", +) + + +def pytest_command(lane: str, extra: list[str]) -> list[str]: + directory = Path("test-results") / lane + command = [sys.executable, "-m", "pytest"] + match lane: + case "test": + command += [ + "tests/", + "-m", + "not e2e", + "--ignore=tests/e2e", + "--ignore=tests/host_smoke", + "--cov-report=xml:" + str(directory / "coverage.xml"), + ] + case "otel": + command += [*OTEL_TESTS, "--no-cov"] + case _: + command += ["tests/e2e/", "-m", "e2e", "--no-cov"] + return command + ["--junitxml=" + str(directory / "junit.xml"), *extra] + + +def execute(lane: str, extra: list[str]) -> int: + directory = Path("test-results") / lane + if directory.exists(): + shutil.rmtree(directory) + (directory / "logs").mkdir(parents=True) + command = pytest_command(lane, extra) + reason, code, blocked = "", 0, False + try: + if lane in {"azure-e2e", "local-e2e"}: + required = ["E2E_BASE_URL", "E2E_EXPECTED_VERSION"] + if lane == "azure-e2e": + required += ["APPINSIGHTS_NAME", "E2E_RESOURCE_GROUP"] + missing = [key for key in required if not os.environ.get(key, "").strip()] + if lane == "azure-e2e" and shutil.which("az") is None: + missing.append("az") + if missing: + blocked, code, reason = True, 2, "Missing prerequisites: " + ", ".join(missing) + if lane in {"host-smoke", "host-matrix"}: + command = ["func", "start", "(managed logging host smoke)"] + if doctor(quality=False): + blocked, code, reason = True, 2, "Missing required integration tools" + else: + code = host_smoke(directory, matrix=lane == "host-matrix") + elif not blocked: + code = run(command, directory / "logs/runner.log") + except KeyboardInterrupt: + code, reason = 130, "Interrupted; owned child processes terminated" + except (OSError, ET.ParseError, ValueError) as error: + code, reason = 2, str(error) + try: + summary = collect(directory, command, code) + except (ET.ParseError, ValueError) as error: + summary = Summary( + 1, + lane, + tuple(command), + 0, + 0, + 0, + 0, + "INCOMPLETE", + 2, + reason=f"Invalid evidence: {error}", + ) + if blocked: + summary = replace(summary, status="INCOMPLETE") + ET.ElementTree(ET.Element("testsuite", name=lane, tests="0")).write( + directory / "junit.xml", encoding="unicode" + ) + summary = save(directory, replace(summary, reason=reason or summary.reason)) + publish(summary) + return summary.exit_code + + +def aggregate(lane: str, outcomes: list[Summary]) -> int: + status: Status = "FAIL" if any(item.status == "FAIL" for item in outcomes) else "PASS" + if status == "PASS" and (not outcomes or any(item.status != "PASS" for item in outcomes)): + status = "INCOMPLETE" + directory = Path("test-results") / lane + directory.mkdir(parents=True, exist_ok=True) + summary = Summary( + 1, + lane, + ("make", lane), + sum(s.selected for s in outcomes), + sum(s.passed for s in outcomes), + sum(s.failed for s in outcomes), + sum(s.skipped for s in outcomes), + status, + int(status != "PASS"), + reason="; ".join(f"{item.lane}={item.status}" for item in outcomes), + otel_instrumentation=next( + ( + s.otel_instrumentation + for s in outcomes + if s.otel_instrumentation != "NOT_APPLICABLE" + ), + "NOT_APPLICABLE", + ), + ) + (directory / "lanes.json").write_text(json.dumps([asdict(s) for s in outcomes], indent=2)) + publish(save(directory, summary)) + return summary.exit_code + + +def integration() -> int: + directory = Path("test-results/integration") + if directory.exists(): + shutil.rmtree(directory) + outcomes = [] + for lane in ("otel", "host-smoke", "host-matrix"): + execute(lane, []) + source = Path("test-results") / lane + outcomes.append(Summary(**json.loads((source / "summary.json").read_text()))) + for pattern, destination in (("**/*junit.xml", "junit"), ("logs/*", "logs")): + for artifact in source.glob(pattern): + target = directory / destination / f"{lane}-{artifact.name}" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(artifact, target) + return aggregate("integration", outcomes) + + +def verify() -> int: + directory = Path("test-results/verify") + if directory.exists(): + shutil.rmtree(directory) + outcomes = [] + for target in ("check", "test", "integration", "wheel-smoke", "docs"): + path = Path("test-results") / target / "summary.json" + path.unlink(missing_ok=True) + code = run(["make", target], directory / "logs" / f"{target}.log", 1200) + if target != "check" and path.exists(): + summary = Summary(**json.loads(path.read_text())) + if code and summary.status == "PASS": + summary = replace(summary, status="FAIL", exit_code=code) + else: + required = target != "check" + summary = Summary( + 1, + target, + ("make", target), + int(not required), + int(not required and code == 0), + int(code != 0), + 0, + "INCOMPLETE" if required else ("FAIL" if code else "PASS"), + code or int(required), + reason="Required lane summary missing" if required else "", + ) + outcomes.append(summary) + return aggregate("verify", outcomes) + + +def focused(lane: str, command: list[str]) -> int: + directory = Path("test-results") / lane + if directory.exists(): + shutil.rmtree(directory) + code = run(command, directory / "logs/runner.log", 1200) + summary = Summary( + 1, + lane, + tuple(command), + 1, + int(code == 0), + int(code != 0), + 0, + "PASS" if code == 0 else "FAIL", + code, + ) + publish(save(directory, summary)) + return code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "lane", + choices=( + "test", + "integration", + "otel", + "host-smoke", + "host-matrix", + "verify", + "azure-e2e", + "local-e2e", + "report", + "doctor", + "clean", + "focused", + ), + ) + args, extra = parser.parse_known_args() + if not (3, 11) <= sys.version_info[:2] < (3, 15): + parser.error("Python >=3.11,<3.15 is required") + match args.lane: + case "report": + return report(Path("test-results")) + case "doctor": + return doctor() + case "clean": + clean(Path.cwd()) + return 0 + case "verify": + return verify() + case "integration": + return integration() + case "focused": + return focused(extra[0], [sys.executable, *extra[1:]]) + case _: + return execute(args.lane, extra) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verification_environment.py b/tools/verification_environment.py new file mode 100644 index 0000000..b4752eb --- /dev/null +++ b/tools/verification_environment.py @@ -0,0 +1,55 @@ +"""Read-only prerequisites and allowlisted generated-artifact cleanup.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import shutil +import sys + + +def doctor(live: bool = True, quality: bool = True) -> int: + healthy = (3, 11) <= sys.version_info[:2] < (3, 15) + print(f"python REQUIRED >=3.11,<3.15: {sys.version.split()[0]} ({sys.executable})") + for module in ("venv", "ruff", "mypy", "pytest", "bandit", "build", "mkdocs"): + found = importlib.util.find_spec(module) is not None + print(f"{module} REQUIRED: {'found' if found else 'MISSING'}") + healthy = healthy and found + for tool in ("func", "azurite"): + path = shutil.which(tool) + print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}") + if live: + healthy = healthy and path is not None + if quality: + mermaid = shutil.which("mmdc") + print(f"mmdc REQUIRED for check/verify: {mermaid or 'MISSING'}") + healthy = healthy and mermaid is not None + return 0 if healthy else 1 + + +def clean(root: Path) -> None: + for name in ( + "test-results", + "dist", + "build", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + ".coverage", + "coverage.xml", + "htmlcov", + "site", + "__pycache__", + ): + path = root / name + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + for tree in ("src", "tests", "tools", "scripts", "examples", "benchmarks"): + for path in (root / tree).rglob("__pycache__"): + if not any(part in {".venv", "venv", "env"} for part in path.parts): + if path.is_symlink(): + path.unlink() + else: + shutil.rmtree(path) diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py new file mode 100644 index 0000000..395dbdb --- /dev/null +++ b/tools/verification_evidence.py @@ -0,0 +1,217 @@ +"""Versioned, redacted evidence shared by local commands and CI.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +from importlib.metadata import PackageNotFoundError, version +import json +import os +from pathlib import Path +import platform +import re +import subprocess +import sys +from typing import Literal +import xml.etree.ElementTree as ET + +Status = Literal["PASS", "FAIL", "INCOMPLETE", "NOT_APPLICABLE"] +APP_INSIGHTS_TEST = "test_structured_log_appears_in_app_insights" +VERSION_TEST = "test_deployed_version_matches_candidate" + + +@dataclass(frozen=True, slots=True) +class Summary: + schema_version: int + lane: str + command: tuple[str, ...] + selected: int + passed: int + failed: int + skipped: int + status: Status + exit_code: int + supporting_files: tuple[str, ...] = () + coverage: float | None = None + reason: str = "" + otel_instrumentation: Status = "NOT_APPLICABLE" + cloud_monitor: Status = "NOT_APPLICABLE" + deployed_version_match: Status = "NOT_APPLICABLE" + telemetry_destination: str = "not configured" + cloud_tests_not_run: tuple[str, ...] = (APP_INSIGHTS_TEST,) + + +def redact(text: str) -> str: + text = re.sub(r"(?im)(authorization[\"']?\s*[:=]\s*)[^\r\n<\"]+", r"\1[REDACTED]", text) + text = re.sub(r"(https?://)[^/@\s]+:[^/@\s]+@", r"\1[REDACTED]@", text) + text = re.sub(r"(?i)(DefaultEndpointsProtocol=)[^\s<\"']+", r"\1[REDACTED]", text) + text = re.sub( + r"(?i)(?= 6 and re.search( + r"(?i)secret|token|password|connection|string|accountkey|functions.key|appinsights", key + ): + text = text.replace(value, "[REDACTED]") + return text + + +def environment(live: bool = False) -> dict[str, str]: + metadata = { + "python": platform.python_version(), + "executable": sys.executable, + "platform": platform.platform(), + "runtime": platform.python_implementation(), + } + for package in ( + "azure-functions", + "azure-functions-logging", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation-logging", + "hatch", + ): + try: + metadata[package] = version(package) + except PackageNotFoundError: + metadata[package] = "not installed" + for key in ("E2E_BASE_URL", "E2E_EXPECTED_VERSION", "GITHUB_SHA"): + if key in os.environ: + metadata[key] = redact(os.environ[key]) + for tool in ("func", "azurite") if live else (): + try: + result = subprocess.run([tool, "--version"], capture_output=True, text=True, timeout=10) + metadata[tool] = redact(result.stdout.strip()) + except (OSError, subprocess.TimeoutExpired): + metadata[tool] = "unavailable" + return metadata + + +def assertion_status(cases: list[ET.Element], name: str, required: bool) -> Status: + matches = [case for case in cases if case.get("name") == name] + if any(case.find("failure") is not None or case.find("error") is not None for case in matches): + return "FAIL" + if matches and all(case.find("skipped") is None for case in matches): + return "PASS" + return "INCOMPLETE" if required else "NOT_APPLICABLE" + + +def collect(directory: Path, command: list[str], exit_code: int) -> Summary: + cases: list[ET.Element] = [] + for path in directory.rglob("*.xml"): + if "junit" in path.name: + cases.extend(ET.parse(path).getroot().iter("testcase")) + skipped = sum(case.find("skipped") is not None for case in cases) + failed = sum( + case.find("failure") is not None or case.find("error") is not None for case in cases + ) + passed = len(cases) - skipped - failed + lane = directory.name + cloud = assertion_status(cases, APP_INSIGHTS_TEST, lane == "azure-e2e") + deployed = assertion_status(cases, VERSION_TEST, lane in {"azure-e2e", "local-e2e"}) + status: Status = "PASS" + required_skips = skipped if lane != "test" else 0 + if lane == "local-e2e": + required_skips -= sum( + case.get("name") == APP_INSIGHTS_TEST and case.find("skipped") is not None + for case in cases + ) + if failed or exit_code not in (0, 5): + status = "FAIL" + elif not cases or not passed or required_skips: + status = "INCOMPLETE" + elif lane == "azure-e2e" and (cloud != "PASS" or deployed != "PASS"): + status = "INCOMPLETE" + elif lane == "local-e2e" and deployed != "PASS": + status = "INCOMPLETE" + elif exit_code: + status = "FAIL" + coverage_path = directory / "coverage.xml" + coverage = None + if coverage_path.exists(): + coverage = float(ET.parse(coverage_path).getroot().get("line-rate", "0")) + return Summary( + 1, + lane, + tuple(command), + len(cases), + passed, + failed, + skipped, + status, + exit_code if exit_code else int(status != "PASS"), + coverage=coverage, + otel_instrumentation=status if lane == "otel" else "NOT_APPLICABLE", + cloud_monitor=cloud, + deployed_version_match=deployed, + telemetry_destination="[REDACTED]" + if os.environ.get("APPINSIGHTS_NAME") + else "not configured", + cloud_tests_not_run=() if cloud == "PASS" else (APP_INSIGHTS_TEST,), + ) + + +def render(summary: Summary) -> str: + return ( + f"{summary.lane}: {summary.status} | selected={summary.selected} " + f"passed={summary.passed} failed={summary.failed} skipped={summary.skipped} " + f"exit={summary.exit_code} | otel={summary.otel_instrumentation} " + f"cloud_monitor={summary.cloud_monitor} deployed_version={summary.deployed_version_match} " + f"telemetry_destination={summary.telemetry_destination} " + f"cloud_tests_not_run={','.join(summary.cloud_tests_not_run) or 'none'}" + + (f" | {summary.reason}" if summary.reason else "") + ) + + +def publish(summary: Summary) -> None: + text = redact(render(summary)) + print(text) + destination = os.environ.get("GITHUB_STEP_SUMMARY") + if destination: + with Path(destination).open("a", encoding="utf-8") as stream: + stream.write(f"\n- {text}\n") + + +def save(directory: Path, summary: Summary) -> Summary: + directory.mkdir(parents=True, exist_ok=True) + for path in directory.rglob("*"): + if path.is_file(): + path.write_text(redact(path.read_text(encoding="utf-8")), encoding="utf-8") + (directory / "environment.json").write_text( + redact( + json.dumps( + environment(live=directory.name in {"integration", "host-smoke", "host-matrix"}), + indent=2, + ) + ) + + "\n", + encoding="utf-8", + ) + summary = replace( + summary, + command=tuple(redact(part) for part in summary.command), + reason=redact(summary.reason), + telemetry_destination=redact(summary.telemetry_destination), + supporting_files=tuple( + str(path) + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != "summary.json" + ), + ) + serialized = redact(json.dumps(asdict(summary), indent=2)) + (directory / "summary.json").write_text(serialized + "\n", encoding="utf-8") + return Summary(**json.loads(serialized)) + + +def report(root: Path) -> int: + paths = sorted(root.glob("*/summary.json")) + if not paths: + print("No saved verification results") + return 1 + for path in paths: + publish(Summary(**json.loads(path.read_text(encoding="utf-8")))) + return 0 diff --git a/tools/verification_host.py b/tools/verification_host.py new file mode 100644 index 0000000..56e7c6f --- /dev/null +++ b/tools/verification_host.py @@ -0,0 +1,152 @@ +"""Real logging host smokes with private Azurite storage and owned processes.""" + +from __future__ import annotations + +import base64 +from contextlib import ExitStack +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import sys +import tempfile +import time +from urllib.error import URLError +from urllib.request import urlopen +import xml.etree.ElementTree as ET + +from scripts.assert_correlation import check +from tools.verification_process import child + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def wait_ready(process: subprocess.Popen[str], url: str) -> None: + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("service exited before readiness") + try: + with urlopen(url, timeout=1) as response: + if response.status == 200: + return + except URLError: + time.sleep(0.25) + raise TimeoutError("service readiness deadline exceeded") + + +def assert_routes(base: str, matrix: bool, work: Path) -> str: + if not matrix: + for route, expected in (("smoke", "smoke"), ("smoke-default", "smoke_default")): + with urlopen(f"{base}/{route}", timeout=5) as response: + assert response.read().decode() == expected + return "" + with urlopen(f"{base}/health", timeout=5) as response: + assert json.load(response)["status"] == "ok" + with urlopen(f"{base}/version", timeout=5) as response: + assert json.load(response)["version"] + with urlopen(f"{base}/correlation", timeout=5) as response: + text: str = response.read().decode() + (work / "correlation-response.json").write_text(text) + return text + + +def host_smoke(directory: Path, matrix: bool = False) -> int: + directory = directory.resolve() + suite = ET.Element("testsuite", name="real-functions-host") + case = ET.SubElement(suite, "testcase", name="correlation" if matrix else "decorated_handlers") + try: + with tempfile.TemporaryDirectory(prefix="logging-host-") as temporary, ExitStack() as stack: + work = Path(temporary) + source = Path("examples/e2e_app" if matrix else "tests/host_smoke") + for name in ("function_app.py", "host.json"): + (work / name).write_bytes((source / name).read_bytes()) + ports: set[int] = set() + while len(ports) < 4: + ports.add(free_port()) + blob, queue, table, port = ports + key = base64.b64encode(b"logging-local-emulator-only" * 2).decode() + emulator = stack.enter_context( + child( + [ + "env", + f"AZURITE_ACCOUNTS=loggingdev:{key}", + "azurite", + "--silent", + "--location", + str(work / "storage"), + "--blobPort", + str(blob), + "--queuePort", + str(queue), + "--tablePort", + str(table), + ], + directory / "logs/azurite.log", + ) + ) + deadline = time.monotonic() + 30 + for service in (blob, queue, table): + while True: + if emulator.poll() is not None or time.monotonic() >= deadline: + raise TimeoutError("Azurite failed readiness") + try: + with socket.create_connection(("127.0.0.1", service), timeout=1): + break + except OSError: + time.sleep(0.25) + (work / "local.settings.json").write_text( + json.dumps( + { + "IsEncrypted": False, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": ( + f"DefaultEndpointsProtocol=http;AccountName=loggingdev;AccountKey={key};" + f"BlobEndpoint=http://127.0.0.1:{blob}/loggingdev;" + f"QueueEndpoint=http://127.0.0.1:{queue}/loggingdev;" + f"TableEndpoint=http://127.0.0.1:{table}/loggingdev;" + ), + }, + } + ) + ) + previous_path = os.environ.get("PATH", "") + os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + previous_path + try: + host = stack.enter_context( + child( + ["func", "start", "--python", "--verbose", "--port", str(port)], + directory / "logs/host.log", + work, + ) + ) + base = f"http://127.0.0.1:{port}/api" + wait_ready(host, base + ("/health" if matrix else "/smoke")) + text = assert_routes(base, matrix, directory) + finally: + os.environ["PATH"] = previous_path + host_text = (directory / "logs/host.log").read_text() + assert not re.search( + r"0 functions (loaded|found \(Worker\))|No job functions found|" + r"Executed 'Functions\..*' \(Failed", + host_text, + ) + if matrix: + failures = check(host_text, text) + assert not failures, "; ".join(failures) + code = 0 + except (OSError, RuntimeError, TimeoutError, AssertionError, ValueError) as error: + ET.SubElement(case, "failure").text = str(error) + code = 1 + finally: + destination = directory / "junit/junit.xml" + destination.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(destination, encoding="unicode") + return code diff --git a/tools/verification_process.py b/tools/verification_process.py new file mode 100644 index 0000000..116fd2e --- /dev/null +++ b/tools/verification_process.py @@ -0,0 +1,81 @@ +"""Bounded subprocess ownership and redacted output capture.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +import os +from pathlib import Path +import signal +import subprocess +import tempfile +from types import FrameType + +from tools.verification_evidence import redact + + +def terminate(process: subprocess.Popen[str]) -> None: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + process.wait() + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + + +@contextmanager +def child( + command: list[str], log: Path, cwd: Path | None = None +) -> Iterator[subprocess.Popen[str]]: + log.parent.mkdir(parents=True, exist_ok=True) + # Raw subprocess output stays anonymous until credential redaction. + with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output: + with subprocess.Popen( + command, + cwd=cwd, + stdout=output, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) as process: + + def interrupted(number: int, frame: FrameType | None) -> None: + terminate(process) + raise KeyboardInterrupt(number) + + previous = { + sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT) + } + try: + yield process + finally: + terminate(process) + for sig, handler in previous.items(): + signal.signal(sig, handler) + output.seek(0) + text = redact(output.read()) + log.write_text(text, encoding="utf-8") + print(text, end="") + + +def run(command: list[str], log: Path, timeout: float = 600) -> int: + try: + with child(command, log) as process: + try: + code = process.wait(timeout=timeout) + return code if code >= 0 else 128 - code + except subprocess.TimeoutExpired: + return 124 + except FileNotFoundError as error: + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(redact(str(error)), encoding="utf-8") + return 127 diff --git a/tools/verification_wheel.py b/tools/verification_wheel.py new file mode 100644 index 0000000..b294782 --- /dev/null +++ b/tools/verification_wheel.py @@ -0,0 +1,60 @@ +"""Check installed wheel identity outside the source checkout.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +import sysconfig +import tempfile + +from tools.verification_process import run + + +def main() -> int: + directory = Path("test-results/wheel-smoke/logs") + with tempfile.TemporaryDirectory(prefix="logging-wheel-") as temporary: + root = Path(temporary) + code = run( + [sys.executable, "-m", "build", "--no-isolation", "--outdir", str(root / "dist")], + directory / "build.log", + ) + if code: + return code + wheels = list((root / "dist").glob("*.whl")) + if len(wheels) != 1: + print("Expected exactly one freshly built wheel in isolated output directory") + return 1 + subprocess.run( + [sys.executable, "-m", "venv", "--system-site-packages", str(root / "venv")], check=True + ) + python = str(root / "venv/bin/python") + packages = ( + root + / "venv/lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" + ) + (packages / "verification-dependencies.pth").write_text( + sysconfig.get_path("purelib") + "\n" + ) + code = run( + [python, "-m", "pip", "install", "--no-deps", "--no-index", str(wheels[0].resolve())], + directory / "wheel-install.log", + ) + if code: + return code + script = ( + "from pathlib import Path; import logging; import azure_functions_logging as a; " + f"assert Path(a.__file__).resolve().is_relative_to(Path({str(root)!r})); " + "assert a.get_logger('artifact-smoke').name == 'artifact-smoke'; " + "record=logging.LogRecord('artifact-smoke',20,'smoke',1," + "'installed wheel smoke',(),None); " + "assert 'installed wheel smoke' in a.JsonFormatter().format(record); " + "print('Installed wheel imports and formats outside checkout')" + ) + return run([python, "-I", "-c", script], directory / "wheel-smoke.log") + + +if __name__ == "__main__": + raise SystemExit(main()) From 67ce9f373866511f0ab4a4c643c6f5e9d039bc33 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 02/15] build: standardize local verification command contracts --- .gitignore | 1 + Makefile | 236 ++++++++++------------------ pyproject.toml | 6 +- tests/test_verification_commands.py | 80 ++++++++++ 4 files changed, 163 insertions(+), 160 deletions(-) create mode 100644 tests/test_verification_commands.py diff --git a/.gitignore b/.gitignore index 3006d94..a4e2bc9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ env/ .ruff_cache/ .coverage coverage.xml +test-results/ htmlcov/ # Build artifacts diff --git a/Makefile b/Makefile index cef3d64..8337c97 100644 --- a/Makefile +++ b/Makefile @@ -1,179 +1,101 @@ VENV_DIR := .venv -PYTHON := $(VENV_DIR)/bin/python -PIP := $(VENV_DIR)/bin/pip -HATCH := $(VENV_DIR)/bin/hatch - -.PHONY: bootstrap -bootstrap: - @if [ ! -d "$(VENV_DIR)" ]; then \ - echo "Creating virtual environment..."; \ - python3 -m venv $(VENV_DIR); \ - fi - @echo "Ensuring Hatch is installed in virtual environment..." - @$(PIP) install --upgrade pip > /dev/null - @$(PIP) install hatch > /dev/null - @echo "Hatch installed at $(HATCH)" - -.PHONY: ensure-hatch -ensure-hatch: bootstrap - -.PHONY: install +PYTHON ?= python +HATCH ?= hatch +ifeq ($(CI),) +RUN := $(HATCH) run python +else +RUN := $(PYTHON) +endif +TREES := src tests tools examples benchmarks scripts + +.PHONY: bootstrap ensure-hatch install install-ci +bootstrap ensure-hatch: + @command -v $(HATCH) >/dev/null || { echo "Install Hatch in an isolated tool environment first"; exit 1; } install: ensure-hatch @$(HATCH) env create - @if [ -n "$$CI" ]; then \ - echo "CI detected: skipping pre-commit hook installation"; \ - else \ - $(MAKE) precommit-install; \ - fi + @if [ -z "$$CI" ]; then $(MAKE) precommit-install; fi +install-ci: + @$(PYTHON) -c 'import sys; assert (3, 11) <= sys.version_info[:2] < (3, 15), sys.version' + @$(PYTHON) -m pip install -e ".[dev,docs]" -.PHONY: shell +.PHONY: shell reset hatch-clean shell: ensure-hatch @$(HATCH) shell - -.PHONY: reset reset: clean-all install - @echo "Project reset complete." - -.PHONY: hatch-clean hatch-clean: ensure-hatch - @$(HATCH) env remove || echo "No hatch environment to remove" - -.PHONY: format -format: ensure-hatch - @$(HATCH) run format - -.PHONY: format-check -format-check: ensure-hatch - @$(HATCH) run format-check - -.PHONY: format-check-changed -format-check-changed: ensure-hatch - @$(HATCH) run python tools/check_pr_format.py "$(BASE)" "$(HEAD)" - -.PHONY: style -style: ensure-hatch - @$(HATCH) run style - -.PHONY: typecheck -typecheck: ensure-hatch - @$(HATCH) run typecheck - -.PHONY: lint -lint: ensure-hatch - @$(HATCH) run lint - -.PHONY: security -security: ensure-hatch - @$(HATCH) run security - -.PHONY: check -check: ensure-hatch - @$(MAKE) lint - @$(MAKE) typecheck - @echo "Lint and type check passed." - -.PHONY: lint-workflows -lint-workflows: ensure-hatch - @$(HATCH) run python tools/lint_release_workflows.py - @$(HATCH) run python tools/lint_workflow_pins.py - @$(HATCH) run python tools/lint_hatch_matrix.py - -.PHONY: lint-doc-links -lint-doc-links: ensure-hatch - @$(HATCH) run python tools/lint_doc_links.py - -.PHONY: check-all -check-all: ensure-hatch - @$(MAKE) lint-workflows - @$(MAKE) lint-doc-links - @$(MAKE) check - @$(MAKE) test - @$(MAKE) security - @echo "All checks passed including tests and security scan." + @$(HATCH) env remove + +.PHONY: format format-check format-check-changed style typecheck lint security +format: + @$(RUN) -m ruff format $(TREES) +format-check: + @$(RUN) -m ruff format --check $(TREES) +format-check-changed: + @$(RUN) tools/check_pr_format.py "$(BASE)" "$(HEAD)" +style: format-check + @$(RUN) -m ruff check $(TREES) +typecheck: + @$(RUN) -m mypy src tests tools/verification*.py +lint: style typecheck +security: + @$(RUN) -m bandit -r src + +.PHONY: check lint-workflows lint-doc-links lint-mermaid check-all +lint-workflows: + @$(RUN) tools/lint_release_workflows.py + @$(RUN) tools/lint_workflow_pins.py + @$(RUN) tools/lint_hatch_matrix.py +lint-doc-links: + @$(RUN) tools/lint_doc_links.py +lint-mermaid: + @$(RUN) scripts/render_mermaid.py +check: style typecheck security lint-workflows lint-doc-links lint-mermaid +check-all: verify + +.PHONY: test integration otel host-smoke host-matrix verify report wheel-smoke +test integration otel host-smoke host-matrix verify report: + @$(RUN) -m tools.verification $@ +wheel-smoke: + @$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel + +.PHONY: cov bench e2e-local e2e-azure build docs docs-serve +cov: + @$(RUN) -m tools.verification test --cov-report=html +bench: + @$(RUN) benchmarks/bench.py +e2e-local: + @E2E_BASE_URL=$${E2E_BASE_URL:-http://localhost:7071} E2E_EXPECTED_VERSION=$${E2E_EXPECTED_VERSION:-$$($(HATCH) version)} $(RUN) -m tools.verification local-e2e +e2e-azure: + @$(RUN) -m tools.verification azure-e2e +build: + @$(RUN) -m tools.verification focused build -m build --no-isolation +docs: + @$(RUN) -m tools.verification focused docs -m mkdocs build --strict +docs-serve: + @$(RUN) -m mkdocs serve -.PHONY: precommit +.PHONY: precommit precommit-install publish-test version precommit: ensure-hatch @$(HATCH) run precommit - -.PHONY: precommit-install precommit-install: ensure-hatch @$(HATCH) run precommit-install - -.PHONY: test -test: ensure-hatch - @echo "Running tests..." - @$(HATCH) run test - -.PHONY: bench -bench: ensure-hatch - @echo "Running micro-benchmarks..." - @$(HATCH) run python benchmarks/bench.py - -.PHONY: cov -cov: ensure-hatch - @$(HATCH) run cov - @echo "Open htmlcov/index.html in your browser to view the coverage report." - @echo "coverage.xml generated for Codecov upload." - -.PHONY: e2e-local -e2e-local: ensure-hatch - @echo "Running e2e tests against local Azurite (E2E_BASE_URL=http://localhost:7071)..." - @echo "E2E_EXPECTED_VERSION defaults to the locally installed candidate version (hatch version); override to certify a different deployed build." - @E2E_BASE_URL=http://localhost:7071 E2E_EXPECTED_VERSION=$${E2E_EXPECTED_VERSION:-$$($(HATCH) version)} $(HATCH) run e2e-azure - -.PHONY: e2e-azure -e2e-azure: ensure-hatch - @echo "Running e2e tests against Azure (E2E_BASE_URL and E2E_EXPECTED_VERSION must both be set)..." - @$(HATCH) run e2e-azure - -.PHONY: build -build: ensure-hatch - @$(HATCH) build - -.PHONY: publish-test publish-test: ensure-hatch @$(HATCH) publish --repo test - -.PHONY: version version: ensure-hatch - @echo "Current version:" @$(HATCH) version -.PHONY: docs -docs: - @if [ -n "$$CI" ]; then \ - echo "CI detected: running MkDocs directly"; \ - python -m pip install --upgrade pip > /dev/null 2>&1 || true; \ - pip install "mkdocs<2.0" "mkdocs-material<10.0" "mkdocstrings[python]<1.0" > /dev/null 2>&1; \ - mkdocs build; \ - else \ - $(MAKE) ensure-hatch > /dev/null; \ - $(HATCH) run mkdocs build; \ - fi - -.PHONY: docs-serve -docs-serve: ensure-hatch - @$(HATCH) run docs - -.PHONY: doctor +.PHONY: doctor clean clean-all help doctor: - @echo "Python version:" && $(PYTHON) --version - @echo "Installed packages:" && $(HATCH) env run pip list || echo "No hatch environment found" - @echo "Azure Function Core Tools version:" && func --version || echo "func not found" - @echo "Pre-commit hook installed:" - @if [ -f .git/hooks/pre-commit ]; then echo yes; else echo no; fi - -.PHONY: clean + @$(RUN) -m tools.verification doctor clean: - @rm -rf *.egg-info dist build __pycache__ .pytest_cache - -.PHONY: clean-all + @$(RUN) -m tools.verification clean clean-all: clean - @find . -type d -name "__pycache__" -exec rm -rf {} + - @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -delete - @rm -rf .mypy_cache .ruff_cache .pytest_cache .coverage coverage.xml htmlcov .DS_Store $(VENV_DIR) site - -.PHONY: help + @find . -type d -name "__pycache__" -not -path "./.git/*" -exec rm -rf {} + + @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -not -path "./.git/*" -delete + @rm -rf .DS_Store $(VENV_DIR) help: - @echo "Available commands:" && \ - grep -E '^\.PHONY: ' Makefile | cut -d ':' -f2 | xargs -n1 echo " - make" + @echo "Canonical: install install-ci doctor format format-check check test integration verify e2e-azure report clean" + @echo "check-all: exact alias of verify (requires func and Azurite)" + @echo "Partial/focused: style lint typecheck security lint-workflows lint-doc-links cov bench otel host-smoke host-matrix build wheel-smoke docs docs-serve format-check-changed" + @echo "Local only, not Azure certification: e2e-local" + @echo "Maintenance: bootstrap ensure-hatch shell reset hatch-clean clean-all precommit precommit-install publish-test version" diff --git a/pyproject.toml b/pyproject.toml index 014e8a6..84e44bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,11 +92,11 @@ features = ["dev", "docs"] format = "ruff format src tests tools examples benchmarks scripts" format-check = "ruff format --check src tests tools examples benchmarks scripts" style = "ruff check src tests tools examples benchmarks scripts && ruff format --check src tests tools examples benchmarks scripts" -typecheck = "mypy src tests" +typecheck = "mypy src tests tools/verification*.py" lint = "hatch run style && hatch run typecheck" -test = "pytest -v tests" +test = "python -m tools.verification test" cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/" -e2e-azure = "pytest -v -m e2e tests/e2e/ --no-cov" +e2e-azure = "python -m tools.verification azure-e2e" docs = "mkdocs serve" precommit = "pre-commit run --all-files" precommit-install = "pre-commit install" diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py new file mode 100644 index 0000000..78fe31a --- /dev/null +++ b/tests/test_verification_commands.py @@ -0,0 +1,80 @@ +"""Public CLI contracts exercised through real subprocesses.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize( + "target", + [ + "install", + "install-ci", + "doctor", + "format", + "format-check", + "check", + "test", + "integration", + "verify", + "e2e-azure", + "report", + "clean", + ], +) +def test_routine_commands_do_not_bootstrap(target: str) -> None: + # Given the public contract, when expanding a recipe without executing it. + result = subprocess.run(["make", "-n", target], cwd=ROOT, capture_output=True, text=True) + # Then setup is explicit, never a pip upgrade hidden in a routine command. + assert result.returncode == 0, result.stderr + assert "install --upgrade pip" not in result.stdout + if target not in {"install", "install-ci"}: + assert "pip install hatch" not in result.stdout + + +def test_check_is_complete_non_mutating_and_types_once() -> None: + # Given the complete quality gate, when expanding its dependencies. + result = subprocess.run(["make", "-n", "check"], cwd=ROOT, capture_output=True, text=True) + # Then every repository policy checker runs without mutation. + assert result.returncode == 0 + assert result.stdout.count("-m mypy") == 1 + for name in ( + "lint_release_workflows.py", + "lint_workflow_pins.py", + "lint_hatch_matrix.py", + "lint_doc_links.py", + ): + assert name in result.stdout + assert "-m bandit" in result.stdout + assert "scripts/render_mermaid.py" in result.stdout + assert "--check" in result.stdout + assert "--fix" not in result.stdout + + +def test_unconfigured_azure_cli_fails_with_saved_incomplete(tmp_path: Path) -> None: + # Given no Azure configuration and an isolated evidence directory. + environment = dict(os.environ, PYTHONPATH=str(ROOT)) + environment.pop("E2E_BASE_URL", None) + # When invoking the real canonical CLI, not a mocked execution function. + result = subprocess.run( + [sys.executable, "-m", "tools.verification", "azure-e2e"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + # Then absence of configuration is evidence, never successful certification. + assert result.returncode != 0 + path = tmp_path / "test-results/azure-e2e/summary.json" + assert path.exists(), result.stderr + data = json.loads(path.read_text()) + assert data["status"] == "INCOMPLETE" + assert data["selected"] == 0 From 6e5ea108ac4cb4357030c2e21385669aec2c4331 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 03/15] ci: reuse canonical verification on selected matrix interpreters --- .github/workflows/ci-test.yml | 99 +++++++++++++++++++---------------- tests/test_verification_ci.py | 64 ++++++++++++++++++++++ 2 files changed, 119 insertions(+), 44 deletions(-) create mode 100644 tests/test_verification_ci.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index beec567..787e990 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -91,22 +91,18 @@ jobs: cache-dependency-path: pyproject.toml - name: Install package with dev extras - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + run: make install-ci - - name: Lint workflows (pin hygiene + release-gate canonical SHAs) - run: | - python tools/lint_release_workflows.py - python tools/lint_workflow_pins.py + - name: Set up Node.js for Mermaid policy checks + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "20" - - name: Hatch/matrix hygiene lint - run: python tools/lint_hatch_matrix.py + - name: Install Mermaid render checker + run: npm install -g @mermaid-js/mermaid-cli - - name: Ruff style check - run: | - ruff check src tests tools examples benchmarks scripts - ruff format --check src tests tools examples benchmarks scripts + - name: Complete non-mutating quality gate + run: make check - name: Ruff format check (changed Python files) if: github.event_name == 'pull_request' @@ -115,8 +111,12 @@ jobs: PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: python tools/check_pr_format.py "$PR_BASE_SHA" "$PR_HEAD_SHA" - - name: Type check (mypy, strict) - run: mypy src tests + - name: Upload quality evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: quality-evidence + path: test-results/** test: name: Test (Py ${{ matrix.python-version }}) @@ -160,9 +160,7 @@ jobs: PY - name: Install package with dev extras (on the matrix interpreter) - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + run: make install-ci - name: Run tests on the matrix interpreter # Raw pytest on the interpreter provisioned above (NOT through the @@ -174,26 +172,36 @@ jobs: IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' }} run: | if [ "$IS_COVERAGE_CELL" = "true" ]; then - pytest tests/ + python -m tools.verification test else - pytest tests/ --no-cov + python -m tools.verification test --no-cov fi - name: Verify coverage output if: matrix.python-version == '3.11' - run: ls -lh coverage.xml + run: ls -lh test-results/test/coverage.xml - name: Upload coverage to Codecov if: matrix.python-version == '3.11' && github.actor != 'dependabot[bot]' uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml + files: ./test-results/test/coverage.xml verbose: true fail_ci_if_error: false env: CODECOV_RETRIES: 3 + - name: Record OTel instrumentation separately from ingestion + run: python -m tools.verification otel + + - name: Upload offline and OTel evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-evidence-py${{ matrix.python-version }} + path: test-results/** + minimum-dependencies: name: Minimum dependencies (Py 3.11) runs-on: ubuntu-latest @@ -228,7 +236,14 @@ jobs: PY - name: Run full suite against minimum runtime dependencies - run: pytest tests/ --no-cov + run: python -m tools.verification test --no-cov + + - name: Upload minimum-dependency evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: minimum-dependencies-evidence + path: test-results/** artifact-build: name: Build compatibility artifacts @@ -371,28 +386,17 @@ jobs: npm install --global azurite - name: Install smoke app dependencies - working-directory: tests/host_smoke - run: pip install -r requirements.txt + run: make install-ci - - name: Start Azurite and Functions host - run: | - azurite --silent > /tmp/azurite.log 2>&1 & - cd tests/host_smoke - func start --python --port 7071 > /tmp/functions-host.log 2>&1 & + - name: Managed decorated-handler and correlation host verification + run: python -m tools.verification integration - - name: Call decorated handlers - run: | - for attempt in {1..30}; do - if response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke)" \ - && default_response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke-default)"; then - test "$response" = "smoke" - test "$default_response" = "smoke_default" - exit 0 - fi - sleep 2 - done - cat /tmp/functions-host.log - exit 1 + - name: Upload local host evidence (not App Insights certification) + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: host-evidence-py${{ matrix.python-version }} + path: test-results/** docs-check: runs-on: ubuntu-latest @@ -421,7 +425,14 @@ jobs: npm install -g @mermaid-js/mermaid-cli - name: Build documentation strictly - run: mkdocs build --strict + run: make docs + + - name: Upload documentation evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docs-evidence + path: test-results/** - name: Lint documentation links run: python tools/lint_doc_links.py diff --git a/tests/test_verification_ci.py b/tests/test_verification_ci.py new file mode 100644 index 0000000..b813869 --- /dev/null +++ b/tests/test_verification_ci.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def test_ci_canonical_runners_preserve_matrix_and_failure_uploads() -> None: + workflows = ROOT / ".github/workflows" + ci = (workflows / "ci-test.yml").read_text() + assert 'python-version: ["3.11", "3.12", "3.13", "3.14"]' in ci + assert 'python-version: ["3.11", "3.12"]' in ci + assert "python -m tools.verification test --no-cov" in ci + assert "python -m tools.verification integration" in ci + assert "python -m tools.verification otel" in ci + assert "make install-ci" in ci + for name in ( + "ci-test.yml", + "e2e-azure.yml", + "publish-pypi.yml", + "otel-nightly.yml", + "e2e-host-matrix.yml", + ): + text = (workflows / name).read_text() + evidence = [ + block for block in text.split(" - name:") if "path: test-results/**" in block + ] + assert evidence, name + assert all("if: always()" in block for block in evidence) + assert 'core_tools: ["4", "4.12.0"]' in (workflows / "e2e-host-matrix.yml").read_text() + + +def test_ci_selects_actual_interpreter_without_hatch() -> None: + result = subprocess.run( + ["make", "--no-print-directory", "-n", "test", "CI=true", "PYTHON=matrix-python"], + cwd=ROOT, + text=True, + capture_output=True, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "matrix-python -m tools.verification test" + + +def test_check_all_exact_alias_and_deep_clean_preserved() -> None: + assert "check-all: verify\n\n" in (ROOT / "Makefile").read_text() + result = subprocess.run(["make", "-n", "clean-all"], cwd=ROOT, text=True, capture_output=True) + assert result.returncode == 0 + assert "-m tools.verification clean" in result.stdout + assert "__pycache__" in result.stdout + assert "rm -rf .DS_Store .venv" in result.stdout + + +def test_release_requires_cloud_monitor_and_candidate_version() -> None: + text = (ROOT / ".github/workflows/e2e-azure.yml").read_text() + assert "python -m tools.verification azure-e2e" in text + for field in ( + "E2E_BASE_URL:", + "E2E_EXPECTED_VERSION:", + "APPINSIGHTS_NAME:", + "E2E_RESOURCE_GROUP:", + ): + assert field in text + assert text.index("tools.verification azure-e2e") < text.index("- name: Record certification") From a86f3ec10a9a96bf132151c1ea8d5213e2923937 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 04/15] ci: record separate OTel instrumentation evidence --- .github/workflows/otel-nightly.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/otel-nightly.yml b/.github/workflows/otel-nightly.yml index d88b15b..8c85de5 100644 --- a/.github/workflows/otel-nightly.yml +++ b/.github/workflows/otel-nightly.yml @@ -39,13 +39,11 @@ jobs: pip show opentelemetry-api opentelemetry-sdk | grep -E "^(Name|Version):" - name: Run OTel-focused test suite - run: | - # This canary runs a four-file subset, but pyproject's addopts force - # --cov over the whole package and fail_under=95 applies. Coverage of - # a subset is meaningless here and was failing the job at ~49%, - # masking the upstream breakage this canary exists to detect. - python -m pytest -q --cov-fail-under=0 \ - tests/test_otel.py \ - tests/test_otel_contract.py \ - tests/test_otel_filters.py \ - tests/test_host_config_otel.py + run: python -m tools.verification otel + + - name: Upload OTel instrumentation evidence (not ingestion certification) + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: otel-latest-evidence + path: test-results/** From fab53c02486c476b7bbd1842c4b13c8786cb7cbd Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 05/15] ci: manage host matrix lifecycle and correlation evidence --- .github/workflows/e2e-host-matrix.yml | 127 ++------------------------ 1 file changed, 10 insertions(+), 117 deletions(-) diff --git a/.github/workflows/e2e-host-matrix.yml b/.github/workflows/e2e-host-matrix.yml index 6305e5a..4a56268 100644 --- a/.github/workflows/e2e-host-matrix.yml +++ b/.github/workflows/e2e-host-matrix.yml @@ -1,20 +1,7 @@ name: e2e-host-matrix -# Scheduled Core Tools / host version matrix smoke (#394). -# -# Boots the local ``examples/e2e_app`` under a real ``func`` host + Azurite -# across multiple Azure Functions Core Tools versions and asserts the host -# starts and serves HTTP. This catches host-behavior drift across Core -# Tools / host releases early, WITHOUT deploying to Azure (no cloud, no -# secrets). It is intentionally **scheduled-only** (plus manual dispatch) so -# it never blocks a PR and its CI cost stays bounded. -# -# NOTE: this is a *host-boot* smoke — it proves the app registers and serves -# under each Core Tools version. It is not a substitute for the real-Azure -# certification in e2e-azure.yml. on: schedule: - # Weekly, Mondays 05:00 UTC. Off the PR path. - cron: "0 5 * * 1" workflow_dispatch: @@ -26,19 +13,8 @@ jobs: name: Host-boot smoke (func ${{ matrix.core_tools }}) runs-on: ubuntu-latest strategy: - # Do not cancel the whole matrix when one Core Tools version fails — - # each entry is an independent drift signal. fail-fast: false matrix: - # Chosen set (documented per the issue acceptance checklist): - # * "4" — the latest published 4.x Core Tools (drift detector: - # surfaces breakage in the newest release before it is - # pinned anywhere). - # * "4.12.0" — the version currently pinned by e2e-azure.yml and - # cookbook-host-smoke (control / known-good baseline). - # Both are guaranteed installable via npm. Expand this list with - # additional explicit versions only after confirming they install and - # boot cleanly here. core_tools: ["4", "4.12.0"] steps: - name: Checkout repository @@ -54,104 +30,21 @@ jobs: with: node-version: "22" - - name: Install Azure Functions Core Tools + Azurite - run: | - npm install -g "azure-functions-core-tools@${{ matrix.core_tools }}" azurite - echo "func version: $(func --version)" - echo "azurite version: $(azurite --version)" - - - name: Install the library and the app's runtime deps - run: | - python -m pip install --upgrade pip - # The worker that `func start` launches uses this Python env, so the - # app's import of azure_functions_logging must resolve here. - pip install -e . - pip install -r examples/e2e_app/requirements.txt + - name: Install Azure Functions Core Tools and Azurite + run: npm install -g "azure-functions-core-tools@${{ matrix.core_tools }}" azurite - - name: Start Azurite + - name: Install selected-interpreter development and app dependencies run: | - mkdir -p /tmp/azurite - azurite --silent --location /tmp/azurite \ - --blobHost 127.0.0.1 --queueHost 127.0.0.1 --tableHost 127.0.0.1 & - echo "AZURITE_PID=$!" >> "$GITHUB_ENV" - - - name: Boot func host and smoke the HTTP endpoints - working-directory: examples/e2e_app - env: - AzureWebJobsStorage: UseDevelopmentStorage=true - FUNCTIONS_WORKER_RUNTIME: python - run: | - set -euo pipefail - # Launch the host in the background, capturing logs for diagnostics. - func start --port 7071 --python > "$RUNNER_TEMP/func-host.log" 2>&1 & - echo "FUNC_PID=$!" >> "$GITHUB_ENV" - - # Poll for readiness (host + worker registration can take a while on - # a cold runner). Fail loudly with the host log if it never comes up. - ready="" - for i in $(seq 1 60); do - if curl -fsS "http://127.0.0.1:7071/api/health" >/dev/null 2>&1; then - ready="yes" - break - fi - sleep 5 - done - if [ -z "$ready" ]; then - echo "::error ::func host did not become ready within timeout (Core Tools ${{ matrix.core_tools }})" - echo "----- func host log -----" - cat "$RUNNER_TEMP/func-host.log" || true - exit 1 - fi - - echo "Host is up. Asserting /api/health payload..." - HEALTH="$(curl -fsS http://127.0.0.1:7071/api/health)" - echo "health response: $HEALTH" - echo "$HEALTH" | grep -q '"status": "ok"' + make install-ci + python -m pip install -r examples/e2e_app/requirements.txt - echo "Asserting /api/version reports the installed package..." - VERSION="$(curl -fsS http://127.0.0.1:7071/api/version)" - echo "version response: $VERSION" - echo "$VERSION" | grep -q '"version"' - - echo "Exercising /api/correlation to certify correlation claims..." - CORR="$(curl -fsS http://127.0.0.1:7071/api/correlation)" - echo "correlation response: $CORR" - printf '%s' "$CORR" > "$RUNNER_TEMP/correlation-response.json" - - # Worker log records reach the host log asynchronously; wait until the - # three in-context markers have flushed (or fail loudly with the log). - markers="" - for i in $(seq 1 24); do - if grep -q 'corr-main-1' "$RUNNER_TEMP/func-host.log" \ - && grep -q 'corr-main-2' "$RUNNER_TEMP/func-host.log" \ - && grep -q 'corr-thread-propagated' "$RUNNER_TEMP/func-host.log"; then - markers="yes" - break - fi - sleep 5 - done - if [ -z "$markers" ]; then - echo "::error ::correlation markers never appeared in the host log (Core Tools ${{ matrix.core_tools }})" - cat "$RUNNER_TEMP/func-host.log" || true - exit 1 - fi - - echo "Asserting correlation claims against the host log and HTTP response..." - python ../../scripts/assert_correlation.py \ - "$RUNNER_TEMP/func-host.log" \ - "$RUNNER_TEMP/correlation-response.json" - - - name: Stop background processes - if: always() - run: | - [ -n "${FUNC_PID:-}" ] && kill "${FUNC_PID}" 2>/dev/null || true - [ -n "${AZURITE_PID:-}" ] && kill "${AZURITE_PID}" 2>/dev/null || true + - name: Managed host HTTP and correlation checks (not cloud ingestion) + run: python -m tools.verification host-matrix - - name: Upload func host log + - name: Upload redacted host evidence if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: func-host-log-${{ matrix.core_tools }} - path: ${{ runner.temp }}/func-host.log - if-no-files-found: ignore + path: test-results/** retention-days: 14 From 8ee29d0a346a626324f34c21a4fc0f61f42fe709 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 06/15] ci: require observed App Insights assertions for certification --- .github/workflows/e2e-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-azure.yml b/.github/workflows/e2e-azure.yml index 9392b3a..ea947e8 100644 --- a/.github/workflows/e2e-azure.yml +++ b/.github/workflows/e2e-azure.yml @@ -173,7 +173,7 @@ jobs: func azure functionapp publish "${{ steps.names.outputs.app }}" --python --build remote - name: Run e2e tests (HTTP + App Insights) - run: pytest -v -m e2e tests/e2e/ --no-cov --html=e2e-report/report.html --self-contained-html + run: python -m tools.verification azure-e2e -v --html=test-results/azure-e2e/report.html --self-contained-html env: E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net E2E_EXPECTED_VERSION: ${{ env.RESOLVED_VERSION }} @@ -218,7 +218,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }} - path: e2e-report/ + path: test-results/** retention-days: 30 cleanup: From 38dd4b363fc53a398e33d174d2ae21fd16ee82db Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 07/15] ci: preserve release evidence and pin the OIDC publisher --- .github/workflows/publish-pypi.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 000aecb..8d544cd 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -101,11 +101,18 @@ jobs: with: python-version: "3.11" - - name: Install Hatch - run: pip install hatch + - name: Install test dependencies on the selected interpreter + run: make install-ci - name: Run test suite - run: hatch run test + run: python -m tools.verification test + + - name: Upload release library evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-library-evidence + path: test-results/** cookbook-smoke: name: Cookbook dogfood smoke (candidate wheel) @@ -278,7 +285,7 @@ jobs: path: dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 # pin-exempt: PyPA official publisher (trusted-publisher OIDC); tracks release/v1 by design + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 notify-cookbook: name: Trigger cookbook e2e re-verification From 84e0009386e4f7b8e280a6e0bacdc548a5237e6b Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:02:12 +0900 Subject: [PATCH 08/15] docs: define verification commands and certification boundaries --- AGENTS.md | 32 ++++++++++----- CONTRIBUTING.md | 89 ++++++++++++++++++++++++++++++++++-------- docs/agent-playbook.md | 16 +++++--- docs/contributing.md | 26 +++++++++--- docs/development.md | 61 +++++++++++++++++++++++++---- docs/testing.md | 18 ++++++++- 6 files changed, 197 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c5aa8ad..d40d5b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ ### Test Coverage - Maintain test coverage at **95% or above** for committed changes and PRs. -- Run `hatch run pytest --cov --cov-report=term-missing -q` to verify before submitting changes. +- Run `make test` to verify the offline suite and coverage before submitting changes. - Any PR that drops coverage below 95% must include additional tests to compensate. - In Azure/Core Tools mode, `setup_logging()` installs `ContextFilter` on the root logger's existing handlers (and the root logger itself for late-attaching handlers) but never adds new handlers or changes the root level. When `use_record_factory=True`, no `ContextFilter` is attached; context is injected via the global `LogRecordFactory` instead. In standalone local mode, the root logger is configured by default (`logger_name=None`); pass an explicit `logger_name` to avoid modifying the root logger. - `azure-functions>=1.21.0` is a required runtime dependency (declared in `pyproject.toml`, added deliberately in #528 to align the toolkit floor). OpenTelemetry support remains an optional `[otel]` extra, imported only when `activate_trace_context=True` is used. @@ -101,10 +101,11 @@ When splitting a large piece of work into focused issues, keep the umbrella open Keep issue creation **open/unrestricted** in this repository (issue tracker enabled, no `interaction-limits`). External bug reports are the only inbound support channel, so restricting issue creation suppresses the signal we most need. Do not enable an interaction limit or restrict issue creation without an explicit, documented reason recorded in an issue first. ## Validation -- `make test` -- `make lint` -- `make typecheck` -- `make build` +- `make doctor` (read-only required prerequisites) +- `make check` (complete non-mutating quality/policy gate) +- `make test` (offline, all privacy regressions, coverage >=95%) +- `make integration` (separate OTel and managed host/Azurite evidence) +- `make verify` (`make check-all` is an exact alias) ## Release Process @@ -181,11 +182,24 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor | Purpose | Command | | --- | --- | | Environment setup | `make install` | +| CI-selected interpreter setup | `make install-ci` | +| Read-only prerequisites | `make doctor` | | Format code | `make format` | -| Lint | `make lint` | -| Type check | `make typecheck` | -| Tests | `make test` | -| Clean | `make clean` | +| Check formatting | `make format-check` | +| Complete non-mutating quality/policies | `make check` | +| Offline tests with 95% coverage | `make test` | +| Managed local OTel/host/Azurite lanes | `make integration` | +| Full local validation | `make verify` (`make check-all` exact alias) | +| Configured real Azure/App Insights certification | `make e2e-azure` | +| Render saved results only | `make report` | +| Generated artifact cleanup only | `make clean` | +| Partial/focused lint/types/security/coverage | `make lint`, `make typecheck`, `make security`, `make cov` | +| Partial/focused docs/build/wheel | `make docs`, `make build`, `make wheel-smoke` | + +See CONTRIBUTING.md for the command/evidence contract. Separate summaries live in +gitignored `test-results/`. Local host or OTel success is not App Insights ingestion +certification; required skipped cloud assertions are INCOMPLETE. `clean-all` retains +deep cleanup including bytecode and the project-local `.venv`. ## Design Constraints diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5204359..db25a1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,10 +34,12 @@ Recommended branch prefixes: ## Local development quickstart -Local development requires Python 3.11, 3.12, 3.13, or 3.14. No repo-specific services or -tools such as Azure Functions Core Tools, Azurite, or Docker are required for the standard -local checks. The `make install` target creates `.venv`, installs Hatch into it, creates the -Hatch environment, and installs the pre-commit hook: +Install Hatch once in an isolated tool environment. The default Hatch environment +requires Python 3.11; CI selects Python 3.11–3.14 explicitly. `make install` creates +the isolated dev/docs environment and installs local hooks. Routine commands never +upgrade pip, reinstall Hatch, or change the global interpreter. Offline quality and +unit checks need no services; full local verification requires `func` and `azurite` +on PATH. Docker and an Azure account are not needed locally. ```bash git clone https://github.com/yeongseon/azure-functions-logging-python.git @@ -49,22 +51,79 @@ Run the standard verification commands from the repository root: ```bash make test -make lint -make typecheck -make build +make check +make verify ``` ## Project Commands -```bash -make format # Format code with ruff -make lint # Lint with ruff -make typecheck # Type check with mypy -make test # Run tests -make cov # Run tests with coverage -make check-all # Run the full local gate +| Command | Contract | +| --- | --- | +| `make install` | Isolated Hatch development/docs environment and local hooks | +| `make install-ci` | Install extras into CI's explicitly selected `PYTHON` | +| `make doctor` | Read-only prerequisites; fail non-zero for missing required tools | +| `make format` / `make format-check` | Mutating Ruff formatting / non-mutating check | +| `make check` | Ruff lint/format, strict mypy once, Bandit, workflow/doc-link policies and Mermaid rendering | +| `make test` | Fully offline unit/mock suite, all privacy/redaction regressions, coverage >=95% | +| `make integration` | Separate OTel instrumentation, decorated real-host and correlation host lanes; managed private Azurite | +| `make verify` | Always check + test + integration + installed wheel smoke + strict docs | +| `make e2e-azure` | Explicit existing deployment and actual App Insights ingestion certification; never provisions | +| `make report` | Render saved JSON only, without running tests | +| `make clean` | Generated artifacts only; preserve environments and shared storage | + +`make check-all` is an exact backward-compatible alias of `make verify`. +`clean-all` retains historical deep cleanup: artifacts, bytecode and local `.venv`. +Other useful targets are partial/focused: `style`, `lint`, `typecheck`, `security`, +`lint-workflows`, `lint-doc-links`, `cov` (HTML coverage), `bench`, `otel`, +`host-smoke`, `host-matrix`, `build`, `wheel-smoke`, `docs`, `docs-serve`, +`format-check-changed`, and maintenance commands. Quality checks require Mermaid +CLI (`mmdc`) and its Chromium installation; rendering uses temporary output only. + +### Verification evidence + +```text +test-results/ + test/{junit.xml,coverage.xml,summary.json,environment.json,logs/} + otel/{junit.xml,summary.json,environment.json,logs/} + host-smoke/{junit/junit.xml,summary.json,environment.json,logs/} + host-matrix/{junit/junit.xml,summary.json,environment.json,logs/} + integration/{summary.json,environment.json,lanes.json,junit/,logs/} + verify/{summary.json,environment.json,lanes.json,logs/} + azure-e2e/{summary.json,environment.json,junit.xml,logs/} ``` +Each summary records schema version, lane/command, selected/passed/failed/skipped +counts, exit status and supporting paths. Actual Python/runtime/SDK/OTel and live +service versions are captured without dumping environment variables. Credentials, +connection strings (including instrumentation keys/endpoints), authorization, +function keys and SAS are masked in logs, XML, saved JSON, console and Job Summary. +CI runs the same runner on its selected interpreter and uploads `test-results/**` +with `if: always()`; it never routes matrix tests through Hatch's Python 3.11. + +- **PASS** requires actual execution and successful required checks. +- **FAIL** means failed tests/checks, non-zero processes or timeouts. +- **INCOMPLETE** means missing prerequisites/evidence, zero tests, all skipped, + or skipped required live assertions. These runs exit non-zero. +- **NOT_APPLICABLE** is only an explicitly unsupported/not-requested capability, + never a fallback for missing required tools or cloud assertions. + +OTel instrumentation and local Functions-host success do **not** certify Azure +Monitor ingestion. `cloud_monitor` is `NOT_APPLICABLE` locally, with the unrun +App Insights assertion listed in `cloud_tests_not_run`. `e2e-local` targets an +already-running app and permits only that explicitly optional cloud assertion to +skip. A required `azure-e2e` run with that assertion skipped or absent is +`INCOMPLETE`, never PASS—even if HTTP tests passed. Deployed-version matching has +its own status derived from the actual JUnit assertion, not from configuration. + +Real Azure requires `E2E_BASE_URL`, `E2E_EXPECTED_VERSION`, `APPINSIGHTS_NAME`, +`E2E_RESOURCE_GROUP`, and an authenticated `az` CLI. The runner never deploys or +provisions; the existing release workflow still owns deployment, exact-ref and +candidate-version records, cleanup, OIDC, and upload of the original built artifact. +Telemetry destination is recorded as configured/redacted, never a connection string. +Host lanes use temporary storage, distinct ephemeral ports and bounded readiness; +only their owned process groups are stopped, including on interruption. Core Tools +may need its extension bundle cached before an offline host run. + ## GitHub Actions Pinning All external `uses:` references in `.github/workflows/` MUST pin to a @@ -98,8 +157,6 @@ context never drifts from the pinned commit. permitted in this repository and are flagged with an inline comment at the call site: -- `pypa/gh-action-pypi-publish@release/v1` — PyPA-maintained stable - release channel; this pinning style is explicitly recommended upstream. - Local composite actions (`uses: ./...`) — versioned with the repo. When adding a new external Action, resolve the SHA with diff --git a/docs/agent-playbook.md b/docs/agent-playbook.md index 9bf0ec1..4de0ad2 100644 --- a/docs/agent-playbook.md +++ b/docs/agent-playbook.md @@ -26,11 +26,17 @@ This also applies when using the API or CLI. 5. Do not broaden supported Python versions or dependency ranges casually. ## Validation -- `make test` -- `make lint` -- `make typecheck` -- `make security` -- `make build` +- `make doctor`: read-only prerequisites and actual interpreter. +- `make check`: complete non-mutating quality/policies, strict types once. +- `make test`: offline, all privacy regressions, coverage >=95%. +- `make integration`: separate OTel and managed real-host/private Azurite lanes. +- `make verify`: full local gate; `make check-all` is an exact alias. +- `make e2e-azure`: configured deployment and real App Insights ingestion, no provisioning. +- `make report`: saved redacted JSON only; `make clean`: generated artifacts only. + +Other targets are focused/partial. `clean-all` retains deep cleanup including local +`.venv`. Local host/OTel success is not cloud certification: skipped required +App Insights assertions are INCOMPLETE. See [Development](development.md#development-commands). ## Review Evidence diff --git a/docs/contributing.md b/docs/contributing.md index bed417a..7c55ce2 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -15,7 +15,7 @@ Thank you for your interest in contributing to `azure-functions-logging`. This g 3. Install development dependencies: ```bash - pip install -e ".[dev]" + make install ``` 4. Verify your setup: @@ -50,15 +50,19 @@ Before pushing, run the full local gate: make check-all ``` -This runs: +`make check-all` is an exact alias of `make verify`, requiring Core Tools and +Azurite. It runs the following non-mutating/local gates (formatting fixes remain +an explicit `make format` action): | Check | Command | | ----- | ------- | -| Code formatting | `make format` (ruff) | +| Code formatting | `make format-check` (ruff, non-mutating) | | Linting | `make style` (ruff) | | Type checking | `make typecheck` (mypy) | | Security scan | `make security` (bandit) | | Tests | `make test` (pytest) | +| OTel instrumentation and managed real host/Azurite | `make integration` | +| Installed wheel smoke and strict docs | `make wheel-smoke`, `make docs` | ### 4. Commit @@ -123,10 +127,22 @@ Titles for issues, pull requests, and commits follow the **Title Convention** in | `make style` | Lint with ruff | | `make typecheck` | Type check with mypy (strict mode) | | `make lint` | Run both style and typecheck | -| `make test` | Run tests with pytest | +| `make test` | Fully offline pytest, all privacy regressions, coverage >=95% | | `make cov` | Run tests and generate coverage report | | `make security` | Run bandit security scan | -| `make check-all` | Run full local gate (all of the above) | +| `make check` | Complete non-mutating quality/policy gate, types once | +| `make integration` | Separate OTel instrumentation and managed real-host/Azurite lanes | +| `make verify` / `make check-all` | Full local gate / exact alias | +| `make e2e-azure` | Explicitly configured real Azure/App Insights certification, no provisioning | +| `make report` | Render saved evidence without execution | +| `make clean` | Generated artifacts only; `clean-all` also removes bytecode/local `.venv` | + +`lint`, `style`, `typecheck`, `security`, `cov`, `build` and `docs` are partial/focused. +Local host/OTel success is not cloud ingestion certification. Required skipped +App Insights assertions are INCOMPLETE/non-zero; optional local omissions are +NOT_APPLICABLE and listed explicitly. See [Development](development.md#evidence-and-certification) +for saved `test-results/` layout, redaction and status semantics. CI uses the same +runner on each actual matrix interpreter and uploads evidence even on failure. ## Pre-commit Hooks diff --git a/docs/development.md b/docs/development.md index 148cca7..9dc7c1c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -7,6 +7,8 @@ This guide explains how to set up a development environment for `azure-functions - Python 3.11-3.14 (`>=3.11,<3.15`) - Git - Make (for running development commands) +- Hatch installed once in an isolated tool environment +- Functions Core Tools (`func`) and Azurite for integration/verify only ## Getting Started @@ -20,13 +22,13 @@ This guide explains how to set up a development environment for `azure-functions 2. Install the package in editable mode with development dependencies: ```bash - pip install -e ".[dev]" + make install ``` 3. Install pre-commit hooks: ```bash - pre-commit install + make precommit-install ``` 4. Verify everything works: @@ -72,10 +74,51 @@ All development tasks are automated via `Makefile` targets: | `make style` | Lint for code quality issues | ruff | | `make typecheck` | Static type checking | mypy (strict) | | `make lint` | Run both `style` and `typecheck` | ruff, mypy | -| `make test` | Run test suite | pytest | +| `make test` | Offline unit/mock suite, all privacy regressions, coverage >=95% | pytest | | `make cov` | Run tests with coverage report | pytest-cov | | `make security` | Security vulnerability scan | bandit | -| `make check-all` | Run all checks (full local gate) | All of the above | +| `make install` / `make install-ci` | Isolated local setup / CI-selected interpreter setup | Hatch / selected Python | +| `make doctor` | Read-only prerequisite check; missing required tools fail | canonical runner | +| `make format-check` | Non-mutating formatting check | Ruff | +| `make check` | Lint, format, strict mypy once, security, workflow/doc-link policies | non-mutating | +| `make integration` | Separate OTel, decorated-host and correlation-host lanes; private Azurite | managed processes | +| `make verify` | Check + test + integration + wheel smoke + strict docs | full local gate | +| `make check-all` | Exact alias of verify | full local gate | +| `make e2e-azure` | Explicit deployed candidate and App Insights ingestion | no provisioning | +| `make report` | Render saved JSON, never rerun tests | canonical runner | +| `make clean` | Generated artifacts only, preserve environments/shared data | allowlisted cleanup | +| `make clean-all` | Deep cleanup including bytecode and local `.venv` | maintenance | + +Other targets (`lint`, `style`, `typecheck`, `security`, `cov`, `bench`, `build`, +`wheel-smoke`, `docs`, `otel`, `host-smoke`, `host-matrix`, workflow/link lints) +are partial/focused. `e2e-local` checks an already-running local app, not Azure. +Routine commands never upgrade pip or reinstall Hatch. CI uses `make install-ci` +and the canonical runner on its selected Python/SDK/OTel versions, never the +Python 3.11 Hatch default. Quality checks include Mermaid rendering via `mmdc` +and its Chromium installation, using temporary output only. + +### Evidence and certification + +Gitignored `test-results/` holds separate `test/`, `otel/`, `host-smoke/`, +`host-matrix/`, `integration/`, `verify/`, and `azure-e2e/` summaries. Unit evidence +includes JUnit and coverage XML; integration copies JUnit/logs while retaining each +lane's summary; verify adds `lanes.json` and `environment.json`. Every summary has +schema version, command, selected/passed/failed/skipped counts, status, exit code +and supporting paths. Actual interpreter/SDK/service versions are recorded. +Secrets are masked before console/Job Summary publication and artifact upload. + +PASS requires execution; failures/timeouts are FAIL; missing prerequisites, +zero/all-skipped tests and required skipped assertions are INCOMPLETE/non-zero. +NOT_APPLICABLE is only a capability not requested/supported, never missing tools. +Local host and OTel instrumentation cannot certify cloud ingestion. A skipped +App Insights assertion is listed in `cloud_tests_not_run`; `cloud_monitor` is +NOT_APPLICABLE for local runs and INCOMPLETE for required Azure certification. +Deployed-version matching is separately derived from its JUnit assertion. + +`make e2e-azure` requires `E2E_BASE_URL`, `E2E_EXPECTED_VERSION`, `APPINSIGHTS_NAME`, +`E2E_RESOURCE_GROUP` and authenticated `az`. It never provisions. Host lanes use +private temporary storage, unique ephemeral ports, bounded readiness, and owned +process-group cleanup. See the repository CONTRIBUTING.md for the full contract. ### Running Individual Commands @@ -83,7 +126,7 @@ All development tasks are automated via `Makefile` targets: make format # Fix formatting issues make style # Check for lint violations make typecheck # Check types -make test # Run all tests +make test # Offline suite only make cov # Run tests with coverage make security # Run security scan make check-all # Run everything @@ -103,7 +146,7 @@ The project uses [Hatch](https://hatch.pypa.io/) as its build system and environ ```bash hatch run test # Run tests in managed environment -hatch run lint:all # Run all linting checks +hatch run lint # Focused lint and type checks hatch run cov # Run tests with coverage ``` @@ -164,7 +207,7 @@ See the [Testing](testing.md) guide for detailed information on the test suite. Quick reference: ```bash -make test # Run all tests +make test # Offline suite only make cov # Run with coverage report pytest tests/test_formatter.py # Run specific test file pytest -k "test_json" # Run tests matching pattern @@ -220,10 +263,12 @@ The hooks check formatting, linting, and type safety. If a hook fails, the commi CI runs on every push and pull request via GitHub Actions. The pipeline: -1. Runs `make check-all` (format, lint, typecheck, security, tests) +1. Runs `make check` (non-mutating format/lint/types/security/policies) 2. Tests across the Python version matrix: 3.11, 3.12, 3.13, 3.14 3. Generates coverage report 4. Uploads coverage to Codecov +5. Runs the same canonical managed host runner and preserves wheel/minimum-dependency gates +6. Uploads `test-results/**` even on failure and appends the same counts/status to Job Summary Pull requests must pass all CI checks before merging. diff --git a/docs/testing.md b/docs/testing.md index af504f9..7a296c1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4,7 +4,7 @@ This guide covers the testing strategy, structure, and practices for `azure-func ## Running Tests -### All Tests +### Offline Tests ```bash make test @@ -16,7 +16,21 @@ Or using Hatch: hatch run test ``` -Both commands run the full test suite via pytest. +Both commands run the offline unit/mock suite with the existing 95% coverage +floor, including every redaction/privacy regression. No Azure, host, Docker or +emulator is needed. Only the dedicated E2E/host directories are excluded. + +`make integration` separately runs OTel instrumentation and real decorated-handler +and correlation host checks with managed private Azurite. `make verify` (exact +alias: `make check-all`) always adds quality/policies, offline tests, integration, +installed wheel smoke and strict docs. Missing tools, zero/all-skipped cases and +required skipped live assertions fail closed. `make report` reads saved JSON only. + +OTel/local-host success does not prove Azure Monitor ingestion. Required Azure +certification requires the App Insights assertion and deployed-version match; +skips or absent assertions are INCOMPLETE. Optional local App Insights omissions +are NOT_APPLICABLE and explicitly listed, never certification. See +[Development](development.md#evidence-and-certification) for the evidence contract. ### With Coverage From 492b307f715bdac922e40ab0aeeb5faf6677b4fb Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 07:10:40 +0900 Subject: [PATCH 09/15] ci: parse generated test reports without semgrep's unsafe xml parse rule --- tools/verification_evidence.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py index 395dbdb..701e5cf 100644 --- a/tools/verification_evidence.py +++ b/tools/verification_evidence.py @@ -19,6 +19,14 @@ VERSION_TEST = "test_deployed_version_matches_candidate" +def read_xml_root(path: Path) -> ET.Element: + """Parse a locally generated report, refusing entity declarations.""" + data = path.read_bytes() + if b" Summary: cases: list[ET.Element] = [] for path in directory.rglob("*.xml"): if "junit" in path.name: - cases.extend(ET.parse(path).getroot().iter("testcase")) + cases.extend(read_xml_root(path).iter("testcase")) skipped = sum(case.find("skipped") is not None for case in cases) failed = sum( case.find("failure") is not None or case.find("error") is not None for case in cases @@ -133,7 +141,7 @@ def collect(directory: Path, command: list[str], exit_code: int) -> Summary: coverage_path = directory / "coverage.xml" coverage = None if coverage_path.exists(): - coverage = float(ET.parse(coverage_path).getroot().get("line-rate", "0")) + coverage = float(read_xml_root(coverage_path).get("line-rate", "0")) return Summary( 1, lane, From b09ad6d25d30de74f3a27142fa7bc649bce1df50 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 07:10:40 +0900 Subject: [PATCH 10/15] ci(security): install the selected interpreter's tools before the bandit job --- .github/workflows/security.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 6f76dbd..be26acc 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -23,10 +23,8 @@ jobs: with: python-version: "3.11" - - name: Install Hatch and dependencies via Makefile - run: | - pip install hatch - make install + - name: Install dependencies on the selected interpreter + run: make install-ci - name: Run Bandit run: make security From 6a418ad54eb629c292a472ff7dae607be77248da Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 07:20:56 +0900 Subject: [PATCH 11/15] test(verification): isolate prerequisite probes from installed modules Mock Python module discovery for the Mermaid prerequisite test and prove each required module still fails closed independently. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tests/test_verification_evidence.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py index 3fda6de..827fda1 100644 --- a/tests/test_verification_evidence.py +++ b/tests/test_verification_evidence.py @@ -1,5 +1,6 @@ from __future__ import annotations +from importlib.machinery import ModuleSpec import json from pathlib import Path import sys @@ -166,6 +167,11 @@ def test_missing_tools_fail_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPat def test_host_prerequisites_do_not_require_mermaid(monkeypatch: pytest.MonkeyPatch) -> None: + # Given all required Python modules and host tools, but no Mermaid executable. + monkeypatch.setattr( + "tools.verification_environment.importlib.util.find_spec", + lambda module: ModuleSpec(module, loader=None), + ) monkeypatch.setattr( "tools.verification_environment.shutil.which", lambda tool: None if tool == "mmdc" else "/tools/" + tool, @@ -174,6 +180,23 @@ def test_host_prerequisites_do_not_require_mermaid(monkeypatch: pytest.MonkeyPat assert doctor() != 0 +@pytest.mark.parametrize("missing", ["venv", "ruff", "mypy", "pytest", "bandit", "build", "mkdocs"]) +def test_host_prerequisites_fail_when_required_module_is_missing( + monkeypatch: pytest.MonkeyPatch, missing: str +) -> None: + # Given exactly one missing module, with every executable available. + monkeypatch.setattr( + "tools.verification_environment.importlib.util.find_spec", + lambda module: None if module == missing else ModuleSpec(module, loader=None), + ) + monkeypatch.setattr( + "tools.verification_environment.shutil.which", lambda tool: "/tools/" + tool + ) + + # When checking host prerequisites, then the missing module must fail closed. + assert doctor(quality=False) == 1 + + def test_offline_selection_keeps_all_privacy_tests() -> None: command = pytest_command("test", []) assert "tests/" in command From e2f6b74bd79757b59fe633d0e26400e2da8e0d15 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:57:33 +0900 Subject: [PATCH 12/15] test(verification): isolate synthetic job summaries --- tests/conftest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..eb72493 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_verification_job_summary( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> None: + if request.module.__name__.startswith("test_verification"): + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) From cbc18fb3276a596be6cbea3b83e885efdd5b4a0b Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:57:33 +0900 Subject: [PATCH 13/15] ci(evidence): redact whole credentials and preserve structured assets --- tests/test_verification_review_regressions.py | 77 +++++++++++++ tools/verification_evidence.py | 105 +++++++++++++----- 2 files changed, 154 insertions(+), 28 deletions(-) create mode 100644 tests/test_verification_review_regressions.py diff --git a/tests/test_verification_review_regressions.py b/tests/test_verification_review_regressions.py new file mode 100644 index 0000000..25571e1 --- /dev/null +++ b/tests/test_verification_review_regressions.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from contextlib import nullcontext +import json +from pathlib import Path +import subprocess +import sys +from unittest.mock import Mock + +import pytest + +from tools.verification_evidence import Summary, redact, save +from tools.verification_host import wait_ready + + +@pytest.mark.parametrize( + "text", + [ + '{"Authorization": "Bearer CANARY"}', + "{'Proxy-Authorization': 'Basic CANARY'}", + '{"password": "alpha CANARY beta"}', + '{"token": "with \\"escaped\\" CANARY"}', + "SharedAccessKey=CANARY;SharedAccessKeyName=Root", + "InstrumentationKey=CANARY;IngestionEndpoint=https://CANARY/;", + 'x', + ], +) +def test_quoted_credentials_are_consumed_whole(text: str) -> None: + assert "CANARY" not in redact(text) + + +def test_save_preserves_structure_binary_and_external_symlink(tmp_path: Path) -> None: + png = b"\x89PNG\r\n\xff\xfe" + (tmp_path / "chart.png").write_bytes(png) + outside = tmp_path.parent / (tmp_path.name + "-outside.log") + outside.write_text("password=CANARY") + (tmp_path / "link.log").symlink_to(outside) + (tmp_path / "extra.json").write_text('{"password": 123, "ok": true}') + save(tmp_path, Summary(1, "test", ('Authorization: "Bearer CANARY"',), 1, 1, 0, 0, "PASS", 0)) + assert "CANARY" not in (tmp_path / "summary.json").read_text() + assert json.loads((tmp_path / "extra.json").read_text())["ok"] + assert (tmp_path / "chart.png").read_bytes() == png + assert outside.read_text() == "password=CANARY" + + +def test_known_secret_replaced_before_patterns(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MY_API_TOKEN", 'pre;fix&CANARY "quoted" value') + assert "CANARY" not in redact(json.dumps({"x": 'pre;fix&CANARY "quoted" value'})) + + +@pytest.mark.parametrize("error", [TimeoutError(), ConnectionError(), OSError()]) +def test_readiness_retries_transient_failure( + error: OSError, monkeypatch: pytest.MonkeyPatch +) -> None: + process = Mock() + process.poll.return_value = None + request = Mock(side_effect=[error, nullcontext(Mock(status=200))]) + monkeypatch.setattr("tools.verification_host.urlopen", request) + monkeypatch.setattr("tools.verification_host.time.sleep", lambda _: None) + wait_ready(process, "http://localhost/health") + assert request.call_count == 2 + + +def test_check_failure_retains_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from tools.verification import focused + + monkeypatch.chdir(tmp_path) + assert focused("check", [sys.executable, "-c", "print('password=CANARY'); exit(7)"]) == 7 + summary = json.loads(Path("test-results/check/summary.json").read_text()) + assert (summary["status"], summary["exit_code"]) == ("FAIL", 7) + assert "CANARY" not in Path("test-results/check/logs/runner.log").read_text() + + +def test_check_wraps_prerequisites_in_evidence() -> None: + result = subprocess.run(["make", "-n", "check", "CI=true"], capture_output=True, text=True) + assert "tools.verification focused check" in result.stdout + assert "tools.verification_check" in result.stdout diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py index 701e5cf..215b600 100644 --- a/tools/verification_evidence.py +++ b/tools/verification_evidence.py @@ -48,24 +48,64 @@ class Summary: cloud_tests_not_run: tuple[str, ...] = (APP_INSIGHTS_TEST,) +_REDACTED = "[REDACTED]" +_QUOTED = r"\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'" +_AUTHORIZATION = re.compile( + r"(?im)(?P(?:proxy-)?authorization[\"']?\s*[:=]\s*)(?P" + _QUOTED + r"|[^\r\n\"<]+)" +) +_URL_CREDENTIALS = re.compile(r"(https?://)[^/@\s\"']+:[^/@\s\"']+@") +_SECRET_FIELD = re.compile( + r"(?i)(?(?:[\w.-]*(?:token|password|passwd|secret|api[_-]?key|" + r"connection[_-]?string)|AccountKey|SharedAccessKey|SharedAccessSignature|" + r"AzureWebJobsStorage|InstrumentationKey|IngestionEndpoint|LiveEndpoint|" + r"x-functions-key|code|sig|sas)" + r"[\"']?\s*[:=]\s*)(?P" + _QUOTED + r"|[^\s;&<>\"',]+)" +) +_SECRET_ENV_NAME = re.compile( + r"(?i)secret|token|password|connection|string|accountkey|functions.key|api.?key|sas|appinsights" +) +_TEXT_SUFFIXES = frozenset({".log", ".txt", ".xml", ".json", ".md"}) + + +def _mask(match: re.Match[str]) -> str: + value = match.group("value") + quote = value[0] if value[:1] in {'"', "'"} else "" + return f"{match.group('key')}{quote}{_REDACTED}{quote}" + + +def _known_secrets() -> list[str]: + values = { + value + for key, value in os.environ.items() + if len(value) >= 6 and _SECRET_ENV_NAME.search(key) + } + escaped = {json.dumps(value)[1:-1] for value in values} + return sorted(values | escaped, key=len, reverse=True) + + def redact(text: str) -> str: - text = re.sub(r"(?im)(authorization[\"']?\s*[:=]\s*)[^\r\n<\"]+", r"\1[REDACTED]", text) - text = re.sub(r"(https?://)[^/@\s]+:[^/@\s]+@", r"\1[REDACTED]@", text) + """Consume whole credentials before regex masking, preserving quoted syntax.""" + for value in _known_secrets(): + text = text.replace(value, _REDACTED) + text = _AUTHORIZATION.sub(_mask, text) + text = _URL_CREDENTIALS.sub(r"\1" + _REDACTED + "@", text) text = re.sub(r"(?i)(DefaultEndpointsProtocol=)[^\s<\"']+", r"\1[REDACTED]", text) - text = re.sub( - r"(?i)(?= 6 and re.search( - r"(?i)secret|token|password|connection|string|accountkey|functions.key|appinsights", key - ): - text = text.replace(value, "[REDACTED]") - return text + return _SECRET_FIELD.sub(_mask, text) + + +def redact_json(value: object) -> object: + """Redact string leaves before serialization, including tuple commands.""" + if isinstance(value, str): + return redact(value) + if isinstance(value, (list, tuple)): + return [redact_json(item) for item in value] + if isinstance(value, dict): + return {redact(str(key)): redact_json(item) for key, item in value.items()} + return value + + +def dump_json(value: object) -> str: + return json.dumps(redact_json(value), indent=2) + "\n" def environment(live: bool = False) -> dict[str, str]: @@ -184,19 +224,29 @@ def publish(summary: Summary) -> None: stream.write(f"\n- {text}\n") +def _redact_file_text(suffix: str, text: str) -> str: + if suffix == ".json": + try: + return dump_json(json.loads(text)) + except ValueError: + return redact(text) + return redact(text) + + def save(directory: Path, summary: Summary) -> Summary: directory.mkdir(parents=True, exist_ok=True) for path in directory.rglob("*"): - if path.is_file(): - path.write_text(redact(path.read_text(encoding="utf-8")), encoding="utf-8") + if path.is_symlink() or not path.is_file() or path.suffix not in _TEXT_SUFFIXES: + continue + try: + original = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + redacted = _redact_file_text(path.suffix, original) + if redacted != original: + path.write_text(redacted, encoding="utf-8") (directory / "environment.json").write_text( - redact( - json.dumps( - environment(live=directory.name in {"integration", "host-smoke", "host-matrix"}), - indent=2, - ) - ) - + "\n", + dump_json(environment(live=directory.name in {"integration", "host-smoke", "host-matrix"})), encoding="utf-8", ) summary = replace( @@ -210,9 +260,8 @@ def save(directory: Path, summary: Summary) -> Summary: if path.is_file() and path.name != "summary.json" ), ) - serialized = redact(json.dumps(asdict(summary), indent=2)) - (directory / "summary.json").write_text(serialized + "\n", encoding="utf-8") - return Summary(**json.loads(serialized)) + (directory / "summary.json").write_text(dump_json(asdict(summary)), encoding="utf-8") + return summary def report(root: Path) -> int: From 7d95d227d203dc73a8449cbdae74224f7b11bef2 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:57:33 +0900 Subject: [PATCH 14/15] ci(verification): capture quality gates without bootstrapping maintenance --- Makefile | 14 ++++++++++---- tests/test_verification_commands.py | 9 ++++++++- tools/maintenance_python.sh | 11 +++++++++++ tools/verification.py | 2 +- tools/verification_check.py | 11 +++++++++++ tools/verification_environment.py | 4 ++++ 6 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tools/maintenance_python.sh create mode 100644 tools/verification_check.py diff --git a/Makefile b/Makefile index 8337c97..e7f9e27 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ else RUN := $(PYTHON) endif TREES := src tests tools examples benchmarks scripts +MAINTENANCE_PYTHON = $(shell sh tools/maintenance_python.sh) .PHONY: bootstrap ensure-hatch install install-ci bootstrap ensure-hatch: @@ -49,12 +50,17 @@ lint-doc-links: @$(RUN) tools/lint_doc_links.py lint-mermaid: @$(RUN) scripts/render_mermaid.py -check: style typecheck security lint-workflows lint-doc-links lint-mermaid +check: + @$(RUN) -m tools.verification focused check -m tools.verification_check +.PHONY: check-steps +check-steps: style typecheck security lint-workflows lint-doc-links lint-mermaid check-all: verify .PHONY: test integration otel host-smoke host-matrix verify report wheel-smoke -test integration otel host-smoke host-matrix verify report: +test integration otel host-smoke host-matrix verify: @$(RUN) -m tools.verification $@ +report: + @$(MAINTENANCE_PYTHON) -m tools.verification report wheel-smoke: @$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel @@ -86,9 +92,9 @@ version: ensure-hatch .PHONY: doctor clean clean-all help doctor: - @$(RUN) -m tools.verification doctor + @$(MAINTENANCE_PYTHON) -m tools.verification doctor clean: - @$(RUN) -m tools.verification clean + @$(MAINTENANCE_PYTHON) -m tools.verification clean clean-all: clean @find . -type d -name "__pycache__" -not -path "./.git/*" -exec rm -rf {} + @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -not -path "./.git/*" -delete diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py index 78fe31a..07bea1a 100644 --- a/tests/test_verification_commands.py +++ b/tests/test_verification_commands.py @@ -42,7 +42,7 @@ def test_routine_commands_do_not_bootstrap(target: str) -> None: def test_check_is_complete_non_mutating_and_types_once() -> None: # Given the complete quality gate, when expanding its dependencies. - result = subprocess.run(["make", "-n", "check"], cwd=ROOT, capture_output=True, text=True) + result = subprocess.run(["make", "-n", "check-steps"], cwd=ROOT, capture_output=True, text=True) # Then every repository policy checker runs without mutation. assert result.returncode == 0 assert result.stdout.count("-m mypy") == 1 @@ -59,6 +59,13 @@ def test_check_is_complete_non_mutating_and_types_once() -> None: assert "--fix" not in result.stdout +def test_maintenance_uses_no_hatch_environment(tmp_path: Path) -> None: + environment = dict(os.environ, HATCH_DATA_DIR=str(tmp_path / "hatch"), CI="") + for target in ("doctor", "report"): + subprocess.run(["make", target], cwd=ROOT, env=environment, capture_output=True, text=True) + assert not list((tmp_path / "hatch").rglob("pyvenv.cfg")) + + def test_unconfigured_azure_cli_fails_with_saved_incomplete(tmp_path: Path) -> None: # Given no Azure configuration and an isolated evidence directory. environment = dict(os.environ, PYTHONPATH=str(ROOT)) diff --git a/tools/maintenance_python.sh b/tools/maintenance_python.sh new file mode 100644 index 0000000..66f7c0f --- /dev/null +++ b/tools/maintenance_python.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Resolve only existing interpreters; never create or synchronize a Hatch environment. +existing=$(hatch env find 2>/dev/null || true) +for candidate in "$existing/bin/python" "${VIRTUAL_ENV:-/nonexistent}/bin/python" .venv/bin/python python3.11 python3 python; do + if "$candidate" -c 'import sys; assert (3, 11) <= sys.version_info[:2] < (3, 15)' 2>/dev/null; then + printf '%s\n' "$candidate" + exit 0 + fi +done +printf '%s\n' 'Python >=3.11,<3.15 required; run make install' >&2 +exit 1 diff --git a/tools/verification.py b/tools/verification.py index 19cdbce..b57b8bd 100644 --- a/tools/verification.py +++ b/tools/verification.py @@ -155,7 +155,7 @@ def verify() -> int: path = Path("test-results") / target / "summary.json" path.unlink(missing_ok=True) code = run(["make", target], directory / "logs" / f"{target}.log", 1200) - if target != "check" and path.exists(): + if path.exists(): summary = Summary(**json.loads(path.read_text())) if code and summary.status == "PASS": summary = replace(summary, status="FAIL", exit_code=code) diff --git a/tools/verification_check.py b/tools/verification_check.py new file mode 100644 index 0000000..774cdf3 --- /dev/null +++ b/tools/verification_check.py @@ -0,0 +1,11 @@ +"""Run every quality prerequisite inside the shared check evidence capture.""" + +import subprocess + + +def main() -> int: + return subprocess.call(["make", "check-steps"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verification_environment.py b/tools/verification_environment.py index b4752eb..3b54498 100644 --- a/tools/verification_environment.py +++ b/tools/verification_environment.py @@ -10,11 +10,15 @@ def doctor(live: bool = True, quality: bool = True) -> int: healthy = (3, 11) <= sys.version_info[:2] < (3, 15) + modules_available = True print(f"python REQUIRED >=3.11,<3.15: {sys.version.split()[0]} ({sys.executable})") for module in ("venv", "ruff", "mypy", "pytest", "bandit", "build", "mkdocs"): found = importlib.util.find_spec(module) is not None print(f"{module} REQUIRED: {'found' if found else 'MISSING'}") healthy = healthy and found + modules_available = modules_available and found + if not modules_available: + print("development environment not created: run make install") for tool in ("func", "azurite"): path = shutil.which(tool) print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}") From 97cf1e458d426bb32e01d05cf908e0d26ab3ddbc Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:57:33 +0900 Subject: [PATCH 15/15] ci(host): retry transient readiness transport errors --- tools/verification_host.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/verification_host.py b/tools/verification_host.py index 56e7c6f..a777e3d 100644 --- a/tools/verification_host.py +++ b/tools/verification_host.py @@ -4,6 +4,7 @@ import base64 from contextlib import ExitStack +from http.client import HTTPException import json import os from pathlib import Path @@ -36,7 +37,7 @@ def wait_ready(process: subprocess.Popen[str], url: str) -> None: with urlopen(url, timeout=1) as response: if response.status == 200: return - except URLError: + except (OSError, URLError, HTTPException): time.sleep(0.25) raise TimeoutError("service readiness deadline exceeded")