From 55e65897d26d97856e366864070b9a2a28cde177 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:40 +0900 Subject: [PATCH 01/17] ci: add versioned redacted verification evidence --- tests/test_verification_evidence.py | 102 ++++++++++++++++++ tools/__init__.py | 0 tools/verification_evidence.py | 159 ++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+) create mode 100644 tests/test_verification_evidence.py create mode 100644 tools/__init__.py create mode 100644 tools/verification_evidence.py diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py new file mode 100644 index 00000000..2221b1a9 --- /dev/null +++ b/tests/test_verification_evidence.py @@ -0,0 +1,102 @@ +"""Offline contracts for the saved verification evidence boundary.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +import pytest + + +def test_report_renders_saved_json_without_running_a_lane(tmp_path: Path) -> None: + # Given no results and an isolated working directory. + runner = Path(__file__).resolve().parents[1] / "tools" / "verification.py" + # When requesting a report, no test process should be started. + result = subprocess.run( + [sys.executable, str(runner), "report"], cwd=tmp_path, capture_output=True, text=True + ) + # Then absence of saved evidence is explicit and non-successful. + assert result.returncode == 1 + assert "No saved" in result.stdout + assert not (tmp_path / "test-results").exists() + + +@pytest.mark.parametrize( + ("cases", "exit_code", "status"), + [ + ("", 0, "INCOMPLETE"), + ("", 0, "INCOMPLETE"), + ("", 1, "FAIL"), + ("", 0, "PASS"), + ], +) +def test_junit_classifies_execution_fail_closed( + tmp_path: Path, cases: str, exit_code: int, status: str +) -> None: + from tools.verification_evidence import collect + + # Given actual JUnit cases, not an optimistic process exit alone. + (tmp_path / "junit.xml").write_text(f"{cases}") + # When collecting evidence. + summary = collect(tmp_path, ["pytest"], exit_code) + # Then zero/all-skipped cases cannot pass. + assert summary.status == status + + +@pytest.mark.parametrize( + "secret", + [ + "AccountKey=VERYSECRET;", + "SharedAccessSignature=VERYSECRET;", + "code=VERYSECRET&x=1", + "x-functions-key: VERYSECRET", + "Authorization: Bearer VERYSECRET", + "password=VERYSECRET", + "token=VERYSECRET", + "AzureWebJobsStorage=VERYSECRET", + ], +) +def test_secrets_are_redacted_from_persisted_evidence(tmp_path: Path, secret: str) -> None: + from tools.verification_evidence import collect, save + + # Given secret-bearing logs and XML attributes/text. + (tmp_path / "junit.xml").write_text( + "" + + secret.replace("&", "&") + + "" + ) + (tmp_path / "output.log").write_text(secret) + # When saving all evidence. + save(tmp_path, collect(tmp_path, ["pytest", secret], 0)) + # Then persisted evidence contains no credential value. + for path in tmp_path.rglob("*"): + if path.is_file(): + assert "VERYSECRET" not in path.read_text() + + +def test_saved_summary_has_versioned_schema_and_supporting_paths(tmp_path: Path) -> None: + from tools.verification_evidence import collect, save + + # Given one executed test and coverage. + (tmp_path / "junit.xml").write_text("") + (tmp_path / "coverage.xml").write_text('') + # When saving a lane. + save(tmp_path, collect(tmp_path, ["pytest"], 0)) + data = json.loads((tmp_path / "summary.json").read_text()) + # Then consumers can identify the contract and its proof. + assert data["schema_version"] == 1 + assert (data["selected"], data["passed"], data["failed"], data["skipped"]) == (1, 1, 0, 0) + assert data["coverage"] == 0.99 + assert str(tmp_path / "junit.xml") in data["supporting_files"] + assert (tmp_path / "environment.json").exists() + + +def test_redaction_preserves_json_syntax(tmp_path: Path) -> None: + from tools.verification_evidence import Summary, save + + summary = Summary(1, "test", ("Authorization: Bearer VERYSECRET",), 1, 1, 0, 0, "PASS", 0) + save(tmp_path, summary) + data = json.loads((tmp_path / "summary.json").read_text()) + assert "VERYSECRET" not in str(data) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py new file mode 100644 index 00000000..75704bc6 --- /dev/null +++ b/tools/verification_evidence.py @@ -0,0 +1,159 @@ +"""Versioned evidence shared by local commands and CI.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +from importlib.metadata import PackageNotFoundError, version +import json +import os +from pathlib import Path +import platform +import re +import subprocess +import sys +from typing import Literal +import xml.etree.ElementTree as ET + +Status = Literal["PASS", "FAIL", "INCOMPLETE", "NOT_APPLICABLE"] + + +@dataclass(frozen=True, slots=True) +class Summary: + schema_version: int + lane: str + command: tuple[str, ...] + selected: int + passed: int + failed: int + skipped: int + status: Status + exit_code: int + supporting_files: tuple[str, ...] = () + coverage: float | None = None + reason: str = "" + + +def redact(text: str) -> str: + # Redact entire authorization values, including the scheme and whitespace. + text = re.sub(r"(?im)(authorization[\"']?\s*[:=]\s*)[^\r\n<\"]+", r"\1[REDACTED]", text) + text = re.sub(r"(https?://)[^/@\s]+:[^/@\s]+@", r"\1[REDACTED]@", text) + text = re.sub( + r"(?i)(?= 6 and re.search( + r"(?i)secret|token|password|connection|string|accountkey|functions.key", key + ): + text = text.replace(value, "[REDACTED]") + return text + + +def environment(live: bool = False) -> dict[str, str]: + metadata = { + "python": platform.python_version(), + "executable": sys.executable, + "platform": platform.platform(), + "runtime": platform.python_implementation(), + } + for package in ("azure-functions", "azure-functions-openapi", "pydantic", "hatch"): + try: + metadata[package] = version(package) + except PackageNotFoundError: + metadata[package] = "not installed" + for key in ("E2E_BASE_URL", "E2E_EXPECTED_VERSION", "GITHUB_SHA"): + if key in os.environ: + metadata[key] = redact(os.environ[key]) + for tool in ("func", "azurite") if live else (): + try: + result = subprocess.run([tool, "--version"], capture_output=True, text=True, timeout=10) + metadata[tool] = redact(result.stdout.strip()) + except (OSError, subprocess.TimeoutExpired): + metadata[tool] = "unavailable" + return metadata + + +def collect(directory: Path, command: list[str], exit_code: int) -> Summary: + cases: list[ET.Element] = [] + for path in directory.rglob("*.xml"): + if "junit" in path.name: + cases.extend(ET.parse(path).getroot().iter("testcase")) + skipped = sum(case.find("skipped") is not None for case in cases) + failed = sum( + case.find("failure") is not None or case.find("error") is not None for case in cases + ) + passed = len(cases) - skipped - failed + status: Status = "PASS" + if exit_code or failed: + status = "FAIL" + elif not cases or not passed or (directory.name != "test" and skipped): + status = "INCOMPLETE" + coverage_path = directory / "coverage.xml" + coverage = None + if coverage_path.exists(): + coverage = float(ET.parse(coverage_path).getroot().get("line-rate", "0")) + return Summary( + 1, + directory.name, + tuple(command), + len(cases), + passed, + failed, + skipped, + status, + exit_code, + coverage=coverage, + ) + + +def render(summary: Summary) -> str: + return ( + f"{summary.lane}: {summary.status} | selected={summary.selected} " + f"passed={summary.passed} failed={summary.failed} skipped={summary.skipped} " + f"exit={summary.exit_code}" + (f" | {summary.reason}" if summary.reason else "") + ) + + +def publish(summary: Summary) -> None: + text = render(summary) + print(text) + destination = os.environ.get("GITHUB_STEP_SUMMARY") + if destination: + with Path(destination).open("a", encoding="utf-8") as stream: + stream.write(f"\n- {text}\n") + + +def save(directory: Path, summary: Summary) -> Summary: + directory.mkdir(parents=True, exist_ok=True) + for path in directory.rglob("*"): + if path.is_file(): + path.write_text(redact(path.read_text(encoding="utf-8")), encoding="utf-8") + (directory / "environment.json").write_text( + redact(json.dumps(environment(live=directory.name == "integration"), indent=2)) + "\n", + encoding="utf-8", + ) + summary = replace( + summary, + supporting_files=tuple( + str(path) + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != "summary.json" + ), + ) + path = directory / "summary.json" + path.write_text(redact(json.dumps(asdict(summary), indent=2)) + "\n", encoding="utf-8") + return summary + + +def report(root: Path) -> int: + paths = sorted(root.glob("*/summary.json")) + if not paths: + print("No saved verification results") + return 1 + for path in paths: + data = json.loads(path.read_text(encoding="utf-8")) + publish(Summary(**data)) + return 0 From a00930ffc5c507aefefc1f3088fa01046d2c43d1 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:42 +0900 Subject: [PATCH 02/17] ci: manage fail-closed verification lanes and process lifecycles --- tests/test_verification_commands.py | 195 +++++++++++++++++++++++++ tools/verification.py | 212 ++++++++++++++++++++++++++++ tools/verification_environment.py | 50 +++++++ tools/verification_host.py | 139 ++++++++++++++++++ tools/verification_process.py | 77 ++++++++++ tools/verification_wheel.py | 62 ++++++++ 6 files changed, 735 insertions(+) create mode 100644 tests/test_verification_commands.py create mode 100644 tools/verification.py create mode 100644 tools/verification_environment.py create mode 100644 tools/verification_host.py create mode 100644 tools/verification_process.py create mode 100644 tools/verification_wheel.py diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py new file mode 100644 index 00000000..3bdea765 --- /dev/null +++ b/tests/test_verification_commands.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +from tools.verification_environment import clean, doctor +from tools.verification_process import run + +ROOT = Path(__file__).resolve().parents[1] +CONTRACT = ( + "install", + "install-ci", + "doctor", + "format", + "format-check", + "check", + "test", + "integration", + "verify", + "e2e-azure", + "report", + "clean", +) + + +def test_process_propagates_exit_code(tmp_path: Path) -> None: + assert run([sys.executable, "-c", "raise SystemExit(7)"], tmp_path / "log") == 7 + + +def test_missing_process_is_non_successful(tmp_path: Path) -> None: + assert run(["/nonexistent/openapi-command"], tmp_path / "log") == 127 + + +def test_owned_child_is_reaped_on_interruption(tmp_path: Path) -> None: + from tools.verification_process import child + + with ( + pytest.raises(KeyboardInterrupt), + child([sys.executable, "-c", "import time; time.sleep(30)"], tmp_path / "log") as process, + ): + raise KeyboardInterrupt + assert process.poll() is not None + + +def test_verify_runs_every_required_lane_even_after_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tools.verification import verify + + commands: list[list[str]] = [] + + def fake_run(command: list[str], log: Path, timeout: float = 600) -> int: + from tools.verification_evidence import Summary, save + + commands.append(command) + if command[1] in {"test", "integration"}: + failed = command[1] == "integration" + save( + Path("test-results") / command[1], + Summary( + 1, + command[1], + tuple(command), + 1, + int(not failed), + int(failed), + 0, + "FAIL" if failed else "PASS", + int(failed), + ), + ) + return 1 if command == ["make", "integration"] else 0 + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("tools.verification.run", fake_run) + assert verify() == 1 + assert [command[1] for command in commands] == [ + "check", + "test", + "integration", + "perf", + "wheel-smoke", + "docs", + ] + summary = json.loads((tmp_path / "test-results/verify/summary.json").read_text()) + assert summary["status"] == "FAIL" + assert (tmp_path / "test-results/verify/environment.json").exists() + + +def test_integration_missing_tools_is_incomplete( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tools.verification import execute + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("tools.verification_environment.shutil.which", lambda _: None) + assert execute("integration", []) != 0 + data = json.loads((tmp_path / "test-results/integration/summary.json").read_text()) + assert data["status"] == "INCOMPLETE" + + +def test_host_readiness_fails_when_child_exits(tmp_path: Path) -> None: + from tools.verification_host import wait_ready + + with subprocess.Popen([sys.executable, "-c", "raise SystemExit(1)"], text=True) as child: + child.wait() + with pytest.raises(RuntimeError, match="before readiness"): + wait_ready(child, "http://127.0.0.1:1") + + +def test_live_lane_rejects_partially_skipped_results(tmp_path: Path) -> None: + from tools.verification_evidence import collect + + (tmp_path / "junit.xml").write_text( + "" + ) + assert collect(tmp_path, ["pytest"], 0).status == "INCOMPLETE" + + +def test_process_times_out_and_redacts_output(tmp_path: Path) -> None: + code = run( + [sys.executable, "-u", "-c", "import time; print('password=SECRET'); time.sleep(30)"], + tmp_path / "log", + 1, + ) + assert code == 124 + assert "SECRET" not in (tmp_path / "log").read_text() + + +def test_clean_preserves_venvs_sources_and_shared_emulator_data(tmp_path: Path) -> None: + for directory in (".venv", "src", "shared-azurite", "test-results", "dist", "build"): + (tmp_path / directory).mkdir() + (tmp_path / directory / "keep").touch() + clean(tmp_path) + assert sorted(p.name for p in tmp_path.iterdir()) == [".venv", "shared-azurite", "src"] + + +def test_clean_unlinks_artifact_symlink_without_following_it(tmp_path: Path) -> None: + (tmp_path / "shared").mkdir() + (tmp_path / "shared/keep").touch() + (tmp_path / "test-results").symlink_to(tmp_path / "shared", target_is_directory=True) + clean(tmp_path) + assert (tmp_path / "shared/keep").exists() + + +def test_doctor_fails_when_required_tool_missing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("tools.verification_environment.importlib.util.find_spec", lambda _: None) + assert doctor() == 1 + + +def test_azure_missing_prerequisites_saves_incomplete( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tools.verification import execute + + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("E2E_BASE_URL", raising=False) + monkeypatch.delenv("E2E_EXPECTED_VERSION", raising=False) + assert execute("azure-e2e", []) != 0 + data = json.loads((tmp_path / "test-results/azure-e2e/summary.json").read_text()) + assert data["status"] == "INCOMPLETE" + assert data["selected"] == 0 + + +@pytest.mark.parametrize("target", CONTRACT) +def test_make_contract_is_install_free_on_routine_runs(target: str) -> None: + result = subprocess.run(["make", "-n", target], cwd=ROOT, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "install --upgrade pip" not in result.stdout + if target not in {"install", "install-ci"}: + assert "pip install hatch" not in result.stdout + + +def test_check_runs_typechecking_once_without_test_execution() -> None: + result = subprocess.run(["make", "-n", "check"], cwd=ROOT, capture_output=True, text=True) + assert result.returncode == 0 + assert result.stdout.count("-m mypy") == 1 + assert "-m pytest" not in result.stdout + + +def test_report_uses_saved_counts_and_job_summary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tools.verification_evidence import Summary, report, save + + directory = tmp_path / "test-results/test" + summary = Summary(1, "test", ("not-a-real-command",), 3, 2, 0, 1, "PASS", 0) + save(directory, summary) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary")) + assert report(tmp_path / "test-results") == 0 + assert "passed=2" in (tmp_path / "job-summary").read_text() diff --git a/tools/verification.py b/tools/verification.py new file mode 100644 index 00000000..d9186b6f --- /dev/null +++ b/tools/verification.py @@ -0,0 +1,212 @@ +"""Canonical stdlib-only verification CLI. Run via make or python -m tools.verification.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +import os +from pathlib import Path +import shutil +import sys +import xml.etree.ElementTree as ET + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tools.verification_environment import clean, doctor +from tools.verification_evidence import Status, Summary, collect, publish, report, save +from tools.verification_host import host_smoke +from tools.verification_process import run + + +def pytest_command(lane: str, extra: list[str]) -> list[str]: + directory = Path("test-results") / lane + command = [sys.executable, "-m", "pytest"] + if lane == "test": + command += [ + "tests/", + "-m", + "not e2e and not perf", + "--ignore=tests/e2e", + "--cov-report=xml:" + str(directory / "coverage.xml"), + ] + else: + command += ["tests/e2e/", "-m", "e2e", "--no-cov"] + return command + ["--junitxml=" + str(directory / "junit.xml"), *extra] + + +def execute(lane: str, extra: list[str]) -> int: + directory = Path("test-results") / lane + if directory.exists(): + shutil.rmtree(directory) + (directory / "logs").mkdir(parents=True) + command = pytest_command(lane, extra) + reason = "" + code = 0 + blocked = False + try: + if lane in {"azure-e2e", "local-e2e"}: + missing = [ + key for key in ("E2E_BASE_URL", "E2E_EXPECTED_VERSION") if not os.environ.get(key) + ] + if missing: + blocked, code, reason = True, 2, "Missing prerequisites: " + ", ".join(missing) + if lane == "integration": + command = ["func", "start", "(managed host smoke)"] + if doctor(live=True): + blocked, code, reason = True, 2, "Missing required integration tools" + else: + code = host_smoke(directory) + elif not blocked: + code = run(command, directory / "logs/runner.log") + except KeyboardInterrupt: + code, reason = 130, "Interrupted; owned child processes terminated" + except (OSError, ET.ParseError, ValueError) as error: + code, reason = 2, str(error) + try: + summary = collect(directory, command, code) + except (ET.ParseError, ValueError) as error: + summary = Summary( + 1, + lane, + tuple(command), + 0, + 0, + 0, + 0, + "INCOMPLETE", + 2, + reason=f"Invalid evidence: {error}", + ) + if blocked: + summary = replace(summary, status="INCOMPLETE") + destination = directory / "junit.xml" + suite = ET.Element("testsuite", name=lane, tests="0") + ET.ElementTree(suite).write(destination, encoding="unicode") + summary = replace(summary, reason=reason or summary.reason) + summary = save(directory, summary) + publish(summary) + return code if code > 0 else (0 if summary.status == "PASS" else 1) + + +def verify() -> int: + directory = Path("test-results/verify") + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir(parents=True) + outcomes: list[Summary] = [] + for target in ("check", "test", "integration", "perf", "wheel-smoke", "docs"): + code = run(["make", target], directory / "logs" / f"{target}.log", 1200) + path = Path("test-results") / target / "summary.json" + if target in {"test", "integration"} and path.exists(): + import json + + summary = Summary(**json.loads(path.read_text())) + if code and summary.status == "PASS": + summary = replace(summary, status="FAIL", exit_code=code) + elif target in {"test", "integration"}: + summary = Summary( + 1, + target, + ("make", target), + 0, + 0, + 0, + 0, + "INCOMPLETE", + code or 1, + reason="Required lane summary missing", + ) + else: + summary = Summary( + 1, + target, + ("make", target), + 1, + int(code == 0), + int(code != 0), + 0, + "PASS" if code == 0 else "FAIL", + code, + ) + outcomes.append(summary) + status: Status = "FAIL" if any(item.status == "FAIL" for item in outcomes) else "PASS" + if status == "PASS" and any(item.status != "PASS" for item in outcomes): + status = "INCOMPLETE" + aggregate = Summary( + 1, + "verify", + ("make", "verify"), + sum(s.selected for s in outcomes), + sum(s.passed for s in outcomes), + sum(s.failed for s in outcomes), + sum(s.skipped for s in outcomes), + status, + 0 if status == "PASS" else 1, + reason="; ".join(f"{item.lane}={item.status}" for item in outcomes), + ) + from dataclasses import asdict + import json + + (directory / "lanes.json").write_text(json.dumps([asdict(s) for s in outcomes], indent=2)) + publish(save(directory, aggregate)) + return aggregate.exit_code + + +def focused(lane: str, command: list[str]) -> int: + directory = Path("test-results") / lane + if directory.exists(): + shutil.rmtree(directory) + code = run(command, directory / "logs/runner.log", 1200) + summary = Summary( + 1, + lane, + tuple(command), + 1, + int(code == 0), + int(code != 0), + 0, + "PASS" if code == 0 else "FAIL", + code, + ) + publish(save(directory, summary)) + return code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "lane", + choices=( + "test", + "integration", + "verify", + "azure-e2e", + "local-e2e", + "report", + "doctor", + "clean", + "focused", + ), + ) + args, extra = parser.parse_known_args() + if tuple(sys.version_info[:2]) < (3, 11): + parser.error("Python >=3.11 is required") + match args.lane: + case "report": + return report(Path("test-results")) + case "doctor": + return doctor() + case "clean": + clean(Path.cwd()) + return 0 + case "verify": + return verify() + case "focused": + return focused(extra[0], [sys.executable, *extra[1:]]) + case _: + return execute(args.lane, extra) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verification_environment.py b/tools/verification_environment.py new file mode 100644 index 00000000..72ccacb5 --- /dev/null +++ b/tools/verification_environment.py @@ -0,0 +1,50 @@ +"""Read-only prerequisites and allowlisted artifact cleanup.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import shutil +import sys + + +def doctor(live: bool = False) -> int: + healthy = sys.version_info >= (3, 11) + print(f"python REQUIRED >=3.11: {sys.version.split()[0]} ({sys.executable})") + for module in ("venv", "ruff", "mypy", "pytest", "bandit"): + found = importlib.util.find_spec(module) is not None + print(f"{module} REQUIRED: {'found' if found else 'MISSING'}") + healthy = healthy and found + for tool in ("func", "azurite"): + path = shutil.which(tool) + print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}") + if live: + healthy = healthy and path is not None + return 0 if healthy else 1 + + +def clean(root: Path) -> None: + for name in ( + "test-results", + "dist", + "build", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + ".coverage", + "coverage.xml", + "htmlcov", + "site", + ): + path = root / name + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + for tree in ("src", "tests", "tools", "scripts"): + for path in (root / tree).rglob("__pycache__"): + if not any(part in {".venv", "venv", "env"} for part in path.parts): + if path.is_symlink(): + path.unlink() + else: + shutil.rmtree(path) diff --git a/tools/verification_host.py b/tools/verification_host.py new file mode 100644 index 00000000..abf02c00 --- /dev/null +++ b/tools/verification_host.py @@ -0,0 +1,139 @@ +"""Real host contract, with isolated emulator storage and owned processes.""" + +from __future__ import annotations + +import base64 +from contextlib import ExitStack +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import sys +import tempfile +import time +from urllib.error import URLError +from urllib.request import urlopen +import xml.etree.ElementTree as ET + +from tools.verification_process import child + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def wait_ready(process: subprocess.Popen[str], url: str) -> None: + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("service exited before readiness") + try: + with urlopen(url, timeout=1) as response: + if response.status < 500: + return + except URLError: + time.sleep(0.25) + raise TimeoutError("service readiness deadline exceeded") + + +def host_smoke(directory: Path) -> int: + directory = directory.resolve() + suite = ET.Element("testsuite", name="real-functions-host") + case = ET.SubElement(suite, "testcase", name="documented_route_matches_live_host") + try: + with tempfile.TemporaryDirectory(prefix="openapi-host-") as temporary, ExitStack() as stack: + work = Path(temporary) + for name in ("function_app.py", "host.json"): + (work / name).write_bytes((Path("tests/host_smoke") / name).read_bytes()) + blob, queue, table, port = (free_port() for _ in range(4)) + key = base64.b64encode(b"openapi-local-emulator-only" * 2).decode() + emulator = stack.enter_context( + child( + [ + "env", + f"AZURITE_ACCOUNTS=openapidev:{key}", + "azurite", + "--silent", + "--location", + str(work / "storage"), + "--blobPort", + str(blob), + "--queuePort", + str(queue), + "--tablePort", + str(table), + ], + directory / "logs/azurite.log", + ) + ) + deadline = time.monotonic() + 30 + for service in (blob, queue, table): + while True: + if emulator.poll() is not None or time.monotonic() >= deadline: + raise TimeoutError("Azurite failed readiness") + try: + with socket.create_connection(("127.0.0.1", service), timeout=1): + break + except OSError: + time.sleep(0.25) + (work / "local.settings.json").write_text( + json.dumps( + { + "IsEncrypted": False, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": ( + f"DefaultEndpointsProtocol=http;AccountName=openapidev;AccountKey={key};" + f"BlobEndpoint=http://127.0.0.1:{blob}/openapidev;" + f"QueueEndpoint=http://127.0.0.1:{queue}/openapidev;" + f"TableEndpoint=http://127.0.0.1:{table}/openapidev;" + ), + }, + } + ) + ) + env_path = str(Path(sys.executable).parent) + previous_path = os.environ.get("PATH", "") + os.environ["PATH"] = env_path + os.pathsep + previous_path + try: + host = stack.enter_context( + child( + ["func", "start", "--verbose", "--port", str(port)], + directory / "logs/host.log", + work, + ) + ) + base = f"http://127.0.0.1:{port}" + wait_ready(host, base + "/api/custom_name") + with urlopen(base + "/api/custom_name", timeout=5) as response: + assert json.load(response) == {"function_name": "custom_name"} + with urlopen(base + "/api/openapi.json", timeout=5) as response: + spec = json.load(response) + assert spec["openapi"] + paths = [p for p in spec["paths"] if p not in ("/api/openapi.json", "/api/docs")] + assert paths == ["/api/custom_name"], paths + with urlopen(base + paths[0], timeout=5) as response: + assert json.load(response) == {"function_name": "custom_name"} + with urlopen(base + "/api/docs", timeout=5) as response: + assert "SwaggerUIBundle" in response.read().decode() + finally: + os.environ["PATH"] = previous_path + text = (directory / "logs/host.log").read_text() + assert not re.search( + r"0 functions (loaded|found \(Worker\))|No job functions found|" + r"Executed 'Functions\..*' \(Failed", + text, + ) + code = 0 + except (OSError, RuntimeError, TimeoutError, AssertionError, ValueError) as error: + ET.SubElement(case, "failure").text = str(error) + code = 1 + finally: + destination = directory / "junit/junit.xml" + destination.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(suite).write(destination, encoding="unicode") + return code diff --git a/tools/verification_process.py b/tools/verification_process.py new file mode 100644 index 00000000..7a243afd --- /dev/null +++ b/tools/verification_process.py @@ -0,0 +1,77 @@ +"""Bounded subprocess ownership; no child survives timeout or interruption.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +import os +from pathlib import Path +import signal +import subprocess +from types import FrameType + +from tools.verification_evidence import redact + + +def terminate(process: subprocess.Popen[str]) -> None: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + + +@contextmanager +def child( + command: list[str], log: Path, cwd: Path | None = None +) -> Iterator[subprocess.Popen[str]]: + log.parent.mkdir(parents=True, exist_ok=True) + # Raw output stays in an anonymous temporary file until it can be redacted. + import tempfile + + with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output: + with subprocess.Popen( + command, + cwd=cwd, + stdout=output, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) as process: + + def interrupted(number: int, frame: FrameType | None) -> None: + terminate(process) + raise KeyboardInterrupt(number) + + previous = { + sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT) + } + try: + yield process + finally: + terminate(process) + for sig, handler in previous.items(): + signal.signal(sig, handler) + output.seek(0) + text = redact(output.read()) + log.write_text(text, encoding="utf-8") + print(text, end="") + + +def run(command: list[str], log: Path, timeout: float = 600) -> int: + try: + with child(command, log) as process: + try: + return process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + return 124 + except FileNotFoundError as error: + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(redact(str(error)), encoding="utf-8") + return 127 diff --git a/tools/verification_wheel.py b/tools/verification_wheel.py new file mode 100644 index 00000000..52d8b2bc --- /dev/null +++ b/tools/verification_wheel.py @@ -0,0 +1,62 @@ +"""Offline wheel-install smoke outside the checkout, using installed dependencies.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +import sysconfig +import tempfile + +from tools.verification_process import run + + +def main() -> int: + directory = Path("test-results/verify/logs") + code = run([sys.executable, "-m", "build", "--no-isolation"], directory / "build.log") + if code: + return code + wheels = list(Path("dist").glob("*.whl")) + if len(wheels) != 1: + print("Expected exactly one wheel; clean stale dist artifacts before verifying") + return 1 + with tempfile.TemporaryDirectory(prefix="openapi-wheel-") as temporary: + root = Path(temporary) + subprocess.run( + [sys.executable, "-m", "venv", "--system-site-packages", str(root / "venv")], check=True + ) + python = str(root / "venv/bin/python") + packages = ( + root + / "venv/lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" + ) + (packages / "verification-dependencies.pth").write_text( + sysconfig.get_path("purelib") + "\n" + ) + code = run( + [python, "-m", "pip", "install", "--no-deps", "--no-index", str(wheels[0].resolve())], + directory / "wheel-install.log", + ) + if code: + return code + script = ( + "from pathlib import Path; import azure.functions as f; " + "import azure_functions_openapi as a; " + f"assert Path(a.__file__).resolve().is_relative_to(Path({str(root)!r})); " + "app=f.FunctionApp(); " + "handler=lambda req: f.HttpResponse('ok'); " + "app.route(route='wheel-smoke', methods=['GET'])(" + "a.openapi(summary='Wheel smoke',responses={200:{'description':'OK'}})(handler)); " + "spec=a.generate_openapi_spec(app=app); assert '/api/wheel-smoke' in spec['paths']; " + "print('Installed wheel generates the documented route outside checkout')" + ) + result = subprocess.run([python, "-c", script], cwd=root, capture_output=True, text=True) + (directory / "wheel-smoke.log").write_text(result.stdout + result.stderr) + print(result.stdout + result.stderr) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) From 31aee7a4ee8434b44e51d390d995b09763720f06 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:42 +0900 Subject: [PATCH 03/17] ci: standardize canonical development and verification commands --- .gitignore | 1 + Makefile | 136 +++++++++++++++++++++---------------------------- pyproject.toml | 4 +- 3 files changed, 62 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 80c3d126..21ef48ee 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ env/ .coverage coverage.xml htmlcov/ +test-results/ # Build artifacts build/ diff --git a/Makefile b/Makefile index 115203b1..2250bc04 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,12 @@ VENV_DIR := .venv -PYTHON := $(VENV_DIR)/bin/python +PYTHON ?= python PIP := $(VENV_DIR)/bin/pip -HATCH := $(VENV_DIR)/bin/hatch +HATCH ?= hatch +ifeq ($(CI),) +RUN := $(HATCH) run python +else +RUN := $(PYTHON) +endif SPEC_PREVIEW_PNG := docs/assets/webhook_receiver_openapi_spec_preview.png PLAYWRIGHT_VERSION := 1.54.1 PLAYWRIGHT_BROWSERS_PATH := $(CURDIR)/.cache/ms-playwright @@ -13,17 +18,11 @@ EXAMPLES_PREVIEW_PORT := 8130 .PHONY: bootstrap bootstrap: - @if [ ! -d "$(VENV_DIR)" ]; then \ - echo "Creating virtual environment..."; \ - python3 -m venv $(VENV_DIR); \ - fi - @echo "Ensuring Hatch is installed in virtual environment..." - @$(PIP) install --upgrade pip > /dev/null - @$(PIP) install hatch > /dev/null - @echo "Hatch installed at $(HATCH)" + @command -v $(HATCH) >/dev/null || { echo "Install Hatch in an isolated tool environment first"; exit 1; } .PHONY: ensure-hatch -ensure-hatch: bootstrap +ensure-hatch: + @command -v $(HATCH) >/dev/null || { echo "Hatch is required for focused Hatch targets"; exit 1; } .PHONY: install install: ensure-hatch @@ -34,6 +33,11 @@ install: ensure-hatch $(MAKE) precommit-install; \ fi +.PHONY: install-ci +install-ci: + @$(PYTHON) -c 'import sys; assert sys.version_info >= (3, 11), sys.version' + @$(PYTHON) -m pip install -e ".[dev,docs]" + .PHONY: shell shell: ensure-hatch @$(HATCH) shell @@ -47,48 +51,49 @@ hatch-clean: ensure-hatch @$(HATCH) env remove || echo "No hatch environment to remove" .PHONY: format -format: ensure-hatch - @$(HATCH) run format +format: + @$(RUN) -m ruff format src tests tools/verification*.py .PHONY: format-check -format-check: ensure-hatch - @$(HATCH) run format-check +format-check: + @$(RUN) -m ruff format --check src tests tools/verification*.py .PHONY: style style: ensure-hatch @$(HATCH) run style .PHONY: typecheck -typecheck: ensure-hatch - @$(HATCH) run typecheck +typecheck: + @$(RUN) -m mypy src tests tools/verification*.py .PHONY: lint -lint: ensure-hatch - @$(HATCH) run lint +lint: style typecheck .PHONY: security -security: ensure-hatch - @$(HATCH) run security +security: + @$(RUN) -m bandit -r src .PHONY: check -check: ensure-hatch - @$(MAKE) lint - @$(MAKE) typecheck - @echo "Lint and type check passed." +check: format-check typecheck security lint-workflows + @$(RUN) -m ruff check src tests tools/verification*.py + @$(RUN) scripts/lint_mermaid.py + @$(RUN) scripts/check_screenshots.py --strict .PHONY: lint-workflows -lint-workflows: ensure-hatch - @$(HATCH) run python tools/lint_release_workflows.py - @$(HATCH) run python tools/lint_workflow_pins.py - @$(HATCH) run python tools/lint_hatch_matrix.py +lint-workflows: + @$(RUN) tools/lint_release_workflows.py + @$(RUN) tools/lint_workflow_pins.py + @$(RUN) tools/lint_hatch_matrix.py .PHONY: check-all -check-all: ensure-hatch - @$(MAKE) lint-workflows - @$(MAKE) check - @$(MAKE) test - @$(MAKE) security - @echo "All checks passed including tests and security scan." +check-all: verify + +.PHONY: integration verify report wheel-smoke +integration verify report: + @$(RUN) -m tools.verification $@ + +wheel-smoke: + @$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel .PHONY: precommit precommit: ensure-hatch @@ -99,9 +104,8 @@ precommit-install: ensure-hatch @$(HATCH) run precommit-install .PHONY: test -test: ensure-hatch - @echo "Running tests..." - @$(HATCH) run test +test: + @$(RUN) -m tools.verification test .PHONY: cov cov: ensure-hatch @@ -110,31 +114,20 @@ cov: ensure-hatch @echo "coverage.xml generated for Codecov upload." .PHONY: perf -perf: ensure-hatch - @$(HATCH) run python scripts/benchmark_spec_generation.py --check +perf: + @$(RUN) -m tools.verification focused perf scripts/benchmark_spec_generation.py --check .PHONY: e2e-local -e2e-local: ensure-hatch - @E2E_EXPECTED_VERSION=$${E2E_EXPECTED_VERSION:-$$($(HATCH) version)}; \ - echo "Running e2e tests against local Azurite (E2E_BASE_URL=http://localhost:7071, E2E_EXPECTED_VERSION=$$E2E_EXPECTED_VERSION)..."; \ - E2E_BASE_URL=http://localhost:7071 E2E_EXPECTED_VERSION="$$E2E_EXPECTED_VERSION" $(HATCH) run e2e-azure +e2e-local: + @E2E_BASE_URL=http://localhost:7071 E2E_EXPECTED_VERSION=$${E2E_EXPECTED_VERSION:-$$($(HATCH) version)} $(RUN) -m tools.verification local-e2e .PHONY: e2e-azure -e2e-azure: ensure-hatch - @if [ -z "$$E2E_BASE_URL" ]; then \ - echo "Error: E2E_BASE_URL is required, e.g.:"; \ - echo " E2E_BASE_URL=https://.azurewebsites.net E2E_EXPECTED_VERSION= make e2e-azure"; \ - echo "Running this target with E2E_BASE_URL unset would collect the e2e"; \ - echo "tests but skip every one of them, exiting 0 -- an unintentionally"; \ - echo "successful-looking run that certified nothing."; \ - exit 1; \ - fi - @echo "Running e2e tests against Azure host $$E2E_BASE_URL (E2E_EXPECTED_VERSION=$${E2E_EXPECTED_VERSION:-})..." - @$(HATCH) run e2e-azure +e2e-azure: + @$(RUN) -m tools.verification azure-e2e .PHONY: build -build: ensure-hatch - @$(HATCH) build +build: + @$(RUN) -m tools.verification focused build -m build --no-isolation .PHONY: publish-test publish-test: ensure-hatch @@ -147,15 +140,7 @@ version: ensure-hatch .PHONY: docs docs: - @if [ -n "$$CI" ]; then \ - echo "CI detected: running MkDocs directly"; \ - python -m pip install --upgrade pip > /dev/null 2>&1 || true; \ - pip install "mkdocs<2.0" "mkdocs-material<10.0" "mkdocstrings[python]<1.0" > /dev/null 2>&1; \ - mkdocs build; \ - else \ - $(MAKE) ensure-hatch > /dev/null; \ - $(HATCH) run mkdocs build; \ - fi + @$(RUN) -m tools.verification focused docs -m mkdocs build --strict .PHONY: docs-serve docs-serve: ensure-hatch @@ -256,23 +241,20 @@ demo-examples: ensure-hatch .PHONY: doctor doctor: - @echo "Python version:" && $(PYTHON) --version - @echo "Installed packages:" && $(HATCH) env run pip list || echo "No hatch environment found" - @echo "Azure Function Core Tools version:" && func --version || echo "func not found" - @echo "Pre-commit hook installed:" - @if [ -f .git/hooks/pre-commit ]; then echo yes; else echo no; fi + @$(RUN) -m tools.verification doctor .PHONY: clean clean: - @rm -rf *.egg-info dist build __pycache__ .pytest_cache + @$(RUN) -m tools.verification clean .PHONY: clean-all clean-all: clean - @find . -type d -name "__pycache__" -exec rm -rf {} + - @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -delete - @rm -rf .mypy_cache .ruff_cache .pytest_cache .coverage coverage.xml htmlcov .DS_Store $(VENV_DIR) site .PHONY: help help: - @echo "Available commands:" && \ - grep -E '^\.PHONY: ' Makefile | cut -d ':' -f2 | xargs -n1 echo " - make" + @echo "Canonical: install install-ci doctor format format-check check test integration verify e2e-azure report clean" + @echo "check-all: backward-compatible alias for verify (full local gate, requires func + Azurite)" + @echo "Partial/focused: style lint typecheck security lint-workflows cov perf build wheel-smoke docs docs-serve" + @echo "Local-emulator only, not Azure certification: e2e-local" + @echo "Partial/demo (not test prerequisites): demo demo-swagger demo-examples" + @echo "Environment/maintenance: bootstrap ensure-hatch shell reset hatch-clean clean-all precommit precommit-install publish-test version" diff --git a/pyproject.toml b/pyproject.toml index 374de973..a13814c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,9 +104,9 @@ features = ["dev", "docs"] format = "ruff format src tests" format-check = "ruff format --check src tests" style = "ruff check src tests && ruff format --check src tests" -typecheck = "mypy src tests" +typecheck = "mypy src tests tools/verification*.py" lint = "hatch run style && hatch run typecheck" -test = "pytest -v tests" +test = "python -m tools.verification test" cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/" e2e-azure = "python scripts/require_e2e_env.py && pytest -v -m e2e --no-cov tests/e2e/" docs = "mkdocs serve" From f9452d1f1a4195b2c464cb201177bc5d0d2bf78f Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:56 +0900 Subject: [PATCH 04/17] ci: share verification runners across interpreter and host lanes --- .github/workflows/ci-test.yml | 162 +++++++++++----------------- tests/test_ci_workflow_hardening.py | 20 ++++ 2 files changed, 80 insertions(+), 102 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 68aa942b..a87ce1a0 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -92,20 +92,10 @@ jobs: cache-dependency-path: pyproject.toml - name: Install package with dev extras - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + run: make install-ci - name: Lint workflows (pin hygiene + release-gate canonical SHAs) - run: | - python tools/lint_release_workflows.py - python tools/lint_workflow_pins.py - python tools/lint_hatch_matrix.py - - - name: Ruff style check - run: | - ruff check src tests - ruff format --check src tests + run: make check - name: Ruff format check (changed Python files) if: github.event_name == 'pull_request' @@ -114,8 +104,12 @@ jobs: PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: python tools/check_pr_format.py "$PR_BASE_SHA" "$PR_HEAD_SHA" - - name: Type check (mypy, strict) - run: mypy src tests + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: quality-evidence + path: test-results/** test: name: Test (Py ${{ matrix.python-version }}, af ${{ matrix.azure-functions-version }}) @@ -169,9 +163,7 @@ jobs: PY - name: Install package with dev extras (on the matrix interpreter) - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + run: make install-ci - name: Install azure-functions ${{ matrix.azure-functions-version }} run: | @@ -206,26 +198,33 @@ jobs: IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' && matrix.azure-functions-version == 'latest' }} run: | if [ "$IS_COVERAGE_CELL" = "true" ]; then - pytest tests/ + python -m tools.verification test else - pytest tests/ --no-cov + python -m tools.verification test --no-cov fi - name: Verify coverage output if: matrix.python-version == '3.11' && matrix.azure-functions-version == 'latest' - run: ls -lh coverage.xml + run: ls -lh test-results/test/coverage.xml - name: Upload coverage to Codecov if: matrix.python-version == '3.11' && matrix.azure-functions-version == 'latest' && github.actor != 'dependabot[bot]' uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml + files: ./test-results/test/coverage.xml verbose: true fail_ci_if_error: false env: CODECOV_RETRIES: 3 + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: test-evidence-py${{ matrix.python-version }}-af${{ matrix.azure-functions-version }} + path: test-results/** + docs-check: name: Documentation checks runs-on: ubuntu-latest @@ -255,7 +254,14 @@ jobs: run: pytest tests/test_screenshot_manifest.py --no-cov - name: Build docs (strict) - run: mkdocs build --strict + run: make docs + + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: docs-evidence + path: test-results/** # Advisory regression lane: visible on PRs/main but intentionally omitted # from ci-required so noisy shared-runner timing cannot block a merge. @@ -281,6 +287,13 @@ jobs: - name: Check spec-generation budgets run: make perf + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: perf-evidence + path: test-results/** + # The single required status check for branch protection. # # Branch protection matches required checks by literal name, so requiring @@ -390,7 +403,14 @@ jobs: PY - name: Run full suite against minimum runtime dependencies - run: pytest tests/ --no-cov + run: python -m tools.verification test --no-cov + + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: minimum-dependencies-evidence + path: test-results/** artifact-build: name: Build compatibility artifacts @@ -540,87 +560,18 @@ jobs: - name: Install Functions Core Tools v4 and smoke app run: | - npm install --global azure-functions-core-tools@4 --unsafe-perm true - python -m venv tests/host_smoke/.venv - tests/host_smoke/.venv/bin/python -m pip install --upgrade pip - cd tests/host_smoke - .venv/bin/python -m pip install -r requirements.txt + npm install --global azure-functions-core-tools@4 azurite --unsafe-perm true + make install-ci - name: Exercise OpenAPI routes on a real Functions host - run: | - npx --yes azurite --silent > "$RUNNER_TEMP/azurite.log" 2>&1 & - azurite_pid=$! - trap 'kill "$azurite_pid" "$host_pid" 2>/dev/null || true' EXIT - - cd tests/host_smoke - export PATH="$PWD/.venv/bin:$PATH" - timeout 120s func start --verbose > "$RUNNER_TEMP/host-smoke.log" 2>&1 & - host_pid=$! - - ready=false - for _ in {1..60}; do - if curl --fail --silent http://127.0.0.1:7071/api/custom_name \ - --output "$RUNNER_TEMP/custom-route.json"; then - ready=true - break - fi - if ! kill -0 "$host_pid" 2>/dev/null; then - break - fi - sleep 2 - done - - if [ "$ready" != true ]; then - cat "$RUNNER_TEMP/host-smoke.log" - echo "::error::The custom-name route did not become ready" - exit 1 - fi - - python -c 'import json, pathlib; assert json.loads(pathlib.Path("'"$RUNNER_TEMP"'/custom-route.json").read_text()) == {"function_name": "custom_name"}' - curl --fail --silent http://127.0.0.1:7071/api/openapi.json \ - --output "$RUNNER_TEMP/openapi.json" - python - <<'PY' - import json - import os - import pathlib - import urllib.request - - spec = json.loads((pathlib.Path(os.environ["RUNNER_TEMP"]) / "openapi.json").read_text()) - assert spec["openapi"], "spec is missing the 'openapi' version key" - paths = spec["paths"] - - # #786: the fixture previously documented the pre-function_name route - # ('/api/python_handler_name') while the host actually served - # '/api/custom_name', and the host-smoke job passed anyway because it - # only checked a hard-coded URL, never the generated document itself. - assert "/api/python_handler_name" not in paths, ( - f"spec still documents the stale pre-function_name route; paths={sorted(paths)}" - ) - app_paths = [p for p in paths if p not in ("/api/openapi.json", "/api/docs")] - assert app_paths == ["/api/custom_name"], ( - f"expected the documented app route to be exactly ['/api/custom_name'], got {app_paths}" - ) - - # Exercise the URL taken FROM the generated document -- not a second - # hard-coded '/api/custom_name' literal -- so a documentation/runtime - # mismatch cannot hide behind two copies of the same string. - documented_path = app_paths[0] - with urllib.request.urlopen(f"http://127.0.0.1:7071{documented_path}") as resp: - body = json.loads(resp.read()) - assert body == {"function_name": "custom_name"}, body - PY - curl --fail --silent http://127.0.0.1:7071/api/docs \ - --output "$RUNNER_TEMP/swagger.html" - grep -q 'SwaggerUIBundle' "$RUNNER_TEMP/swagger.html" + run: python -m tools.verification integration - kill "$host_pid" 2>/dev/null || true - wait "$host_pid" 2>/dev/null || true - cat "$RUNNER_TEMP/host-smoke.log" - - if grep -Eq "0 functions (loaded|found \(Worker\))|No job functions found|Executed 'Functions\..*' \(Failed" "$RUNNER_TEMP/host-smoke.log"; then - echo "::error::The Functions host reported indexing or invocation errors" - exit 1 - fi + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: host-evidence-py${{ matrix.python-version }} + path: test-results/** # azure-functions 2.x compatibility proof lane (issues #488 / #528), # following the sibling proof-lane pattern (azure-functions-validation #351 / @@ -697,4 +648,11 @@ jobs: # `FunctionBuilder._function` private-attribute tripwires also run, # proving the adapter surface holds under azure-functions 2.x. run: | - pytest tests/ --no-cov -q -o "pythonpath=." + python -m tools.verification test --no-cov -q -o "pythonpath=." + + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: wheel-2x-evidence-py${{ matrix.python-version }} + path: test-results/** diff --git a/tests/test_ci_workflow_hardening.py b/tests/test_ci_workflow_hardening.py index c8e1d277..db2b0518 100644 --- a/tests/test_ci_workflow_hardening.py +++ b/tests/test_ci_workflow_hardening.py @@ -92,3 +92,23 @@ def test_gate_accepts_a_successful_unknown_dependency() -> None: jobs["future-job"] = {"result": "success", "outputs": {}} assert run_gate(jobs).returncode == 0 + + +@pytest.mark.parametrize( + "job", ["test", "minimum-dependencies", "azure-functions-2x", "host-smoke"] +) +def test_execution_lanes_share_runner_and_always_upload_evidence(job: str) -> None: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = workflow["jobs"][job]["steps"] + commands = "\n".join(step.get("run", "") for step in steps) + assert "python -m tools.verification " in commands + assert "hatch run" not in commands + uploads = [step for step in steps if step.get("with", {}).get("path") == "test-results/**"] + assert uploads + assert all(step.get("if") == "always()" for step in uploads) + + +def test_quality_uses_complete_canonical_gate() -> None: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + steps = workflow["jobs"]["quality"]["steps"] + assert any(step.get("run") == "make check" for step in steps) From fa0920290613ae55910dcc12cf173d7e35e14a7f Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:56 +0900 Subject: [PATCH 05/17] ci: retain standardized evidence through release certification gates --- .github/workflows/e2e-azure.yml | 9 +++++---- .github/workflows/publish-pypi.yml | 13 ++++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-azure.yml b/.github/workflows/e2e-azure.yml index d20b4224..ab7044bf 100644 --- a/.github/workflows/e2e-azure.yml +++ b/.github/workflows/e2e-azure.yml @@ -194,8 +194,7 @@ jobs: - name: Run e2e tests run: | - pytest tests/e2e -v -m e2e --no-cov \ - --junitxml=e2e-report/junit.xml \ + python -m tools.verification azure-e2e -v \ --html=e2e-report/report.html --self-contained-html env: E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net @@ -211,7 +210,7 @@ jobs: import sys import xml.etree.ElementTree as ET - root = ET.parse("e2e-report/junit.xml").getroot() + root = ET.parse("test-results/azure-e2e/junit.xml").getroot() suite = root if root.tag == "testsuite" else root.find("testsuite") tests = int(suite.get("tests", 0)) skipped = int(suite.get("skipped", 0)) @@ -304,7 +303,9 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }} - path: e2e-report/ + path: | + e2e-report/ + test-results/** retention-days: 30 cleanup: diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index a0465e63..5c9b91db 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -103,11 +103,18 @@ jobs: with: python-version: "3.11" - - name: Install Hatch - run: pip install hatch + - name: Install test dependencies on the selected interpreter + run: make install-ci - name: Run test suite - run: hatch run test + run: python -m tools.verification test + + - name: Upload verification evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-library-evidence + path: test-results/** cookbook-smoke: name: Cookbook dogfood smoke (candidate wheel) From 4fc1b3baaa0a27fc9f14031f615537b31caef401 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:56 +0900 Subject: [PATCH 06/17] test: type optional validation decorators without stale suppressions --- .../test_validate_http_shorthand_conflict.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/test_validate_http_shorthand_conflict.py b/tests/test_validate_http_shorthand_conflict.py index 44827b8e..3b7795a2 100644 --- a/tests/test_validate_http_shorthand_conflict.py +++ b/tests/test_validate_http_shorthand_conflict.py @@ -14,6 +14,7 @@ from __future__ import annotations +from collections.abc import Callable import importlib from typing import Any @@ -34,6 +35,14 @@ except ImportError: HAS_VALIDATION = False + +def _validator() -> Callable[..., Callable[[Callable[..., Any]], Callable[..., Any]]]: + decorator: Callable[..., Callable[[Callable[..., Any]], Callable[..., Any]]] = ( + importlib.import_module("azure_functions_validation").validate_http + ) + return decorator + + pytestmark = pytest.mark.skipif( not HAS_VALIDATION, reason="azure-functions-validation not installed", @@ -63,7 +72,7 @@ class _OtherValidated(BaseModel): "openapi_above_route", [True, False], ids=["openapi-above", "openapi-below"] ) def test_conflicting_request_model_shorthand_raises(openapi_above_route: bool) -> None: - from azure_functions_validation import validate_http + validate_http = _validator() app = func.FunctionApp() @@ -71,14 +80,14 @@ def test_conflicting_request_model_shorthand_raises(openapi_above_route: bool) - @openapi(requests=_Authored) @app.route(route="orders", methods=["POST"]) - @validate_http(body=_Validated) # type: ignore[untyped-decorator] + @validate_http(body=_Validated) def orders(req: Any, body: Any) -> Any: return body else: @app.route(route="orders", methods=["POST"]) @openapi(requests=_Authored) - @validate_http(body=_Validated) # type: ignore[untyped-decorator] + @validate_http(body=_Validated) def orders(req: Any, body: Any) -> Any: # noqa: F811 return body @@ -87,13 +96,13 @@ def orders(req: Any, body: Any) -> Any: # noqa: F811 def test_conflicting_response_model_shorthand_raises() -> None: - from azure_functions_validation import validate_http + validate_http = _validator() app = func.FunctionApp() @openapi(responses=_Authored) @app.route(route="orders", methods=["GET"]) - @validate_http(response_model=_Validated) # type: ignore[untyped-decorator] + @validate_http(response_model=_Validated) def orders(req: Any) -> Any: ... with pytest.raises(OpenAPISpecConfigError, match="Conflicting status 200"): @@ -101,13 +110,13 @@ def orders(req: Any) -> Any: ... def test_request_conflict_detected_even_when_response_matches() -> None: - from azure_functions_validation import validate_http + validate_http = _validator() app = func.FunctionApp() @openapi(requests=_Authored, responses=_Validated) @app.route(route="orders", methods=["POST"]) - @validate_http(body=_Validated, response_model=_Validated) # type: ignore[untyped-decorator] + @validate_http(body=_Validated, response_model=_Validated) def orders(req: Any, body: Any) -> Any: return body @@ -116,13 +125,13 @@ def orders(req: Any, body: Any) -> Any: def test_response_conflict_detected_even_when_request_matches() -> None: - from azure_functions_validation import validate_http + validate_http = _validator() app = func.FunctionApp() @openapi(requests=_Validated, responses=_Authored) @app.route(route="orders", methods=["POST"]) - @validate_http(body=_Validated, response_model=_OtherValidated) # type: ignore[untyped-decorator] + @validate_http(body=_Validated, response_model=_OtherValidated) def orders(req: Any, body: Any) -> Any: return body @@ -131,13 +140,13 @@ def orders(req: Any, body: Any) -> Any: def test_equivalent_model_declared_through_both_decorators_merges() -> None: - from azure_functions_validation import validate_http + validate_http = _validator() app = func.FunctionApp() @openapi(requests=_Validated, responses=_Validated) @app.route(route="orders", methods=["POST"]) - @validate_http(body=_Validated, response_model=_Validated) # type: ignore[untyped-decorator] + @validate_http(body=_Validated, response_model=_Validated) def orders(req: Any, body: Any) -> Any: return body @@ -150,13 +159,13 @@ def orders(req: Any, body: Any) -> Any: def test_inferred_response_is_still_superseded_by_validation_metadata() -> None: - from azure_functions_validation import validate_http + validate_http = _validator() app = func.FunctionApp() @openapi() @app.route(route="orders", methods=["GET"]) - @validate_http(response_model=_Validated) # type: ignore[untyped-decorator] + @validate_http(response_model=_Validated) def orders(req: Any) -> _Authored: raise NotImplementedError From df7a319d5355cd658a172b2dcd8d64c996dfcf45 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:57 +0900 Subject: [PATCH 07/17] docs: define canonical verification commands and evidence meanings --- AGENTS.md | 29 ++++++++++-------- CONTRIBUTING.md | 80 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index efe75f68..03cbe16e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -190,10 +190,11 @@ This repository is **issue-based, not milestone-based**. Track and group work us - Before adding any organizational structure, check whether the repository already has an established convention. A category being empty or unused (zero milestones, no `epic:*` labels) is evidence to follow the existing pattern, not to introduce a new one. ## Validation -- `make test` -- `make lint` -- `make typecheck` -- `make build` +- `make doctor` +- `make check` +- `make test` (offline, 95% coverage) +- `make integration` (requires func + Azurite) +- `make verify` (`make check-all` is an exact backward-compatible alias) ## Release Process @@ -275,15 +276,19 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor | Purpose | Command | | --- | --- | | Environment setup | `make install` | +| CI-selected interpreter setup | `make install-ci` | +| Read-only prerequisite diagnostics | `make doctor` | | Format code | `make format` | | Check formatting (`src`, `tests`) | `make format-check` | -| Lint | `make lint` | -| Type check | `make typecheck` | -| Tests | `make test` | -| Coverage | `make cov` | -| Full validation | `make check-all` | -| Docs build | `make docs` | -| Package build | `make build` | +| Complete non-mutating quality gate | `make check` | +| Offline tests with 95% coverage | `make test` | +| Managed local host/emulator tests | `make integration` | +| Full local validation | `make verify` (`make check-all` alias) | +| Configured Azure candidate certification | `make e2e-azure` | +| Render saved evidence | `make report` | +| Generated artifact cleanup only | `make clean` | +| Focused lint/types/coverage | `make lint`, `make typecheck`, `make cov` | +| Focused docs/package/performance | `make docs`, `make build`, `make perf` | ## Compatibility Rules @@ -297,7 +302,7 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor - Public APIs require tests. - Bug fixes require regression tests. - Representative and complex examples must remain smoke-tested. -- `make check-all` is the minimum merge gate. +- `make verify` is the minimum local merge gate; `make check-all` aliases it exactly. ## Commit Rules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a2c3701..0a4d97d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,17 +43,19 @@ your machine's default interpreter is newer. From the repository root, run: make install ``` -This creates the repository's `.venv` when needed, installs Hatch, creates the -Hatch environment, and installs the development dependencies. No Azure account -or deployed Function App is required for the standard local checks. +Install Hatch once in an isolated tool environment before running this command. +It creates the Hatch environment and installs development/docs dependencies and +local pre-commit hooks; it never upgrades pip or modifies the global interpreter. +Routine checks do not bootstrap tools. No Azure account is needed locally. +CI uses `make install-ci PYTHON=python` to install into the interpreter selected by +`actions/setup-python`, never the Python 3.11 Hatch environment. Run the same checks used by the project locally: ```bash make test -make lint -make typecheck -make build +make check +make verify ``` The default test command excludes real-Azure end-to-end tests. Those tests need @@ -61,19 +63,67 @@ an `E2E_BASE_URL`; use `make e2e-azure` only when that environment is available. ## Project Commands -```bash -make format # Format code with ruff -make lint # Lint with ruff -make typecheck # Type check with mypy -make test # Run tests -make cov # Run tests with coverage -make perf # Check spec-generation performance budgets -make check-all # Run the full local gate +| Command | Contract | +| --- | --- | +| `make install` | Isolated Hatch development environment and local hooks | +| `make install-ci` | Install extras into CI's explicitly selected `PYTHON` | +| `make doctor` | Read-only prerequisites; non-zero when required tools are missing | +| `make format` | Mutating Ruff formatting | +| `make format-check` | Non-mutating formatting check | +| `make check` | Ruff lint/format, strict mypy once, Bandit, workflow and repository policy checks | +| `make test` | Offline suite, `not e2e and not perf`, existing 95% coverage floor | +| `make integration` | Real Functions host plus isolated Azurite, bounded readiness and cleanup | +| `make verify` | Always check + test + integration + perf + wheel build/install smoke + strict docs | +| `make e2e-azure` | Existing deployment only; requires `E2E_BASE_URL` and `E2E_EXPECTED_VERSION` | +| `make report` | Render saved evidence without executing any lane | +| `make clean` | Remove generated evidence/build/cache artifacts; preserve `.venv` and shared data | + +`make check-all` is a backward-compatible alias for **exactly `make verify`**. +Full local verification requires Core Tools (`func`) and Azurite on PATH. Doctor +lists these as required-for-integration/verify, not requirements for offline tests. +Integration uses temporary storage and ephemeral ports; it never uses shared emulator +volumes or provisions Azure resources. Core Tools may need its extension bundle cached. +The import-only regression tests in `tests/host_smoke/test_fixture_routes.py` remain +offline; only `make integration` starts the live host. + +Focused/partial targets remain available: `lint`, `style`, `typecheck`, `security`, +`lint-workflows`, `cov` (HTML coverage), `perf`, `build`, `wheel-smoke`, `docs`, and +`demo*`. Demo/screenshots are never offline-test prerequisites. `e2e-local` targets +an already-running local emulator app on port 7071 and is **not Azure certification**. + +### Verification evidence + +Generated, gitignored evidence lives under: + +```text +test-results/ + test/{junit.xml,coverage.xml,summary.json,environment.json,logs/} + integration/{summary.json,environment.json,junit/junit.xml,logs/} + verify/{summary.json,environment.json,lanes.json,logs/} + azure-e2e/{summary.json,environment.json,junit.xml,logs/} ``` +Every summary records schema version, lane/command, selected/passed/failed/skipped +counts, status, exit code and supporting paths. Console and Actions Job Summary +render the same saved fields. CI uploads evidence with `if: always()`. +Environment metadata records the actual Python/runtime and installed SDK/package +versions, plus the explicitly configured Azure candidate version. Credentials and +connection strings are redacted from saved summaries, XML and logs. + +- **PASS**: required checks succeeded and tests actually executed. +- **FAIL**: process/check failure, test failure or timeout. +- **INCOMPLETE**: missing prerequisites, missing/invalid JUnit, zero execution, + all skipped, or any skipped required live test. Optional offline skips remain visible. +- **NOT_APPLICABLE**: reserved for explicitly unsupported lanes, never a fallback + for missing tools. No required local lane is currently unsupported. + +`verify/lanes.json` retains separate host, perf and wheel outcomes. Local success +does not imply Azure certification: only the configured Azure suite proves the +deployed candidate version. A non-successful required lane exits non-zero. + Before opening a PR, run `ruff format --check ` (or `ruff format ` to fix them). `make format-check` checks -`src` and `tests` without changing files. CI checks those same trees and also +`src`, `tests`, and the verification helpers without changing files. CI checks those trees and also reports formatting failures on changed Python files in a dedicated step. ## GitHub Actions Pinning From d6668fa1f06026cbd73799d90622a2ff75d455f7 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:06:57 +0900 Subject: [PATCH 08/17] docs: align developer command tables with full local verification --- docs/contributing.md | 15 +++++++++++---- docs/development.md | 33 +++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 4306e181..03a7a151 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -53,7 +53,8 @@ git checkout -b feat/add-cookie-parameter-support make check-all ``` -This runs linting (Ruff), type checking (mypy), and the full test suite. +This aliases `make verify`: quality + offline tests + managed host/Azurite + perf + +wheel build/install smoke + strict docs. Core Tools and Azurite are required. 4. **Commit changes** using [Conventional Commits](https://www.conventionalcommits.org/) format: @@ -120,14 +121,20 @@ Describe what changed and why. Reference related issues with `Fixes #N` or `Clos | `make lint` | Run Ruff + mypy | | `make typecheck` | Run mypy type checking | | `make security` | Run Bandit security scan | -| `make test` | Run pytest | +| `make test` | Offline pytest, 95% coverage and saved JUnit/coverage/summary | | `make cov` | Run tests with coverage (gated at 95%) | | `make lint-workflows` | Lint release workflows, action pins, Hatch matrix | -| `make check` | Run lint + typecheck | -| `make check-all` | Run lint-workflows + check + test + security (full gate) | +| `make check` | Non-mutating lint, format, types once, security and policy checks | +| `make integration` | Managed real Functions host and isolated Azurite | +| `make verify` | Full deterministic local gate | +| `make check-all` | Exact backward-compatible alias of `verify` | +| `make report` | Render saved evidence only | +| `make clean` | Generated artifacts only; preserve environments/shared data | | `make docs` | Build the MkDocs site | See [Development Guide](development.md#makefile-targets) for the complete list. +`style`, `lint`, `typecheck`, `security`, `cov`, `lint-workflows` and `docs` are +focused/partial checks, not substitutes for `verify`. ### Pre-commit hooks diff --git a/docs/development.md b/docs/development.md index 074d2e02..2368bea5 100644 --- a/docs/development.md +++ b/docs/development.md @@ -136,13 +136,17 @@ Use these as the **golden commands** for local validation and CI parity. Prefer | `make lint` | Run linter (ruff + mypy) | | `make typecheck` | Run mypy type checking | | `make security` | Run Bandit security scan | -| `make test` | Run pytest | +| `make test` | Offline pytest with 95% coverage, saved JUnit/coverage/summary | | `make cov` | Run tests with coverage (gate: `fail_under = 95`) | | `make e2e-local` | Run e2e tests against local Azurite on `:7071` | | `make e2e-azure` | Run e2e tests against Azure (`E2E_BASE_URL` required) | | `make lint-workflows` | Lint release workflows, action pins, and the Hatch matrix | -| `make check` | Run lint + typecheck | -| `make check-all` | Run lint-workflows + check + test + security | +| `make install-ci` | Install extras on CI's explicitly selected interpreter | +| `make check` | Non-mutating lint, format, types once, Bandit and policy checks | +| `make integration` | Real Functions host + isolated Azurite, managed readiness/cleanup | +| `make verify` | Check + test + integration + perf + wheel smoke + strict docs | +| `make check-all` | Exact backward-compatible alias for `verify` | +| `make report` | Render last saved evidence without rerunning tests | | `make docs` | Build the MkDocs site | | `make docs-serve` | Serve the docs locally with live reload | | `make demo` | Run the Swagger UI and example demos | @@ -155,12 +159,33 @@ Use these as the **golden commands** for local validation and CI parity. Prefer | `make reset` | Deep clean, then reinstall the env | | `make hatch-clean` | Remove the Hatch environment | | `make clean` | Remove build artifacts | -| `make clean-all` | Deep clean (caches, coverage, venv) | +| `make clean-all` | Alias of artifact-only `clean`; never deletes `.venv` | | `make help` | List available targets | Coverage is enforced at **95%** (`fail_under = 95` in `pyproject.toml`); a drop below that fails the build. +`make doctor` is read-only and fails for missing required tools. It reports the +actual interpreter; func/Azurite are required only for integration/verify. +Local commands use Hatch's Python 3.11; CI uses its selected `PYTHON` directly. +Install Hatch once in an isolated tool environment; routine commands never upgrade +pip or reinstall Hatch. `install` does not change the global interpreter. + +All targets besides the canonical contract (`install`, `install-ci`, `doctor`, +`format`, `format-check`, `check`, `test`, `integration`, `verify`, `e2e-azure`, +`report`, `clean`) are focused/partial or maintenance commands. Demos are optional. +`e2e-local` is a local-emulator check, never real Azure certification. + +Evidence is saved under gitignored `test-results/`: `test/` contains `junit.xml`, +`coverage.xml`, `summary.json`; `integration/` contains `junit/junit.xml`, logs and +summary; `verify/` contains aggregate summary, `lanes.json` and environment metadata; +`azure-e2e/` contains JUnit, logs and summary with the expected deployed version. +Each lane also saves `environment.json`. Console and Actions summaries use the same +JSON counts/status. PASS requires execution; zero/all-skipped/missing prerequisites +are INCOMPLETE, errors are FAIL, and NOT_APPLICABLE is reserved for unsupported lanes. +`verify` never silently omits a required lane. `e2e-azure` requires both +`E2E_BASE_URL` and `E2E_EXPECTED_VERSION` and never provisions resources. + > For the full release workflow, see [Release Process](release_process.md). --- From f0502eef0716ae0d89255a189279d40df2b56415 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:07:03 +0900 Subject: [PATCH 09/17] docs: distinguish offline host and Azure verification lanes --- docs/agent-playbook.md | 14 ++++++++++---- docs/testing.md | 26 +++++++++++++++++--------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/agent-playbook.md b/docs/agent-playbook.md index 376b1070..7a2d9fec 100644 --- a/docs/agent-playbook.md +++ b/docs/agent-playbook.md @@ -26,8 +26,14 @@ This also applies when using the API or CLI. 5. Record review evidence per [docs/review-evidence-policy.md](review-evidence-policy.md) in the PR's "Review Evidence" section before merging — advisory, not a required check. ## Validation +- `make doctor` checks tools read-only; func/Azurite are required for live local lanes. +- `make check` runs complete non-mutating lint, format, types, security and policy checks. - `make test` -- `make lint` -- `make typecheck` -- `make security` -- `make build` +- `make integration` starts the real host and an isolated emulator with cleanup. +- `make verify` is the full local gate; `make check-all` is an exact alias. +- `make e2e-azure` separately certifies an explicitly configured deployed candidate. +- `make report` renders saved `test-results/` evidence without rerunning checks. +- `make clean` removes generated artifacts, never `.venv` or shared emulator data. + +See [Development Guide](development.md#makefile-targets) for the command contract; +`lint`, `typecheck`, `security`, `build`, `docs`, `perf` and demos are focused/partial. diff --git a/docs/testing.md b/docs/testing.md index 0e2dbe73..0ec95f77 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,12 +26,18 @@ tests/ ## Running Tests -Run the full test suite: +Run the offline suite (no host/emulator/cloud), with the existing 95% floor: ```bash make test ``` +`make integration` separately owns a real Functions host and isolated Azurite. +`make verify` (also `make check-all`) always runs quality, offline tests, host smoke, +performance budgets, wheel build/install smoke and strict docs. Missing live tools +fail closed. `make report` renders saved evidence; see the +[Development Guide](development.md#makefile-targets) for layout and status meanings. + Run tests with coverage reporting: ```bash @@ -96,6 +102,7 @@ Ensure invalid inputs raise `ValueError` with descriptive messages: ```python def test_invalid_route_raises(): with pytest.raises(ValueError, match="Invalid route path"): + @openapi(summary="Bad", route="../../etc/passwd") def bad_route(req): pass @@ -138,6 +145,7 @@ class ItemResponse(BaseModel): id: int name: str + def test_response_model_schema(): @openapi( summary="Get item", @@ -164,9 +172,7 @@ def test_security_scheme_in_components(): summary="Protected", route="/api/protected", security=[{"BearerAuth": []}], - security_scheme={ - "BearerAuth": {"type": "http", "scheme": "bearer"} - }, + security_scheme={"BearerAuth": {"type": "http", "scheme": "bearer"}}, ) def protected(req): pass @@ -207,6 +213,7 @@ exists and the OpenAPI registry is populated: ```python def test_webhook_receiver_example_registers_metadata(): import importlib + mod = importlib.import_module("examples.webhook_receiver.function_app") registry = get_openapi_registry() assert len(registry) > 0 @@ -246,12 +253,13 @@ Examples: The CI workflow runs: -1. `make lint` -- Ruff linting -2. `make typecheck` -- mypy type checking -3. `make test` -- pytest with coverage -4. `make security` -- Bandit security scan +1. `make check` -- Ruff, strict mypy once, Bandit and repository policy checks +2. `python -m tools.verification test` -- the same runner as `make test`, on each matrix interpreter +3. `python -m tools.verification integration` -- the same managed real-host smoke as local +4. Existing wheel, lowest-dependency, docs, and Azure release gates remain separate -All four steps must pass for a PR to be mergeable. The matrix covers +Required jobs must pass for a PR to be mergeable. Evidence artifacts upload even on failure. +The matrix covers Python 3.11, 3.12, 3.13, and 3.14. ## Real Azure E2E Tests From 4168e558afcfd383809a8fcae2b46e5c2e7f9ec2 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Thu, 22 Oct 2026 23:09:23 +0900 Subject: [PATCH 10/17] ci: keep clean-all as the deep maintenance clean and name the quality step accurately --- .github/workflows/ci-test.yml | 2 +- Makefile | 5 +++++ docs/development.md | 2 +- tools/verification.py | 8 ++------ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index a87ce1a0..adb30f36 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -94,7 +94,7 @@ jobs: - name: Install package with dev extras run: make install-ci - - name: Lint workflows (pin hygiene + release-gate canonical SHAs) + - name: Quality gate (make check) run: make check - name: Ruff format check (changed Python files) diff --git a/Makefile b/Makefile index 2250bc04..35cf63f7 100644 --- a/Makefile +++ b/Makefile @@ -247,8 +247,13 @@ doctor: clean: @$(RUN) -m tools.verification clean +# Deep clean (maintenance, not part of the canonical contract): artifacts plus the +# project-local virtual environment. `make clean` never touches `.venv`. .PHONY: clean-all clean-all: clean + @find . -type d -name "__pycache__" -not -path "./.git/*" -exec rm -rf {} + + @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -not -path "./.git/*" -delete + @rm -rf .DS_Store $(VENV_DIR) .PHONY: help help: diff --git a/docs/development.md b/docs/development.md index 2368bea5..627125fd 100644 --- a/docs/development.md +++ b/docs/development.md @@ -159,7 +159,7 @@ Use these as the **golden commands** for local validation and CI parity. Prefer | `make reset` | Deep clean, then reinstall the env | | `make hatch-clean` | Remove the Hatch environment | | `make clean` | Remove build artifacts | -| `make clean-all` | Alias of artifact-only `clean`; never deletes `.venv` | +| `make clean-all` | `clean` plus bytecode caches and the project-local `.venv` (maintenance only) | | `make help` | List available targets | Coverage is enforced at **95%** (`fail_under = 95` in `pyproject.toml`); a drop diff --git a/tools/verification.py b/tools/verification.py index d9186b6f..8412a541 100644 --- a/tools/verification.py +++ b/tools/verification.py @@ -3,7 +3,8 @@ from __future__ import annotations import argparse -from dataclasses import replace +from dataclasses import asdict, replace +import json import os from pathlib import Path import shutil @@ -99,8 +100,6 @@ def verify() -> int: code = run(["make", target], directory / "logs" / f"{target}.log", 1200) path = Path("test-results") / target / "summary.json" if target in {"test", "integration"} and path.exists(): - import json - summary = Summary(**json.loads(path.read_text())) if code and summary.status == "PASS": summary = replace(summary, status="FAIL", exit_code=code) @@ -145,9 +144,6 @@ def verify() -> int: 0 if status == "PASS" else 1, reason="; ".join(f"{item.lane}={item.status}" for item in outcomes), ) - from dataclasses import asdict - import json - (directory / "lanes.json").write_text(json.dumps([asdict(s) for s in outcomes], indent=2)) publish(save(directory, aggregate)) return aggregate.exit_code From 6f1e8ca8163c64d618e353bb8b101294f6c388fb Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 07:10:36 +0900 Subject: [PATCH 11/17] ci: parse generated test reports without semgrep's unsafe xml parse rule --- tools/verification_evidence.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py index 75704bc6..7700f68d 100644 --- a/tools/verification_evidence.py +++ b/tools/verification_evidence.py @@ -17,6 +17,14 @@ Status = Literal["PASS", "FAIL", "INCOMPLETE", "NOT_APPLICABLE"] +def read_xml_root(path: Path) -> ET.Element: + """Parse a locally generated report, refusing entity declarations.""" + data = path.read_bytes() + if b" Summary: cases: list[ET.Element] = [] for path in directory.rglob("*.xml"): if "junit" in path.name: - cases.extend(ET.parse(path).getroot().iter("testcase")) + cases.extend(read_xml_root(path).iter("testcase")) skipped = sum(case.find("skipped") is not None for case in cases) failed = sum( case.find("failure") is not None or case.find("error") is not None for case in cases @@ -94,7 +102,7 @@ def collect(directory: Path, command: list[str], exit_code: int) -> Summary: coverage_path = directory / "coverage.xml" coverage = None if coverage_path.exists(): - coverage = float(ET.parse(coverage_path).getroot().get("line-rate", "0")) + coverage = float(read_xml_root(coverage_path).get("line-rate", "0")) return Summary( 1, directory.name, From bc9e2bd9b3eaf23bb55a91d9a0e35337378c1355 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 07:10:36 +0900 Subject: [PATCH 12/17] ci(security): install the selected interpreter's tools before the bandit job --- .github/workflows/security.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 6f76dbd3..be26acc9 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -23,10 +23,8 @@ jobs: with: python-version: "3.11" - - name: Install Hatch and dependencies via Makefile - run: | - pip install hatch - make install + - name: Install dependencies on the selected interpreter + run: make install-ci - name: Run Bandit run: make security From a94d522a6b91127b320d0c24c58718225042e8c8 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:21:09 +0900 Subject: [PATCH 13/17] ci: redact quoted and connection-string credentials without corrupting evidence --- tests/test_verification_evidence.py | 104 +++++++++++++++++++++++++++ tools/verification_evidence.py | 106 ++++++++++++++++++++++------ 2 files changed, 189 insertions(+), 21 deletions(-) diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py index 2221b1a9..9aafb395 100644 --- a/tests/test_verification_evidence.py +++ b/tests/test_verification_evidence.py @@ -100,3 +100,107 @@ def test_redaction_preserves_json_syntax(tmp_path: Path) -> None: save(tmp_path, summary) data = json.loads((tmp_path / "summary.json").read_text()) assert "VERYSECRET" not in str(data) + + +@pytest.mark.parametrize( + ("text", "leaked"), + [ + ('{"Authorization": "Bearer CANARY_a1"}', "CANARY_a1"), + ("{'Authorization': 'Bearer CANARY_a2'}", "CANARY_a2"), + ("Authorization: Bearer CANARY_a3", "CANARY_a3"), + ("Authorization: Basic CANARY_a4==", "CANARY_a4"), + ("authorization=SharedKey acct:CANARY_a5", "CANARY_a5"), + ("Proxy-Authorization: Bearer CANARY_a6", "CANARY_a6"), + ('{"password": "alpha CANARY_b1 beta"}', "CANARY_b1"), + ('{"token": "with \\"escaped\\" CANARY_b2"}', "CANARY_b2"), + ("password=CANARY_b3&x=1", "CANARY_b3"), + ( + "connection_string=Endpoint=sb://ns/;SharedAccessKeyName=Root;SharedAccessKey=CANARY_c1", + "CANARY_c1", + ), + ( + "DefaultEndpointsProtocol=https;AccountName=a;AccountKey=CANARY_c2;EndpointSuffix=x", + "CANARY_c2", + ), + ("InstrumentationKey=CANARY_c3;IngestionEndpoint=https://x/", "CANARY_c3"), + ("https://user:CANARY_d1@example.test/path", "CANARY_d1"), + ("GET /api?code=CANARY_d2&x=1", "CANARY_d2"), + ('x', "CANARY_e1"), + ], +) +def test_redact_masks_every_credential_shape(text: str, leaked: str) -> None: + from tools.verification_evidence import redact + + assert leaked not in redact(text) + + +@pytest.mark.parametrize( + "text", + [ + '{"Authorization": "Bearer CANARY_f1", "ok": true}', + '{"password": "alpha beta", "n": 1}', + 'text', + ], +) +def test_redact_keeps_json_and_xml_well_formed(text: str) -> None: + import xml.etree.ElementTree as ET + + from tools.verification_evidence import redact + + redacted = redact(text) + if text.startswith("{"): + json.loads(redacted) + else: + ET.fromstring(redacted) + + +def test_redact_leaves_ordinary_text_and_names_alone() -> None: + from tools.verification_evidence import redact + + text = "token_count=3 author=ada SharedAccessKeyName=Root status: ok" + assert redact(text) == text + + +def test_known_environment_secret_is_replaced_whole_before_patterns( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from tools.verification_evidence import redact + + monkeypatch.setenv("MY_API_TOKEN", "pre;fix&CANARY_g1 value") + assert "CANARY_g1" not in redact("seen pre;fix&CANARY_g1 value here") + assert "CANARY_g1" not in redact('{"x": "pre;fix&CANARY_g1 value"}') + + +def test_nested_secret_fields_are_redacted_before_json_is_written(tmp_path: Path) -> None: + from tools.verification_evidence import Summary, save + + summary = Summary( + 1, "test", ("pytest", 'Authorization: "Bearer CANARY_h1"'), 1, 1, 0, 0, "PASS", 0 + ) + save(tmp_path, summary) + text = (tmp_path / "summary.json").read_text() + assert "CANARY_h1" not in text + assert json.loads(text)["lane"] == "test" + + +def test_save_preserves_binary_assets_and_does_not_follow_symlinks(tmp_path: Path) -> None: + from tools.verification_evidence import Summary, save + + png = b"\x89PNG\r\n\x1a\n\xff\xfe\x00binary" + (tmp_path / "chart.png").write_bytes(png) + outside = tmp_path.parent / f"{tmp_path.name}-outside.log" + outside.write_text("password=CANARY_i1") + (tmp_path / "link.log").symlink_to(outside) + + save(tmp_path, Summary(1, "test", ("pytest",), 1, 1, 0, 0, "PASS", 0)) + + assert (tmp_path / "chart.png").read_bytes() == png + assert outside.read_text() == "password=CANARY_i1" + + +def test_structured_json_evidence_stays_valid_when_a_value_is_not_a_string(tmp_path: Path) -> None: + from tools.verification_evidence import Summary, save + + (tmp_path / "extra.json").write_text('{"password": 12345, "name": "x"}') + save(tmp_path, Summary(1, "test", ("pytest",), 1, 1, 0, 0, "PASS", 0)) + assert json.loads((tmp_path / "extra.json").read_text())["name"] == "x" diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py index 7700f68d..8d20008a 100644 --- a/tools/verification_evidence.py +++ b/tools/verification_evidence.py @@ -41,23 +41,70 @@ class Summary: reason: str = "" +_REDACTED = "[REDACTED]" +_QUOTED = r"\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'" +_AUTHORIZATION = re.compile( + r"(?im)(?P(?:proxy-)?authorization[\"']?\s*[:=]\s*)(?P" + _QUOTED + r"|[^\r\n\"<]+)" +) +_URL_CREDENTIALS = re.compile(r"(https?://)[^/@\s\"']+:[^/@\s\"']+@") +_SECRET_FIELD = re.compile( + r"(?i)(?(?:[\w.-]*(?:token|password|passwd|secret|api[_-]?key|" + r"connection[_-]?string)|AccountKey|SharedAccessKey|SharedAccessSignature|" + r"AzureWebJobsStorage|InstrumentationKey|x-functions-key|code|sig)" + r"[\"']?\s*[:=]\s*)(?P" + _QUOTED + r"|[^\s;&<>\"',]+)" +) +_SECRET_ENV_NAME = re.compile( + r"(?i)secret|token|password|connection|string|accountkey|functions.key|api.?key|sas" +) +_TEXT_SUFFIXES = frozenset({".log", ".txt", ".xml", ".json", ".md"}) + + +def _mask(match: re.Match[str]) -> str: + value = match.group("value") + quote = value[0] if value[:1] in {'"', "'"} else "" + return f"{match.group('key')}{quote}{_REDACTED}{quote}" + + +def _known_secrets() -> list[str]: + values = { + value + for key, value in os.environ.items() + if len(value) >= 6 and _SECRET_ENV_NAME.search(key) + } + # Longest first so a secret that contains another secret is replaced whole, and the JSON + # string-escaped spelling is covered because saved logs often embed serialized JSON. + escaped = {json.dumps(value)[1:-1] for value in values} + return sorted(values | escaped, key=len, reverse=True) + + def redact(text: str) -> str: - # Redact entire authorization values, including the scheme and whitespace. - text = re.sub(r"(?im)(authorization[\"']?\s*[:=]\s*)[^\r\n<\"]+", r"\1[REDACTED]", text) - text = re.sub(r"(https?://)[^/@\s]+:[^/@\s]+@", r"\1[REDACTED]@", text) - text = re.sub( - r"(?i)(?= 6 and re.search( - r"(?i)secret|token|password|connection|string|accountkey|functions.key", key - ): - text = text.replace(value, "[REDACTED]") - return text + """Mask credentials in free text without changing its surrounding syntax. + + Complete known secrets are replaced first so a partial pattern match cannot change a + value before the exact-match pass sees it. Quoted values are consumed whole (including + whitespace and escaped quotes) and keep their quotes, so JSON and XML stay well formed. + """ + for value in _known_secrets(): + text = text.replace(value, _REDACTED) + text = _AUTHORIZATION.sub(_mask, text) + text = _URL_CREDENTIALS.sub(r"\1" + _REDACTED + "@", text) + return _SECRET_FIELD.sub(_mask, text) + + +def redact_json(value: object) -> object: + """Redact every string leaf of a JSON-compatible value, leaving its structure intact.""" + if isinstance(value, str): + return redact(value) + if isinstance(value, (list, tuple)): + return [redact_json(item) for item in value] + if isinstance(value, dict): + return {redact(str(key)): redact_json(item) for key, item in value.items()} + return value + + +def dump_json(value: object) -> str: + """Serialize redacted JSON so credentials never reach disk and the file always parses.""" + return json.dumps(redact_json(value), indent=2) + "\n" def environment(live: bool = False) -> dict[str, str]: @@ -134,14 +181,31 @@ def publish(summary: Summary) -> None: stream.write(f"\n- {text}\n") +def _redact_file_text(suffix: str, text: str) -> str: + if suffix == ".json": + try: + return dump_json(json.loads(text)) + except ValueError: + pass + return redact(text) + + def save(directory: Path, summary: Summary) -> Summary: directory.mkdir(parents=True, exist_ok=True) for path in directory.rglob("*"): - if path.is_file(): - path.write_text(redact(path.read_text(encoding="utf-8")), encoding="utf-8") + # Only textual evidence is rewritten: binary assets stay byte-for-byte, and symlinks are + # never followed out of the evidence tree. + if path.is_symlink() or not path.is_file() or path.suffix not in _TEXT_SUFFIXES: + continue + try: + original = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + redacted = _redact_file_text(path.suffix, original) + if redacted != original: + path.write_text(redacted, encoding="utf-8") (directory / "environment.json").write_text( - redact(json.dumps(environment(live=directory.name == "integration"), indent=2)) + "\n", - encoding="utf-8", + dump_json(environment(live=directory.name == "integration")), encoding="utf-8" ) summary = replace( summary, @@ -152,7 +216,7 @@ def save(directory: Path, summary: Summary) -> Summary: ), ) path = directory / "summary.json" - path.write_text(redact(json.dumps(asdict(summary), indent=2)) + "\n", encoding="utf-8") + path.write_text(dump_json(asdict(summary)), encoding="utf-8") return summary From 7f643097060395f79e44ba413a2c630234bbfd4e Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:29:53 +0900 Subject: [PATCH 14/17] test(evidence): cover overlapping escaped credentials --- tests/test_verification_evidence.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py index 9aafb395..c1d0b8ce 100644 --- a/tests/test_verification_evidence.py +++ b/tests/test_verification_evidence.py @@ -204,3 +204,17 @@ def test_structured_json_evidence_stays_valid_when_a_value_is_not_a_string(tmp_p (tmp_path / "extra.json").write_text('{"password": 12345, "name": "x"}') save(tmp_path, Summary(1, "test", ("pytest",), 1, 1, 0, 0, "PASS", 0)) assert json.loads((tmp_path / "extra.json").read_text())["name"] == "x" + + +def test_overlapping_escaped_environment_secrets_are_replaced_longest_first( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from tools.verification_evidence import dump_json, redact + + monkeypatch.setenv("SHORT_TOKEN", "CANARY_overlap") + secret = 'CANARY_overlap;password=alpha "quoted" beta&tail' + monkeypatch.setenv("LONG_TOKEN", secret) + assert redact(json.dumps(secret)) == '"[REDACTED]"' + payload = {"nested": ({"command": 'Authorization: "Bearer CANARY_nested"'}, secret)} + data = json.loads(dump_json(payload)) + assert data == {"nested": [{"command": 'Authorization: "[REDACTED]"'}, "[REDACTED]"]} From 7555b1dcaab1ba88b4c0daa5fc511a44aa9d5a3f Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:29:54 +0900 Subject: [PATCH 15/17] test(evidence): isolate synthetic job summaries --- tests/conftest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..eba48551 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +"""Keep synthetic verification evidence out of the enclosing Actions summary.""" + +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_verification_summary( + request: pytest.FixtureRequest, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + if request.node.path.name.startswith("test_verification"): + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "isolated-job-summary")) From 926eba6d4c0dc95754fad6734b0fbe420c040801 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:29:54 +0900 Subject: [PATCH 16/17] ci(verification): retain quality evidence and avoid maintenance provisioning --- Makefile | 19 ++++- tests/test_verification_commands.py | 4 +- tests/test_verification_review.py | 120 ++++++++++++++++++++++++++++ tools/verification.py | 3 + tools/verification_environment.py | 2 + tools/verification_host.py | 3 +- 6 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 tests/test_verification_review.py diff --git a/Makefile b/Makefile index 35cf63f7..fd1a7f40 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,10 @@ RUN := $(HATCH) run python else RUN := $(PYTHON) endif +MAINTENANCE_PYTHON = $$(for candidate in "$${VIRTUAL_ENV}/bin/python" ".venv/bin/python" "$$($(HATCH) env find 2>/dev/null)/bin/python" python3.11 python3 $(PYTHON); do \ + if "$$candidate" -c 'import sys; sys.exit(sys.version_info < (3, 11))' >/dev/null 2>&1; then \ + printf '%s' "$$candidate"; break; \ + fi; done) SPEC_PREVIEW_PNG := docs/assets/webhook_receiver_openapi_spec_preview.png PLAYWRIGHT_VERSION := 1.54.1 PLAYWRIGHT_BROWSERS_PATH := $(CURDIR)/.cache/ms-playwright @@ -74,7 +78,11 @@ security: @$(RUN) -m bandit -r src .PHONY: check -check: format-check typecheck security lint-workflows +check: + @$(RUN) -m tools.verification check + +.PHONY: check-commands +check-commands: format-check typecheck security lint-workflows @$(RUN) -m ruff check src tests tools/verification*.py @$(RUN) scripts/lint_mermaid.py @$(RUN) scripts/check_screenshots.py --strict @@ -89,9 +97,12 @@ lint-workflows: check-all: verify .PHONY: integration verify report wheel-smoke -integration verify report: +integration verify: @$(RUN) -m tools.verification $@ +report: + @$(MAINTENANCE_PYTHON) -m tools.verification report + wheel-smoke: @$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel @@ -241,11 +252,11 @@ demo-examples: ensure-hatch .PHONY: doctor doctor: - @$(RUN) -m tools.verification doctor + @$(MAINTENANCE_PYTHON) -m tools.verification doctor .PHONY: clean clean: - @$(RUN) -m tools.verification clean + @$(MAINTENANCE_PYTHON) -m tools.verification clean # Deep clean (maintenance, not part of the canonical contract): artifacts plus the # project-local virtual environment. `make clean` never touches `.venv`. diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py index 3bdea765..06db5a4f 100644 --- a/tests/test_verification_commands.py +++ b/tests/test_verification_commands.py @@ -176,7 +176,9 @@ def test_make_contract_is_install_free_on_routine_runs(target: str) -> None: def test_check_runs_typechecking_once_without_test_execution() -> None: - result = subprocess.run(["make", "-n", "check"], cwd=ROOT, capture_output=True, text=True) + result = subprocess.run( + ["make", "-n", "check-commands"], cwd=ROOT, capture_output=True, text=True + ) assert result.returncode == 0 assert result.stdout.count("-m mypy") == 1 assert "-m pytest" not in result.stdout diff --git a/tests/test_verification_review.py b/tests/test_verification_review.py new file mode 100644 index 00000000..7a38d051 --- /dev/null +++ b/tests/test_verification_review.py @@ -0,0 +1,120 @@ +"""Regression contracts for review findings on verification commands.""" + +from __future__ import annotations + +from contextlib import nullcontext +from http.client import HTTPException +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +from unittest.mock import Mock + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("error", [TimeoutError(), ConnectionError(), OSError(), HTTPException()]) +def test_readiness_retries_transient_failure( + error: Exception, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given a live process and one transient request failure followed by success. + from tools.verification_host import wait_ready + + request = Mock(side_effect=[error, nullcontext(Mock(status=200))]) + monkeypatch.setattr("tools.verification_host.urlopen", request) + monkeypatch.setattr("tools.verification_host.time.sleep", lambda _: None) + # When waiting for host readiness. + wait_ready(Mock(poll=Mock(return_value=None)), "http://localhost/api") + # Then startup is retried rather than aborted. + assert request.call_count == 2 + + +@pytest.mark.parametrize("failure", [False, True]) +def test_check_saves_prerequisite_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: bool +) -> None: + # Given a real Makefile and deterministic quality prerequisites. + makefile = (ROOT / "Makefile").read_text() + start = makefile.index("check-commands:") + end = makefile.index(".PHONY: lint-workflows", start) + makefile = ( + makefile[:start] + + ( + "check-commands: review-prerequisite\n\t@true\n" + "review-prerequisite:\n" + + ("\t@echo 'password=CANARY_quality'; exit 7\n\n" if failure else "\t@true\n\n") + ) + + makefile[end:] + ) + (tmp_path / "Makefile").write_text(makefile) + shutil.copytree(ROOT / "tools", tmp_path / "tools") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary")) + # When executing the public quality gate including failing prerequisites. + result = subprocess.run( + [ + "make", + "check", + "CI=1", + f"PYTHON={sys.executable}", + ], + cwd=tmp_path, + capture_output=True, + text=True, + ) + # Then logs and machine-readable failure status survive the make failure. + summary = json.loads((tmp_path / "test-results/check/summary.json").read_text()) + assert summary["status"] == ("FAIL" if failure else "PASS") + assert bool(result.returncode) == failure + assert "CANARY_quality" not in (tmp_path / "test-results/check/logs/runner.log").read_text() + assert f"check: {summary['status']}" in (tmp_path / "job-summary").read_text() + + +@pytest.mark.parametrize("target", ["doctor", "report", "clean"]) +def test_maintenance_never_bootstraps_hatch(tmp_path: Path, target: str) -> None: + # Given a fresh checkout mirror and empty Hatch data directory, with CI unset. + shutil.copy(ROOT / "Makefile", tmp_path) + shutil.copy(ROOT / "pyproject.toml", tmp_path) + shutil.copytree(ROOT / "tools", tmp_path / "tools") + environment = dict(os.environ, HATCH_DATA_DIR=str(tmp_path / "hatch-data")) + environment.pop("CI", None) + environment.pop("VIRTUAL_ENV", None) + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(tmp_path / ".venv")], check=True + ) + # When running a maintenance command without any development environment. + result = subprocess.run( + ["make", target], cwd=tmp_path, env=environment, capture_output=True, text=True + ) + # Then no environment or dependency synchronization occurs. + assert "Creating environment" not in result.stderr + assert not list((tmp_path / "hatch-data").rglob("pyvenv.cfg")) + if target == "doctor": + assert result.returncode != 0 + assert "development environment not created: run make install" in result.stdout + + +def test_verification_tests_leave_outer_summary_untouched(tmp_path: Path) -> None: + outer = tmp_path / "outer-summary" + outer.write_text("real job evidence\n") + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "tests/test_verification_commands.py", + "-k", + "verify_runs_every_required_lane or azure_missing_prerequisites", + "--no-cov", + "-q", + ], + cwd=ROOT, + env=dict(os.environ, GITHUB_STEP_SUMMARY=str(outer)), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert outer.read_text() == "real job evidence\n" diff --git a/tools/verification.py b/tools/verification.py index 8412a541..d2acd622 100644 --- a/tools/verification.py +++ b/tools/verification.py @@ -175,6 +175,7 @@ def main() -> int: "lane", choices=( "test", + "check", "integration", "verify", "azure-e2e", @@ -189,6 +190,8 @@ def main() -> int: if tuple(sys.version_info[:2]) < (3, 11): parser.error("Python >=3.11 is required") match args.lane: + case "check": + return focused("check", ["make", "check-commands"]) case "report": return report(Path("test-results")) case "doctor": diff --git a/tools/verification_environment.py b/tools/verification_environment.py index 72ccacb5..49234268 100644 --- a/tools/verification_environment.py +++ b/tools/verification_environment.py @@ -20,6 +20,8 @@ def doctor(live: bool = False) -> int: print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}") if live: healthy = healthy and path is not None + if not healthy: + print("development environment not created: run make install") return 0 if healthy else 1 diff --git a/tools/verification_host.py b/tools/verification_host.py index abf02c00..9bc33064 100644 --- a/tools/verification_host.py +++ b/tools/verification_host.py @@ -4,6 +4,7 @@ import base64 from contextlib import ExitStack +from http.client import HTTPException import json import os from pathlib import Path @@ -35,7 +36,7 @@ def wait_ready(process: subprocess.Popen[str], url: str) -> None: with urlopen(url, timeout=1) as response: if response.status < 500: return - except URLError: + except (OSError, URLError, HTTPException): time.sleep(0.25) raise TimeoutError("service readiness deadline exceeded") From b1cc3ed19f5ef1301ff3737ad974cc794deba316 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 09:29:54 +0900 Subject: [PATCH 17/17] style(scripts): sort screenshot checker imports --- scripts/check_screenshots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_screenshots.py b/scripts/check_screenshots.py index da3fb5c2..12089381 100644 --- a/scripts/check_screenshots.py +++ b/scripts/check_screenshots.py @@ -29,8 +29,8 @@ import argparse import hashlib -import re from pathlib import Path +import re import sys from typing import Any