From d1eeaac7ae1a82e3a727448a792fe0ccba47243f Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 01/14] ci: add fail-closed verification lanes and redacted
 evidence

---
 tests/test_verification_evidence.py | 240 ++++++++++++++++++++++++++
 tools/__init__.py                   |   0
 tools/verification.py               | 258 ++++++++++++++++++++++++++++
 tools/verification_contracts.py     |  21 +++
 tools/verification_environment.py   |  51 ++++++
 tools/verification_evidence.py      | 165 ++++++++++++++++++
 tools/verification_host.py          | 131 ++++++++++++++
 tools/verification_process.py       |  81 +++++++++
 tools/verification_wheel.py         |  72 ++++++++
 9 files changed, 1019 insertions(+)
 create mode 100644 tests/test_verification_evidence.py
 create mode 100644 tools/__init__.py
 create mode 100644 tools/verification.py
 create mode 100644 tools/verification_contracts.py
 create mode 100644 tools/verification_environment.py
 create mode 100644 tools/verification_evidence.py
 create mode 100644 tools/verification_host.py
 create mode 100644 tools/verification_process.py
 create mode 100644 tools/verification_wheel.py

diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py
new file mode 100644
index 0000000..084d49d
--- /dev/null
+++ b/tests/test_verification_evidence.py
@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+
+import pytest
+
+from tools.verification import execute, pytest_command
+from tools.verification_environment import clean, doctor
+from tools.verification_evidence import Summary, collect, redact, report, save
+from tools.verification_process import child, run, terminate
+
+
+@pytest.mark.parametrize(
+    ("cases", "exit_code", "status"),
+    [
+        ("", 0, "INCOMPLETE"),
+        ("", 5, "INCOMPLETE"),
+        ("<testcase><skipped/></testcase>", 0, "INCOMPLETE"),
+        ("<testcase/><testcase><skipped/></testcase>", 0, "INCOMPLETE"),
+        ("<testcase><failure/></testcase>", 1, "FAIL"),
+        ("<testcase/>", 0, "PASS"),
+        ("<testcase/>", 124, "FAIL"),
+    ],
+)
+def test_junit_classifies_execution_fail_closed(
+    tmp_path: Path, cases: str, exit_code: int, status: str
+) -> None:
+    (tmp_path / "junit.xml").write_text(f"<testsuite>{cases}</testsuite>")
+    summary = collect(tmp_path, ["pytest"], exit_code)
+    assert summary.status == status
+    assert (summary.exit_code == 0) == (status == "PASS")
+
+
+@pytest.mark.parametrize(
+    "secret",
+    [
+        "AccountKey=VERYSECRET;",
+        "SharedAccessSignature=VERYSECRET;",
+        "code=VERYSECRET&x=1",
+        "x-functions-key: VERYSECRET",
+        "Authorization: Bearer VERYSECRET",
+        "password=VERYSECRET",
+        "token=VERYSECRET",
+        "AzureWebJobsStorage=VERYSECRET",
+        "sas=VERYSECRET",
+        "api_key=VERYSECRET",
+        "DefaultEndpointsProtocol=https;AccountName=PRIVATE;AccountKey=VERYSECRET;",
+    ],
+)
+def test_secrets_are_redacted_from_all_saved_evidence(tmp_path: Path, secret: str) -> None:
+    (tmp_path / "junit.xml").write_text(
+        "<testsuite><testcase><system-out>"
+        + secret.replace("&", "&amp;")
+        + "</system-out></testcase></testsuite>"
+    )
+    (tmp_path / "output.log").write_text(secret)
+    save(tmp_path, collect(tmp_path, ["pytest", secret], 0))
+    for path in tmp_path.rglob("*"):
+        if path.is_file():
+            assert "VERYSECRET" not in path.read_text()
+    json.loads((tmp_path / "summary.json").read_text())
+
+
+def test_environment_secrets_are_removed(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("CUSTOM_ACCESS_TOKEN", "opaque-sensitive-value")
+    assert redact("diagnostic opaque-sensitive-value") == "diagnostic [REDACTED]"
+
+
+def test_summary_schema_and_saved_report(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    directory = tmp_path / "test"
+    directory.mkdir()
+    (directory / "junit.xml").write_text("<testsuite><testcase/></testsuite>")
+    (directory / "coverage.xml").write_text('<coverage line-rate="0.99"/>')
+    save(directory, collect(directory, ["not-an-executable"], 0))
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary"))
+    assert report(tmp_path) == 0
+    data = json.loads((directory / "summary.json").read_text())
+    assert data["schema_version"] == 1
+    assert (data["selected"], data["passed"], data["failed"], data["skipped"]) == (1, 1, 0, 0)
+    assert data["coverage"] == 0.99
+    assert str(directory / "junit.xml") in data["supporting_files"]
+    assert "passed=1" in (tmp_path / "job-summary").read_text()
+
+
+def test_report_without_results_does_not_create_evidence(tmp_path: Path) -> None:
+    assert report(tmp_path) == 1
+    assert list(tmp_path.iterdir()) == []
+
+
+def test_required_tools_fail_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr("tools.verification_environment.shutil.which", lambda _: None)
+    assert doctor() != 0
+    assert execute("host-smoke", []) != 0
+    data = json.loads((tmp_path / "test-results/host-smoke/summary.json").read_text())
+    assert data["status"] == "INCOMPLETE"
+
+
+def test_sdk_spike_is_opt_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.delenv("AZFUNC_2X_SPIKE", raising=False)
+    assert execute("sdk-compat", []) != 0
+    data = json.loads((tmp_path / "test-results/sdk-compat/summary.json").read_text())
+    assert data["status"] == "INCOMPLETE"
+
+
+def test_offline_and_worker_selection_are_explicit() -> None:
+    unit = pytest_command("test", [])
+    assert "--ignore=tests/e2e" in unit
+    assert "--ignore=tests/host_smoke" in unit
+    assert "not e2e and not compat2x" in unit
+    assert "tests/test_trigger_arg_name_worker_compat.py" in pytest_command("worker-compat", [])
+    assert "pythonpath=" in pytest_command("sdk-compat", [])
+
+
+def test_clean_preserves_environments_and_shared_storage(tmp_path: Path) -> None:
+    for name in (".venv", "src", "shared-azurite", "test-results", "dist", "build"):
+        (tmp_path / name).mkdir()
+        (tmp_path / name / "keep").touch()
+    clean(tmp_path)
+    assert sorted(p.name for p in tmp_path.iterdir()) == [".venv", "shared-azurite", "src"]
+
+
+def test_clean_unlinks_artifact_symlink_without_following(tmp_path: Path) -> None:
+    (tmp_path / "shared").mkdir()
+    (tmp_path / "shared/keep").touch()
+    (tmp_path / "test-results").symlink_to(tmp_path / "shared", target_is_directory=True)
+    clean(tmp_path)
+    assert (tmp_path / "shared/keep").exists()
+
+
+def test_process_failure_is_preserved(tmp_path: Path) -> None:
+    assert run([sys.executable, "-c", "raise SystemExit(7)"], tmp_path / "log") == 7
+    assert run(["/nonexistent/verification-command"], tmp_path / "missing") == 127
+
+
+def test_owned_process_is_reaped_after_interrupt(tmp_path: Path) -> None:
+    with (
+        pytest.raises(KeyboardInterrupt),
+        child([sys.executable, "-c", "import time; time.sleep(30)"], tmp_path / "log") as process,
+    ):
+        raise KeyboardInterrupt
+    assert process.poll() is not None
+
+
+def test_timeout_reaps_process_and_redacts_logs(tmp_path: Path) -> None:
+    code = run(
+        [sys.executable, "-u", "-c", "import time; print('password=VERYSECRET'); time.sleep(30)"],
+        tmp_path / "log",
+        1,
+    )
+    assert code == 124
+    assert "VERYSECRET" not in (tmp_path / "log").read_text()
+
+
+def test_cleanup_tolerates_process_exit_race(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    def exited_group(pid: int, number: int) -> None:
+        raise ProcessLookupError(pid)
+
+    with child([sys.executable, "-c", "raise SystemExit(0)"], tmp_path / "log") as process:
+        monkeypatch.setattr("tools.verification_process.os.killpg", exited_group)
+        terminate(process)
+    assert process.returncode == 0
+
+
+def test_wheel_smoke_rejects_stale_dist_when_build_produces_nothing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from tools.verification_wheel import main
+
+    (tmp_path / "dist").mkdir()
+    (tmp_path / "dist/stale.whl").write_bytes(b"stale")
+
+    def empty_build(command: list[str], log: Path, timeout: float = 600) -> int:
+        assert "--outdir" in command
+        assert str(tmp_path / "dist") not in command
+        return 0
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr("tools.verification_wheel.run", empty_build)
+    assert main() != 0
+
+
+def test_integration_preserves_separate_required_lanes(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from tools.verification import integration
+
+    calls: list[str] = []
+
+    def fake_execute(lane: str, extra: list[str]) -> int:
+        calls.append(lane)
+        save(
+            Path("test-results") / lane,
+            Summary(
+                1,
+                lane,
+                ("test",),
+                0,
+                0,
+                0,
+                0,
+                "INCOMPLETE",
+                1,
+            ),
+        )
+        return 1
+
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr("tools.verification.execute", fake_execute)
+    assert integration() != 0
+    assert calls == ["worker-compat", "host-smoke"]
+    assert (
+        json.loads(Path("test-results/integration/summary.json").read_text())["status"]
+        == "INCOMPLETE"
+    )
+
+
+def test_verify_runs_all_lanes_after_failure(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from tools.verification import verify
+
+    calls: list[str] = []
+
+    def fake_run(command: list[str], log: Path, timeout: float = 600) -> int:
+        calls.append(command[1])
+        return 1
+
+    monkeypatch.chdir(tmp_path)
+    save(Path("test-results/test"), Summary(1, "test", ("old",), 1, 1, 0, 0, "PASS", 0))
+    monkeypatch.setattr("tools.verification.run", fake_run)
+    assert verify() != 0
+    assert calls == ["check", "test", "integration", "wheel-smoke", "docs"]
+    lanes = json.loads(Path("test-results/verify/lanes.json").read_text())
+    assert lanes[1]["status"] == "INCOMPLETE"
diff --git a/tools/__init__.py b/tools/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/verification.py b/tools/verification.py
new file mode 100644
index 0000000..72b98d8
--- /dev/null
+++ b/tools/verification.py
@@ -0,0 +1,258 @@
+"""Canonical verification CLI: python -m tools.verification, normally through Make."""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import asdict, replace
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import xml.etree.ElementTree as ET
+
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from tools.verification_environment import clean, doctor
+from tools.verification_evidence import Status, Summary, collect, publish, report, save
+from tools.verification_host import host_smoke
+from tools.verification_process import run
+
+WORKER_TESTS = (
+    "tests/test_decorator.py",
+    "tests/test_integration.py",
+    "tests/test_trigger_arg_name_worker_compat.py",
+)
+CONTRACT_TESTS = (
+    "tests/test_endpoint_schema.py",
+    "tests/test_metadata_helpers_drift.py",
+    "tests/test_metadata_contract.py",
+    "tests/test_toolkit_metadata.py",
+    "tests/test_metadata.py",
+    "tests/test_check_wheel_schemas.py",
+)
+
+
+def pytest_command(lane: str, extra: list[str]) -> list[str]:
+    directory = Path("test-results") / lane
+    command = [sys.executable, "-m", "pytest"]
+    match lane:
+        case "test":
+            command += [
+                "tests/",
+                "-m",
+                "not e2e and not compat2x",
+                "--ignore=tests/e2e",
+                "--ignore=tests/host_smoke",
+                "--cov-report=xml:" + str(directory / "coverage.xml"),
+            ]
+        case "worker-compat":
+            command += [*WORKER_TESTS, "--no-cov", "-m", "not e2e and not compat2x"]
+        case "sdk-compat":
+            command += [
+                "tests/test_worker_compat_2x_spike.py",
+                "-m",
+                "compat2x",
+                "-o",
+                "addopts=",
+                "-o",
+                "pythonpath=",
+            ]
+        case "contracts":
+            command += [*CONTRACT_TESTS, "--no-cov"]
+        case _:
+            command += ["tests/e2e/", "-m", "e2e", "--no-cov"]
+    return command + ["--junitxml=" + str(directory / "junit.xml"), *extra]
+
+
+def execute(lane: str, extra: list[str]) -> int:
+    directory = Path("test-results") / lane
+    if directory.exists():
+        shutil.rmtree(directory)
+    (directory / "logs").mkdir(parents=True)
+    command = pytest_command(lane, extra)
+    reason = ""
+    code = 0
+    blocked = False
+    try:
+        if lane in {"azure-e2e", "local-e2e"} and not os.environ.get("E2E_BASE_URL"):
+            blocked, code, reason = True, 2, "Missing prerequisite: E2E_BASE_URL"
+        if lane == "sdk-compat" and os.environ.get("AZFUNC_2X_SPIKE") != "1":
+            blocked, code, reason = True, 2, "Opt-in SDK spike requires AZFUNC_2X_SPIKE=1"
+        if lane == "host-smoke":
+            command = ["func", "start", "(managed validation host smoke)"]
+            if doctor():
+                blocked, code, reason = True, 2, "Missing required integration tools"
+            else:
+                code = host_smoke(directory)
+        elif not blocked:
+            code = run(command, directory / "logs/runner.log")
+    except KeyboardInterrupt:
+        code, reason = 130, "Interrupted; owned child processes terminated"
+    except (OSError, ET.ParseError, ValueError) as error:
+        code, reason = 2, str(error)
+    try:
+        summary = collect(directory, command, code)
+    except (ET.ParseError, ValueError) as error:
+        summary = Summary(
+            1,
+            lane,
+            tuple(command),
+            0,
+            0,
+            0,
+            0,
+            "INCOMPLETE",
+            2,
+            reason=f"Invalid evidence: {error}",
+        )
+    if blocked:
+        summary = replace(summary, status="INCOMPLETE")
+        ET.ElementTree(ET.Element("testsuite", name=lane, tests="0")).write(
+            directory / "junit.xml", encoding="unicode"
+        )
+    if lane == "sdk-compat":
+        reason = (
+            reason + "; " if reason else ""
+        ) + "SDK compatibility spike, NOT Azure certification"
+    summary = save(directory, replace(summary, reason=reason or summary.reason))
+    publish(summary)
+    return summary.exit_code
+
+
+def aggregate(lane: str, outcomes: list[Summary]) -> int:
+    status: Status = "FAIL" if any(item.status == "FAIL" for item in outcomes) else "PASS"
+    if status == "PASS" and any(item.status != "PASS" for item in outcomes):
+        status = "INCOMPLETE"
+    directory = Path("test-results") / lane
+    directory.mkdir(parents=True, exist_ok=True)
+    summary = Summary(
+        1,
+        lane,
+        ("make", lane),
+        sum(s.selected for s in outcomes),
+        sum(s.passed for s in outcomes),
+        sum(s.failed for s in outcomes),
+        sum(s.skipped for s in outcomes),
+        status,
+        int(status != "PASS"),
+        reason="; ".join(f"{item.lane}={item.status}" for item in outcomes),
+    )
+    (directory / "lanes.json").write_text(json.dumps([asdict(s) for s in outcomes], indent=2))
+    publish(save(directory, summary))
+    return summary.exit_code
+
+
+def integration() -> int:
+    directory = Path("test-results/integration")
+    if directory.exists():
+        shutil.rmtree(directory)
+    outcomes = []
+    for lane in ("worker-compat", "host-smoke"):
+        execute(lane, [])
+        source = Path("test-results") / lane
+        path = source / "summary.json"
+        outcomes.append(Summary(**json.loads(path.read_text())))
+        for pattern, destination in (("**/*junit.xml", "junit"), ("logs/*", "logs")):
+            for artifact in source.glob(pattern):
+                target = directory / destination / f"{lane}-{artifact.name}"
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(artifact, target)
+    return aggregate("integration", outcomes)
+
+
+def verify() -> int:
+    directory = Path("test-results/verify")
+    if directory.exists():
+        shutil.rmtree(directory)
+    outcomes = []
+    for target in ("check", "test", "integration", "wheel-smoke", "docs"):
+        path = Path("test-results") / target / "summary.json"
+        path.unlink(missing_ok=True)
+        code = run(["make", target], directory / "logs" / f"{target}.log", 1200)
+        if target in {"test", "integration", "wheel-smoke", "docs"} and path.exists():
+            summary = Summary(**json.loads(path.read_text()))
+            if code and summary.status == "PASS":
+                summary = replace(summary, status="FAIL", exit_code=code)
+        else:
+            required = target != "check"
+            summary = Summary(
+                1,
+                target,
+                ("make", target),
+                int(not required),
+                int(not required and code == 0),
+                int(code != 0),
+                0,
+                "INCOMPLETE" if required else ("FAIL" if code else "PASS"),
+                code or int(required),
+                reason="Required lane summary missing" if required else "",
+            )
+        outcomes.append(summary)
+    return aggregate("verify", outcomes)
+
+
+def focused(lane: str, command: list[str]) -> int:
+    directory = Path("test-results") / lane
+    if directory.exists():
+        shutil.rmtree(directory)
+    code = run(command, directory / "logs/runner.log", 1200)
+    summary = Summary(
+        1,
+        lane,
+        tuple(command),
+        1,
+        int(code == 0),
+        int(code != 0),
+        0,
+        "PASS" if code == 0 else "FAIL",
+        code,
+    )
+    publish(save(directory, summary))
+    return code
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "lane",
+        choices=(
+            "test",
+            "integration",
+            "worker-compat",
+            "host-smoke",
+            "sdk-compat",
+            "contracts",
+            "verify",
+            "azure-e2e",
+            "local-e2e",
+            "report",
+            "doctor",
+            "clean",
+            "focused",
+        ),
+    )
+    args, extra = parser.parse_known_args()
+    if not (3, 11) <= sys.version_info[:2] < (3, 15):
+        parser.error("Python >=3.11,<3.15 is required")
+    match args.lane:
+        case "report":
+            return report(Path("test-results"))
+        case "doctor":
+            return doctor()
+        case "clean":
+            clean(Path.cwd())
+            return 0
+        case "verify":
+            return verify()
+        case "integration":
+            return integration()
+        case "focused":
+            return focused(extra[0], [sys.executable, *extra[1:]])
+        case _:
+            return execute(args.lane, extra)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/tools/verification_contracts.py b/tools/verification_contracts.py
new file mode 100644
index 0000000..a739e7f
--- /dev/null
+++ b/tools/verification_contracts.py
@@ -0,0 +1,21 @@
+"""Non-mutating schema-pin and metadata-contract checks."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def schema_hash(root: Path) -> int:
+    directory = root / "src/azure_functions_validation/schemas"
+    expected = (directory / "endpoint.schema.sha256").read_text().split()[0]
+    actual = hashlib.sha256((directory / "endpoint.schema.json").read_bytes()).hexdigest()
+    if actual != expected:
+        print(f"endpoint.schema.json digest drift: pinned={expected} actual={actual}")
+        return 1
+    print("endpoint.schema.json digest matches pin.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(schema_hash(Path.cwd()))
diff --git a/tools/verification_environment.py b/tools/verification_environment.py
new file mode 100644
index 0000000..0eab1ed
--- /dev/null
+++ b/tools/verification_environment.py
@@ -0,0 +1,51 @@
+"""Read-only prerequisites and allowlisted generated-artifact cleanup."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+import shutil
+import sys
+
+
+def doctor(live: bool = True) -> int:
+    healthy = (3, 11) <= sys.version_info[:2] < (3, 15)
+    print(f"python REQUIRED >=3.11,<3.15: {sys.version.split()[0]} ({sys.executable})")
+    for module in ("venv", "ruff", "mypy", "pytest", "bandit", "build", "mkdocs"):
+        found = importlib.util.find_spec(module) is not None
+        print(f"{module} REQUIRED: {'found' if found else 'MISSING'}")
+        healthy = healthy and found
+    for tool in ("func", "azurite"):
+        path = shutil.which(tool)
+        print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}")
+        if live:
+            healthy = healthy and path is not None
+    return 0 if healthy else 1
+
+
+def clean(root: Path) -> None:
+    for name in (
+        "test-results",
+        "dist",
+        "build",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".coverage",
+        "coverage.xml",
+        "htmlcov",
+        "site",
+        "__pycache__",
+    ):
+        path = root / name
+        if path.is_symlink() or path.is_file():
+            path.unlink()
+        elif path.is_dir():
+            shutil.rmtree(path)
+    for tree in ("src", "tests", "tools", "scripts", "examples"):
+        for path in (root / tree).rglob("__pycache__"):
+            if not any(part in {".venv", "venv", "env"} for part in path.parts):
+                if path.is_symlink():
+                    path.unlink()
+                else:
+                    shutil.rmtree(path)
diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py
new file mode 100644
index 0000000..ae2af87
--- /dev/null
+++ b/tools/verification_evidence.py
@@ -0,0 +1,165 @@
+"""Versioned, redacted evidence shared by local commands and CI."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, replace
+from importlib.metadata import PackageNotFoundError, version
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import subprocess
+import sys
+from typing import Literal
+import xml.etree.ElementTree as ET
+
+Status = Literal["PASS", "FAIL", "INCOMPLETE", "NOT_APPLICABLE"]
+
+
+@dataclass(frozen=True, slots=True)
+class Summary:
+    schema_version: int
+    lane: str
+    command: tuple[str, ...]
+    selected: int
+    passed: int
+    failed: int
+    skipped: int
+    status: Status
+    exit_code: int
+    supporting_files: tuple[str, ...] = ()
+    coverage: float | None = None
+    reason: str = ""
+
+
+def redact(text: str) -> str:
+    text = re.sub(r"(?im)(authorization[\"']?\s*[:=]\s*)[^\r\n<\"]+", r"\1[REDACTED]", text)
+    text = re.sub(r"(https?://)[^/@\s]+:[^/@\s]+@", r"\1[REDACTED]@", text)
+    # Connection strings are one credential, not just their AccountKey component.
+    text = re.sub(r"(?i)(DefaultEndpointsProtocol=)[^\s<\"']+", r"\1[REDACTED]", text)
+    text = re.sub(
+        r"(?i)(?<![\w-])((?:[\w-]*(?:token|password|secret|connectionstring)|AccountKey|"
+        r"SharedAccessSignature|AzureWebJobsStorage|x-functions-key|code|sig|sas|api[_-]?key)"
+        r"[\"']?\s*[:=]\s*[\"']?)[^\s;&<\"']+",
+        r"\1[REDACTED]",
+        text,
+    )
+    for key, value in os.environ.items():
+        if len(value) >= 6 and re.search(
+            r"(?i)secret|token|password|connection|string|accountkey|functions.key", key
+        ):
+            text = text.replace(value, "[REDACTED]")
+    return text
+
+
+def environment(live: bool = False) -> dict[str, str]:
+    metadata = {
+        "python": platform.python_version(),
+        "executable": sys.executable,
+        "platform": platform.platform(),
+        "runtime": platform.python_implementation(),
+    }
+    for package in ("azure-functions", "azure-functions-validation", "pydantic", "hatch"):
+        try:
+            metadata[package] = version(package)
+        except PackageNotFoundError:
+            metadata[package] = "not installed"
+    for key in ("E2E_BASE_URL", "E2E_EXPECTED_VERSION", "GITHUB_SHA", "AZFUNC_2X_SPIKE"):
+        if key in os.environ:
+            metadata[key] = redact(os.environ[key])
+    for tool in ("func", "azurite") if live else ():
+        try:
+            result = subprocess.run([tool, "--version"], capture_output=True, text=True, timeout=10)
+            metadata[tool] = redact(result.stdout.strip())
+        except (OSError, subprocess.TimeoutExpired):
+            metadata[tool] = "unavailable"
+    return metadata
+
+
+def collect(directory: Path, command: list[str], exit_code: int) -> Summary:
+    cases: list[ET.Element] = []
+    for path in directory.rglob("*.xml"):
+        if "junit" in path.name:
+            cases.extend(ET.parse(path).getroot().iter("testcase"))
+    skipped = sum(case.find("skipped") is not None for case in cases)
+    failed = sum(
+        case.find("failure") is not None or case.find("error") is not None for case in cases
+    )
+    passed = len(cases) - skipped - failed
+    status: Status = "PASS"
+    if failed or exit_code not in (0, 5):
+        status = "FAIL"
+    elif not cases or not passed or (directory.name != "test" and skipped):
+        status = "INCOMPLETE"
+    elif exit_code:
+        status = "FAIL"
+    coverage_path = directory / "coverage.xml"
+    coverage = None
+    if coverage_path.exists():
+        coverage = float(ET.parse(coverage_path).getroot().get("line-rate", "0"))
+    return Summary(
+        1,
+        directory.name,
+        tuple(command),
+        len(cases),
+        passed,
+        failed,
+        skipped,
+        status,
+        exit_code if exit_code else int(status != "PASS"),
+        coverage=coverage,
+    )
+
+
+def render(summary: Summary) -> str:
+    return (
+        f"{summary.lane}: {summary.status} | selected={summary.selected} "
+        f"passed={summary.passed} failed={summary.failed} skipped={summary.skipped} "
+        f"exit={summary.exit_code}" + (f" | {summary.reason}" if summary.reason else "")
+    )
+
+
+def publish(summary: Summary) -> None:
+    text = redact(render(summary))
+    print(text)
+    destination = os.environ.get("GITHUB_STEP_SUMMARY")
+    if destination:
+        with Path(destination).open("a", encoding="utf-8") as stream:
+            stream.write(f"\n- {text}\n")
+
+
+def save(directory: Path, summary: Summary) -> Summary:
+    directory.mkdir(parents=True, exist_ok=True)
+    for path in directory.rglob("*"):
+        if path.is_file():
+            path.write_text(redact(path.read_text(encoding="utf-8")), encoding="utf-8")
+    (directory / "environment.json").write_text(
+        redact(
+            json.dumps(environment(live=directory.name in {"integration", "host-smoke"}), indent=2)
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    summary = replace(
+        summary,
+        supporting_files=tuple(
+            str(path)
+            for path in sorted(directory.rglob("*"))
+            if path.is_file() and path.name != "summary.json"
+        ),
+    )
+    (directory / "summary.json").write_text(
+        redact(json.dumps(asdict(summary), indent=2)) + "\n", encoding="utf-8"
+    )
+    return summary
+
+
+def report(root: Path) -> int:
+    paths = sorted(root.glob("*/summary.json"))
+    if not paths:
+        print("No saved verification results")
+        return 1
+    for path in paths:
+        publish(Summary(**json.loads(path.read_text(encoding="utf-8"))))
+    return 0
diff --git a/tools/verification_host.py b/tools/verification_host.py
new file mode 100644
index 0000000..a22285d
--- /dev/null
+++ b/tools/verification_host.py
@@ -0,0 +1,131 @@
+"""Real validation host smoke with private Azurite storage and owned processes."""
+
+from __future__ import annotations
+
+import base64
+from contextlib import ExitStack
+import json
+import os
+from pathlib import Path
+import re
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from urllib.error import URLError
+from urllib.request import urlopen
+import xml.etree.ElementTree as ET
+
+from tools.verification_process import child
+
+
+def free_port() -> int:
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
+def wait_ready(process: subprocess.Popen[str], url: str) -> None:
+    deadline = time.monotonic() + 90
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError("service exited before readiness")
+        try:
+            with urlopen(url, timeout=1) as response:
+                if response.status < 500:
+                    return
+        except URLError:
+            time.sleep(0.25)
+    raise TimeoutError("service readiness deadline exceeded")
+
+
+def host_smoke(directory: Path) -> int:
+    directory = directory.resolve()
+    suite = ET.Element("testsuite", name="real-functions-host")
+    case = ET.SubElement(suite, "testcase", name="validated_context_and_response_filtering")
+    try:
+        with (
+            tempfile.TemporaryDirectory(prefix="validation-host-") as temporary,
+            ExitStack() as stack,
+        ):
+            work = Path(temporary)
+            for name in ("function_app.py", "host.json"):
+                (work / name).write_bytes((Path("tests/host_smoke") / name).read_bytes())
+            blob, queue, table, port = (free_port() for _ in range(4))
+            key = base64.b64encode(b"validation-local-emulator-only" * 2).decode()
+            emulator = stack.enter_context(
+                child(
+                    [
+                        "env",
+                        f"AZURITE_ACCOUNTS=validationdev:{key}",
+                        "azurite",
+                        "--silent",
+                        "--location",
+                        str(work / "storage"),
+                        "--blobPort",
+                        str(blob),
+                        "--queuePort",
+                        str(queue),
+                        "--tablePort",
+                        str(table),
+                    ],
+                    directory / "logs/azurite.log",
+                )
+            )
+            deadline = time.monotonic() + 30
+            for service in (blob, queue, table):
+                while True:
+                    if emulator.poll() is not None or time.monotonic() >= deadline:
+                        raise TimeoutError("Azurite failed readiness")
+                    try:
+                        with socket.create_connection(("127.0.0.1", service), timeout=1):
+                            break
+                    except OSError:
+                        time.sleep(0.25)
+            (work / "local.settings.json").write_text(
+                json.dumps(
+                    {
+                        "IsEncrypted": False,
+                        "Values": {
+                            "FUNCTIONS_WORKER_RUNTIME": "python",
+                            "AzureWebJobsStorage": (
+                                f"DefaultEndpointsProtocol=http;AccountName=validationdev;AccountKey={key};"
+                                f"BlobEndpoint=http://127.0.0.1:{blob}/validationdev;"
+                                f"QueueEndpoint=http://127.0.0.1:{queue}/validationdev;"
+                                f"TableEndpoint=http://127.0.0.1:{table}/validationdev;"
+                            ),
+                        },
+                    }
+                )
+            )
+            previous_path = os.environ.get("PATH", "")
+            os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + previous_path
+            try:
+                host = stack.enter_context(
+                    child(
+                        ["func", "start", "--python", "--verbose", "--port", str(port)],
+                        directory / "logs/host.log",
+                        work,
+                    )
+                )
+                url = f"http://127.0.0.1:{port}/api/smoke"
+                wait_ready(host, url)
+                with urlopen(url, timeout=5) as response:
+                    assert json.load(response) == {"function_name": "smoke"}
+            finally:
+                os.environ["PATH"] = previous_path
+        assert not re.search(
+            r"0 functions (loaded|found \(Worker\))|No job functions found|"
+            r"Executed 'Functions\..*' \(Failed",
+            (directory / "logs/host.log").read_text(),
+        )
+        code = 0
+    except (OSError, RuntimeError, TimeoutError, AssertionError, ValueError) as error:
+        ET.SubElement(case, "failure").text = str(error)
+        code = 1
+    finally:
+        destination = directory / "junit/junit.xml"
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        ET.ElementTree(suite).write(destination, encoding="unicode")
+    return code
diff --git a/tools/verification_process.py b/tools/verification_process.py
new file mode 100644
index 0000000..e7da4ee
--- /dev/null
+++ b/tools/verification_process.py
@@ -0,0 +1,81 @@
+"""Bounded subprocess ownership and redacted output capture."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+from types import FrameType
+
+from tools.verification_evidence import redact
+
+
+def terminate(process: subprocess.Popen[str]) -> None:
+    if process.poll() is None:
+        try:
+            os.killpg(process.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            process.wait()
+            return
+        try:
+            process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            os.killpg(process.pid, signal.SIGKILL)
+            process.wait()
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        return
+
+
+@contextmanager
+def child(
+    command: list[str], log: Path, cwd: Path | None = None
+) -> Iterator[subprocess.Popen[str]]:
+    log.parent.mkdir(parents=True, exist_ok=True)
+    # Keep raw output anonymous until credentials have been removed.
+    with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output:
+        with subprocess.Popen(
+            command,
+            cwd=cwd,
+            stdout=output,
+            stderr=subprocess.STDOUT,
+            text=True,
+            start_new_session=True,
+        ) as process:
+
+            def interrupted(number: int, frame: FrameType | None) -> None:
+                terminate(process)
+                raise KeyboardInterrupt(number)
+
+            previous = {
+                sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT)
+            }
+            try:
+                yield process
+            finally:
+                terminate(process)
+                for sig, handler in previous.items():
+                    signal.signal(sig, handler)
+                output.seek(0)
+                text = redact(output.read())
+                log.write_text(text, encoding="utf-8")
+                print(text, end="")
+
+
+def run(command: list[str], log: Path, timeout: float = 600) -> int:
+    try:
+        with child(command, log) as process:
+            try:
+                code = process.wait(timeout=timeout)
+                return code if code >= 0 else 128 - code
+            except subprocess.TimeoutExpired:
+                return 124
+    except FileNotFoundError as error:
+        log.parent.mkdir(parents=True, exist_ok=True)
+        log.write_text(redact(str(error)), encoding="utf-8")
+        return 127
diff --git a/tools/verification_wheel.py b/tools/verification_wheel.py
new file mode 100644
index 0000000..065772f
--- /dev/null
+++ b/tools/verification_wheel.py
@@ -0,0 +1,72 @@
+"""Install the built wheel and check schema/import identity outside the checkout."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+import sys
+import sysconfig
+import tempfile
+
+from scripts.check_wheel_schemas import verify_wheel
+from tools.verification_evidence import redact
+from tools.verification_process import run
+
+
+def main() -> int:
+    directory = Path("test-results/wheel-smoke/logs")
+    with tempfile.TemporaryDirectory(prefix="validation-wheel-") as temporary:
+        root = Path(temporary)
+        code = run(
+            [sys.executable, "-m", "build", "--no-isolation", "--outdir", str(root / "dist")],
+            directory / "build.log",
+        )
+        if code:
+            return code
+        wheels = list((root / "dist").glob("*.whl"))
+        if len(wheels) != 1:
+            print("Expected exactly one freshly built wheel in isolated output directory")
+            return 1
+        verify_wheel(wheels[0])
+        subprocess.run(
+            [sys.executable, "-m", "venv", "--system-site-packages", str(root / "venv")], check=True
+        )
+        python = str(root / "venv/bin/python")
+        packages = (
+            root
+            / "venv/lib"
+            / f"python{sys.version_info.major}.{sys.version_info.minor}"
+            / "site-packages"
+        )
+        (packages / "verification-dependencies.pth").write_text(
+            sysconfig.get_path("purelib") + "\n"
+        )
+        code = run(
+            [python, "-m", "pip", "install", "--no-deps", "--no-index", str(wheels[0].resolve())],
+            directory / "wheel-install.log",
+        )
+        if code:
+            return code
+        script = (
+            "from pathlib import Path; from importlib import resources; import hashlib; "
+            "import azure_functions_validation as a; "
+            f"assert Path(a.__file__).resolve().is_relative_to(Path({str(root)!r})); "
+            "assert callable(a.validate_http); "
+            "p=resources.files('azure_functions_validation.schemas'); "
+            "assert hashlib.sha256(p.joinpath('endpoint.schema.json').read_bytes()).hexdigest() "
+            "== p.joinpath('endpoint.schema.sha256').read_text().split()[0]; "
+            "print('Installed wheel imports outside checkout; schema matches pin')"
+        )
+        with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output:
+            result = subprocess.run(
+                [python, "-I", "-c", script], cwd=root, stdout=output, stderr=subprocess.STDOUT
+            )
+            output.seek(0)
+            text = redact(output.read())
+        (directory / "wheel-smoke.log").write_text(text)
+        print(text)
+        return result.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From e4bb4c055efd0069db44261b6f34a47f9fea0fda Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 02/14] ci: standardize local verification command contracts

---
 .gitignore                          |   1 +
 Makefile                            | 236 ++++++++++------------------
 pyproject.toml                      |  12 +-
 tests/test_verification_commands.py |  91 +++++++++++
 4 files changed, 180 insertions(+), 160 deletions(-)
 create mode 100644 tests/test_verification_commands.py

diff --git a/.gitignore b/.gitignore
index 2ff99df..7a001a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ env/
 .coverage
 coverage.xml
 htmlcov/
+test-results/
 
 # Build artifacts
 build/
diff --git a/Makefile b/Makefile
index c781c5f..86dc93f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,177 +1,105 @@
 VENV_DIR := .venv
-PYTHON := $(VENV_DIR)/bin/python
-PIP := $(VENV_DIR)/bin/pip
-HATCH := $(VENV_DIR)/bin/hatch
-
-.PHONY: bootstrap
-bootstrap:
-	@if [ ! -d "$(VENV_DIR)" ]; then \
-		echo "Creating virtual environment..."; \
-		python3 -m venv $(VENV_DIR); \
-	fi
-	@echo "Ensuring Hatch is installed in virtual environment..."
-	@$(PIP) install --upgrade pip > /dev/null
-	@$(PIP) install hatch > /dev/null
-	@echo "Hatch installed at $(HATCH)"
-
-.PHONY: ensure-hatch
-ensure-hatch: bootstrap
+PYTHON ?= python
+HATCH ?= hatch
+ifeq ($(CI),)
+RUN := $(HATCH) run python
+else
+RUN := $(PYTHON)
+endif
+
+.PHONY: bootstrap ensure-hatch
+bootstrap ensure-hatch:
+	@command -v $(HATCH) >/dev/null || { echo "Install Hatch in an isolated tool environment first"; exit 1; }
 
 .PHONY: install
 install: ensure-hatch
 	@$(HATCH) env create
-	@if [ -n "$$CI" ]; then \
-		echo "CI detected: skipping pre-commit hook installation"; \
-	else \
-		$(MAKE) precommit-install; \
-	fi
+	@if [ -z "$$CI" ]; then $(MAKE) precommit-install; fi
+
+.PHONY: install-ci
+install-ci:
+	@$(PYTHON) -c 'import sys; assert (3, 11) <= sys.version_info[:2] < (3, 15), sys.version'
+	@$(PYTHON) -m pip install -e ".[dev,docs]"
 
-.PHONY: shell
+.PHONY: shell reset hatch-clean
 shell: ensure-hatch
 	@$(HATCH) shell
-
-.PHONY: reset
 reset: clean-all install
-	@echo "Project reset complete."
-
-.PHONY: hatch-clean
 hatch-clean: ensure-hatch
-	@$(HATCH) env remove || echo "No hatch environment to remove"
-
-.PHONY: format
-format: ensure-hatch
-	@$(HATCH) run format
-
-.PHONY: format-check
-format-check: ensure-hatch
-	@$(HATCH) run format-check
-
-.PHONY: style
-style: ensure-hatch
-	@$(HATCH) run style
-
-.PHONY: typecheck
-typecheck: ensure-hatch
-	@$(HATCH) run typecheck
-
-.PHONY: lint
-lint: ensure-hatch
-	@$(HATCH) run lint
-
-.PHONY: security
-security: ensure-hatch
-	@$(HATCH) run security
-
-.PHONY: check-schema-hash
-check-schema-hash: ensure-hatch
-	@$(HATCH) run python -c "import hashlib, pathlib, sys; d = pathlib.Path('src/azure_functions_validation/schemas'); expected = (d / 'endpoint.schema.sha256').read_text().split()[0]; actual = hashlib.sha256((d / 'endpoint.schema.json').read_bytes()).hexdigest(); print('endpoint.schema.json digest matches pin.') if expected == actual else sys.exit('endpoint.schema.json digest drift:\n  pinned:  ' + expected + '\n  actual:  ' + actual + '\nUpdate endpoint.schema.sha256 if this change is intentional.')"
-
-.PHONY: lint-workflows
-lint-workflows: ensure-hatch
-	@$(HATCH) run python tools/lint_release_workflows.py
-	@$(HATCH) run python tools/lint_workflow_pins.py
-	@$(HATCH) run python tools/lint_hatch_matrix.py
-
-.PHONY: check
-check: ensure-hatch
-	@$(MAKE) lint
-	@$(MAKE) typecheck
-	@echo "Lint and type check passed."
-
-.PHONY: check-all
-check-all: ensure-hatch
-	@$(MAKE) check-schema-hash
-	@$(MAKE) lint-workflows
-	@$(MAKE) check
-	@$(MAKE) test
-	@$(MAKE) security
-	@echo "All checks passed including tests and security scan."
-
-.PHONY: screenshots-check
-screenshots-check: ensure-hatch
-	@$(HATCH) run python scripts/check_screenshots.py --strict
-
-.PHONY: screenshots-update
-screenshots-update: ensure-hatch
-	@$(HATCH) run python scripts/check_screenshots.py --update
-
-.PHONY: precommit
+	@$(HATCH) env remove
+
+.PHONY: format format-check style typecheck lint security
+format:
+	@$(RUN) -m ruff format src tests tools/verification*.py
+format-check:
+	@$(RUN) -m ruff format --check src tests tools/verification*.py
+style: format-check
+	@$(RUN) -m ruff check src tests tools/verification*.py
+typecheck:
+	@$(RUN) -m mypy src tests tools/verification*.py
+lint: style typecheck
+security:
+	@$(RUN) -m bandit -r src
+
+.PHONY: check-schema-hash contracts lint-workflows check check-all
+check-schema-hash:
+	@$(RUN) -m tools.verification_contracts schema-hash
+contracts:
+	@$(RUN) -m tools.verification contracts
+lint-workflows:
+	@$(RUN) tools/lint_release_workflows.py
+	@$(RUN) tools/lint_workflow_pins.py
+	@$(RUN) tools/lint_hatch_matrix.py
+check: style typecheck security lint-workflows check-schema-hash contracts
+check-all: verify
+
+.PHONY: test integration worker-compat sdk-compat verify report wheel-smoke
+test integration worker-compat sdk-compat verify report:
+	@$(RUN) -m tools.verification $@
+wheel-smoke:
+	@$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel
+
+.PHONY: cov e2e-local e2e-azure build docs docs-serve
+cov:
+	@$(RUN) -m tools.verification test --cov-report=html
+e2e-local:
+	@E2E_BASE_URL=$${E2E_BASE_URL:-http://localhost:7071} $(RUN) -m tools.verification local-e2e
+e2e-azure:
+	@$(RUN) -m tools.verification azure-e2e
+build:
+	@$(RUN) -m tools.verification focused build -m build --no-isolation
+docs:
+	@$(RUN) -m tools.verification focused docs -m mkdocs build --strict
+docs-serve:
+	@$(RUN) -m mkdocs serve
+
+.PHONY: screenshots-check screenshots-update precommit precommit-install publish-test version
+screenshots-check:
+	@$(RUN) scripts/check_screenshots.py --strict
+screenshots-update:
+	@$(RUN) scripts/check_screenshots.py --update
 precommit: ensure-hatch
 	@$(HATCH) run precommit
-
-.PHONY: precommit-install
 precommit-install: ensure-hatch
 	@$(HATCH) run precommit-install
-
-.PHONY: test
-test: ensure-hatch
-	@echo "Running tests..."
-	@$(HATCH) run test
-
-.PHONY: cov
-cov: ensure-hatch
-	@$(HATCH) run cov
-	@echo "Open htmlcov/index.html in your browser to view the coverage report."
-	@echo "coverage.xml generated for Codecov upload."
-
-.PHONY: e2e-local
-e2e-local: ensure-hatch
-	@echo "Running e2e tests against local Azurite (E2E_BASE_URL=http://localhost:7071)..."
-	@E2E_BASE_URL=http://localhost:7071 $(HATCH) run e2e-azure
-
-.PHONY: e2e-azure
-e2e-azure: ensure-hatch
-	@echo "Running e2e tests against Azure (E2E_BASE_URL must be set)..."
-	@$(HATCH) run e2e-azure
-
-.PHONY: build
-build: ensure-hatch
-	@$(HATCH) build
-
-.PHONY: publish-test
 publish-test: ensure-hatch
 	@$(HATCH) publish --repo test
-
-.PHONY: version
 version: ensure-hatch
-	@echo "Current version:"
 	@$(HATCH) version
 
-.PHONY: docs
-docs:
-	@if [ -n "$$CI" ]; then \
-		echo "CI detected: running MkDocs directly"; \
-		python -m pip install --upgrade pip > /dev/null 2>&1 || true; \
-		pip install "mkdocs<2.0" "mkdocs-material<10.0" "mkdocstrings[python]<1.0" > /dev/null 2>&1; \
-		mkdocs build; \
-	else \
-		$(MAKE) ensure-hatch > /dev/null; \
-		$(HATCH) run mkdocs build; \
-	fi
-
-.PHONY: docs-serve
-docs-serve: ensure-hatch
-	@$(HATCH) run docs
-
-.PHONY: doctor
+.PHONY: doctor clean clean-all help
 doctor:
-	@echo "Python version:" && $(PYTHON) --version
-	@echo "Installed packages:" && $(HATCH) env run pip list || echo "No hatch environment found"
-	@echo "Azure Function Core Tools version:" && func --version || echo "func not found"
-	@echo "Pre-commit hook installed:"
-	@if [ -f .git/hooks/pre-commit ]; then echo yes; else echo no; fi
-
-.PHONY: clean
+	@$(RUN) -m tools.verification doctor
 clean:
-	@rm -rf *.egg-info dist build __pycache__ .pytest_cache
-
-.PHONY: clean-all
+	@$(RUN) -m tools.verification clean
+# Historical deep cleanup is intentionally separate from artifact-only clean.
 clean-all: clean
-	@find . -type d -name "__pycache__" -exec rm -rf {} +
-	@find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -delete
-	@rm -rf .mypy_cache .ruff_cache .pytest_cache .coverage coverage.xml htmlcov .DS_Store $(VENV_DIR) site
-
-.PHONY: help
+	@find . -type d -name "__pycache__" -not -path "./.git/*" -exec rm -rf {} +
+	@find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -not -path "./.git/*" -delete
+	@rm -rf .DS_Store $(VENV_DIR)
 help:
-	@echo "Available commands:" && \
-	grep -E '^\.PHONY: ' Makefile | cut -d ':' -f2 | xargs -n1 echo "  - make"
+	@echo "Canonical: install install-ci doctor format format-check check test integration verify e2e-azure report clean"
+	@echo "check-all: exact backward-compatible alias of verify (requires func and Azurite)"
+	@echo "Partial/focused: style lint typecheck security check-schema-hash contracts lint-workflows cov worker-compat sdk-compat build wheel-smoke docs docs-serve screenshots-check screenshots-update"
+	@echo "Local only, not Azure certification: e2e-local; SDK spike only: sdk-compat"
+	@echo "Maintenance: bootstrap ensure-hatch shell reset hatch-clean clean-all precommit precommit-install publish-test version"
diff --git a/pyproject.toml b/pyproject.toml
index 1a62864..dd11514 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -104,14 +104,14 @@ python = "3.11"
 features = ["dev", "docs"]
 
 [tool.hatch.envs.default.scripts]
-format = "ruff format src tests"
-format-check = "ruff format --check src tests"
-style = "ruff check src tests && ruff format --check src tests"
-typecheck = "mypy src tests"
+format = "ruff format src tests tools/verification*.py"
+format-check = "ruff format --check src tests tools/verification*.py"
+style = "ruff check src tests tools/verification*.py && ruff format --check src tests tools/verification*.py"
+typecheck = "mypy src tests tools/verification*.py"
 lint = "hatch run style && hatch run typecheck"
-test = "pytest -v tests"
+test = "python -m tools.verification test"
 cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/"
-e2e-azure = "pytest -v -m e2e tests/e2e/ --no-cov"
+e2e-azure = "python -m tools.verification azure-e2e"
 docs = "mkdocs serve"
 precommit = "pre-commit run --all-files"
 precommit-install = "pre-commit install"
diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py
new file mode 100644
index 0000000..743e995
--- /dev/null
+++ b/tests/test_verification_commands.py
@@ -0,0 +1,91 @@
+"""Regression contracts for the canonical developer commands."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+CONTRACT = (
+    "install",
+    "install-ci",
+    "doctor",
+    "format",
+    "format-check",
+    "check",
+    "test",
+    "integration",
+    "verify",
+    "e2e-azure",
+    "report",
+    "clean",
+)
+
+
+@pytest.mark.parametrize("target", CONTRACT)
+def test_make_contract_is_install_free_on_routine_runs(target: str) -> None:
+    # Given the public command contract, when dry-running a command.
+    result = subprocess.run(["make", "-n", target], cwd=ROOT, capture_output=True, text=True)
+    # Then routine runs never upgrade pip or bootstrap Hatch.
+    assert result.returncode == 0, result.stderr
+    assert "install --upgrade pip" not in result.stdout
+    if target not in {"install", "install-ci"}:
+        assert "pip install hatch" not in result.stdout
+
+
+def test_check_runs_types_once_and_read_only_contract_checks() -> None:
+    # Given the complete quality command, when expanding its recipe.
+    result = subprocess.run(["make", "-n", "check"], cwd=ROOT, capture_output=True, text=True)
+    # Then types run once and pinned contracts are checked, never regenerated.
+    assert result.returncode == 0
+    assert result.stdout.count("-m mypy") == 1
+    assert "schema-hash" in result.stdout
+    assert "contracts" in result.stdout
+    assert "--update" not in result.stdout
+
+
+def test_schema_mismatch_does_not_rewrite_pin(tmp_path: Path) -> None:
+    # Given a checkout-shaped directory with an intentionally stale schema pin.
+    schema = tmp_path / "src/azure_functions_validation/schemas"
+    schema.mkdir(parents=True)
+    (schema / "endpoint.schema.json").write_text("{}\n")
+    pin = schema / "endpoint.schema.sha256"
+    before = b"incorrect-pin  endpoint.schema.json\n"
+    pin.write_bytes(before)
+    environment = dict(os.environ, PYTHONPATH=str(ROOT))
+    # When invoking the same non-mutating checker used by make check.
+    result = subprocess.run(
+        [sys.executable, "-m", "tools.verification_contracts", "schema-hash"],
+        cwd=tmp_path,
+        env=environment,
+        capture_output=True,
+        text=True,
+    )
+    # Then drift fails without changing pinned metadata bytes.
+    assert result.returncode != 0
+    assert "digest drift" in result.stdout
+    assert pin.read_bytes() == before
+
+
+def test_unconfigured_azure_saves_incomplete(tmp_path: Path) -> None:
+    # Given no Azure URL and an isolated evidence root.
+    environment = dict(os.environ, PYTHONPATH=str(ROOT))
+    environment.pop("E2E_BASE_URL", None)
+    # When using the canonical CLI surface.
+    result = subprocess.run(
+        [sys.executable, "-m", "tools.verification", "azure-e2e"],
+        cwd=tmp_path,
+        env=environment,
+        capture_output=True,
+        text=True,
+    )
+    # Then configuration absence is saved as INCOMPLETE, not certification.
+    assert result.returncode != 0
+    data = json.loads((tmp_path / "test-results/azure-e2e/summary.json").read_text())
+    assert data["status"] == "INCOMPLETE"
+    assert data["selected"] == 0

From a608bc1f175a622fe208f1734913bd890188c5c4 Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 03/14] ci: reuse canonical runners across matrix and worker
 checks

---
 .github/workflows/ci-test.yml        | 107 ++++++++++++++-------------
 .github/workflows/worker-nightly.yml |  11 ++-
 tests/test_verification_ci.py        |  62 ++++++++++++++++
 3 files changed, 127 insertions(+), 53 deletions(-)
 create mode 100644 tests/test_verification_ci.py

diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index 53fbe38..b827fb8 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -93,28 +93,17 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: Install package with dev extras
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
-
-      - name: Check endpoint schema hash pin
-        run: pytest --no-cov tests/test_endpoint_schema.py::TestSchemaHashPin
-
-      - name: Lint workflows (pin hygiene + release-gate canonical SHAs)
-        run: |
-          python tools/lint_release_workflows.py
-          python tools/lint_workflow_pins.py
-
-      - name: Hatch/matrix hygiene lint
-        run: python tools/lint_hatch_matrix.py
+        run: make install-ci
 
-      - name: Ruff style check
-        run: |
-          ruff check src tests
-          ruff format --check src tests
+      - name: Complete non-mutating quality and contract gate
+        run: make check
 
-      - name: Type check (mypy, strict)
-        run: mypy src tests
+      - name: Upload quality evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: quality-evidence
+          path: test-results/**
 
   test:
     name: Test (Py ${{ matrix.python-version }})
@@ -157,9 +146,7 @@ jobs:
           PY
 
       - name: Install package with dev extras (on the matrix interpreter)
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+        run: make install-ci
 
       - name: Run tests on the matrix interpreter
         # Raw pytest on the interpreter provisioned above (NOT through the
@@ -171,26 +158,36 @@ jobs:
           IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' }}
         run: |
           if [ "$IS_COVERAGE_CELL" = "true" ]; then
-            pytest tests/
+            python -m tools.verification test
           else
-            pytest tests/ --no-cov
+            python -m tools.verification test --no-cov
           fi
 
       - name: Verify coverage output
         if: matrix.python-version == '3.11'
-        run: ls -lh coverage.xml
+        run: ls -lh test-results/test/coverage.xml
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.11' && github.actor != 'dependabot[bot]'
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.xml
+          files: ./test-results/test/coverage.xml
           verbose: true
           fail_ci_if_error: false
         env:
           CODECOV_RETRIES: 3
 
+      - name: Required worker signature and registration evidence
+        run: python -m tools.verification worker-compat
+
+      - name: Upload test and worker evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-evidence-py${{ matrix.python-version }}
+          path: test-results/**
+
   docs-check:
     name: Documentation checks
     needs: changes
@@ -211,7 +208,14 @@ jobs:
         run: pip install -e ".[docs]" pytest pytest-cov PyYAML
 
       - name: Build docs (strict)
-        run: mkdocs build --strict
+        run: make docs
+
+      - name: Upload docs evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: docs-evidence
+          path: test-results/**
 
       - name: Check README examples and screenshot manifest
         run: |
@@ -297,7 +301,14 @@ jobs:
           PY
 
       - name: Run full suite against minimum runtime dependencies
-        run: pytest tests/ --no-cov
+        run: python -m tools.verification test --no-cov
+
+      - name: Upload minimum-dependency evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: minimum-dependencies-evidence
+          path: test-results/**
 
   artifact-build:
     name: Build compatibility artifacts
@@ -444,26 +455,17 @@ jobs:
           npm install --global azurite
 
       - name: Install smoke app dependencies
-        working-directory: tests/host_smoke
-        run: pip install -r requirements.txt
+        run: make install-ci
 
-      - name: Start Azurite and Functions host
-        run: |
-          azurite --silent > /tmp/azurite.log 2>&1 &
-          cd tests/host_smoke
-          func start --python --port 7071 > /tmp/functions-host.log 2>&1 &
+      - name: Managed real Functions host and worker signature smoke
+        run: python -m tools.verification integration
 
-      - name: Call validated handler
-        run: |
-          for attempt in {1..30}; do
-            if response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke)"; then
-              python -c 'import json, sys; assert json.loads(sys.argv[1]) == {"function_name": "smoke"}' "$response"
-              exit 0
-            fi
-            sleep 2
-          done
-          cat /tmp/functions-host.log
-          exit 1
+      - name: Upload host and worker evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: host-evidence-py${{ matrix.python-version }}
+          path: test-results/**
 
   # azure-functions 2.x compatibility proof lane (issue #351), modeled on
   # langgraph d433b63. This is a SINGLE representative 2.x lane, not a matrix:
@@ -528,6 +530,11 @@ jobs:
         # pythonpath so the installed WHEEL is exercised, not the source tree.
         env:
           AZFUNC_2X_SPIKE: "1"
-        run: |
-          pytest -m compat2x -o addopts= -o pythonpath= \
-            tests/test_worker_compat_2x_spike.py -v
+        run: python -m tools.verification sdk-compat -v
+
+      - name: Upload SDK compatibility spike evidence (not Azure certification)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: sdk-2x-evidence
+          path: test-results/**
diff --git a/.github/workflows/worker-nightly.yml b/.github/workflows/worker-nightly.yml
index 67cb6b5..44411fb 100644
--- a/.github/workflows/worker-nightly.yml
+++ b/.github/workflows/worker-nightly.yml
@@ -61,9 +61,14 @@ jobs:
           python -c "import azure.functions as f; print('azure-functions', f.__version__)"
 
       - name: Run WorkerCompat suite against pre-release worker library
-        run: |
-          python -m pytest tests/test_decorator.py tests/test_integration.py \
-            -v --no-header -p no:cacheprovider --no-cov
+        run: python -m tools.verification worker-compat -v --no-header -p no:cacheprovider
+
+      - name: Upload pre-release worker evidence (not Azure certification)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: worker-prerelease-py${{ matrix.python-version }}
+          path: test-results/**
 
       - name: Explain a WorkerCompat break
         if: failure()
diff --git a/tests/test_verification_ci.py b/tests/test_verification_ci.py
new file mode 100644
index 0000000..a4b8f91
--- /dev/null
+++ b/tests/test_verification_ci.py
@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_ci_uses_canonical_runners_and_failure_artifacts() -> None:
+    workflows = ROOT / ".github/workflows"
+    ci = (workflows / "ci-test.yml").read_text()
+    assert 'python-version: ["3.11", "3.12", "3.13", "3.14"]' in ci
+    assert "python -m tools.verification test --no-cov" in ci
+    assert "python -m tools.verification integration" in ci
+    assert "python -m tools.verification sdk-compat" in ci
+    assert "make install-ci" in ci
+    for name in ("ci-test.yml", "e2e-azure.yml", "publish-pypi.yml", "worker-nightly.yml"):
+        text = (workflows / name).read_text()
+        assert "path: test-results/**" in text
+        blocks = text.split("      - name:")
+        evidence = [block for block in blocks if "path: test-results/**" in block]
+        assert evidence
+        assert all("if: always()" in block for block in evidence)
+
+
+def test_check_all_is_exact_verify_alias() -> None:
+    text = (ROOT / "Makefile").read_text()
+    assert "check-all: verify\n\n" in text
+    result = subprocess.run(
+        ["make", "--no-print-directory", "-n", "test", "CI=true", "PYTHON=matrix-python"],
+        cwd=ROOT,
+        text=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+    assert result.stdout.strip() == "matrix-python -m tools.verification test"
+
+
+def test_clean_all_retains_deep_cleanup_contract() -> None:
+    result = subprocess.run(
+        ["make", "-n", "clean-all"],
+        cwd=ROOT,
+        text=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+    assert "-m tools.verification clean" in result.stdout
+    assert "__pycache__" in result.stdout
+    assert "rm -rf .DS_Store .venv" in result.stdout
+
+
+def test_schema_and_wheel_commands_are_non_mutating() -> None:
+    result = subprocess.run(
+        ["make", "-n", "check-schema-hash", "wheel-smoke"],
+        cwd=ROOT,
+        text=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+    assert "--update" not in result.stdout
+    assert "tools.verification_contracts schema-hash" in result.stdout
+    assert "tools.verification_wheel" in result.stdout

From 681f3e185e47e5a4ce1fd3300e77f71f18fb3be7 Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 04/14] ci: retain canonical release and azure test evidence

---
 .github/workflows/e2e-azure.yml    |  4 ++--
 .github/workflows/publish-pypi.yml | 13 ++++++++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e-azure.yml b/.github/workflows/e2e-azure.yml
index 21e12b8..9c671ac 100644
--- a/.github/workflows/e2e-azure.yml
+++ b/.github/workflows/e2e-azure.yml
@@ -218,7 +218,7 @@ jobs:
           echo "=== Live logstream (20s) ==="
           timeout 20 az webapp log tail --name "$APP" --resource-group "$RG" 2>&1 | head -200 || true
       - name: Run e2e tests
-        run: pytest -v -m e2e tests/e2e/ --no-cov --html=e2e-report/report.html --self-contained-html
+        run: python -m tools.verification azure-e2e -v --html=test-results/azure-e2e/report.html --self-contained-html
         env:
           E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net
 
@@ -287,7 +287,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
-          path: e2e-report/
+          path: test-results/**
           retention-days: 30
 
   cleanup:
diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
index f513d85..eabfa6b 100644
--- a/.github/workflows/publish-pypi.yml
+++ b/.github/workflows/publish-pypi.yml
@@ -108,11 +108,18 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch
-        run: pip install hatch
+      - name: Install test dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run test suite
-        run: hatch run test
+        run: python -m tools.verification test
+
+      - name: Upload release library evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: release-library-evidence
+          path: test-results/**
 
   cookbook-smoke:
     name: Cookbook dogfood smoke (candidate wheel)

From 7399849f9f7db14181dbf8150da2259d3541b3ab Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 05/14] docs: align agent validation and golden commands

---
 AGENTS.md | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 83144be..25fe886 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -18,7 +18,7 @@
 
 ### Test Coverage
 - Maintain test coverage at **95% or above** for committed changes and PRs.
-- Run `hatch run pytest --cov --cov-report=term-missing -q` to verify before submitting changes.
+- Run `make test` to verify the offline suite and coverage before submitting changes.
 - Any PR that drops coverage below 95% must include additional tests to compensate.
 - Runtime code must remain compatible with Python 3.11-3.14 (`requires-python = ">=3.11,<3.15"`).
 - Public APIs must be fully typed.
@@ -106,10 +106,11 @@ This repository is **issue-based, not milestone-based**. Track and group work us
 - Before adding any organizational structure, check whether the repository already has an established convention. A category being empty or unused (zero milestones, no `epic:*` labels) is evidence to follow the existing pattern, not to introduce a new one.
 
 ## Validation
-- `make test`
-- `make lint`
-- `make typecheck`
-- `make build`
+- `make doctor` (read-only prerequisites, actual interpreter)
+- `make check` (complete non-mutating quality and schema/metadata contracts)
+- `make test` (offline, coverage >=95%)
+- `make integration` (worker signatures and real host/private Azurite)
+- `make verify` (`make check-all` is an exact alias)
 
 ## Release Process
 
@@ -186,22 +187,31 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor
 | Purpose | Command |
 | --- | --- |
 | Environment setup | `make install` |
+| CI-selected interpreter setup | `make install-ci` |
+| Read-only prerequisite diagnostics | `make doctor` |
 | Format code | `make format` |
 | Check formatting (`src`, `tests`) | `make format-check` |
-| Lint | `make lint` |
-| Type check | `make typecheck` |
-| Tests | `make test` |
-| Coverage | `make cov` |
-| Full validation | `make check-all` |
-| Docs build | `make docs` |
-| Package build | `make build` |
+| Complete non-mutating quality/contracts | `make check` |
+| Offline tests, coverage >=95% | `make test` |
+| Managed worker/host/emulator lanes | `make integration` |
+| Full local validation | `make verify` (`make check-all` exact alias) |
+| Configured real Azure certification | `make e2e-azure` |
+| Render saved evidence only | `make report` |
+| Generated artifacts only | `make clean` |
+| Partial/focused lint/types/coverage | `make lint`, `make typecheck`, `make cov` |
+| Partial/focused docs/package/wheel | `make docs`, `make build`, `make wheel-smoke` |
+
+See CONTRIBUTING.md for evidence/status semantics. Required worker and host lanes
+have separate summaries under gitignored `test-results/`. SDK spikes are not Azure
+certification. Verification must never rewrite schema/hash pins. `clean-all` is
+maintenance-only deep cleanup, including bytecode and the project-local `.venv`.
 
 ## Testing Rules
 
 - Public APIs require tests.
 - Bug fixes require regression tests.
 - Representative and complex examples must remain smoke-tested.
-- `make check-all` is the minimum merge gate.
+- `make verify` is the minimum local merge gate; `make check-all` aliases it exactly.
 
 ## Commit Rules
 

From ec7bf661ac33486134f7850bfc98384908a7c7bd Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 06/14] docs: define verification commands and evidence
 semantics

---
 CONTRIBUTING.md | 85 +++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1c1aef2..eb0af7f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,11 +6,12 @@ We welcome contributions to the `azure-functions-validation` project.
 
 The package supports Python 3.11, 3.12, 3.13, and 3.14. Local development
 requires Python 3.11 because the default Hatch environment is pinned to that
-interpreter. No repository-specific services or infrastructure are required
-for the standard checks.
+interpreter. Offline tests and quality checks need no services. Full local
+verification additionally requires Functions Core Tools (`func`) and Azurite.
 
-Clone the repository, enter it, and let the Makefile create a local virtual
-environment and install Hatch:
+Install Hatch once in an isolated tool environment, then clone the repository
+and create the development environment. Routine checks never upgrade pip or
+reinstall Hatch, and never change the global interpreter:
 
 ```bash
 git clone https://github.com/yeongseon/azure-functions-validation-python.git
@@ -22,12 +23,13 @@ Run the standard verification commands before opening a pull request:
 
 ```bash
 make test
-make lint
-make typecheck
-make build
+make check
+make verify
 ```
 
-`make check-all` runs the complete local quality gate used for contributions.
+`make check-all` is an exact backward-compatible alias of `make verify`.
+CI uses `make install-ci PYTHON=python` on the explicitly selected matrix
+interpreter, not Hatch's pinned Python 3.11 environment.
 
 ## Branch Strategy
 
@@ -61,17 +63,66 @@ Recommended branch prefixes:
 
 ## Project Commands
 
-```bash
-make format      # Format code with ruff
-make lint        # Lint with ruff
-make typecheck   # Type check with mypy
-make test        # Run tests
-make cov         # Run tests with coverage
-make check-all   # Run the full local gate
+| Command | Contract |
+| --- | --- |
+| `make install` | Create isolated Hatch dev/docs environment and install local hooks |
+| `make install-ci` | Install extras into CI's selected `PYTHON`, preserving the matrix cell |
+| `make doctor` | Read-only prerequisite check; fail for missing required tools |
+| `make format` / `make format-check` | Mutating format action / non-mutating check |
+| `make check` | Ruff lint/format, strict mypy once, Bandit, workflow policy and pinned metadata/schema contracts |
+| `make test` | Offline unit/mock suite only, existing 95% coverage floor |
+| `make integration` | Required worker signatures/registration plus managed real host and private Azurite |
+| `make verify` | Always check + test + integration + wheel/schema install smoke + strict docs |
+| `make e2e-azure` | Separate real Azure run against explicit `E2E_BASE_URL`; never provisions resources |
+| `make report` | Render saved results without rerunning tests |
+| `make clean` | Remove generated artifacts only; preserve environments/shared emulator storage |
+
+`make check-all` means exactly `make verify`. `make clean-all` retains its historical
+deep-clean meaning: artifacts, bytecode and the project-local `.venv`.
+Focused/partial targets remain: `style`, `lint`, `typecheck`, `security`, `cov`,
+`lint-workflows`, `check-schema-hash`, `contracts`, `worker-compat`, `sdk-compat`,
+`build`, `wheel-smoke`, `docs`, screenshot checks and maintenance commands.
+`sdk-compat` requires `AZFUNC_2X_SPIKE=1` and tests an installed wheel without source
+path injection; it is a compatibility spike, never general Azure certification.
+`e2e-local` targets an already-running local app and is not Azure certification.
+
+### Verification evidence
+
+Generated evidence is gitignored:
+
+```text
+test-results/
+  test/{junit.xml,coverage.xml,summary.json,environment.json,logs/}
+  worker-compat/{junit.xml,summary.json,environment.json,logs/}
+  host-smoke/{junit/junit.xml,summary.json,environment.json,logs/}
+  integration/{summary.json,environment.json,lanes.json}
+  verify/{summary.json,environment.json,lanes.json,logs/}
+  azure-e2e/{junit.xml,summary.json,environment.json,logs/}
 ```
 
-Before opening a PR, run `make format-check` to check `src` and `tests` without
-changing files. Run `make format` to fix those trees. CI also checks every
+Each summary has a schema version, lane/command, selected/passed/failed/skipped
+counts, exit status and supporting paths. Actual Python/runtime/SDK versions are
+recorded; credentials, connection strings, authorization headers, SAS and tokens
+are redacted. Console and Actions Job Summary use the same saved fields; CI uploads
+`test-results/**` with `if: always()`.
+
+- **PASS** requires actual execution and successful required checks.
+- **FAIL** means a process/check/test failed or timed out.
+- **INCOMPLETE** means missing prerequisites, missing/invalid JUnit, zero tests,
+  all skipped, or a skipped required live/worker test. These exit non-zero.
+- **NOT_APPLICABLE** is reserved for explicitly unsupported lanes, never missing tools.
+
+Host smoke uses temporary storage, unique ephemeral ports and bounded readiness;
+it stops only processes it owns, including on interruption. Core Tools may need
+its extension bundle cached. Worker signature evidence and host evidence stay
+separate. Local verification never implies real Azure certification. Azure release
+certification keeps the existing exact-ref/source-version record; this repository's
+live endpoint does not currently expose a deployed package-version assertion.
+Schema/hash checks never rewrite pinned metadata. Wheel smoke verifies packaged
+schema bytes and installed imports outside the checkout, separately from source tests.
+
+Before opening a PR, run `make format-check` to check `src`, `tests` and verification
+helpers without changing files. Run `make format` to fix those trees. CI also checks every
 changed Python file, including files outside `src` and `tests`.
 
 ## GitHub Actions Pinning

From 01bde5e2bf0659dba960df19d663c3ade4f655cf Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 07/14] docs: document local and ci verification parity

---
 docs/development.md | 41 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/docs/development.md b/docs/development.md
index f0a6e9e..f98bfb4 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -118,22 +118,53 @@ Use these as the **golden commands** for local validation and CI parity. Prefer
 | Target | Description |
 |--------|-------------|
 | `make install` | Create Hatch env and install pre-commit hooks |
+| `make install-ci` | Install extras into CI's explicitly selected interpreter |
 | `make format` | Format code (ruff) |
 | `make lint` | Run linter (ruff + mypy) |
 | `make typecheck` | Run mypy type checking |
 | `make security` | Run Bandit security scan |
-| `make test` | Run pytest |
+| `make test` | Offline unit/mock pytest, coverage >=95%, saved JUnit/coverage/summary |
 | `make cov` | Run tests with coverage |
-| `make check` | Run lint + typecheck |
-| `make check-all` | Run lint + typecheck + test |
+| `make check` | Non-mutating Ruff lint/format, mypy once, Bandit, workflow and pinned schema/metadata contracts |
+| `make integration` | Required worker signatures plus managed real host/private Azurite |
+| `make verify` | Check + test + integration + wheel/schema install smoke + strict docs |
+| `make check-all` | Exact backward-compatible alias of verify |
+| `make e2e-azure` | Real Azure suite against explicit E2E_BASE_URL, no provisioning |
+| `make report` | Render saved evidence, never rerun tests |
 | `make build` | Build package |
 | `make publish-test` | Publish to TestPyPI |
 | `make precommit` | Run all pre-commit hooks |
 | `make precommit-install` | Install pre-commit hooks |
-| `make doctor` | Show environment diagnostic info |
-| `make clean` | Remove build artifacts |
+| `make doctor` | Read-only prerequisites; missing required tools fail non-zero |
+| `make clean` | Generated artifacts only, preserve environments/shared storage |
 | `make clean-all` | Deep clean (caches, coverage, venv) |
 
+Install Hatch once in an isolated tool environment. Routine commands never upgrade
+pip or reinstall Hatch. Local commands use Hatch's Python 3.11; CI runs on its
+selected Python directly, preserving Python/SDK variants. Full local verification
+requires `func` and `azurite` on PATH; offline tests require neither.
+
+The canonical contract is `install`, `install-ci`, `doctor`, `format`, `format-check`,
+`check`, `test`, `integration`, `verify`, `e2e-azure`, `report`, `clean`. Other targets
+are focused/partial or maintenance. `sdk-compat` requires `AZFUNC_2X_SPIKE=1` and an
+installed wheel; neither it nor `e2e-local` is Azure certification.
+
+Evidence lives in gitignored `test-results/`: `test/` has JUnit and coverage XML;
+`worker-compat/` and `host-smoke/` each have their own summary, JUnit and logs;
+`integration/` aggregates those lanes; `verify/` adds lane results/environment;
+`azure-e2e/` has separate live evidence. Each summary includes schema version,
+command, selected/passed/failed/skipped counts, status and supporting paths.
+Actual interpreter/SDK versions are captured and secrets are redacted.
+Console and Actions Job Summary render the same fields; uploads run even on failure.
+
+PASS requires execution; failed checks/timeouts are FAIL; zero/all-skipped/missing
+prerequisites are INCOMPLETE and non-zero. NOT_APPLICABLE is only for explicitly
+unsupported lanes. Required live/worker skips cannot pass. Integration owns temporary
+storage and processes on ephemeral ports and never uses shared emulator volumes.
+Schema/metadata checks never rewrite pins. Wheel imports/schema resources are checked
+outside the source tree. Azure certification retains its exact-ref/version record,
+not an inferred result from local success. See the root CONTRIBUTING.md for details.
+
 ---
 
 ## Tips

From 36b455397b924b04f64453a59f37b5323589498a Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:07 +0900
Subject: [PATCH 08/14] docs: distinguish offline host and azure verification

---
 docs/contributing.md |  8 +++++++-
 docs/testing.md      | 11 ++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/docs/contributing.md b/docs/contributing.md
index 8f2fd46..276d8ac 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -61,7 +61,13 @@ We maintain strict quality standards to ensure the reliability of the validation
 Tool versions are pinned in the `dev` dependency group of `pyproject.toml` — that
 file is the single source of truth, so this guide does not restate the numbers.
 
-Run `make check-all` to execute all these tools locally before pushing your changes.
+Run `make verify` (exact alias: `make check-all`) before pushing. It always includes
+quality/contracts, offline tests, worker signature/registration, managed real host
+and private Azurite, wheel/schema install smoke and strict docs. Core Tools and
+Azurite are required for the full gate; `make check` and `make test` stay offline.
+Other targets such as `lint`, `typecheck`, `security`, `cov`, `build` and `docs` are
+focused/partial. `make report` renders saved redacted `test-results/`; see
+[Development Guide](development.md#makefile-targets) for commands and status semantics.
 
 ## Testing Requirements
 
diff --git a/docs/testing.md b/docs/testing.md
index 3d6bf92..7caa3d7 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -79,7 +79,7 @@ The project uses `pytest` as its primary testing framework. You can run the test
 The simplest way to run tests is using the provided `Makefile` targets:
 
 ```bash
-# Run all tests
+# Run offline unit/mock tests (no Azure, host or emulator)
 make test
 
 # Run tests and generate a coverage report
@@ -88,6 +88,15 @@ make cov
 
 The `make cov` command generates a terminal report, an XML report for CI integration, and a detailed HTML report in the `htmlcov/` directory.
 
+`make integration` separately runs required worker signature/registration checks
+and the real validation handler on a managed Functions host with private Azurite.
+Each lane has its own `test-results/` summary. `make verify` (exact alias:
+`make check-all`) always adds quality/schema/metadata contracts, offline coverage,
+integration, installed wheel/schema smoke and strict docs. `make report` reads saved
+evidence only. Missing prerequisites, zero execution and skipped required lanes fail
+closed; local success and opt-in SDK compatibility never certify real Azure.
+See [Development Guide](development.md#makefile-targets) for the full contract.
+
 ### Direct Commands
 
 If you prefer to run commands manually via `hatch`:

From 9203fdb531d07149303408f70a2ce93f4a83fcb8 Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 00:11:08 +0900
Subject: [PATCH 09/14] docs: connect schema and agent guidance to verification
 gates

---
 docs/METADATA_SPEC.md  |  2 +-
 docs/agent-playbook.md | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/docs/METADATA_SPEC.md b/docs/METADATA_SPEC.md
index baac284..f891c96 100644
--- a/docs/METADATA_SPEC.md
+++ b/docs/METADATA_SPEC.md
@@ -50,7 +50,7 @@ its own local conformance tests against the versioned dict convention.
   own their own local conformance tests against the same versioned dict
   convention (same philosophy as the vendored `_metadata_helpers.py`).
 - Local drift is caught mechanically: `endpoint.schema.sha256` pins the file
-  digest and `make check-schema-hash` (wired into `make check-all`) fails on
+digest and `make check-schema-hash` (wired into `make check` and therefore `make verify` / `make check-all`) fails on
   any change. When you intentionally change the schema, update the pin and
   bump the `version` field if the change is breaking.
 
diff --git a/docs/agent-playbook.md b/docs/agent-playbook.md
index 2520f50..6b33b9b 100644
--- a/docs/agent-playbook.md
+++ b/docs/agent-playbook.md
@@ -32,8 +32,13 @@ advisory, not a required check — it does not change the zero-native-approval m
 required CI contexts in `AGENTS.md`.
 
 ## Validation
-- `make test`
-- `make lint`
-- `make typecheck`
-- `make security`
-- `make build`
+- `make doctor`: read-only prerequisites and actual Python interpreter.
+- `make check`: complete non-mutating quality/schema/metadata contracts.
+- `make test`: offline unit/mock suite with 95% coverage.
+- `make integration`: separate worker signatures and managed host/Azurite evidence.
+- `make verify`: full local gate; `make check-all` is an exact alias.
+- `make e2e-azure`: separately configured Azure certification, never provisioning.
+- `make report`: render saved `test-results/` without rerunning tests.
+- `make clean`: generated artifacts only; `clean-all` also removes local `.venv`.
+
+Other useful targets are focused/partial. See [Development Guide](development.md#makefile-targets).

From 336687c688601f771e5e4ce30bcc599985917abd Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 07:10:38 +0900
Subject: [PATCH 10/14] ci: parse generated test reports without semgrep's
 unsafe xml parse rule

---
 tools/verification_evidence.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py
index ae2af87..866f38f 100644
--- a/tools/verification_evidence.py
+++ b/tools/verification_evidence.py
@@ -17,6 +17,14 @@
 Status = Literal["PASS", "FAIL", "INCOMPLETE", "NOT_APPLICABLE"]
 
 
+def read_xml_root(path: Path) -> ET.Element:
+    """Parse a locally generated report, refusing entity declarations."""
+    data = path.read_bytes()
+    if b"<!ENTITY" in data.upper():
+        raise ValueError("XML reports must not declare entities")
+    return ET.fromstring(data)
+
+
 @dataclass(frozen=True, slots=True)
 class Summary:
     schema_version: int
@@ -81,7 +89,7 @@ def collect(directory: Path, command: list[str], exit_code: int) -> Summary:
     cases: list[ET.Element] = []
     for path in directory.rglob("*.xml"):
         if "junit" in path.name:
-            cases.extend(ET.parse(path).getroot().iter("testcase"))
+            cases.extend(read_xml_root(path).iter("testcase"))
     skipped = sum(case.find("skipped") is not None for case in cases)
     failed = sum(
         case.find("failure") is not None or case.find("error") is not None for case in cases
@@ -97,7 +105,7 @@ def collect(directory: Path, command: list[str], exit_code: int) -> Summary:
     coverage_path = directory / "coverage.xml"
     coverage = None
     if coverage_path.exists():
-        coverage = float(ET.parse(coverage_path).getroot().get("line-rate", "0"))
+        coverage = float(read_xml_root(coverage_path).get("line-rate", "0"))
     return Summary(
         1,
         directory.name,

From 473cdc2cb8a97e9de946926fe0c75be6a7ea3ff8 Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 07:10:38 +0900
Subject: [PATCH 11/14] ci(security): install the selected interpreter's tools
 before the bandit job

---
 .github/workflows/security.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
index 6f76dbd..be26acc 100644
--- a/.github/workflows/security.yml
+++ b/.github/workflows/security.yml
@@ -23,10 +23,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch and dependencies via Makefile
-        run: |
-          pip install hatch
-          make install
+      - name: Install dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run Bandit
         run: make security

From 2865a7ab03f018be44dddfdcdceaab7877b0762c Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 09:42:45 +0900
Subject: [PATCH 12/14] ci(evidence): redact complete credentials and retry
 host readiness

---
 tests/test_verification_review.py | 109 ++++++++++++++++++++++++++++++
 tools/verification_evidence.py    |  98 ++++++++++++++++++++-------
 tools/verification_host.py        |   4 +-
 3 files changed, 184 insertions(+), 27 deletions(-)
 create mode 100644 tests/test_verification_review.py

diff --git a/tests/test_verification_review.py b/tests/test_verification_review.py
new file mode 100644
index 0000000..337fc56
--- /dev/null
+++ b/tests/test_verification_review.py
@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from contextlib import nullcontext
+import json
+import os
+from pathlib import Path
+import subprocess
+from unittest.mock import Mock
+
+import pytest
+
+from tools.verification import focused
+from tools.verification_evidence import Summary, redact, save
+from tools.verification_host import wait_ready
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        '{"Authorization": "Bearer CANARY"}',
+        "{'Authorization': 'Bearer CANARY'}",
+        "Proxy-Authorization: Bearer CANARY",
+        '{"password": "alpha CANARY beta"}',
+        '{"token": "with \\"escaped\\" CANARY"}',
+        "connection_string=Endpoint=sb://ns/;SharedAccessKey=CANARY",
+        "InstrumentationKey=CANARY;IngestionEndpoint=https://x/",
+        "https://user:CANARY@example.test/path",
+        '<failure message="Authorization: Bearer CANARY">x</failure>',
+    ],
+)
+def test_quoted_credentials_are_consumed_whole(text: str) -> None:
+    # Given credential-bearing text, when redacting, then no suffix leaks.
+    assert "CANARY" not in redact(text)
+
+
+def test_save_keeps_structured_evidence_valid_and_binary_untouched(tmp_path: Path) -> None:
+    # Given JSON, binary and a symlink outside the evidence tree.
+    png = b"\x89PNG\r\n\x1a\n\xff\xfe\x00binary"
+    (tmp_path / "chart.png").write_bytes(png)
+    outside = tmp_path.parent / f"{tmp_path.name}-outside.log"
+    outside.write_text("password=CANARY")
+    (tmp_path / "link.log").symlink_to(outside)
+    (tmp_path / "extra.json").write_text('{"nested": ["Authorization: Bearer CANARY"]}')
+    # When saving evidence including tuple-valued command arguments.
+    save(tmp_path, Summary(1, "test", ('Authorization: "Bearer CANARY"',), 1, 1, 0, 0, "PASS", 0))
+    # Then JSON parses, secrets are absent, and non-text/outside assets stay intact.
+    for name in ("summary.json", "extra.json"):
+        assert "CANARY" not in str(json.loads((tmp_path / name).read_text()))
+    assert (tmp_path / "chart.png").read_bytes() == png
+    assert outside.read_text() == "password=CANARY"
+
+
+def test_known_secret_is_replaced_before_partial_patterns(monkeypatch: pytest.MonkeyPatch) -> None:
+    # Given a complete secret containing separators and escaped quotes.
+    monkeypatch.setenv("MY_API_TOKEN", 'pre;fix&CANARY "value"')
+    # When redacting raw and JSON-escaped spellings, then the full value disappears.
+    for value in (os.environ["MY_API_TOKEN"], json.dumps(os.environ["MY_API_TOKEN"])[1:-1]):
+        assert "CANARY" not in redact(value)
+
+
+@pytest.mark.parametrize("error", [TimeoutError(), ConnectionError(), OSError()])
+def test_host_readiness_retries_transient_errors(
+    error: OSError, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # Given a transient read failure followed by a ready host.
+    request = Mock(side_effect=[error, nullcontext(Mock(status=200))])
+    monkeypatch.setattr("tools.verification_host.urlopen", request)
+    monkeypatch.setattr("tools.verification_host.time.sleep", lambda _: None)
+    # When awaiting readiness, then the second probe succeeds.
+    wait_ready(Mock(poll=Mock(return_value=None)), "http://localhost/")
+    assert request.call_count == 2
+
+
+def test_failed_quality_command_saves_redacted_log_and_summary(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    import sys
+
+    # Given a failing quality command and a private evidence directory.
+    monkeypatch.chdir(tmp_path)
+    # When executing through the shared evidence runner.
+    code = focused("check", [sys.executable, "-c", "print('password=CANARY');exit(7)"])
+    # Then the exit code and failure survive, with a safe log.
+    assert code == 7
+    data = json.loads(Path("test-results/check/summary.json").read_text())
+    assert data["status"] == "FAIL" and data["exit_code"] == 7
+    assert "CANARY" not in Path("test-results/check/logs/runner.log").read_text()
+
+
+@pytest.mark.parametrize("target", ["doctor", "report", "clean"])
+def test_maintenance_never_bootstraps_hatch(target: str, tmp_path: Path) -> None:
+    # Given an empty Hatch data directory and a Hatch command that cannot run.
+    root = Path(__file__).resolve().parents[1]
+    environment = dict(os.environ, HATCH_DATA_DIR=str(tmp_path / "hatch"))
+    environment.pop("CI", None)
+    # When running a maintenance command, then no Hatch environment is created.
+    (tmp_path / "Makefile").write_bytes((root / "Makefile").read_bytes())
+    environment["PYTHONPATH"] = str(root)
+    result = subprocess.run(
+        ["make", target, "HATCH=false"],
+        cwd=tmp_path,
+        env=environment,
+        capture_output=True,
+        text=True,
+    )
+    assert "false run" not in result.stdout + result.stderr
+    assert not (tmp_path / "hatch").exists()
+    if target == "clean":
+        assert result.returncode == 0, result.stderr
diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py
index 866f38f..5ea3c9a 100644
--- a/tools/verification_evidence.py
+++ b/tools/verification_evidence.py
@@ -41,24 +41,61 @@ class Summary:
     reason: str = ""
 
 
+_REDACTED = "[REDACTED]"
+_QUOTED = r"\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'"
+_AUTHORIZATION = re.compile(
+    r"(?im)(?P<key>(?:proxy-)?authorization[\"']?\s*[:=]\s*)(?P<value>" + _QUOTED + r"|[^\r\n\"<]+)"
+)
+_URL_CREDENTIALS = re.compile(r"(https?://)[^/@\s\"']+:[^/@\s\"']+@")
+_SECRET_FIELD = re.compile(
+    r"(?i)(?<![\w-])(?P<key>(?:[\w.-]*(?:token|password|passwd|secret|api[_-]?key|"
+    r"connection[_-]?string)|AccountKey|SharedAccessKey|SharedAccessSignature|"
+    r"AzureWebJobsStorage|InstrumentationKey|x-functions-key|code|sig|sas)"
+    r"[\"']?\s*[:=]\s*)(?P<value>" + _QUOTED + r"|[^\s;&<>\"',]+)"
+)
+_SECRET_ENV_NAME = re.compile(
+    r"(?i)secret|token|password|connection|string|accountkey|functions.key|api.?key|sas"
+)
+_TEXT_SUFFIXES = frozenset({".log", ".txt", ".xml", ".json", ".md"})
+
+
+def _mask(match: re.Match[str]) -> str:
+    value = match.group("value")
+    quote = value[0] if value[:1] in {'"', "'"} else ""
+    return f"{match.group('key')}{quote}{_REDACTED}{quote}"
+
+
+def _known_secrets() -> list[str]:
+    values = {
+        value
+        for key, value in os.environ.items()
+        if len(value) >= 6 and _SECRET_ENV_NAME.search(key)
+    }
+    escaped = {json.dumps(value)[1:-1] for value in values}
+    return sorted(values | escaped, key=len, reverse=True)
+
+
 def redact(text: str) -> str:
-    text = re.sub(r"(?im)(authorization[\"']?\s*[:=]\s*)[^\r\n<\"]+", r"\1[REDACTED]", text)
-    text = re.sub(r"(https?://)[^/@\s]+:[^/@\s]+@", r"\1[REDACTED]@", text)
-    # Connection strings are one credential, not just their AccountKey component.
+    for value in _known_secrets():
+        text = text.replace(value, _REDACTED)
+    text = _AUTHORIZATION.sub(_mask, text)
+    text = _URL_CREDENTIALS.sub(r"\1" + _REDACTED + "@", text)
     text = re.sub(r"(?i)(DefaultEndpointsProtocol=)[^\s<\"']+", r"\1[REDACTED]", text)
-    text = re.sub(
-        r"(?i)(?<![\w-])((?:[\w-]*(?:token|password|secret|connectionstring)|AccountKey|"
-        r"SharedAccessSignature|AzureWebJobsStorage|x-functions-key|code|sig|sas|api[_-]?key)"
-        r"[\"']?\s*[:=]\s*[\"']?)[^\s;&<\"']+",
-        r"\1[REDACTED]",
-        text,
-    )
-    for key, value in os.environ.items():
-        if len(value) >= 6 and re.search(
-            r"(?i)secret|token|password|connection|string|accountkey|functions.key", key
-        ):
-            text = text.replace(value, "[REDACTED]")
-    return text
+    return _SECRET_FIELD.sub(_mask, text)
+
+
+def redact_json(value: object) -> object:
+    if isinstance(value, str):
+        return redact(value)
+    if isinstance(value, (list, tuple)):
+        return [redact_json(item) for item in value]
+    if isinstance(value, dict):
+        return {redact(str(key)): redact_json(item) for key, item in value.items()}
+    return value
+
+
+def dump_json(value: object) -> str:
+    return json.dumps(redact_json(value), indent=2) + "\n"
 
 
 def environment(live: bool = False) -> dict[str, str]:
@@ -137,16 +174,29 @@ def publish(summary: Summary) -> None:
             stream.write(f"\n- {text}\n")
 
 
+def _redact_file_text(suffix: str, text: str) -> str:
+    if suffix == ".json":
+        try:
+            return dump_json(json.loads(text))
+        except ValueError:
+            return redact(text)
+    return redact(text)
+
+
 def save(directory: Path, summary: Summary) -> Summary:
     directory.mkdir(parents=True, exist_ok=True)
     for path in directory.rglob("*"):
-        if path.is_file():
-            path.write_text(redact(path.read_text(encoding="utf-8")), encoding="utf-8")
+        if path.is_symlink() or not path.is_file() or path.suffix not in _TEXT_SUFFIXES:
+            continue
+        try:
+            original = path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            continue
+        redacted = _redact_file_text(path.suffix, original)
+        if redacted != original:
+            path.write_text(redacted, encoding="utf-8")
     (directory / "environment.json").write_text(
-        redact(
-            json.dumps(environment(live=directory.name in {"integration", "host-smoke"}), indent=2)
-        )
-        + "\n",
+        dump_json(environment(live=directory.name in {"integration", "host-smoke"})),
         encoding="utf-8",
     )
     summary = replace(
@@ -157,9 +207,7 @@ def save(directory: Path, summary: Summary) -> Summary:
             if path.is_file() and path.name != "summary.json"
         ),
     )
-    (directory / "summary.json").write_text(
-        redact(json.dumps(asdict(summary), indent=2)) + "\n", encoding="utf-8"
-    )
+    (directory / "summary.json").write_text(dump_json(asdict(summary)), encoding="utf-8")
     return summary
 
 
diff --git a/tools/verification_host.py b/tools/verification_host.py
index a22285d..06ad5a7 100644
--- a/tools/verification_host.py
+++ b/tools/verification_host.py
@@ -4,6 +4,7 @@
 
 import base64
 from contextlib import ExitStack
+from http.client import HTTPException
 import json
 import os
 from pathlib import Path
@@ -13,7 +14,6 @@
 import sys
 import tempfile
 import time
-from urllib.error import URLError
 from urllib.request import urlopen
 import xml.etree.ElementTree as ET
 
@@ -35,7 +35,7 @@ def wait_ready(process: subprocess.Popen[str], url: str) -> None:
             with urlopen(url, timeout=1) as response:
                 if response.status < 500:
                     return
-        except URLError:
+        except (OSError, HTTPException):
             time.sleep(0.25)
     raise TimeoutError("service readiness deadline exceeded")
 

From 8b0cdfb4c3b523f18c9216f415311a7cd140c4f1 Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 09:42:45 +0900
Subject: [PATCH 13/14] ci(verification): retain quality evidence without
 bootstrapping maintenance

---
 Makefile                            | 14 ++++++++++----
 tests/test_verification_commands.py | 22 +++++++++++++++++++++-
 tools/verification.py               |  5 ++++-
 tools/verification_environment.py   |  2 ++
 4 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 86dc93f..4d22010 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,7 @@ RUN := $(HATCH) run python
 else
 RUN := $(PYTHON)
 endif
+MAINTENANCE_PYTHON ?= $(shell for p in "$$VIRTUAL_ENV/bin/python" "$(VENV_DIR)/bin/python" python3.11 python3 python; do if "$$p" -c 'import sys; sys.exit(not (3,11) <= sys.version_info[:2] < (3,15))' 2>/dev/null; then command -v "$$p"; break; fi; done)
 
 .PHONY: bootstrap ensure-hatch
 bootstrap ensure-hatch:
@@ -50,12 +51,17 @@ lint-workflows:
 	@$(RUN) tools/lint_release_workflows.py
 	@$(RUN) tools/lint_workflow_pins.py
 	@$(RUN) tools/lint_hatch_matrix.py
-check: style typecheck security lint-workflows check-schema-hash contracts
+.PHONY: check-steps
+check:
+	@$(RUN) -m tools.verification check
+check-steps: style typecheck security lint-workflows check-schema-hash contracts
 check-all: verify
 
 .PHONY: test integration worker-compat sdk-compat verify report wheel-smoke
-test integration worker-compat sdk-compat verify report:
+test integration worker-compat sdk-compat verify:
 	@$(RUN) -m tools.verification $@
+report:
+	@$(MAINTENANCE_PYTHON) -m tools.verification report
 wheel-smoke:
 	@$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel
 
@@ -89,9 +95,9 @@ version: ensure-hatch
 
 .PHONY: doctor clean clean-all help
 doctor:
-	@$(RUN) -m tools.verification doctor
+	@$(MAINTENANCE_PYTHON) -m tools.verification doctor
 clean:
-	@$(RUN) -m tools.verification clean
+	@$(MAINTENANCE_PYTHON) -m tools.verification clean
 # Historical deep cleanup is intentionally separate from artifact-only clean.
 clean-all: clean
 	@find . -type d -name "__pycache__" -not -path "./.git/*" -exec rm -rf {} +
diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py
index 743e995..6a419ce 100644
--- a/tests/test_verification_commands.py
+++ b/tests/test_verification_commands.py
@@ -40,7 +40,7 @@ def test_make_contract_is_install_free_on_routine_runs(target: str) -> None:
 
 def test_check_runs_types_once_and_read_only_contract_checks() -> None:
     # Given the complete quality command, when expanding its recipe.
-    result = subprocess.run(["make", "-n", "check"], cwd=ROOT, capture_output=True, text=True)
+    result = subprocess.run(["make", "-n", "check-steps"], cwd=ROOT, capture_output=True, text=True)
     # Then types run once and pinned contracts are checked, never regenerated.
     assert result.returncode == 0
     assert result.stdout.count("-m mypy") == 1
@@ -89,3 +89,23 @@ def test_unconfigured_azure_saves_incomplete(tmp_path: Path) -> None:
     data = json.loads((tmp_path / "test-results/azure-e2e/summary.json").read_text())
     assert data["status"] == "INCOMPLETE"
     assert data["selected"] == 0
+
+
+def test_make_check_retains_failed_prerequisite_evidence(tmp_path: Path) -> None:
+    # Given a failing lint prerequisite in a checkout-shaped Makefile.
+    makefile = (ROOT / "Makefile").read_text()
+    (tmp_path / "Makefile").write_text(makefile + "\nstyle:\n\t@echo password=CANARY; exit 7\n")
+    environment = dict(os.environ, CI="1", PYTHONPATH=str(ROOT))
+    # When invoking the actual public quality entry point.
+    result = subprocess.run(
+        ["make", "check", f"PYTHON={sys.executable}"],
+        cwd=tmp_path,
+        env=environment,
+        capture_output=True,
+        text=True,
+    )
+    # Then Make fails and the shared runner retains redacted failure evidence.
+    assert result.returncode != 0
+    data = json.loads((tmp_path / "test-results/check/summary.json").read_text())
+    assert data["lane"] == "check" and data["status"] == "FAIL"
+    assert "CANARY" not in (tmp_path / "test-results/check/logs/runner.log").read_text()
diff --git a/tools/verification.py b/tools/verification.py
index 72b98d8..9d22413 100644
--- a/tools/verification.py
+++ b/tools/verification.py
@@ -171,7 +171,7 @@ def verify() -> int:
         path = Path("test-results") / target / "summary.json"
         path.unlink(missing_ok=True)
         code = run(["make", target], directory / "logs" / f"{target}.log", 1200)
-        if target in {"test", "integration", "wheel-smoke", "docs"} and path.exists():
+        if path.exists():
             summary = Summary(**json.loads(path.read_text()))
             if code and summary.status == "PASS":
                 summary = replace(summary, status="FAIL", exit_code=code)
@@ -219,6 +219,7 @@ def main() -> int:
         "lane",
         choices=(
             "test",
+            "check",
             "integration",
             "worker-compat",
             "host-smoke",
@@ -246,6 +247,8 @@ def main() -> int:
             return 0
         case "verify":
             return verify()
+        case "check":
+            return focused("check", ["make", "check-steps"])
         case "integration":
             return integration()
         case "focused":
diff --git a/tools/verification_environment.py b/tools/verification_environment.py
index 0eab1ed..45d86f7 100644
--- a/tools/verification_environment.py
+++ b/tools/verification_environment.py
@@ -15,6 +15,8 @@ def doctor(live: bool = True) -> int:
         found = importlib.util.find_spec(module) is not None
         print(f"{module} REQUIRED: {'found' if found else 'MISSING'}")
         healthy = healthy and found
+    if not healthy:
+        print("development environment not created: run make install")
     for tool in ("func", "azurite"):
         path = shutil.which(tool)
         print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}")

From 59cc6494013d3cb72ab36524d7c9347def323a1e Mon Sep 17 00:00:00 2001
From: Yeongseon Choe <yeongseon.choe@gmail.com>
Date: Fri, 23 Oct 2026 09:42:45 +0900
Subject: [PATCH 14/14] test(evidence): isolate synthetic job summaries

---
 tests/conftest.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/conftest.py b/tests/conftest.py
index 291f8cf..5284b19 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from pathlib import Path
 from unittest.mock import Mock
 
 from azure.functions import HttpRequest
@@ -42,3 +43,11 @@ def _create_request(
         return mock_req
 
     return _create_request
+
+
+@pytest.fixture(autouse=True)
+def isolate_verification_summary(
+    request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    if request.module.__name__.split(".")[-1].startswith("test_verification"):
+        monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary"))