From 60c46371c53db8302734779b61776203be6605f3 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 01/13] ci(security): add redacted evidence and expiring exception primitives Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tests/test_security_report.py | 87 ++++++++++++++++++++++++ tools/__init__.py | 0 tools/security_exceptions.py | 49 ++++++++++++++ tools/security_report.py | 120 ++++++++++++++++++++++++++++++++++ 4 files changed, 256 insertions(+) create mode 100644 tests/test_security_report.py create mode 100644 tools/__init__.py create mode 100644 tools/security_exceptions.py create mode 100644 tools/security_report.py diff --git a/tests/test_security_report.py b/tests/test_security_report.py new file mode 100644 index 0000000..f90db3a --- /dev/null +++ b/tests/test_security_report.py @@ -0,0 +1,87 @@ +from datetime import date +import json +from pathlib import Path + +import pytest + +from tools.security_exceptions import parse_exceptions +from tools.security_report import ( + Check, + EvidenceError, + Status, + identifier, + integer, + load, + sequence, + text, + write_report, +) + + +@pytest.mark.parametrize("raw", ["null", "[]", '"text"', "true"]) +def test_json_boundary_requires_object(raw: str) -> None: + # Given non-object input; When parsed; Then the evidence shape is rejected. + with pytest.raises(EvidenceError): + load(raw) + + +@pytest.mark.parametrize("value", [None, 0, True, ""]) +def test_text_requires_nonempty_string(value: None | int | str) -> None: + # Given invalid text; When parsed; Then it cannot enter an identifier. + with pytest.raises(EvidenceError): + text(value) + + +@pytest.mark.parametrize("value", [None, True, -1, "1"]) +def test_count_requires_nonnegative_integer(value: None | int | str) -> None: + # Given invalid counts; When parsed; Then malformed evidence cannot pass. + with pytest.raises(EvidenceError): + integer(value) + + +@pytest.mark.parametrize("value", [None, "[]", 0]) +def test_array_requires_sequence(value: None | str | int) -> None: + # Given a nonarray; When parsed; Then the boundary rejects it. + with pytest.raises(EvidenceError): + sequence(value) + + +@pytest.mark.parametrize( + "value", + [ + "/tmp/file.py", + "../secret", + "https://example.com", + "Authorization: Bearer CANARY_SECRET", + "x?code=CANARY_SECRET", + ], +) +def test_sensitive_metadata_is_rejected(value: str) -> None: + # Given unsafe metadata; When parsed; Then no original credential enters evidence. + with pytest.raises(EvidenceError): + identifier(value) + + +def test_duplicate_exception_is_rejected() -> None: + # Given duplicate policy entries; When parsed; Then exception accounting is unambiguous. + rule = { + "id": "CVE-1", + "package": "example", + "kind": "not-affected", + "reason": "reviewed", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + } + parsed = load(json.dumps({"exceptions": [rule, rule]})) + with pytest.raises(EvidenceError): + parse_exceptions(parsed["exceptions"], date(2026, 10, 22)) + + +def test_summary_appends_to_github_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given an existing job summary; When evidence is written; Then prior evidence is preserved. + summary = tmp_path / "summary.md" + summary.write_text("existing\n") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + assert write_report([Check("scanner", (), Status.PASS, 0)], tmp_path / "out") == 0 + assert summary.read_text().startswith("existing\n") diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/security_exceptions.py b/tools/security_exceptions.py new file mode 100644 index 0000000..2162a63 --- /dev/null +++ b/tools/security_exceptions.py @@ -0,0 +1,49 @@ +"""Owned, exact exceptions with a maximum 90-day validity window.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date + +from tools.security_report import JSON, EvidenceError, identifier, integer, mapping, sequence, text + + +@dataclass(frozen=True, slots=True) +class ExceptionRule: + id: str + package: str + file: str + line: int + kind: str + expires: date + added: date + + +def parse_exceptions(data: JSON, today: date) -> tuple[ExceptionRule, ...]: + rules: list[ExceptionRule] = [] + for entry in sequence(data): + item = mapping(entry) + text(item["reason"]) + text(item["owner"]) + added = date.fromisoformat(text(item["added"])) + expires = date.fromisoformat(text(item["expires"])) + if not 0 < (expires - added).days <= 90 or added > today: + raise EvidenceError("invalid exception validity window") + rule = ExceptionRule( + identifier(item["id"]), + identifier(item["package"]) if "package" in item else "", + identifier(item["file"]) if "file" in item else "", + integer(item.get("line", 0)), + text(item["kind"]), + expires, + added, + ) + if rule.kind not in {"fix-not-available", "not-affected", "reviewed-static"}: + raise EvidenceError("unknown exception kind") + if any( + (r.id, r.package, r.file, r.line) == (rule.id, rule.package, rule.file, rule.line) + for r in rules + ): + raise EvidenceError("duplicate exception") + rules.append(rule) + return tuple(rules) diff --git a/tools/security_report.py b/tools/security_report.py new file mode 100644 index 0000000..0692c5f --- /dev/null +++ b/tools/security_report.py @@ -0,0 +1,120 @@ +"""Allow-listed security evidence; raw scanner diagnostics are never retained.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from enum import StrEnum +import json +import os +from pathlib import Path +import re +from typing import TypeAlias, assert_never + +JSON: TypeAlias = "None | bool | int | float | str | list[JSON] | dict[str, JSON]" + + +class EvidenceError(ValueError): + """Invalid evidence; messages contain no external input.""" + + +def mapping(value: JSON) -> dict[str, JSON]: + if not isinstance(value, dict): + raise EvidenceError("expected JSON object") + return value + + +def sequence(value: JSON) -> list[JSON]: + if not isinstance(value, list): + raise EvidenceError("expected JSON array") + return value + + +def text(value: JSON) -> str: + if not isinstance(value, str) or not value: + raise EvidenceError("expected nonempty string") + return value + + +def integer(value: JSON) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise EvidenceError("expected nonnegative integer") + return value + + +def load(raw: str) -> dict[str, JSON]: + parsed: JSON = json.loads(raw) + return mapping(parsed) + + +def identifier(value: JSON) -> str: + candidate = text(value) + if not re.fullmatch(r"[A-Za-z0-9_.:/@+-]{1,240}", candidate): + raise EvidenceError("unsafe evidence identifier") + if candidate.startswith(("/", "http:", "https:")) or ".." in candidate: + raise EvidenceError("unsafe evidence path") + return candidate + + +class Status(StrEnum): + PASS = "PASS" + FAIL = "FAIL" + TOOL_ERROR = "TOOL_ERROR" + INCOMPLETE = "INCOMPLETE" + + +@dataclass(frozen=True, slots=True) +class Finding: + id: str + file: str = "" + line: int = 0 + severity: str = "UNKNOWN" + confidence: str = "UNKNOWN" + package: str = "" + + +@dataclass(frozen=True, slots=True) +class Check: + name: str + scope: tuple[str, ...] + status: Status + exit_status: int | None + counts: dict[str, int] = field(default_factory=dict) + suppressions: int = 0 + exceptions_applied: tuple[str, ...] = () + tool_versions: dict[str, str] = field(default_factory=dict) + findings: tuple[Finding, ...] = () + message: str = "" + + @property + def returncode(self) -> int: + match self.status: + case Status.PASS: + return 0 + case Status.FAIL: + return 1 + case Status.TOOL_ERROR: + return 2 + case Status.INCOMPLETE: + return 3 + case unreachable: + assert_never(unreachable) + + +def write_report(checks: list[Check], output: Path, summary: Path | None = None) -> int: + output.mkdir(parents=True, exist_ok=True) + (output / "summary.json").write_text( + json.dumps({"schema_version": 1, "checks": [asdict(c) for c in checks]}, indent=2) + "\n", + encoding="utf-8", + ) + markdown = "## Security evidence\n\n| Check | Status | Counts |\n| --- | --- | --- |\n" + for check in checks: + markdown += f"| {check.name} | {check.status} | {json.dumps(check.counts)} |\n" + markdown += f"\n{check.message}\n\n" + if summary is None and os.environ.get("GITHUB_STEP_SUMMARY"): + summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) + if summary is not None: + with summary.open("a", encoding="utf-8") as stream: + stream.write(markdown) + for check in checks: + print(f"{check.name}: {check.status} {json.dumps(check.counts)} {check.message}") + return max((c.returncode for c in checks), default=3) From 3a74a24f59deda1a5a3fdd32f4d7ecb51e47df40 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 02/13] ci(security): classify isolated runtime dependency audits fail closed Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- security/dependency-audit-policy.json | 5 + tests/test_dependency_audit_policy.py | 135 +++++++++++++++++++++++ tools/dependency_audit_policy.py | 153 ++++++++++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 security/dependency-audit-policy.json create mode 100644 tests/test_dependency_audit_policy.py create mode 100644 tools/dependency_audit_policy.py diff --git a/security/dependency-audit-policy.json b/security/dependency-audit-policy.json new file mode 100644 index 0000000..edf5af5 --- /dev/null +++ b/security/dependency-audit-policy.json @@ -0,0 +1,5 @@ +{ + "schema_version": 1, + "blocking": "all-known-runtime", + "exceptions": [] +} diff --git a/tests/test_dependency_audit_policy.py b/tests/test_dependency_audit_policy.py new file mode 100644 index 0000000..12f1cbc --- /dev/null +++ b/tests/test_dependency_audit_policy.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from datetime import date +import json + +import pytest + +from tools.dependency_audit_policy import classify_audit +from tools.security_exceptions import parse_exceptions +from tools.security_report import EvidenceError, Status, load + +TODAY = date(2026, 10, 22) +POLICY = '{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}' + + +def payload(vulnerable: bool = False) -> str: + return json.dumps( + { + "dependencies": [ + { + "name": "example", + "version": "1", + "vulns": [ + {"id": "CVE-2026-12345", "fix_versions": [], "description": "CANARY_SECRET"} + ] + if vulnerable + else [], + } + ] + } + ) + + +@pytest.mark.parametrize("vulnerable,expected", [(False, Status.PASS), (True, Status.FAIL)]) +def test_any_known_runtime_advisory_blocks(vulnerable: bool, expected: Status) -> None: + # Given resolved packages; When audited; Then even UNKNOWN severity blocks. + result = classify_audit(payload(vulnerable), int(vulnerable), policy_raw=POLICY, today=TODAY) + assert result.status == expected + assert "CANARY_SECRET" not in repr(result) + + +@pytest.mark.parametrize("raw", ["", "invalid", "{}", '{"dependencies":[]}']) +def test_invalid_or_zero_package_audit_is_tool_error(raw: str) -> None: + # Given unusable audit output; When classified; Then it is never a clean scan. + assert classify_audit(raw, 0, policy_raw=POLICY, today=TODAY).status == Status.TOOL_ERROR + + +def test_skipped_package_is_incomplete() -> None: + # Given a package the tool skipped; When classified; Then coverage is incomplete. + raw = '{"dependencies":[{"name":"x","version":"1","skip_reason":"CANARY_SECRET"}]}' + assert classify_audit(raw, 0, policy_raw=POLICY, today=TODAY).status == Status.INCOMPLETE + + +@pytest.mark.parametrize( + "expires,expected", [("2026-10-21", Status.FAIL), ("2026-11-21", Status.PASS)] +) +def test_exception_expiry_and_affectedness(expires: str, expected: Status) -> None: + # Given an owned exception; When evaluated; Then only current/no-fix exceptions apply. + policy = json.loads(POLICY) + policy["exceptions"] = [ + { + "id": "CVE-2026-12345", + "package": "example", + "kind": "fix-not-available", + "reason": "no patched release", + "owner": "maintainer", + "added": "2026-10-01", + "expires": expires, + } + ] + result = classify_audit(payload(True), 1, policy_raw=json.dumps(policy), today=TODAY) + assert result.status == expected + assert bool(result.exceptions_applied) == (expected == Status.PASS) + raw = json.loads(payload(True)) + raw["dependencies"][0]["vulns"][0]["fix_versions"] = ["2"] + assert ( + classify_audit(json.dumps(raw), 1, policy_raw=json.dumps(policy), today=TODAY).status + == Status.FAIL + ) + + +def test_stale_exception_is_reported() -> None: + # Given a current unused exception; When evaluated; Then it is visible for removal. + policy = json.loads(POLICY) + policy["exceptions"] = [ + { + "id": "CVE-2026-12345", + "package": "example", + "kind": "not-affected", + "reason": "reviewed unreachable path", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + } + ] + result = classify_audit(payload(), 0, policy_raw=json.dumps(policy), today=TODAY) + assert result.status == Status.PASS + assert result.counts["stale_exceptions"] == 1 + + +@pytest.mark.parametrize( + "patch", + [ + {"expires": "2027-10-22"}, + {"added": "2026-11-01"}, + {"kind": "ignore-all"}, + {"line": -1}, + {"line": True}, + {"owner": ""}, + {"reason": ""}, + ], +) +def test_invalid_exception_policy_fails(patch: dict[str, str | int]) -> None: + # Given malformed policy; When parsed; Then no exception can hide a finding. + rule = { + "id": "CVE-1", + "package": "example", + "kind": "not-affected", + "reason": "reviewed", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + **patch, + } + with pytest.raises((EvidenceError, ValueError)): + parse_exceptions(load(json.dumps({"exceptions": [rule]}))["exceptions"], TODAY) + + +@pytest.mark.parametrize("code", [0, 2]) +def test_inconsistent_exit_status_fails(code: int) -> None: + # Given findings inconsistent with exit status; When classified; Then tooling failed. + assert ( + classify_audit(payload(True), code, policy_raw=POLICY, today=TODAY).status + == Status.TOOL_ERROR + ) diff --git a/tools/dependency_audit_policy.py b/tools/dependency_audit_policy.py new file mode 100644 index 0000000..f695576 --- /dev/null +++ b/tools/dependency_audit_policy.py @@ -0,0 +1,153 @@ +"""Audit only an isolated wheel environment, with fail-closed redacted evidence.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +from datetime import date +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +import re +import subprocess +import sys + +from tools.security_exceptions import parse_exceptions +from tools.security_report import ( + Check, + EvidenceError, + Finding, + Status, + identifier, + load, + mapping, + sequence, + text, + write_report, +) + + +def classify_audit(raw: str, exit_status: int, *, policy_raw: str, today: date) -> Check: + result = Check("pip-audit", ("resolved-runtime-wheel",), Status.TOOL_ERROR, exit_status) + try: + policy = load(policy_raw) + if policy["schema_version"] != 1 or policy["blocking"] != "all-known-runtime": + raise EvidenceError("unsupported policy") + rules = parse_exceptions(policy["exceptions"], today) + if any(r.file or r.line or not r.package or r.kind == "reviewed-static" for r in rules): + raise EvidenceError("invalid dependency exception") + dependencies = sequence(load(raw)["dependencies"]) + if exit_status not in {0, 1} or not dependencies: + raise EvidenceError("tool crash or zero packages") + if any("skip_reason" in mapping(d) for d in dependencies): + return replace(result, status=Status.INCOMPLETE, message="Skipped audit scope.") + findings: list[Finding] = [] + applied: set[str] = set() + blocking = 0 + for dependency in dependencies: + item = mapping(dependency) + package = re.sub(r"[-_.]+", "-", identifier(item["name"]).lower()) + text(item["version"]) + for advisory in sequence(item["vulns"]): + vuln = mapping(advisory) + advisory_id = identifier(vuln["id"]) + severity = identifier(vuln.get("severity", "UNKNOWN")) + findings.append(Finding(advisory_id, severity=severity, package=package)) + matches = [ + r + for r in rules + if r.id == advisory_id + and re.sub(r"[-_.]+", "-", r.package.lower()) == package + and r.expires >= today + ] + if matches and ( + matches[0].kind != "fix-not-available" or not sequence(vuln["fix_versions"]) + ): + applied.add(f"{package}:{advisory_id}") + else: + blocking += 1 + if bool(findings) != (exit_status == 1): + raise EvidenceError("exit status disagrees with findings") + expired = sum(r.expires < today for r in rules) + return replace( + result, + status=Status.FAIL if blocking or expired else Status.PASS, + counts={ + "packages": len(dependencies), + "findings": len(findings), + "blocking": blocking, + "expired_exceptions": expired, + "stale_exceptions": len(rules) - len(applied), + }, + findings=tuple(findings), + exceptions_applied=tuple(sorted(applied)), + message="All known runtime advisories block without a current applicable exception.", + ) + except (ValueError, KeyError, TypeError): + return replace(result, message="Invalid audit/tool/policy evidence; release blocked.") + + +def run_audit(site_packages: Path, policy: Path) -> Check: + try: + if not site_packages.is_dir() or not list(site_packages.glob("*.dist-info")): + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.INCOMPLETE, + None, + message="Candidate installation inventory missing.", + ) + process = subprocess.run( + [ + sys.executable, + "-m", + "pip_audit", + "--path", + str(site_packages), + "--format", + "json", + "--desc", + "off", + "--progress-spinner", + "off", + "--timeout", + "30", + ], + capture_output=True, + text=True, + check=False, + timeout=300, + ) + result = classify_audit( + process.stdout, + process.returncode, + policy_raw=policy.read_text(encoding="utf-8"), + today=date.today(), + ) + return replace(result, tool_versions={"pip-audit": version("pip-audit")}) + except (OSError, subprocess.TimeoutExpired, PackageNotFoundError): + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.TOOL_ERROR, + None, + message="Audit execution failed; release blocked.", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--site-packages", type=Path, required=True) + parser.add_argument( + "--policy", type=Path, default=Path("security/dependency-audit-policy.json") + ) + parser.add_argument("--output", type=Path, default=Path("security-results")) + parser.add_argument("--development", action="store_true") + args = parser.parse_args() + check = run_audit(args.site_packages, args.policy) + if args.development: + check = replace(check, name="pip-audit-development", scope=("runtime-and-development",)) + return write_report([check], args.output) + + +if __name__ == "__main__": + raise SystemExit(main()) From 0185d217f9735a9e12b043f438c9790cde32463e Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 03/13] ci(security): audit executable surfaces and reviewed inline suppressions Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- security/bandit-policy.json | 50 ++++++ tests/test_security_evidence.py | 148 ++++++++++++++++ tests/test_security_tool_execution.py | 145 ++++++++++++++++ tools/security_evidence.py | 233 ++++++++++++++++++++++++++ 4 files changed, 576 insertions(+) create mode 100644 security/bandit-policy.json create mode 100644 tests/test_security_evidence.py create mode 100644 tests/test_security_tool_execution.py create mode 100644 tools/security_evidence.py diff --git a/security/bandit-policy.json b/security/bandit-policy.json new file mode 100644 index 0000000..569fabd --- /dev/null +++ b/security/bandit-policy.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "inline_suppressions": [ + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":207,"kind":"reviewed-static","reason":"Existing context cleanup is deliberately non-raising; no runtime change in this automation port.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":231,"kind":"reviewed-static","reason":"Existing context propagation is deliberately best-effort and non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":236,"kind":"reviewed-static","reason":"Existing worker metadata propagation must not crash application execution.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":245,"kind":"reviewed-static","reason":"Existing context cleanup must not crash application execution.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":251,"kind":"reviewed-static","reason":"Existing worker thread-local cleanup is deliberately silent on failure.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":371,"kind":"reviewed-static","reason":"Existing context metadata inspection is best-effort and never raises.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":388,"kind":"reviewed-static","reason":"Existing propagation failure must not replace the application result.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":397,"kind":"reviewed-static","reason":"Existing executor context restoration is deliberately non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":401,"kind":"reviewed-static","reason":"Existing executor worker metadata restoration is deliberately silent.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":577,"kind":"reviewed-static","reason":"Existing binding metadata cleanup is best-effort and never raises.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":199,"kind":"reviewed-static","reason":"Existing unreadable host configuration is non-fatal by design.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":310,"kind":"reviewed-static","reason":"Existing host configuration override inspection is non-fatal.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":143,"kind":"reviewed-static","reason":"Existing filter sampling must not raise into application logging.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":348,"kind":"reviewed-static","reason":"Existing per-field masking resilience retained; runtime hardening is a separate change, not certified by this review.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":352,"kind":"reviewed-static","reason":"Existing filter never-raise contract retained; this exception does not establish privacy guarantees.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":379,"kind":"reviewed-static","reason":"Existing message masking never-raise path retained; separate runtime hardening owns fail-closed behavior.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":401,"kind":"reviewed-static","reason":"Existing exception masking resilience retained without claiming fail-closed runtime behavior.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":494,"kind":"reviewed-static","reason":"Existing per-field flattening is deliberately non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":496,"kind":"reviewed-static","reason":"Existing flattening filter never-raise contract retained.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":49,"kind":"reviewed-static","reason":"Optional tracing dependency failures degrade gracefully by design.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":107,"kind":"reviewed-static","reason":"Optional trace activation failures must not crash the handler.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":142,"kind":"reviewed-static","reason":"Existing tracing context cleanup is deliberately non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":151,"kind":"reviewed-static","reason":"Existing optional OTel context cleanup is deliberately silent.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_instance.py","line":60,"kind":"reviewed-static","reason":"Existing best-effort host metadata resolution must never crash logging.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B112","file":"src/azure_functions_logging/_redaction.py","line":170,"kind":"reviewed-static","reason":"Existing broken-pattern isolation retained; runtime redaction hardening is separate and not certified here.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B307","file":"src/azure_functions_logging/_decorator.py","line":83,"kind":"reviewed-static","reason":"Existing evaluation uses the handler author's annotation and globals, not HTTP input; no runtime edit permitted in this port.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"} + ], + "exceptions": [ + {"id":"B105","file":"benchmarks/bench.py","line":65,"kind":"reviewed-static","reason":"Synthetic benchmark fixture, not a credential or an authentication decision.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B105","file":"examples/otel_app/function_app.py","line":70,"kind":"reviewed-static","reason":"Synthetic masking demonstration; never use real credentials in this example.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B101","file":"scripts/assert_correlation.py","line":132,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; not authorization or input validation.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B101","file":"scripts/assert_correlation.py","line":133,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; observable certification checks remain ordinary conditionals.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B404","file":"scripts/render_mermaid.py","line":26,"kind":"reviewed-static","reason":"Documentation renderer invokes a shell-free CLI on repository Markdown.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B603","file":"scripts/render_mermaid.py","line":89,"kind":"reviewed-static","reason":"Fixed mmdc command with temporary paths as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B404","file":"tools/check_pr_format.py","line":5,"kind":"reviewed-static","reason":"Contributor formatter uses shell-free git and ruff subprocesses.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B607","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"git resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B603","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"Revision range is one argument and git options end with --, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B607","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"ruff resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B603","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"Filenames follow -- as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B404","file":"tools/dependency_audit_policy.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free pip-audit command uses sys.executable.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B603","file":"tools/dependency_audit_policy.py","line":99,"kind":"reviewed-static","reason":"Fixed pip-audit module with isolated path passed as separate argument, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B310","file":"tools/lint_doc_links.py","line":103,"kind":"reviewed-static","reason":"Caller constructs HTTPS GitHub/raw.githubusercontent URLs from parsed documentation links; fixed HEAD request, no user-request input.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B404","file":"tools/security_evidence.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free Bandit command uses sys.executable.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B603","file":"tools/security_evidence.py","line":129,"kind":"reviewed-static","reason":"Fixed Bandit command over explicit repository-controlled surfaces, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, + {"id":"B105","file":"tools/security_report.py","line":59,"kind":"reviewed-static","reason":"TOOL_ERROR is a public status enum, not a credential.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"} + ] +} diff --git a/tests/test_security_evidence.py b/tests/test_security_evidence.py new file mode 100644 index 0000000..6c2399d --- /dev/null +++ b/tests/test_security_evidence.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from datetime import date +import json +from pathlib import Path + +import pytest + +from tools.security_evidence import apply_bandit_policy, classify_bandit, semgrep_evidence +from tools.security_report import Check, Status, write_report + + +def payload() -> str: + return json.dumps( + { + "errors": [], + "metrics": {"_totals": {"loc": 10}, "src/x.py": {"loc": 10}}, + "results": [], + } + ) + + +@pytest.mark.parametrize( + "raw,code,status", + [ + ("", 2, Status.TOOL_ERROR), + ("invalid", 0, Status.TOOL_ERROR), + ('{"errors":[],"metrics":{"_totals":{}},"results":[]}', 0, Status.INCOMPLETE), + ], +) +def test_missing_evidence_fails_closed(raw: str, code: int, status: Status) -> None: + # Given missing evidence; When classified; Then it cannot pass. + assert classify_bandit(raw, code, floor=1).status == status + + +def test_complete_scan_has_real_counts() -> None: + # Given one scanned file; When classified; Then its inventory is reported. + result = classify_bandit(payload(), 0, floor=1) + assert result.status == Status.PASS + assert result.counts == {"files": 1, "loc": 10, "errors": 0} + + +@pytest.mark.parametrize("severity", ["LOW", "MEDIUM", "HIGH"]) +def test_findings_block_without_raw_diagnostics(severity: str, tmp_path: Path) -> None: + # Given secret-bearing diagnostics; When rendered; Then only metadata survives. + data = json.loads(payload()) + data["results"] = [ + { + "test_id": "B105", + "filename": "src/x.py", + "line_number": 3, + "issue_severity": severity, + "issue_confidence": "HIGH", + "code": "Authorization: Bearer CANARY_SECRET", + "issue_text": "AccountKey=CANARY_SECRET", + } + ] + result = classify_bandit(json.dumps(data), 1, floor=1) + summary = tmp_path / "summary.md" + assert write_report([result], tmp_path, summary) == 1 + saved = (tmp_path / "summary.json").read_text() + assert "CANARY_SECRET" not in saved + summary.read_text() + assert result.counts[f"severity.{severity}"] == 1 + parsed = json.loads(saved) + assert parsed["schema_version"] == 1 + assert set(parsed["checks"][0]) == { + "name", + "scope", + "status", + "exit_status", + "counts", + "suppressions", + "exceptions_applied", + "tool_versions", + "findings", + "message", + } + + +@pytest.mark.parametrize("code", [0, 2]) +def test_error_files_are_tool_errors(code: int) -> None: + # Given a scanner error; When classified; Then it is not a vulnerability result. + data = json.loads(payload()) + data["errors"] = [{"reason": "CANARY_SECRET"}] + assert classify_bandit(json.dumps(data), code, floor=1).status == Status.TOOL_ERROR + + +@pytest.mark.parametrize("field", ["nosec", "skipped_tests"]) +def test_suppression_overflow_is_incomplete(field: str) -> None: + # Given an unexpected scanner skip; When classified; Then completeness fails. + data = json.loads(payload()) + data["metrics"]["_totals"][field] = 1 + assert classify_bandit(json.dumps(data), 0, floor=1).status == Status.INCOMPLETE + + +def test_static_policy_matches_exact_location_and_expires() -> None: + # Given an exact reviewed exception; When evaluated; Then expiry/location matter. + data = json.loads(payload()) + data["results"] = [ + { + "test_id": "B404", + "filename": "src/x.py", + "line_number": 1, + "issue_severity": "LOW", + "issue_confidence": "HIGH", + } + ] + check = classify_bandit(json.dumps(data), 1, floor=1) + rule = { + "id": "B404", + "file": "src/x.py", + "line": 1, + "kind": "reviewed-static", + "reason": "fixed shell-free command", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + } + policy = json.dumps({"schema_version": 1, "exceptions": [rule]}) + applied = apply_bandit_policy(check, policy, date(2026, 10, 22)) + assert applied.status == Status.PASS + assert applied.exceptions_applied == ("B404:src/x.py:1",) + assert apply_bandit_policy(check, policy, date(2026, 11, 2)).status == Status.FAIL + rule["line"] = 2 + stale = apply_bandit_policy( + check, json.dumps({"schema_version": 1, "exceptions": [rule]}), date(2026, 10, 22) + ) + assert stale.status == Status.FAIL + assert stale.counts["stale_exceptions"] == 1 + assert apply_bandit_policy(check, "invalid", date(2026, 10, 22)).status == Status.TOOL_ERROR + + +@pytest.mark.parametrize( + "outcome,status", + [("success", Status.PASS), ("failure", Status.TOOL_ERROR), ("skipped", Status.INCOMPLETE)], +) +def test_semgrep_outcome_is_not_a_vulnerability(outcome: str, status: Status) -> None: + # Given action outcome only; When recorded; Then finding counts stay unknown. + assert semgrep_evidence(outcome).status == status + + +@pytest.mark.parametrize( + "status,code", + [(Status.PASS, 0), (Status.FAIL, 1), (Status.TOOL_ERROR, 2), (Status.INCOMPLETE, 3)], +) +def test_status_exit_codes_are_distinct(status: Status, code: int) -> None: + # Given a status; When returned to CI; Then its code is unambiguous. + assert Check("test", (), status, 0).returncode == code diff --git a/tests/test_security_tool_execution.py b/tests/test_security_tool_execution.py new file mode 100644 index 0000000..8ab4151 --- /dev/null +++ b/tests/test_security_tool_execution.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from datetime import date +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +from tools import dependency_audit_policy, security_evidence +from tools.security_report import Check, Status + + +def test_bandit_inventory_matches_disk(monkeypatch: pytest.MonkeyPatch) -> None: + # Given a complete fake scan; When inventory loses a file; Then coverage fails closed. + metrics = { + p.as_posix(): {"loc": 1} + for surface in security_evidence.SCOPE + for p in Path(surface).rglob("*.py") + } + metrics["_totals"] = {"loc": len(metrics)} + raw = json.dumps({"errors": [], "metrics": metrics, "results": []}) + + def complete(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: + assert "--ignore-nosec" in command + return subprocess.CompletedProcess(command, 0, stdout=raw, stderr="CANARY_SECRET") + + monkeypatch.setattr(subprocess, "run", complete) + monkeypatch.setattr(security_evidence, "version", lambda name: "1.9.4") + assert security_evidence.run_bandit().status == Status.PASS + del metrics[next(iter(metrics))] + raw = json.dumps({"errors": [], "metrics": metrics, "results": []}) + assert security_evidence.run_bandit().status == Status.INCOMPLETE + + +@pytest.mark.parametrize("marker", ["# nosec", "# nosec B999", "# nosec B110"]) +def test_unlisted_suppression_is_incomplete( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, marker: str +) -> None: + # Given a new inline marker; When scanned; Then its exact location needs review. + (tmp_path / "src").mkdir() + (tmp_path / "src/x.py").write_text(f"value = 1 {marker}\n") + (tmp_path / "security").mkdir() + (tmp_path / "security/bandit-policy.json").write_text( + '{"schema_version":1,"exceptions":[],"inline_suppressions":[]}' + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(security_evidence, "SCOPE", ("src",)) + raw = json.dumps( + { + "errors": [], + "metrics": {"_totals": {"loc": 40}, **{f"src/{i}.py": {} for i in range(40)}}, + "results": [], + } + ) + + def complete(*args: str, **kwargs: str) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess([], 0, stdout=raw, stderr="") + + monkeypatch.setattr(subprocess, "run", complete) + monkeypatch.setattr(security_evidence, "version", lambda name: "1.9.4") + assert security_evidence.run_bandit().status == Status.INCOMPLETE + + +@pytest.mark.parametrize("failure", [OSError(), subprocess.TimeoutExpired("scanner", 1)]) +def test_execution_errors_never_pass( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: OSError | subprocess.TimeoutExpired +) -> None: + # Given unavailable tools; When invoked; Then no raw diagnostic is retained. + (tmp_path / "example.dist-info").mkdir() + + def fail(*args: str, **kwargs: str) -> subprocess.CompletedProcess[str]: + raise failure + + monkeypatch.setattr(subprocess, "run", fail) + assert security_evidence.run_bandit().status == Status.TOOL_ERROR + assert dependency_audit_policy.run_audit(tmp_path, tmp_path).status == Status.TOOL_ERROR + + +def test_audit_uses_only_candidate_installation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given an isolated inventory; When audited; Then tooling cannot contaminate its closure. + (tmp_path / "example.dist-info").mkdir() + policy = tmp_path / "policy.json" + policy.write_text('{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}') + + def complete(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: + assert command[command.index("--path") + 1] == str(tmp_path) + return subprocess.CompletedProcess( + command, + 0, + stdout='{"dependencies":[{"name":"example","version":"1","vulns":[]}]}', + stderr="CANARY_SECRET", + ) + + monkeypatch.setattr(subprocess, "run", complete) + monkeypatch.setattr(dependency_audit_policy, "version", lambda name: "2.10.1") + assert dependency_audit_policy.run_audit(tmp_path, policy).status == Status.PASS + + +@pytest.mark.parametrize("development", [False, True]) +@pytest.mark.parametrize( + "status,code", [(Status.FAIL, 1), (Status.TOOL_ERROR, 2), (Status.INCOMPLETE, 3)] +) +def test_audit_cli_preserves_failure_codes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, development: bool, status: Status, code: int +) -> None: + # Given a classified failure; When CLI writes evidence; Then the same failure reaches CI. + args = ["audit", "--site-packages", str(tmp_path), "--output", str(tmp_path)] + if development: + args.append("--development") + monkeypatch.setattr(sys, "argv", args) + monkeypatch.setattr( + dependency_audit_policy, "run_audit", lambda path, policy: Check("pip-audit", (), status, 1) + ) + assert dependency_audit_policy.main() == code + assert json.loads((tmp_path / "summary.json").read_text())["checks"][0]["status"] == status + + +def test_real_cli_writes_stdlib_only_evidence(tmp_path: Path) -> None: + # Given the real CLI; When action evidence is missing; Then it writes INCOMPLETE and exits 3. + result = subprocess.run( + [ + sys.executable, + "-m", + "tools.security_evidence", + "semgrep", + "--outcome", + "cancelled", + "--output", + str(tmp_path), + ], + capture_output=True, + check=False, + ) + assert result.returncode == 3 + assert json.loads((tmp_path / "summary.json").read_text())["schema_version"] == 1 + + +def test_incomplete_scan_cannot_be_excepted() -> None: + # Given missing coverage; When exceptions are applied; Then they cannot waive completeness. + check = Check("bandit", (), Status.INCOMPLETE, 0) + assert security_evidence.apply_bandit_policy(check, "{}", date(2026, 10, 22)) == check diff --git a/tools/security_evidence.py b/tools/security_evidence.py new file mode 100644 index 0000000..0e433b2 --- /dev/null +++ b/tools/security_evidence.py @@ -0,0 +1,233 @@ +"""Execute Bandit's explicit product inventory and record Semgrep action evidence.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +from datetime import date +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +import re +import subprocess +import sys +import tokenize + +from tools.security_exceptions import parse_exceptions +from tools.security_report import ( + Check, + EvidenceError, + Finding, + Status, + identifier, + integer, + load, + mapping, + sequence, + text, + write_report, +) + +SCOPE = ("src", "examples", "scripts", "tools", "benchmarks") +FILE_FLOOR = 38 + + +def classify_bandit(raw: str, exit_status: int, *, floor: int = FILE_FLOOR) -> Check: + result = Check("bandit", SCOPE, Status.TOOL_ERROR, exit_status) + try: + data = load(raw) + metrics = mapping(data["metrics"]) + totals = mapping(metrics["_totals"]) + files = tuple(identifier(f.removeprefix("./")) for f in metrics if f != "_totals") + findings: list[Finding] = [] + counts = { + "files": len(files), + "loc": integer(totals.get("loc", 0)), + "errors": len(sequence(data["errors"])), + } + for entry in sequence(data["results"]): + item = mapping(entry) + severity, confidence = text(item["issue_severity"]), text(item["issue_confidence"]) + if severity not in {"LOW", "MEDIUM", "HIGH", "UNDEFINED"} or confidence not in { + "LOW", + "MEDIUM", + "HIGH", + "UNDEFINED", + }: + raise EvidenceError("invalid finding classification") + findings.append( + Finding( + identifier(item["test_id"]), + identifier(text(item["filename"]).removeprefix("./")), + integer(item["line_number"]), + severity, + confidence, + ) + ) + for key in (f"severity.{severity}", f"confidence.{confidence}"): + counts[key] = counts.get(key, 0) + 1 + suppressions = integer(totals.get("nosec", 0)) + integer(totals.get("skipped_tests", 0)) + result = replace(result, counts=counts, suppressions=suppressions, findings=tuple(findings)) + if exit_status not in {0, 1} or counts["errors"]: + return replace(result, message="Scanner failure/error files; scan unusable.") + if len(files) < floor or counts["loc"] == 0 or suppressions: + return replace( + result, status=Status.INCOMPLETE, message="Scan floor/skip policy failed." + ) + if bool(findings) != (exit_status == 1): + return replace(result, message="Scanner exit status disagrees with findings.") + return replace(result, status=Status.FAIL if findings else Status.PASS) + except (ValueError, KeyError, TypeError): + return replace(result, message="Invalid Bandit JSON; raw diagnostics discarded.") + + +def apply_bandit_policy(check: Check, policy_raw: str, today: date) -> Check: + if check.status not in {Status.PASS, Status.FAIL}: + return check + try: + policy = load(policy_raw) + if policy["schema_version"] != 1: + raise EvidenceError("unsupported policy") + rules = parse_exceptions( + sequence(policy["exceptions"]) + sequence(policy.get("inline_suppressions", [])), today + ) + if any( + r.expires < today + or not r.file + or not r.line + or r.package + or r.kind != "reviewed-static" + for r in rules + ): + return replace( + check, status=Status.FAIL, message="Expired or invalid static exception." + ) + applied: list[str] = [] + blocking = 0 + for finding in check.findings: + if any( + (r.id, r.file, r.line) == (finding.id, finding.file, finding.line) for r in rules + ): + applied.append(f"{finding.id}:{finding.file}:{finding.line}") + else: + blocking += 1 + return replace( + check, + status=Status.FAIL if blocking else Status.PASS, + exceptions_applied=tuple(applied), + counts={ + **check.counts, + "blocking": blocking, + "stale_exceptions": len(rules) - len(set(applied)), + }, + ) + except (ValueError, KeyError, TypeError): + return replace(check, status=Status.TOOL_ERROR, message="Invalid Bandit exception policy.") + + +def run_bandit() -> Check: + try: + process = subprocess.run( + [sys.executable, "-m", "bandit", "-r", *SCOPE, "-f", "json", "--ignore-nosec"], + capture_output=True, + text=True, + check=False, + timeout=180, + ) + result = classify_bandit(process.stdout, process.returncode) + expected = {p.as_posix() for surface in SCOPE for p in Path(surface).rglob("*.py")} + scanned = set(mapping(load(process.stdout)["metrics"])) - {"_totals"} + if result.status != Status.TOOL_ERROR and ( + {p.removeprefix("./") for p in scanned} != expected + or any(not Path(surface).is_dir() for surface in SCOPE) + ): + result = replace( + result, + status=Status.INCOMPLETE, + message="Scanned inventory differs from declared surfaces.", + ) + policy = Path("security/bandit-policy.json").read_text(encoding="utf-8") + result = apply_bandit_policy(result, policy, date.today()) + rules = parse_exceptions(load(policy)["inline_suppressions"], date.today()) + observed: set[tuple[str, str, int]] = set() + suppressions = 0 + for filename in expected: + with tokenize.open(filename) as stream: + for token in tokenize.generate_tokens(stream.readline): + if token.type != tokenize.COMMENT or "nosec" not in token.string: + continue + ids = re.findall(r"B\d{3}", token.string) + suppressions += 1 + observed.update((rule_id, filename, token.start[0]) for rule_id in ids) + if not ids: + observed.add(("blanket", filename, token.start[0])) + allowed = {(r.id, r.file, r.line) for r in rules} + if observed != allowed and result.status != Status.TOOL_ERROR: + result = replace( + result, + status=Status.INCOMPLETE, + message="Inline suppression inventory differs from reviewed allow-list.", + ) + if any(r.expires < date.today() for r in rules) and result.status in { + Status.PASS, + Status.FAIL, + }: + result = replace( + result, status=Status.FAIL, message="Expired inline suppression review." + ) + return replace( + result, + suppressions=suppressions, + tool_versions={"bandit": version("bandit")}, + counts={**result.counts, "inline_suppressions_reviewed": len(allowed)}, + message=result.message or "Existing inline suppressions audited with ignore-nosec.", + ) + except ( + OSError, + subprocess.TimeoutExpired, + ValueError, + KeyError, + TypeError, + PackageNotFoundError, + tokenize.TokenError, + SyntaxError, + ): + return Check( + "bandit", + SCOPE, + Status.TOOL_ERROR, + None, + message="Bandit execution/evidence collection failed.", + ) + + +def semgrep_evidence(outcome: str) -> Check: + code: int | None + match outcome: + case "success": + status, code = Status.PASS, 0 + case "failure": + status, code = Status.TOOL_ERROR, 1 + case _: + status, code = Status.INCOMPLETE, None + return Check( + "semgrep", + ("p/python", *SCOPE), + status, + code, + tool_versions={"action": "713efdd345f3035192eaa63f56867b88e63e4e5d"}, + message="Action outcome only; finding count unknown; failures may include findings.", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("check", choices=("bandit", "semgrep")) + parser.add_argument("--outcome", default="") + parser.add_argument("--output", type=Path, default=Path("security-results")) + args = parser.parse_args() + check = run_bandit() if args.check == "bandit" else semgrep_evidence(args.outcome) + return write_report([check], args.output) + + +if __name__ == "__main__": + raise SystemExit(main()) From dc383bb7609944e1c1c9e06213c33d7dd12e2924 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 04/13] ci(security): retain scanner evidence and typecheck security tools Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/security.yml | 35 +++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 6f76dbd..c405347 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -7,6 +7,7 @@ on: branches: [main] schedule: - cron: "0 2 * * 1" + workflow_dispatch: permissions: contents: read @@ -29,7 +30,18 @@ jobs: make install - name: Run Bandit - run: make security + run: hatch run python -m tools.security_evidence bandit + + - name: Type check security tools + run: hatch run security-typecheck + + - name: Upload redacted security evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: security-bandit + path: security-results/** + if-no-files-found: error semgrep: runs-on: ubuntu-latest @@ -38,6 +50,27 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Run Semgrep + id: scan uses: semgrep/semgrep-action@713efdd345f3035192eaa63f56867b88e63e4e5d # v1 with: config: p/python + + - name: Set up evidence interpreter + if: always() + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Record Semgrep execution evidence + if: always() + env: + SCAN_OUTCOME: ${{ steps.scan.outcome }} + run: python -m tools.security_evidence semgrep --outcome "$SCAN_OUTCOME" + + - name: Upload redacted security evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: security-semgrep + path: security-results/** + if-no-files-found: error diff --git a/pyproject.toml b/pyproject.toml index 014e8a6..3d79ec3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ format = "ruff format src tests tools examples benchmarks scripts" format-check = "ruff format --check src tests tools examples benchmarks scripts" style = "ruff check src tests tools examples benchmarks scripts && ruff format --check src tests tools examples benchmarks scripts" typecheck = "mypy src tests" +security-typecheck = "mypy --strict --explicit-package-bases tools/security_report.py tools/security_exceptions.py tools/security_evidence.py tools/dependency_audit_policy.py" lint = "hatch run style && hatch run typecheck" test = "pytest -v tests" cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/" From c669cdc23a2e3259c28eeb67acc36011900a7bc2 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 05/13] ci(security): report isolated base otel and development audits Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/dependency-audit.yml | 39 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index da523a2..e3ec86f 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -21,22 +21,43 @@ jobs: with: python-version: "3.14" - - name: Install audit tool and runtime dependencies + - name: Build wheel and isolated reporting environments run: | - python -m pip install --upgrade pip - python -m pip install "pip-audit==2.10.1" . + python -m pip install 'pip-audit==2.10.1' 'build==1.5.0' + python -m build --wheel + wheels=(dist/*.whl) + test "${#wheels[@]}" -eq 1 + python -m venv --without-pip "$RUNNER_TEMP/runtime" + python -m pip --python "$RUNNER_TEMP/runtime/bin/python" install "${wheels[0]}" + python -m venv --without-pip "$RUNNER_TEMP/otel" + python -m pip --python "$RUNNER_TEMP/otel/bin/python" install "${wheels[0]}[otel]" + python -m venv --without-pip "$RUNNER_TEMP/development" + python -m pip --python "$RUNNER_TEMP/development/bin/python" install '.[dev]' - name: Audit runtime dependencies continue-on-error: true run: | - echo "## Runtime dependency audit" >> "$GITHUB_STEP_SUMMARY" - python -m pip_audit 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + site_packages=$("$RUNNER_TEMP/runtime/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + python -m tools.dependency_audit_policy --site-packages "$site_packages" --output security-results/runtime - - name: Install development dependencies - run: python -m pip install ".[dev]" + - name: Audit OTel runtime dependencies + if: always() + continue-on-error: true + run: | + site_packages=$("$RUNNER_TEMP/otel/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + python -m tools.dependency_audit_policy --site-packages "$site_packages" --output security-results/otel - name: Audit runtime and development dependencies + if: always() continue-on-error: true run: | - echo "## Runtime and development dependency audit" >> "$GITHUB_STEP_SUMMARY" - python -m pip_audit 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + site_packages=$("$RUNNER_TEMP/development/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + python -m tools.dependency_audit_policy --site-packages "$site_packages" --development --output security-results/development + + - name: Upload redacted reporting evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dependency-security-report + path: security-results/** + if-no-files-found: error From a52d6434656daa3a37bd3e43d00806262a0483bc Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 06/13] ci(security): gate exact-wheel releases on base and otel dependency audits Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/publish-pypi.yml | 50 ++++++++++++++++- tests/test_security_workflows.py | 88 ++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 tests/test_security_workflows.py diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 000aecb..351eef4 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -263,9 +263,57 @@ jobs: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + runtime-dependency-audit: + name: Runtime dependency audit gate + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + interpreter: ["3.11", "3.12", "3.13", "3.14"] + closure: ["base", "otel"] + steps: + - name: Checkout security policy and tooling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up audit interpreter + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.interpreter }} + - name: Download the exact candidate artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dist + path: candidate-dist + - name: Install only the candidate runtime closure + env: + CLOSURE: ${{ matrix.closure }} + run: | + python -m pip install 'pip-audit==2.10.1' + python -m venv --without-pip "$RUNNER_TEMP/candidate" + wheels=(candidate-dist/*.whl) + test "${#wheels[@]}" -eq 1 + if [ "$CLOSURE" = "otel" ]; then + python -m pip --python "$RUNNER_TEMP/candidate/bin/python" install "${wheels[0]}[otel]" + else + python -m pip --python "$RUNNER_TEMP/candidate/bin/python" install "${wheels[0]}" + fi + - name: Enforce runtime dependency policy + run: | + site_packages=$("$RUNNER_TEMP/candidate/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + python -m tools.dependency_audit_policy --site-packages "$site_packages" --output security-results/${{ matrix.closure }} + - name: Upload redacted release security evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-security-${{ matrix.interpreter }}-${{ matrix.closure }} + path: security-results/** + if-no-files-found: error + publish: name: Publish to PyPI - needs: [build, lib-tests, cookbook-smoke, cookbook-host-smoke, azure-e2e] + needs: [build, lib-tests, cookbook-smoke, cookbook-host-smoke, azure-e2e, runtime-dependency-audit] runs-on: ubuntu-latest environment: pypi permissions: diff --git a/tests/test_security_workflows.py b/tests/test_security_workflows.py new file mode 100644 index 0000000..80d1e79 --- /dev/null +++ b/tests/test_security_workflows.py @@ -0,0 +1,88 @@ +from pathlib import Path +import tomllib + +import yaml + +from tools.lint_release_workflows import check_pins, check_publish_needs +from tools.security_evidence import SCOPE + +ROOT = Path(__file__).resolve().parents[1] + + +def test_release_gate_preserves_existing_tiers_and_candidate() -> None: + # Given release configuration; When inspected; Then the audit adds to every existing gate. + text = (ROOT / ".github/workflows/publish-pypi.yml").read_text() + jobs = yaml.safe_load(text)["jobs"] + assert check_publish_needs(text, "publish-pypi.yml") == [] + assert set(jobs["publish"]["needs"]) == { + "build", + "lib-tests", + "cookbook-smoke", + "cookbook-host-smoke", + "azure-e2e", + "runtime-dependency-audit", + } + gate = jobs["runtime-dependency-audit"] + assert gate["permissions"] == {"contents": "read"} + assert gate["needs"] == "build" + assert gate["strategy"]["matrix"]["interpreter"] == ["3.11", "3.12", "3.13", "3.14"] + assert gate["strategy"]["matrix"]["closure"] == ["base", "otel"] + assert not gate.get("continue-on-error") + assert all(not s.get("continue-on-error") for s in gate["steps"]) + download = next(s for s in gate["steps"] if "download-artifact@" in s.get("uses", "")) + assert download["with"]["name"] == "dist" + commands = "\n".join(s.get("run", "") for s in gate["steps"]) + assert "pip-audit==2.10.1" in commands + assert "--without-pip" in commands + assert '"${wheels[0]}[otel]"' in commands + assert "tools.dependency_audit_policy" in commands + assert "build --wheel" not in commands + + +def test_evidence_uploads_always_run_with_canonical_pins() -> None: + # Given every evidence workflow; When inspected; Then failed scans retain redacted evidence. + for filename in ("security.yml", "dependency-audit.yml", "publish-pypi.yml"): + text = (ROOT / ".github/workflows" / filename).read_text() + workflow = yaml.safe_load(text) + assert workflow["permissions"] == {"contents": "read"} + assert check_pins(text, filename) == [] + uploads = [ + s + for job in workflow["jobs"].values() + for s in job.get("steps", []) + if "upload-artifact@" in s.get("uses", "") + and s.get("with", {}).get("path", "").startswith("security-results") + ] + assert uploads + assert all( + s["if"] == "always()" and s["with"]["if-no-files-found"] == "error" for s in uploads + ) + + +def test_security_scans_declared_executable_surfaces() -> None: + # Given declared surfaces; When workflow runs; Then neither source nor examples are omitted. + workflow = yaml.safe_load((ROOT / ".github/workflows/security.yml").read_text()) + assert SCOPE == ("src", "examples", "scripts", "tools", "benchmarks") + steps = workflow["jobs"]["bandit"]["steps"] + assert any("tools.security_evidence bandit" in s.get("run", "") for s in steps) + assert any("security-typecheck" in s.get("run", "") for s in steps) + semgrep = workflow["jobs"]["semgrep"]["steps"] + action = next(s for s in semgrep if "semgrep/semgrep-action@" in s.get("uses", "")) + assert action["uses"] == "semgrep/semgrep-action@713efdd345f3035192eaa63f56867b88e63e4e5d" + assert action["with"]["config"] == "p/python" + assert not action.get("continue-on-error") + assert any(s.get("if") == "always()" and "--outcome" in s.get("run", "") for s in semgrep) + scripts = tomllib.loads((ROOT / "pyproject.toml").read_text())["tool"]["hatch"]["envs"][ + "default" + ]["scripts"] + assert "--strict" in scripts["security-typecheck"] + + +def test_scheduled_audits_reuse_policy_as_reporting_only() -> None: + # Given scheduled audit lanes; When inspected; Then runtime/OTel/dev use one classifier. + workflow = yaml.safe_load((ROOT / ".github/workflows/dependency-audit.yml").read_text()) + steps = workflow["jobs"]["pip-audit"]["steps"] + audits = [s for s in steps if "tools.dependency_audit_policy" in s.get("run", "")] + assert len(audits) == 3 + assert all(s["continue-on-error"] for s in audits) + assert any("--development" in s["run"] for s in audits) From 4cfbfeb6871626effeed6c9119f20e064a95e13f Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 01:39:36 +0900 Subject: [PATCH 07/13] docs(security): define evidence policy and production deployment boundaries Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .gitignore | 1 + AGENTS.md | 3 + SECURITY.md | 140 +++++++++++++++++++++++++++++++++++++++++++++ examples/README.md | 10 ++++ 4 files changed, 154 insertions(+) diff --git a/.gitignore b/.gitignore index 3006d94..3e33816 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ env/ .ruff_cache/ .coverage coverage.xml +security-results/ htmlcov/ # Build artifacts diff --git a/AGENTS.md b/AGENTS.md index c5aa8ad..8d94b96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,9 @@ ## Working Rules +Security gates, evidence redaction, production boundaries and expiring exceptions: +see [SECURITY.md](SECURITY.md). + ### Test Coverage - Maintain test coverage at **95% or above** for committed changes and PRs. - Run `hatch run pytest --cov --cov-report=term-missing -q` to verify before submitting changes. diff --git a/SECURITY.md b/SECURITY.md index 081cb24..dff38bb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,3 +33,143 @@ If you prefer email, contact: yeongseon.choe@gmail.com | ------- | --------- | | Latest release | :white_check_mark: | | Older releases | :x: | + +## Automated controls and permissions + +| Control | Runs where | Enforcement | +| --- | --- | --- | +| Bandit 1.9.4 | PR, main, weekly, dispatch (`security.yml`) | Blocking scan; every severity reviewed | +| SHA-pinned Semgrep `p/python` | Same workflow | Existing blocking action; execution evidence added | +| CodeQL `security-extended` | PR, main, weekly | Existing analysis/code-scanning control, unchanged | +| Runtime pip-audit 2.10.1 | Exact candidate wheel, Python 3.11–3.14, base and `otel` closures | Blocking dependency of `publish` | +| Runtime/OTel/development audit | Weekly/dispatch (`dependency-audit.yml`) | Reporting-only, shared classifier | +| SBOM | Weekly/dispatch | Existing installed-wheel inventory, unchanged | + +Branch-protection requirements are repository settings, not established by these +workflow files. Existing merge checks, runtime/host smoke and real-Azure certification +remain intact. This change adds no write permission: security workflows and the new +release audit job use only `contents: read`. CodeQL retains `contents: read` and +`security-events: write`; Azure certification retains `contents: read` and +`id-token: write`; PyPI publishing retains `id-token: write` and its trusted-publisher +exception. Publishing still uploads the exact `build` artifact, without rebuilding. + +## Static scope and reviewed suppressions + +`hatch run python -m tools.security_evidence bandit` scans every Python file under +`src/`, `examples/` (including `e2e_app` and `otel_app`), `scripts/`, `tools/`, and +`benchmarks/`. Benchmarks are included even when not executed by a particular CI +lane. There are no per-file exclusions within these surfaces. Tests/fixtures are +excluded from this product/tool scan. Bytecode and non-Python artifacts are not +Bandit inputs. Saved host logs and deployment configuration require separate review; +this scan does not claim to analyze Bicep, workflows, or arbitrary log contents. +Inventory must equal the on-disk Python inventory, contain at least 38 files and +have positive LOC. Missing surfaces, errors and unlisted suppressions cannot pass. + +Bandit uses `--ignore-nosec`, so existing markers cannot hide findings. The separate +`inline_suppressions` allow-list in `security/bandit-policy.json` requires exact +ID/file/line equality with all markers, including markers that Bandit does not +currently flag. The baseline has 26 markers across seven source files: + +| Source module | Count | Rules | +| --- | ---: | --- | +| `_context.py` | 10 | B110 | +| `_filters.py` | 7 | B110 | +| `_host_config.py` | 2 | B110 | +| `_otel.py` | 4 | B110 | +| `_host_instance.py` | 1 | B110 | +| `_redaction.py` | 1 | B112 | +| `_decorator.py` | 1 | B307 | + +These are existing best-effort/never-raise contracts and handler-owned annotation +evaluation, not new runtime changes. Retaining a masking-resilience exception is +not proof that runtime failures redact safely; runtime hardening is separate. +The evidence reports marker counts and retains all unsuppressed findings. Other +baseline reviews cover synthetic fixtures, type-narrowing assertions, shell-free +tool invocations, a fixed-host documentation URL checker, and a status-enum false +positive. No global rule exclusions replace review. `make security` retains its +existing local src-only command; CI uses the expanded helper. Security tools receive +a separate strict mypy invocation (`hatch run security-typecheck`), additive to the +existing project typecheck rather than changing its scope. + +## Dependency and exception policy + +Every known advisory in the resolved candidate runtime closure blocks release, +regardless of severity. Missing severity is `UNKNOWN`, never assumed low. This is +stricter than a high/critical-only gate and avoids missing-CVSS false greens. +The base closure contains the wheel and required dependencies; a separate matrix +leg audits the wheel's `otel` extra. That extra declares `opentelemetry-api`, not the +example application's separately installed Azure Monitor exporter. Development, +docs and example-only dependencies are not part of the release base/OTel closure. +Each supported interpreter is audited because environment markers affect resolution. +Candidate environments use `venv --without-pip`; pip-audit runs outside with `--path`. +No audit-tool dependencies contaminate the candidate installation. + +Both policy files require maintainer review, exact identifiers, `reason`, `owner`, +`added` and `expires` (ISO dates), and a positive lifetime at most 90 days. Static +exceptions require `reviewed-static` with exact file/line. Dependency exceptions +require exact package/advisory and `not-affected` (independently verified evidence) +or `fix-not-available` (only while no fixed version is listed). Package separators +are normalized; advisory aliases are not silently ignored. Expired entries block +even when unused; stale entries are counted for removal. The dependency policy +starts empty. Do not add blanket skips or new inline markers instead of review. +When runtime changes move existing lines, re-review the exact allow-list rather +than broadening it; automation and runtime branches must reconcile that inventory. + +## Failure semantics and recovery + +Helpers exit PASS=0, FAIL=1 (unexcepted findings/expired exceptions), TOOL_ERROR=2 +(crash, timeout, missing/invalid JSON, invalid policy, zero audited packages or +tool/database failure), INCOMPLETE=3 (empty static scan, missing coverage, skipped +packages, unlisted inline suppressions or absent execution evidence). Every non-PASS +blocks release. Scheduled reporting preserves the same status but does not gate a +release. Semgrep exposes only action outcome: success PASS, failure conservatively +TOOL_ERROR with unknown findings count, skipped/cancelled INCOMPLETE. Its original +failing step remains blocking; an action failure alone is not called a vulnerability. + +Advisory database outages can delay publication: this is an intentional availability +tradeoff for fail-closed verification. Bounded timeouts limit hangs. Never bypass +the gate. Retry the same unchanged tagged candidate with +`gh workflow run publish-pypi.yml --ref vX.Y.Z`. Dispatch takes no inputs; use the +existing tag as its ref, not main. Corrections to code/workflows must fix forward +to a new release candidate; never move/reuse the old tag. + +## Evidence and production boundaries + +Helpers emit schema-version-1 `security-results/**/summary.json` containing scope, +check/status, tool exit status, counts, tool versions, suppression counts, applied +exception identifiers and allow-listed finding IDs, relative paths/lines, severities, +confidence and package identifiers. Human-readable status/counts go to Job Summary. +Uploads run `if: always()` with unique artifact names and fail if evidence is absent. +Installation/setup failures can prevent report creation; missing artifacts remain +an explicit failure, never PASS. Generated local evidence is gitignored. + +Security evidence must never contain credentials, connection strings, authorization +headers or original user payloads. Helpers do not retain raw scanner stdout/stderr, +source snippets, advisory descriptions or policy reasons; malformed diagnostics +produce fixed messages. Deterministic canary tests guard JSON and Markdown output. +This guarantee covers these evidence helpers, not all pre-existing host/exporter logs. +A finding is a **code/config review finding until runtime impact is independently +demonstrated**; a clean scan does not prove the absence of vulnerabilities. + +Examples are local-only synthetic demonstrations except `e2e_app`, which is also +ephemeral public Azure certification infrastructure. Anonymous routes and disabled +sampling are test conveniences, not production defaults. Query parameters such as +`correlation_id` and `order_id` reach logs: use synthetic identifiers only, never +real payloads, credentials, headers or PII. The OTel masking example uses a dummy +value; do not rely on it as a complete privacy profile. For secure logging +configuration, see the redaction documentation. This branch adds no runtime API. + +Production deployments need explicit authentication/authorization, restricted +ingress, HTTPS/minimum TLS, private blobs, managed identities and storage RBAC, +restricted telemetry access/retention, and appropriate sampling. Review of examples +found no request-body/header dump or printed connection string/instrumentation key; +user-controlled identifiers still require care. `infra/main.bicep` enforces HTTPS, +storage TLS 1.2 and private blobs, but retains shared-key storage connection strings, +public network reachability, no declared managed identity, and an Application +Insights connection-string deployment output. The Azure workflow emits deployment +outputs, so that output is a telemetry-configuration disclosure risk; do not treat +deployment logs as sanitized security evidence. Removing outputs, moving storage to +identity/RBAC, private endpoints, explicit app/SCM TLS and authenticated certification +routes require real-Azure validation. They are follow-ups, not changes in this port. +Host diagnostics/saved logs also need an independently verified allow-list and +retention review before accepting real user data. Use synthetic data only in CI. diff --git a/examples/README.md b/examples/README.md index 5ac1cd3..5c6ba06 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,15 @@ # Examples +These are local-only demonstrations using synthetic data, not production security +templates. `e2e_app` is additionally deployed as ephemeral public Azure certification +infrastructure; its anonymous routes must remain compatible with certification. +`otel_app` has anonymous routes for local demonstration. Configure authentication, +restricted ingress and telemetry access before production deployment. Both apps +disable sampling for demonstration/test visibility, not as production guidance. +Query identifiers (`correlation_id`, `order_id`) are logged: never send credentials, +headers, request payloads or PII. Connection strings belong in deployment environment +configuration and must not be printed. See [security boundaries](../SECURITY.md). + `azure-functions-logging` includes standalone example scripts that demonstrate every public API feature. Each script runs without Azure Functions installed. From 9b80f4771684deadd760608ebbba943b0dfae173 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 07:15:55 +0900 Subject: [PATCH 08/13] ci(security): date policy entries in UTC so CI does not read them as future-dated --- security/bandit-policy.json | 86 ++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/security/bandit-policy.json b/security/bandit-policy.json index 569fabd..4702bf8 100644 --- a/security/bandit-policy.json +++ b/security/bandit-policy.json @@ -1,50 +1,50 @@ { "schema_version": 1, "inline_suppressions": [ - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":207,"kind":"reviewed-static","reason":"Existing context cleanup is deliberately non-raising; no runtime change in this automation port.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":231,"kind":"reviewed-static","reason":"Existing context propagation is deliberately best-effort and non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":236,"kind":"reviewed-static","reason":"Existing worker metadata propagation must not crash application execution.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":245,"kind":"reviewed-static","reason":"Existing context cleanup must not crash application execution.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":251,"kind":"reviewed-static","reason":"Existing worker thread-local cleanup is deliberately silent on failure.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":371,"kind":"reviewed-static","reason":"Existing context metadata inspection is best-effort and never raises.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":388,"kind":"reviewed-static","reason":"Existing propagation failure must not replace the application result.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":397,"kind":"reviewed-static","reason":"Existing executor context restoration is deliberately non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":401,"kind":"reviewed-static","reason":"Existing executor worker metadata restoration is deliberately silent.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_context.py","line":577,"kind":"reviewed-static","reason":"Existing binding metadata cleanup is best-effort and never raises.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":199,"kind":"reviewed-static","reason":"Existing unreadable host configuration is non-fatal by design.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":310,"kind":"reviewed-static","reason":"Existing host configuration override inspection is non-fatal.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":143,"kind":"reviewed-static","reason":"Existing filter sampling must not raise into application logging.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":348,"kind":"reviewed-static","reason":"Existing per-field masking resilience retained; runtime hardening is a separate change, not certified by this review.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":352,"kind":"reviewed-static","reason":"Existing filter never-raise contract retained; this exception does not establish privacy guarantees.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":379,"kind":"reviewed-static","reason":"Existing message masking never-raise path retained; separate runtime hardening owns fail-closed behavior.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":401,"kind":"reviewed-static","reason":"Existing exception masking resilience retained without claiming fail-closed runtime behavior.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":494,"kind":"reviewed-static","reason":"Existing per-field flattening is deliberately non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":496,"kind":"reviewed-static","reason":"Existing flattening filter never-raise contract retained.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":49,"kind":"reviewed-static","reason":"Optional tracing dependency failures degrade gracefully by design.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":107,"kind":"reviewed-static","reason":"Optional trace activation failures must not crash the handler.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":142,"kind":"reviewed-static","reason":"Existing tracing context cleanup is deliberately non-raising.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":151,"kind":"reviewed-static","reason":"Existing optional OTel context cleanup is deliberately silent.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B110","file":"src/azure_functions_logging/_host_instance.py","line":60,"kind":"reviewed-static","reason":"Existing best-effort host metadata resolution must never crash logging.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B112","file":"src/azure_functions_logging/_redaction.py","line":170,"kind":"reviewed-static","reason":"Existing broken-pattern isolation retained; runtime redaction hardening is separate and not certified here.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B307","file":"src/azure_functions_logging/_decorator.py","line":83,"kind":"reviewed-static","reason":"Existing evaluation uses the handler author's annotation and globals, not HTTP input; no runtime edit permitted in this port.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"} + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":207,"kind":"reviewed-static","reason":"Existing context cleanup is deliberately non-raising; no runtime change in this automation port.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":231,"kind":"reviewed-static","reason":"Existing context propagation is deliberately best-effort and non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":236,"kind":"reviewed-static","reason":"Existing worker metadata propagation must not crash application execution.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":245,"kind":"reviewed-static","reason":"Existing context cleanup must not crash application execution.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":251,"kind":"reviewed-static","reason":"Existing worker thread-local cleanup is deliberately silent on failure.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":371,"kind":"reviewed-static","reason":"Existing context metadata inspection is best-effort and never raises.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":388,"kind":"reviewed-static","reason":"Existing propagation failure must not replace the application result.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":397,"kind":"reviewed-static","reason":"Existing executor context restoration is deliberately non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":401,"kind":"reviewed-static","reason":"Existing executor worker metadata restoration is deliberately silent.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":577,"kind":"reviewed-static","reason":"Existing binding metadata cleanup is best-effort and never raises.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":199,"kind":"reviewed-static","reason":"Existing unreadable host configuration is non-fatal by design.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":310,"kind":"reviewed-static","reason":"Existing host configuration override inspection is non-fatal.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":143,"kind":"reviewed-static","reason":"Existing filter sampling must not raise into application logging.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":348,"kind":"reviewed-static","reason":"Existing per-field masking resilience retained; runtime hardening is a separate change, not certified by this review.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":352,"kind":"reviewed-static","reason":"Existing filter never-raise contract retained; this exception does not establish privacy guarantees.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":379,"kind":"reviewed-static","reason":"Existing message masking never-raise path retained; separate runtime hardening owns fail-closed behavior.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":401,"kind":"reviewed-static","reason":"Existing exception masking resilience retained without claiming fail-closed runtime behavior.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":494,"kind":"reviewed-static","reason":"Existing per-field flattening is deliberately non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":496,"kind":"reviewed-static","reason":"Existing flattening filter never-raise contract retained.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":49,"kind":"reviewed-static","reason":"Optional tracing dependency failures degrade gracefully by design.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":107,"kind":"reviewed-static","reason":"Optional trace activation failures must not crash the handler.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":142,"kind":"reviewed-static","reason":"Existing tracing context cleanup is deliberately non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":151,"kind":"reviewed-static","reason":"Existing optional OTel context cleanup is deliberately silent.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_instance.py","line":60,"kind":"reviewed-static","reason":"Existing best-effort host metadata resolution must never crash logging.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B112","file":"src/azure_functions_logging/_redaction.py","line":170,"kind":"reviewed-static","reason":"Existing broken-pattern isolation retained; runtime redaction hardening is separate and not certified here.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B307","file":"src/azure_functions_logging/_decorator.py","line":83,"kind":"reviewed-static","reason":"Existing evaluation uses the handler author's annotation and globals, not HTTP input; no runtime edit permitted in this port.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"} ], "exceptions": [ - {"id":"B105","file":"benchmarks/bench.py","line":65,"kind":"reviewed-static","reason":"Synthetic benchmark fixture, not a credential or an authentication decision.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B105","file":"examples/otel_app/function_app.py","line":70,"kind":"reviewed-static","reason":"Synthetic masking demonstration; never use real credentials in this example.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B101","file":"scripts/assert_correlation.py","line":132,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; not authorization or input validation.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B101","file":"scripts/assert_correlation.py","line":133,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; observable certification checks remain ordinary conditionals.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B404","file":"scripts/render_mermaid.py","line":26,"kind":"reviewed-static","reason":"Documentation renderer invokes a shell-free CLI on repository Markdown.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B603","file":"scripts/render_mermaid.py","line":89,"kind":"reviewed-static","reason":"Fixed mmdc command with temporary paths as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B404","file":"tools/check_pr_format.py","line":5,"kind":"reviewed-static","reason":"Contributor formatter uses shell-free git and ruff subprocesses.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B607","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"git resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B603","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"Revision range is one argument and git options end with --, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B607","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"ruff resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B603","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"Filenames follow -- as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B404","file":"tools/dependency_audit_policy.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free pip-audit command uses sys.executable.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B603","file":"tools/dependency_audit_policy.py","line":99,"kind":"reviewed-static","reason":"Fixed pip-audit module with isolated path passed as separate argument, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B310","file":"tools/lint_doc_links.py","line":103,"kind":"reviewed-static","reason":"Caller constructs HTTPS GitHub/raw.githubusercontent URLs from parsed documentation links; fixed HEAD request, no user-request input.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B404","file":"tools/security_evidence.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free Bandit command uses sys.executable.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B603","file":"tools/security_evidence.py","line":129,"kind":"reviewed-static","reason":"Fixed Bandit command over explicit repository-controlled surfaces, no shell.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"}, - {"id":"B105","file":"tools/security_report.py","line":59,"kind":"reviewed-static","reason":"TOOL_ERROR is a public status enum, not a credential.","owner":"yeongseon","added":"2026-10-23","expires":"2027-01-20"} + {"id":"B105","file":"benchmarks/bench.py","line":65,"kind":"reviewed-static","reason":"Synthetic benchmark fixture, not a credential or an authentication decision.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B105","file":"examples/otel_app/function_app.py","line":70,"kind":"reviewed-static","reason":"Synthetic masking demonstration; never use real credentials in this example.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B101","file":"scripts/assert_correlation.py","line":132,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; not authorization or input validation.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B101","file":"scripts/assert_correlation.py","line":133,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; observable certification checks remain ordinary conditionals.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"scripts/render_mermaid.py","line":26,"kind":"reviewed-static","reason":"Documentation renderer invokes a shell-free CLI on repository Markdown.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"scripts/render_mermaid.py","line":89,"kind":"reviewed-static","reason":"Fixed mmdc command with temporary paths as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"tools/check_pr_format.py","line":5,"kind":"reviewed-static","reason":"Contributor formatter uses shell-free git and ruff subprocesses.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B607","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"git resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"Revision range is one argument and git options end with --, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B607","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"ruff resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"Filenames follow -- as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"tools/dependency_audit_policy.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free pip-audit command uses sys.executable.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/dependency_audit_policy.py","line":99,"kind":"reviewed-static","reason":"Fixed pip-audit module with isolated path passed as separate argument, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B310","file":"tools/lint_doc_links.py","line":103,"kind":"reviewed-static","reason":"Caller constructs HTTPS GitHub/raw.githubusercontent URLs from parsed documentation links; fixed HEAD request, no user-request input.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"tools/security_evidence.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free Bandit command uses sys.executable.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/security_evidence.py","line":129,"kind":"reviewed-static","reason":"Fixed Bandit command over explicit repository-controlled surfaces, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B105","file":"tools/security_report.py","line":59,"kind":"reviewed-static","reason":"TOOL_ERROR is a public status enum, not a credential.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"} ] } From a53aad3bf78c779e515b92bb1248d502f1409601 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 10:05:14 +0900 Subject: [PATCH 09/13] ci(audit): verify candidate identity before auditing frozen third parties --- security/bandit-policy.json | 2 +- tests/test_candidate_audit.py | 66 ++++++++++++++++++ tests/test_security_tool_execution.py | 27 ++++++-- tools/candidate_inventory.py | 39 +++++++++++ tools/dependency_audit_policy.py | 99 +++++++++++++++++++-------- 5 files changed, 197 insertions(+), 36 deletions(-) create mode 100644 tests/test_candidate_audit.py create mode 100644 tools/candidate_inventory.py diff --git a/security/bandit-policy.json b/security/bandit-policy.json index 4702bf8..c0565c0 100644 --- a/security/bandit-policy.json +++ b/security/bandit-policy.json @@ -41,7 +41,7 @@ {"id":"B607","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"ruff resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, {"id":"B603","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"Filenames follow -- as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, {"id":"B404","file":"tools/dependency_audit_policy.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free pip-audit command uses sys.executable.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, - {"id":"B603","file":"tools/dependency_audit_policy.py","line":99,"kind":"reviewed-static","reason":"Fixed pip-audit module with isolated path passed as separate argument, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/dependency_audit_policy.py","line":117,"kind":"reviewed-static","reason":"Fixed pip-audit module with verified frozen inventory passed as a temporary requirements path, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, {"id":"B310","file":"tools/lint_doc_links.py","line":103,"kind":"reviewed-static","reason":"Caller constructs HTTPS GitHub/raw.githubusercontent URLs from parsed documentation links; fixed HEAD request, no user-request input.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, {"id":"B404","file":"tools/security_evidence.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free Bandit command uses sys.executable.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, {"id":"B603","file":"tools/security_evidence.py","line":129,"kind":"reviewed-static","reason":"Fixed Bandit command over explicit repository-controlled surfaces, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, diff --git a/tests/test_candidate_audit.py b/tests/test_candidate_audit.py new file mode 100644 index 0000000..74bede9 --- /dev/null +++ b/tests/test_candidate_audit.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +from zipfile import ZipFile + +import pytest + +from tools.dependency_audit_policy import run_audit +from tools.security_evidence import semgrep_evidence +from tools.security_report import Status + + +@pytest.mark.parametrize("mode", ["clean", "advisory", "skip", "mismatch", "missing"]) +def test_unpublished_candidate_audits_exact_third_party_closure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: str +) -> None: + wheel = tmp_path / "candidate.whl" + with ZipFile(wheel, "w") as archive: + archive.writestr( + "candidate.dist-info/METADATA", "Name: azure-functions-logging\nVersion: 99.0\n" + ) + for name, version in [ + ("azure-functions-logging", "98.0" if mode == "mismatch" else "99.0"), + ("dependency", "1.0"), + ]: + dist = tmp_path / f"{name.replace('-', '_')}-{version}.dist-info" + dist.mkdir() + (dist / "METADATA").write_text(f"Name: {name}\nVersion: {version}\n") + policy = tmp_path / "policy.json" + policy.write_text('{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}') + + def audit(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: + requirements = Path(command[command.index("--requirement") + 1]).read_text() + assert requirements == "dependency==1.0\n" + assert "--no-deps" in command and "--disable-pip" in command + item: dict[str, object] = {"name": "dependency", "version": "1.0", "vulns": []} + if mode == "skip": + item = {"name": "dependency", "skip_reason": "not on PyPI"} + if mode == "advisory": + item["vulns"] = [{"id": "CVE-2026-12345", "fix_versions": []}] + return subprocess.CompletedProcess( + command, + int(mode == "advisory"), + json.dumps({"dependencies": [] if mode == "missing" else [item]}), + "", + ) + + monkeypatch.setattr(subprocess, "run", audit) + monkeypatch.setattr("tools.dependency_audit_policy.version", lambda _: "2.10.1") + result = run_audit(tmp_path, policy, wheel) + assert ( + result.status + == { + "clean": Status.PASS, + "advisory": Status.FAIL, + "skip": Status.INCOMPLETE, + "mismatch": Status.INCOMPLETE, + "missing": Status.TOOL_ERROR, + }[mode] + ) + + +def test_semgrep_action_success_is_incomplete() -> None: + assert semgrep_evidence("success").status == Status.INCOMPLETE diff --git a/tests/test_security_tool_execution.py b/tests/test_security_tool_execution.py index 8ab4151..4cc3b0f 100644 --- a/tests/test_security_tool_execution.py +++ b/tests/test_security_tool_execution.py @@ -5,6 +5,7 @@ from pathlib import Path import subprocess import sys +from zipfile import ZipFile import pytest @@ -75,7 +76,7 @@ def fail(*args: str, **kwargs: str) -> subprocess.CompletedProcess[str]: monkeypatch.setattr(subprocess, "run", fail) assert security_evidence.run_bandit().status == Status.TOOL_ERROR - assert dependency_audit_policy.run_audit(tmp_path, tmp_path).status == Status.TOOL_ERROR + assert dependency_audit_policy.run_audit(tmp_path, tmp_path).status == Status.INCOMPLETE def test_audit_uses_only_candidate_installation( @@ -83,11 +84,17 @@ def test_audit_uses_only_candidate_installation( ) -> None: # Given an isolated inventory; When audited; Then tooling cannot contaminate its closure. (tmp_path / "example.dist-info").mkdir() + (tmp_path / "example.dist-info/METADATA").write_text("Name: example\nVersion: 1\n") + (tmp_path / "candidate.dist-info").mkdir() + (tmp_path / "candidate.dist-info/METADATA").write_text("Name: candidate\nVersion: 99\n") + wheel = tmp_path / "candidate.whl" + with ZipFile(wheel, "w") as archive: + archive.writestr("candidate.dist-info/METADATA", "Name: candidate\nVersion: 99\n") policy = tmp_path / "policy.json" policy.write_text('{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}') def complete(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: - assert command[command.index("--path") + 1] == str(tmp_path) + assert Path(command[command.index("--requirement") + 1]).read_text() == "example==1\n" return subprocess.CompletedProcess( command, 0, @@ -97,7 +104,7 @@ def complete(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[s monkeypatch.setattr(subprocess, "run", complete) monkeypatch.setattr(dependency_audit_policy, "version", lambda name: "2.10.1") - assert dependency_audit_policy.run_audit(tmp_path, policy).status == Status.PASS + assert dependency_audit_policy.run_audit(tmp_path, policy, wheel).status == Status.PASS @pytest.mark.parametrize("development", [False, True]) @@ -108,12 +115,22 @@ def test_audit_cli_preserves_failure_codes( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, development: bool, status: Status, code: int ) -> None: # Given a classified failure; When CLI writes evidence; Then the same failure reaches CI. - args = ["audit", "--site-packages", str(tmp_path), "--output", str(tmp_path)] + args = [ + "audit", + "--site-packages", + str(tmp_path), + "--output", + str(tmp_path), + "--candidate-wheel", + "candidate.whl", + ] if development: args.append("--development") monkeypatch.setattr(sys, "argv", args) monkeypatch.setattr( - dependency_audit_policy, "run_audit", lambda path, policy: Check("pip-audit", (), status, 1) + dependency_audit_policy, + "run_audit", + lambda path, policy, wheel: Check("pip-audit", (), status, 1), ) assert dependency_audit_policy.main() == code assert json.loads((tmp_path / "summary.json").read_text())["checks"][0]["status"] == status diff --git a/tools/candidate_inventory.py b/tools/candidate_inventory.py new file mode 100644 index 0000000..82a84ab --- /dev/null +++ b/tools/candidate_inventory.py @@ -0,0 +1,39 @@ +"""Verify wheel identity against isolated installed metadata before excluding it.""" + +from email.parser import BytesParser +from importlib.metadata import distributions +from pathlib import Path +import re +from zipfile import ZipFile + + +def normalize(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def inventory(site_packages: Path, wheel: Path) -> tuple[tuple[str, str], dict[str, str]]: + with ZipFile(wheel) as archive: + names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] + if len(names) != 1: + raise ValueError("Ambiguous wheel metadata") + metadata = BytesParser().parsebytes(archive.read(names[0])) + name, version = metadata.get("Name", ""), metadata.get("Version", "") + if not name or not version: + raise ValueError("Missing candidate identity") + candidate = (normalize(name), version) + installed: dict[str, str] = {} + for dist in distributions(path=[str(site_packages)]): + package = normalize(dist.metadata["Name"] or "") + release = dist.version + if ( + not re.fullmatch(r"[a-z0-9][a-z0-9-]*", package) + or not release + or not re.fullmatch(r"[a-zA-Z0-9.!+_-]+", release) + ): + raise ValueError("Invalid installed identity") + if package in installed: + raise ValueError("Duplicate installed identity") + installed[package] = release + if installed.pop(candidate[0], None) != candidate[1] or not installed: + raise ValueError("Candidate mismatch or empty third-party inventory") + return candidate, installed diff --git a/tools/dependency_audit_policy.py b/tools/dependency_audit_policy.py index f695576..16b5357 100644 --- a/tools/dependency_audit_policy.py +++ b/tools/dependency_audit_policy.py @@ -10,7 +10,10 @@ import re import subprocess import sys +import tempfile +from zipfile import BadZipFile +from tools.candidate_inventory import inventory, normalize from tools.security_exceptions import parse_exceptions from tools.security_report import ( Check, @@ -86,43 +89,78 @@ def classify_audit(raw: str, exit_status: int, *, policy_raw: str, today: date) return replace(result, message="Invalid audit/tool/policy evidence; release blocked.") -def run_audit(site_packages: Path, policy: Path) -> Check: +def run_audit(site_packages: Path, policy: Path, candidate_wheel: Path | None = None) -> Check: + if candidate_wheel is None: + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.INCOMPLETE, + None, + message="Verified candidate wheel required.", + ) try: - if not site_packages.is_dir() or not list(site_packages.glob("*.dist-info")): - return Check( - "pip-audit", - ("resolved-runtime-wheel",), - Status.INCOMPLETE, - None, - message="Candidate installation inventory missing.", - ) - process = subprocess.run( - [ - sys.executable, - "-m", - "pip_audit", - "--path", - str(site_packages), - "--format", - "json", - "--desc", - "off", - "--progress-spinner", - "off", - "--timeout", - "30", - ], - capture_output=True, - text=True, - check=False, - timeout=300, + candidate, installed = inventory(site_packages, candidate_wheel) + except (OSError, ValueError, BadZipFile, KeyError): + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.INCOMPLETE, + None, + message="Candidate identity or third-party inventory invalid.", ) + try: + with tempfile.TemporaryDirectory(prefix="runtime-audit-") as temporary: + requirements = Path(temporary) / "requirements.txt" + requirements.write_text( + "".join(f"{name}=={release}\n" for name, release in sorted(installed.items())) + ) + process = subprocess.run( + [ + sys.executable, + "-m", + "pip_audit", + "--requirement", + str(requirements), + "--no-deps", + "--disable-pip", + "--format", + "json", + "--desc", + "off", + "--progress-spinner", + "off", + "--timeout", + "30", + ], + capture_output=True, + text=True, + check=False, + timeout=300, + ) result = classify_audit( process.stdout, process.returncode, policy_raw=policy.read_text(encoding="utf-8"), today=date.today(), ) + if result.status in {Status.PASS, Status.FAIL}: + observed: dict[str, str] = {} + for dependency in sequence(load(process.stdout)["dependencies"]): + item = mapping(dependency) + name = normalize(text(item["name"])) + release = text(item["version"]) + if name in observed or (name == candidate[0] and release != candidate[1]): + return replace( + result, status=Status.INCOMPLETE, message="Audit identity mismatch." + ) + observed[name] = release + observed.pop(candidate[0], None) + if observed != installed: + return replace( + result, + status=Status.INCOMPLETE, + message="Audit omitted installed dependencies.", + ) return replace(result, tool_versions={"pip-audit": version("pip-audit")}) except (OSError, subprocess.TimeoutExpired, PackageNotFoundError): return Check( @@ -142,8 +180,9 @@ def main() -> int: ) parser.add_argument("--output", type=Path, default=Path("security-results")) parser.add_argument("--development", action="store_true") + parser.add_argument("--candidate-wheel", type=Path, required=True) args = parser.parse_args() - check = run_audit(args.site_packages, args.policy) + check = run_audit(args.site_packages, args.policy, args.candidate_wheel) if args.development: check = replace(check, name="pip-audit-development", scope=("runtime-and-development",)) return write_report([check], args.output) From f8725719df36a4f15b2c0e3281d052e127df5953 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 10:05:14 +0900 Subject: [PATCH 10/13] ci(semgrep): report action-only evidence as incomplete --- tests/test_security_evidence.py | 6 +++++- tools/security_evidence.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_security_evidence.py b/tests/test_security_evidence.py index 6c2399d..a663d46 100644 --- a/tests/test_security_evidence.py +++ b/tests/test_security_evidence.py @@ -132,7 +132,11 @@ def test_static_policy_matches_exact_location_and_expires() -> None: @pytest.mark.parametrize( "outcome,status", - [("success", Status.PASS), ("failure", Status.TOOL_ERROR), ("skipped", Status.INCOMPLETE)], + [ + ("success", Status.INCOMPLETE), + ("failure", Status.TOOL_ERROR), + ("skipped", Status.INCOMPLETE), + ], ) def test_semgrep_outcome_is_not_a_vulnerability(outcome: str, status: Status) -> None: # Given action outcome only; When recorded; Then finding counts stay unknown. diff --git a/tools/security_evidence.py b/tools/security_evidence.py index 0e433b2..5c8cabf 100644 --- a/tools/security_evidence.py +++ b/tools/security_evidence.py @@ -204,7 +204,7 @@ def semgrep_evidence(outcome: str) -> Check: code: int | None match outcome: case "success": - status, code = Status.PASS, 0 + status, code = Status.INCOMPLETE, 0 case "failure": status, code = Status.TOOL_ERROR, 1 case _: @@ -215,7 +215,7 @@ def semgrep_evidence(outcome: str) -> Check: status, code, tool_versions={"action": "713efdd345f3035192eaa63f56867b88e63e4e5d"}, - message="Action outcome only; finding count unknown; failures may include findings.", + message="action outcome only; finding count unknown; failures may include findings.", ) From 02f0e1b479820d81e3d74172e22540c246f9c8e1 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 10:05:14 +0900 Subject: [PATCH 11/13] ci(security): route shared evidence gates through make targets --- .github/workflows/dependency-audit.yml | 9 ++++++--- .github/workflows/publish-pypi.yml | 3 ++- .github/workflows/security.yml | 7 ++++--- Makefile | 19 ++++++++++++++++++- tests/test_security_workflows.py | 13 +++++++++---- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index e3ec86f..86629dd 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -38,21 +38,24 @@ jobs: continue-on-error: true run: | site_packages=$("$RUNNER_TEMP/runtime/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') - python -m tools.dependency_audit_policy --site-packages "$site_packages" --output security-results/runtime + wheels=(dist/*.whl) + make security-runtime-report AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --output security-results/runtime" - name: Audit OTel runtime dependencies if: always() continue-on-error: true run: | site_packages=$("$RUNNER_TEMP/otel/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') - python -m tools.dependency_audit_policy --site-packages "$site_packages" --output security-results/otel + wheels=(dist/*.whl) + make security-runtime-report AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --output security-results/otel" - name: Audit runtime and development dependencies if: always() continue-on-error: true run: | site_packages=$("$RUNNER_TEMP/development/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') - python -m tools.dependency_audit_policy --site-packages "$site_packages" --development --output security-results/development + wheels=(dist/*.whl) + make security-runtime-report AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --development --output security-results/development" - name: Upload redacted reporting evidence if: always() diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 351eef4..eb66e3d 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -302,7 +302,8 @@ jobs: - name: Enforce runtime dependency policy run: | site_packages=$("$RUNNER_TEMP/candidate/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') - python -m tools.dependency_audit_policy --site-packages "$site_packages" --output security-results/${{ matrix.closure }} + wheels=(candidate-dist/*.whl) + make security-runtime AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --output security-results/${{ matrix.closure }}" - name: Upload redacted release security evidence if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index c405347..b259047 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -30,10 +30,11 @@ jobs: make install - name: Run Bandit - run: hatch run python -m tools.security_evidence bandit + if: always() + run: make security-bandit - name: Type check security tools - run: hatch run security-typecheck + run: make security-typecheck - name: Upload redacted security evidence if: always() @@ -65,7 +66,7 @@ jobs: if: always() env: SCAN_OUTCOME: ${{ steps.scan.outcome }} - run: python -m tools.security_evidence semgrep --outcome "$SCAN_OUTCOME" + run: make security-semgrep - name: Upload redacted security evidence if: always() diff --git a/Makefile b/Makefile index cef3d64..b6704ab 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,24 @@ lint: ensure-hatch .PHONY: security security: ensure-hatch - @$(HATCH) run security + @$(MAKE) security-inventory + +# Security evidence entry points; kept separate from verification standardization. +SECURITY_PYTHON ?= python +AUDIT_ARGS ?= +SCAN_OUTCOME ?= +.PHONY: security-inventory security-bandit security-semgrep security-runtime security-runtime-report security-typecheck security-host +security-inventory security-bandit: ensure-hatch + @$(HATCH) run python -m tools.security_evidence bandit +security-semgrep: + @$(SECURITY_PYTHON) -m tools.security_evidence semgrep --outcome "$(SCAN_OUTCOME)"; code=$$?; [ $$code -eq 3 ] || exit $$code +security-runtime: + @$(SECURITY_PYTHON) -m tools.dependency_audit_policy $(AUDIT_ARGS) +security-runtime-report: + @$(SECURITY_PYTHON) -m tools.dependency_audit_policy $(AUDIT_ARGS) +security-typecheck: ensure-hatch + @$(HATCH) run security-typecheck +security-host: e2e-local .PHONY: check check: ensure-hatch diff --git a/tests/test_security_workflows.py b/tests/test_security_workflows.py index 80d1e79..a7afae5 100644 --- a/tests/test_security_workflows.py +++ b/tests/test_security_workflows.py @@ -35,7 +35,8 @@ def test_release_gate_preserves_existing_tiers_and_candidate() -> None: assert "pip-audit==2.10.1" in commands assert "--without-pip" in commands assert '"${wheels[0]}[otel]"' in commands - assert "tools.dependency_audit_policy" in commands + assert "make security-runtime" in commands + assert "--candidate-wheel" in commands assert "build --wheel" not in commands @@ -64,14 +65,18 @@ def test_security_scans_declared_executable_surfaces() -> None: workflow = yaml.safe_load((ROOT / ".github/workflows/security.yml").read_text()) assert SCOPE == ("src", "examples", "scripts", "tools", "benchmarks") steps = workflow["jobs"]["bandit"]["steps"] - assert any("tools.security_evidence bandit" in s.get("run", "") for s in steps) + assert any( + s.get("if") == "always()" and "make security-bandit" in s.get("run", "") for s in steps + ) assert any("security-typecheck" in s.get("run", "") for s in steps) semgrep = workflow["jobs"]["semgrep"]["steps"] action = next(s for s in semgrep if "semgrep/semgrep-action@" in s.get("uses", "")) assert action["uses"] == "semgrep/semgrep-action@713efdd345f3035192eaa63f56867b88e63e4e5d" assert action["with"]["config"] == "p/python" assert not action.get("continue-on-error") - assert any(s.get("if") == "always()" and "--outcome" in s.get("run", "") for s in semgrep) + assert any( + s.get("if") == "always()" and "make security-semgrep" in s.get("run", "") for s in semgrep + ) scripts = tomllib.loads((ROOT / "pyproject.toml").read_text())["tool"]["hatch"]["envs"][ "default" ]["scripts"] @@ -82,7 +87,7 @@ def test_scheduled_audits_reuse_policy_as_reporting_only() -> None: # Given scheduled audit lanes; When inspected; Then runtime/OTel/dev use one classifier. workflow = yaml.safe_load((ROOT / ".github/workflows/dependency-audit.yml").read_text()) steps = workflow["jobs"]["pip-audit"]["steps"] - audits = [s for s in steps if "tools.dependency_audit_policy" in s.get("run", "")] + audits = [s for s in steps if "make security-runtime-report" in s.get("run", "")] assert len(audits) == 3 assert all(s["continue-on-error"] for s in audits) assert any("--development" in s["run"] for s in audits) From cac7cbd798b6ae518d4187bb3a706b984cf09c74 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 10:05:14 +0900 Subject: [PATCH 12/13] ci(types): check candidate inventory with security tools --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3d79ec3..e230ba0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ format = "ruff format src tests tools examples benchmarks scripts" format-check = "ruff format --check src tests tools examples benchmarks scripts" style = "ruff check src tests tools examples benchmarks scripts && ruff format --check src tests tools examples benchmarks scripts" typecheck = "mypy src tests" -security-typecheck = "mypy --strict --explicit-package-bases tools/security_report.py tools/security_exceptions.py tools/security_evidence.py tools/dependency_audit_policy.py" +security-typecheck = "mypy --strict --explicit-package-bases tools/security_report.py tools/security_exceptions.py tools/security_evidence.py tools/dependency_audit_policy.py tools/candidate_inventory.py" lint = "hatch run style && hatch run typecheck" test = "pytest -v tests" cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/" From 02ad6b6f84cedb2c369b88eee1e9622e25276bd6 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Fri, 23 Oct 2026 10:05:14 +0900 Subject: [PATCH 13/13] docs(security): explain verified candidate and action-only evidence --- AGENTS.md | 5 ++++- SECURITY.md | 25 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d94b96..af33722 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,10 @@ ## Working Rules Security gates, evidence redaction, production boundaries and expiring exceptions: -see [SECURITY.md](SECURITY.md). +see [SECURITY.md](SECURITY.md). Use `make security`, `make security-typecheck`, +`make security-semgrep`, `make security-runtime` (blocking), +`make security-runtime-report` (reporting workflow), and `make security-host` +(opt-in host) as the shared security entry points. ### Test Coverage - Maintain test coverage at **95% or above** for committed changes and PRs. diff --git a/SECURITY.md b/SECURITY.md index dff38bb..9fbaea4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -55,7 +55,7 @@ exception. Publishing still uploads the exact `build` artifact, without rebuildi ## Static scope and reviewed suppressions -`hatch run python -m tools.security_evidence bandit` scans every Python file under +`make security` (also `make security-inventory` / `make security-bandit`) scans every Python file under `src/`, `examples/` (including `e2e_app` and `otel_app`), `scripts/`, `tools/`, and `benchmarks/`. Benchmarks are included even when not executed by a particular CI lane. There are no per-file exclusions within these surfaces. Tests/fixtures are @@ -86,9 +86,9 @@ not proof that runtime failures redact safely; runtime hardening is separate. The evidence reports marker counts and retains all unsuppressed findings. Other baseline reviews cover synthetic fixtures, type-narrowing assertions, shell-free tool invocations, a fixed-host documentation URL checker, and a status-enum false -positive. No global rule exclusions replace review. `make security` retains its -existing local src-only command; CI uses the expanded helper. Security tools receive -a separate strict mypy invocation (`hatch run security-typecheck`), additive to the +positive. No global rule exclusions replace review. Local full validation and CI +use the same expanded inventory. Security tools receive +a separate strict mypy invocation (`make security-typecheck`), additive to the existing project typecheck rather than changing its scope. ## Dependency and exception policy @@ -101,8 +101,17 @@ leg audits the wheel's `otel` extra. That extra declares `opentelemetry-api`, no example application's separately installed Azure Monitor exporter. Development, docs and example-only dependencies are not part of the release base/OTel closure. Each supported interpreter is audited because environment markers affect resolution. -Candidate environments use `venv --without-pip`; pip-audit runs outside with `--path`. +Candidate environments use `venv --without-pip`; audit tools run outside them. +`make security-runtime AUDIT_ARGS="--site-packages --candidate-wheel "` +verifies wheel METADATA name/version against the installed distribution before +excluding only that first-party candidate. Every installed third-party distribution +is frozen as `name==version` and audited with `--requirement`, `--no-deps`, and +`--disable-pip`; returned inventory must match the full installed closure. +This applies independently to base and `otel` environments. Missing/mismatched +candidate identity, skipped third parties or omitted dependencies block release. No audit-tool dependencies contaminate the candidate installation. +Scheduled reporting uses `make security-runtime-report` with the same arguments +and preserves non-PASS evidence in reporting-only workflow steps. Both policy files require maintainer review, exact identifiers, `reason`, `owner`, `added` and `expires` (ISO dates), and a positive lifetime at most 90 days. Static @@ -122,9 +131,13 @@ Helpers exit PASS=0, FAIL=1 (unexcepted findings/expired exceptions), TOOL_ERROR tool/database failure), INCOMPLETE=3 (empty static scan, missing coverage, skipped packages, unlisted inline suppressions or absent execution evidence). Every non-PASS blocks release. Scheduled reporting preserves the same status but does not gate a -release. Semgrep exposes only action outcome: success PASS, failure conservatively +release. `make security-semgrep SCAN_OUTCOME=success` records INCOMPLETE with +"action outcome only", not PASS, because no structured results or analysed +inventory are available. The target tolerates exactly evidence exit 3 while the +SHA-pinned Semgrep action remains the gate. Failure is conservatively TOOL_ERROR with unknown findings count, skipped/cancelled INCOMPLETE. Its original failing step remains blocking; an action failure alone is not called a vulnerability. +`make security-host` is the opt-in local host test; it requires the e2e host prerequisites. Advisory database outages can delay publication: this is an intentional availability tradeoff for fail-closed verification. Bounded timeouts limit hangs. Never bypass