diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml index da523a2..86629dd 100644 --- a/.github/workflows/dependency-audit.yml +++ b/.github/workflows/dependency-audit.yml @@ -21,22 +21,46 @@ jobs: with: python-version: "3.14" - - name: Install audit tool and runtime dependencies + - name: Build wheel and isolated reporting environments run: | - python -m pip install --upgrade pip - python -m pip install "pip-audit==2.10.1" . + python -m pip install 'pip-audit==2.10.1' 'build==1.5.0' + python -m build --wheel + wheels=(dist/*.whl) + test "${#wheels[@]}" -eq 1 + python -m venv --without-pip "$RUNNER_TEMP/runtime" + python -m pip --python "$RUNNER_TEMP/runtime/bin/python" install "${wheels[0]}" + python -m venv --without-pip "$RUNNER_TEMP/otel" + python -m pip --python "$RUNNER_TEMP/otel/bin/python" install "${wheels[0]}[otel]" + python -m venv --without-pip "$RUNNER_TEMP/development" + python -m pip --python "$RUNNER_TEMP/development/bin/python" install '.[dev]' - name: Audit runtime dependencies continue-on-error: true run: | - echo "## Runtime dependency audit" >> "$GITHUB_STEP_SUMMARY" - python -m pip_audit 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + site_packages=$("$RUNNER_TEMP/runtime/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + wheels=(dist/*.whl) + make security-runtime-report AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --output security-results/runtime" - - name: Install development dependencies - run: python -m pip install ".[dev]" + - name: Audit OTel runtime dependencies + if: always() + continue-on-error: true + run: | + site_packages=$("$RUNNER_TEMP/otel/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + wheels=(dist/*.whl) + make security-runtime-report AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --output security-results/otel" - name: Audit runtime and development dependencies + if: always() continue-on-error: true run: | - echo "## Runtime and development dependency audit" >> "$GITHUB_STEP_SUMMARY" - python -m pip_audit 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + site_packages=$("$RUNNER_TEMP/development/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + wheels=(dist/*.whl) + make security-runtime-report AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --development --output security-results/development" + + - name: Upload redacted reporting evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dependency-security-report + path: security-results/** + if-no-files-found: error diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 000aecb..eb66e3d 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -263,9 +263,58 @@ jobs: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + runtime-dependency-audit: + name: Runtime dependency audit gate + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + interpreter: ["3.11", "3.12", "3.13", "3.14"] + closure: ["base", "otel"] + steps: + - name: Checkout security policy and tooling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up audit interpreter + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.interpreter }} + - name: Download the exact candidate artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dist + path: candidate-dist + - name: Install only the candidate runtime closure + env: + CLOSURE: ${{ matrix.closure }} + run: | + python -m pip install 'pip-audit==2.10.1' + python -m venv --without-pip "$RUNNER_TEMP/candidate" + wheels=(candidate-dist/*.whl) + test "${#wheels[@]}" -eq 1 + if [ "$CLOSURE" = "otel" ]; then + python -m pip --python "$RUNNER_TEMP/candidate/bin/python" install "${wheels[0]}[otel]" + else + python -m pip --python "$RUNNER_TEMP/candidate/bin/python" install "${wheels[0]}" + fi + - name: Enforce runtime dependency policy + run: | + site_packages=$("$RUNNER_TEMP/candidate/bin/python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))') + wheels=(candidate-dist/*.whl) + make security-runtime AUDIT_ARGS="--site-packages $site_packages --candidate-wheel ${wheels[0]} --output security-results/${{ matrix.closure }}" + - name: Upload redacted release security evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-security-${{ matrix.interpreter }}-${{ matrix.closure }} + path: security-results/** + if-no-files-found: error + publish: name: Publish to PyPI - needs: [build, lib-tests, cookbook-smoke, cookbook-host-smoke, azure-e2e] + needs: [build, lib-tests, cookbook-smoke, cookbook-host-smoke, azure-e2e, runtime-dependency-audit] runs-on: ubuntu-latest environment: pypi permissions: diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 6f76dbd..b259047 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -7,6 +7,7 @@ on: branches: [main] schedule: - cron: "0 2 * * 1" + workflow_dispatch: permissions: contents: read @@ -29,7 +30,19 @@ jobs: make install - name: Run Bandit - run: make security + if: always() + run: make security-bandit + + - name: Type check security tools + run: make security-typecheck + + - name: Upload redacted security evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: security-bandit + path: security-results/** + if-no-files-found: error semgrep: runs-on: ubuntu-latest @@ -38,6 +51,27 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Run Semgrep + id: scan uses: semgrep/semgrep-action@713efdd345f3035192eaa63f56867b88e63e4e5d # v1 with: config: p/python + + - name: Set up evidence interpreter + if: always() + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Record Semgrep execution evidence + if: always() + env: + SCAN_OUTCOME: ${{ steps.scan.outcome }} + run: make security-semgrep + + - name: Upload redacted security evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: security-semgrep + path: security-results/** + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 3006d94..3e33816 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ env/ .ruff_cache/ .coverage coverage.xml +security-results/ htmlcov/ # Build artifacts diff --git a/AGENTS.md b/AGENTS.md index c5aa8ad..af33722 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,12 @@ ## Working Rules +Security gates, evidence redaction, production boundaries and expiring exceptions: +see [SECURITY.md](SECURITY.md). Use `make security`, `make security-typecheck`, +`make security-semgrep`, `make security-runtime` (blocking), +`make security-runtime-report` (reporting workflow), and `make security-host` +(opt-in host) as the shared security entry points. + ### Test Coverage - Maintain test coverage at **95% or above** for committed changes and PRs. - Run `hatch run pytest --cov --cov-report=term-missing -q` to verify before submitting changes. diff --git a/Makefile b/Makefile index cef3d64..b6704ab 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,24 @@ lint: ensure-hatch .PHONY: security security: ensure-hatch - @$(HATCH) run security + @$(MAKE) security-inventory + +# Security evidence entry points; kept separate from verification standardization. +SECURITY_PYTHON ?= python +AUDIT_ARGS ?= +SCAN_OUTCOME ?= +.PHONY: security-inventory security-bandit security-semgrep security-runtime security-runtime-report security-typecheck security-host +security-inventory security-bandit: ensure-hatch + @$(HATCH) run python -m tools.security_evidence bandit +security-semgrep: + @$(SECURITY_PYTHON) -m tools.security_evidence semgrep --outcome "$(SCAN_OUTCOME)"; code=$$?; [ $$code -eq 3 ] || exit $$code +security-runtime: + @$(SECURITY_PYTHON) -m tools.dependency_audit_policy $(AUDIT_ARGS) +security-runtime-report: + @$(SECURITY_PYTHON) -m tools.dependency_audit_policy $(AUDIT_ARGS) +security-typecheck: ensure-hatch + @$(HATCH) run security-typecheck +security-host: e2e-local .PHONY: check check: ensure-hatch diff --git a/SECURITY.md b/SECURITY.md index 081cb24..9fbaea4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,3 +33,156 @@ If you prefer email, contact: yeongseon.choe@gmail.com | ------- | --------- | | Latest release | :white_check_mark: | | Older releases | :x: | + +## Automated controls and permissions + +| Control | Runs where | Enforcement | +| --- | --- | --- | +| Bandit 1.9.4 | PR, main, weekly, dispatch (`security.yml`) | Blocking scan; every severity reviewed | +| SHA-pinned Semgrep `p/python` | Same workflow | Existing blocking action; execution evidence added | +| CodeQL `security-extended` | PR, main, weekly | Existing analysis/code-scanning control, unchanged | +| Runtime pip-audit 2.10.1 | Exact candidate wheel, Python 3.11–3.14, base and `otel` closures | Blocking dependency of `publish` | +| Runtime/OTel/development audit | Weekly/dispatch (`dependency-audit.yml`) | Reporting-only, shared classifier | +| SBOM | Weekly/dispatch | Existing installed-wheel inventory, unchanged | + +Branch-protection requirements are repository settings, not established by these +workflow files. Existing merge checks, runtime/host smoke and real-Azure certification +remain intact. This change adds no write permission: security workflows and the new +release audit job use only `contents: read`. CodeQL retains `contents: read` and +`security-events: write`; Azure certification retains `contents: read` and +`id-token: write`; PyPI publishing retains `id-token: write` and its trusted-publisher +exception. Publishing still uploads the exact `build` artifact, without rebuilding. + +## Static scope and reviewed suppressions + +`make security` (also `make security-inventory` / `make security-bandit`) scans every Python file under +`src/`, `examples/` (including `e2e_app` and `otel_app`), `scripts/`, `tools/`, and +`benchmarks/`. Benchmarks are included even when not executed by a particular CI +lane. There are no per-file exclusions within these surfaces. Tests/fixtures are +excluded from this product/tool scan. Bytecode and non-Python artifacts are not +Bandit inputs. Saved host logs and deployment configuration require separate review; +this scan does not claim to analyze Bicep, workflows, or arbitrary log contents. +Inventory must equal the on-disk Python inventory, contain at least 38 files and +have positive LOC. Missing surfaces, errors and unlisted suppressions cannot pass. + +Bandit uses `--ignore-nosec`, so existing markers cannot hide findings. The separate +`inline_suppressions` allow-list in `security/bandit-policy.json` requires exact +ID/file/line equality with all markers, including markers that Bandit does not +currently flag. The baseline has 26 markers across seven source files: + +| Source module | Count | Rules | +| --- | ---: | --- | +| `_context.py` | 10 | B110 | +| `_filters.py` | 7 | B110 | +| `_host_config.py` | 2 | B110 | +| `_otel.py` | 4 | B110 | +| `_host_instance.py` | 1 | B110 | +| `_redaction.py` | 1 | B112 | +| `_decorator.py` | 1 | B307 | + +These are existing best-effort/never-raise contracts and handler-owned annotation +evaluation, not new runtime changes. Retaining a masking-resilience exception is +not proof that runtime failures redact safely; runtime hardening is separate. +The evidence reports marker counts and retains all unsuppressed findings. Other +baseline reviews cover synthetic fixtures, type-narrowing assertions, shell-free +tool invocations, a fixed-host documentation URL checker, and a status-enum false +positive. No global rule exclusions replace review. Local full validation and CI +use the same expanded inventory. Security tools receive +a separate strict mypy invocation (`make security-typecheck`), additive to the +existing project typecheck rather than changing its scope. + +## Dependency and exception policy + +Every known advisory in the resolved candidate runtime closure blocks release, +regardless of severity. Missing severity is `UNKNOWN`, never assumed low. This is +stricter than a high/critical-only gate and avoids missing-CVSS false greens. +The base closure contains the wheel and required dependencies; a separate matrix +leg audits the wheel's `otel` extra. That extra declares `opentelemetry-api`, not the +example application's separately installed Azure Monitor exporter. Development, +docs and example-only dependencies are not part of the release base/OTel closure. +Each supported interpreter is audited because environment markers affect resolution. +Candidate environments use `venv --without-pip`; audit tools run outside them. +`make security-runtime AUDIT_ARGS="--site-packages --candidate-wheel "` +verifies wheel METADATA name/version against the installed distribution before +excluding only that first-party candidate. Every installed third-party distribution +is frozen as `name==version` and audited with `--requirement`, `--no-deps`, and +`--disable-pip`; returned inventory must match the full installed closure. +This applies independently to base and `otel` environments. Missing/mismatched +candidate identity, skipped third parties or omitted dependencies block release. +No audit-tool dependencies contaminate the candidate installation. +Scheduled reporting uses `make security-runtime-report` with the same arguments +and preserves non-PASS evidence in reporting-only workflow steps. + +Both policy files require maintainer review, exact identifiers, `reason`, `owner`, +`added` and `expires` (ISO dates), and a positive lifetime at most 90 days. Static +exceptions require `reviewed-static` with exact file/line. Dependency exceptions +require exact package/advisory and `not-affected` (independently verified evidence) +or `fix-not-available` (only while no fixed version is listed). Package separators +are normalized; advisory aliases are not silently ignored. Expired entries block +even when unused; stale entries are counted for removal. The dependency policy +starts empty. Do not add blanket skips or new inline markers instead of review. +When runtime changes move existing lines, re-review the exact allow-list rather +than broadening it; automation and runtime branches must reconcile that inventory. + +## Failure semantics and recovery + +Helpers exit PASS=0, FAIL=1 (unexcepted findings/expired exceptions), TOOL_ERROR=2 +(crash, timeout, missing/invalid JSON, invalid policy, zero audited packages or +tool/database failure), INCOMPLETE=3 (empty static scan, missing coverage, skipped +packages, unlisted inline suppressions or absent execution evidence). Every non-PASS +blocks release. Scheduled reporting preserves the same status but does not gate a +release. `make security-semgrep SCAN_OUTCOME=success` records INCOMPLETE with +"action outcome only", not PASS, because no structured results or analysed +inventory are available. The target tolerates exactly evidence exit 3 while the +SHA-pinned Semgrep action remains the gate. Failure is conservatively +TOOL_ERROR with unknown findings count, skipped/cancelled INCOMPLETE. Its original +failing step remains blocking; an action failure alone is not called a vulnerability. +`make security-host` is the opt-in local host test; it requires the e2e host prerequisites. + +Advisory database outages can delay publication: this is an intentional availability +tradeoff for fail-closed verification. Bounded timeouts limit hangs. Never bypass +the gate. Retry the same unchanged tagged candidate with +`gh workflow run publish-pypi.yml --ref vX.Y.Z`. Dispatch takes no inputs; use the +existing tag as its ref, not main. Corrections to code/workflows must fix forward +to a new release candidate; never move/reuse the old tag. + +## Evidence and production boundaries + +Helpers emit schema-version-1 `security-results/**/summary.json` containing scope, +check/status, tool exit status, counts, tool versions, suppression counts, applied +exception identifiers and allow-listed finding IDs, relative paths/lines, severities, +confidence and package identifiers. Human-readable status/counts go to Job Summary. +Uploads run `if: always()` with unique artifact names and fail if evidence is absent. +Installation/setup failures can prevent report creation; missing artifacts remain +an explicit failure, never PASS. Generated local evidence is gitignored. + +Security evidence must never contain credentials, connection strings, authorization +headers or original user payloads. Helpers do not retain raw scanner stdout/stderr, +source snippets, advisory descriptions or policy reasons; malformed diagnostics +produce fixed messages. Deterministic canary tests guard JSON and Markdown output. +This guarantee covers these evidence helpers, not all pre-existing host/exporter logs. +A finding is a **code/config review finding until runtime impact is independently +demonstrated**; a clean scan does not prove the absence of vulnerabilities. + +Examples are local-only synthetic demonstrations except `e2e_app`, which is also +ephemeral public Azure certification infrastructure. Anonymous routes and disabled +sampling are test conveniences, not production defaults. Query parameters such as +`correlation_id` and `order_id` reach logs: use synthetic identifiers only, never +real payloads, credentials, headers or PII. The OTel masking example uses a dummy +value; do not rely on it as a complete privacy profile. For secure logging +configuration, see the redaction documentation. This branch adds no runtime API. + +Production deployments need explicit authentication/authorization, restricted +ingress, HTTPS/minimum TLS, private blobs, managed identities and storage RBAC, +restricted telemetry access/retention, and appropriate sampling. Review of examples +found no request-body/header dump or printed connection string/instrumentation key; +user-controlled identifiers still require care. `infra/main.bicep` enforces HTTPS, +storage TLS 1.2 and private blobs, but retains shared-key storage connection strings, +public network reachability, no declared managed identity, and an Application +Insights connection-string deployment output. The Azure workflow emits deployment +outputs, so that output is a telemetry-configuration disclosure risk; do not treat +deployment logs as sanitized security evidence. Removing outputs, moving storage to +identity/RBAC, private endpoints, explicit app/SCM TLS and authenticated certification +routes require real-Azure validation. They are follow-ups, not changes in this port. +Host diagnostics/saved logs also need an independently verified allow-list and +retention review before accepting real user data. Use synthetic data only in CI. diff --git a/examples/README.md b/examples/README.md index 5ac1cd3..5c6ba06 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,15 @@ # Examples +These are local-only demonstrations using synthetic data, not production security +templates. `e2e_app` is additionally deployed as ephemeral public Azure certification +infrastructure; its anonymous routes must remain compatible with certification. +`otel_app` has anonymous routes for local demonstration. Configure authentication, +restricted ingress and telemetry access before production deployment. Both apps +disable sampling for demonstration/test visibility, not as production guidance. +Query identifiers (`correlation_id`, `order_id`) are logged: never send credentials, +headers, request payloads or PII. Connection strings belong in deployment environment +configuration and must not be printed. See [security boundaries](../SECURITY.md). + `azure-functions-logging` includes standalone example scripts that demonstrate every public API feature. Each script runs without Azure Functions installed. diff --git a/pyproject.toml b/pyproject.toml index 014e8a6..e230ba0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ format = "ruff format src tests tools examples benchmarks scripts" format-check = "ruff format --check src tests tools examples benchmarks scripts" style = "ruff check src tests tools examples benchmarks scripts && ruff format --check src tests tools examples benchmarks scripts" typecheck = "mypy src tests" +security-typecheck = "mypy --strict --explicit-package-bases tools/security_report.py tools/security_exceptions.py tools/security_evidence.py tools/dependency_audit_policy.py tools/candidate_inventory.py" lint = "hatch run style && hatch run typecheck" test = "pytest -v tests" cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/" diff --git a/security/bandit-policy.json b/security/bandit-policy.json new file mode 100644 index 0000000..c0565c0 --- /dev/null +++ b/security/bandit-policy.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "inline_suppressions": [ + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":207,"kind":"reviewed-static","reason":"Existing context cleanup is deliberately non-raising; no runtime change in this automation port.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":231,"kind":"reviewed-static","reason":"Existing context propagation is deliberately best-effort and non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":236,"kind":"reviewed-static","reason":"Existing worker metadata propagation must not crash application execution.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":245,"kind":"reviewed-static","reason":"Existing context cleanup must not crash application execution.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":251,"kind":"reviewed-static","reason":"Existing worker thread-local cleanup is deliberately silent on failure.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":371,"kind":"reviewed-static","reason":"Existing context metadata inspection is best-effort and never raises.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":388,"kind":"reviewed-static","reason":"Existing propagation failure must not replace the application result.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":397,"kind":"reviewed-static","reason":"Existing executor context restoration is deliberately non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":401,"kind":"reviewed-static","reason":"Existing executor worker metadata restoration is deliberately silent.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_context.py","line":577,"kind":"reviewed-static","reason":"Existing binding metadata cleanup is best-effort and never raises.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":199,"kind":"reviewed-static","reason":"Existing unreadable host configuration is non-fatal by design.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_config.py","line":310,"kind":"reviewed-static","reason":"Existing host configuration override inspection is non-fatal.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":143,"kind":"reviewed-static","reason":"Existing filter sampling must not raise into application logging.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":348,"kind":"reviewed-static","reason":"Existing per-field masking resilience retained; runtime hardening is a separate change, not certified by this review.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":352,"kind":"reviewed-static","reason":"Existing filter never-raise contract retained; this exception does not establish privacy guarantees.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":379,"kind":"reviewed-static","reason":"Existing message masking never-raise path retained; separate runtime hardening owns fail-closed behavior.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":401,"kind":"reviewed-static","reason":"Existing exception masking resilience retained without claiming fail-closed runtime behavior.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":494,"kind":"reviewed-static","reason":"Existing per-field flattening is deliberately non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_filters.py","line":496,"kind":"reviewed-static","reason":"Existing flattening filter never-raise contract retained.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":49,"kind":"reviewed-static","reason":"Optional tracing dependency failures degrade gracefully by design.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":107,"kind":"reviewed-static","reason":"Optional trace activation failures must not crash the handler.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":142,"kind":"reviewed-static","reason":"Existing tracing context cleanup is deliberately non-raising.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_otel.py","line":151,"kind":"reviewed-static","reason":"Existing optional OTel context cleanup is deliberately silent.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B110","file":"src/azure_functions_logging/_host_instance.py","line":60,"kind":"reviewed-static","reason":"Existing best-effort host metadata resolution must never crash logging.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B112","file":"src/azure_functions_logging/_redaction.py","line":170,"kind":"reviewed-static","reason":"Existing broken-pattern isolation retained; runtime redaction hardening is separate and not certified here.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B307","file":"src/azure_functions_logging/_decorator.py","line":83,"kind":"reviewed-static","reason":"Existing evaluation uses the handler author's annotation and globals, not HTTP input; no runtime edit permitted in this port.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"} + ], + "exceptions": [ + {"id":"B105","file":"benchmarks/bench.py","line":65,"kind":"reviewed-static","reason":"Synthetic benchmark fixture, not a credential or an authentication decision.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B105","file":"examples/otel_app/function_app.py","line":70,"kind":"reviewed-static","reason":"Synthetic masking demonstration; never use real credentials in this example.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B101","file":"scripts/assert_correlation.py","line":132,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; not authorization or input validation.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B101","file":"scripts/assert_correlation.py","line":133,"kind":"reviewed-static","reason":"Type narrowing after explicit missing-record failures; observable certification checks remain ordinary conditionals.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"scripts/render_mermaid.py","line":26,"kind":"reviewed-static","reason":"Documentation renderer invokes a shell-free CLI on repository Markdown.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"scripts/render_mermaid.py","line":89,"kind":"reviewed-static","reason":"Fixed mmdc command with temporary paths as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"tools/check_pr_format.py","line":5,"kind":"reviewed-static","reason":"Contributor formatter uses shell-free git and ruff subprocesses.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B607","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"git resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/check_pr_format.py","line":10,"kind":"reviewed-static","reason":"Revision range is one argument and git options end with --, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B607","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"ruff resolves from trusted CI/contributor PATH, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/check_pr_format.py","line":37,"kind":"reviewed-static","reason":"Filenames follow -- as separate arguments, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"tools/dependency_audit_policy.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free pip-audit command uses sys.executable.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/dependency_audit_policy.py","line":117,"kind":"reviewed-static","reason":"Fixed pip-audit module with verified frozen inventory passed as a temporary requirements path, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B310","file":"tools/lint_doc_links.py","line":103,"kind":"reviewed-static","reason":"Caller constructs HTTPS GitHub/raw.githubusercontent URLs from parsed documentation links; fixed HEAD request, no user-request input.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B404","file":"tools/security_evidence.py","line":11,"kind":"reviewed-static","reason":"Bounded shell-free Bandit command uses sys.executable.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B603","file":"tools/security_evidence.py","line":129,"kind":"reviewed-static","reason":"Fixed Bandit command over explicit repository-controlled surfaces, no shell.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"}, + {"id":"B105","file":"tools/security_report.py","line":59,"kind":"reviewed-static","reason":"TOOL_ERROR is a public status enum, not a credential.","owner":"yeongseon","added":"2026-10-22","expires":"2027-01-20"} + ] +} diff --git a/security/dependency-audit-policy.json b/security/dependency-audit-policy.json new file mode 100644 index 0000000..edf5af5 --- /dev/null +++ b/security/dependency-audit-policy.json @@ -0,0 +1,5 @@ +{ + "schema_version": 1, + "blocking": "all-known-runtime", + "exceptions": [] +} diff --git a/tests/test_candidate_audit.py b/tests/test_candidate_audit.py new file mode 100644 index 0000000..74bede9 --- /dev/null +++ b/tests/test_candidate_audit.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +from zipfile import ZipFile + +import pytest + +from tools.dependency_audit_policy import run_audit +from tools.security_evidence import semgrep_evidence +from tools.security_report import Status + + +@pytest.mark.parametrize("mode", ["clean", "advisory", "skip", "mismatch", "missing"]) +def test_unpublished_candidate_audits_exact_third_party_closure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: str +) -> None: + wheel = tmp_path / "candidate.whl" + with ZipFile(wheel, "w") as archive: + archive.writestr( + "candidate.dist-info/METADATA", "Name: azure-functions-logging\nVersion: 99.0\n" + ) + for name, version in [ + ("azure-functions-logging", "98.0" if mode == "mismatch" else "99.0"), + ("dependency", "1.0"), + ]: + dist = tmp_path / f"{name.replace('-', '_')}-{version}.dist-info" + dist.mkdir() + (dist / "METADATA").write_text(f"Name: {name}\nVersion: {version}\n") + policy = tmp_path / "policy.json" + policy.write_text('{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}') + + def audit(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: + requirements = Path(command[command.index("--requirement") + 1]).read_text() + assert requirements == "dependency==1.0\n" + assert "--no-deps" in command and "--disable-pip" in command + item: dict[str, object] = {"name": "dependency", "version": "1.0", "vulns": []} + if mode == "skip": + item = {"name": "dependency", "skip_reason": "not on PyPI"} + if mode == "advisory": + item["vulns"] = [{"id": "CVE-2026-12345", "fix_versions": []}] + return subprocess.CompletedProcess( + command, + int(mode == "advisory"), + json.dumps({"dependencies": [] if mode == "missing" else [item]}), + "", + ) + + monkeypatch.setattr(subprocess, "run", audit) + monkeypatch.setattr("tools.dependency_audit_policy.version", lambda _: "2.10.1") + result = run_audit(tmp_path, policy, wheel) + assert ( + result.status + == { + "clean": Status.PASS, + "advisory": Status.FAIL, + "skip": Status.INCOMPLETE, + "mismatch": Status.INCOMPLETE, + "missing": Status.TOOL_ERROR, + }[mode] + ) + + +def test_semgrep_action_success_is_incomplete() -> None: + assert semgrep_evidence("success").status == Status.INCOMPLETE diff --git a/tests/test_dependency_audit_policy.py b/tests/test_dependency_audit_policy.py new file mode 100644 index 0000000..12f1cbc --- /dev/null +++ b/tests/test_dependency_audit_policy.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from datetime import date +import json + +import pytest + +from tools.dependency_audit_policy import classify_audit +from tools.security_exceptions import parse_exceptions +from tools.security_report import EvidenceError, Status, load + +TODAY = date(2026, 10, 22) +POLICY = '{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}' + + +def payload(vulnerable: bool = False) -> str: + return json.dumps( + { + "dependencies": [ + { + "name": "example", + "version": "1", + "vulns": [ + {"id": "CVE-2026-12345", "fix_versions": [], "description": "CANARY_SECRET"} + ] + if vulnerable + else [], + } + ] + } + ) + + +@pytest.mark.parametrize("vulnerable,expected", [(False, Status.PASS), (True, Status.FAIL)]) +def test_any_known_runtime_advisory_blocks(vulnerable: bool, expected: Status) -> None: + # Given resolved packages; When audited; Then even UNKNOWN severity blocks. + result = classify_audit(payload(vulnerable), int(vulnerable), policy_raw=POLICY, today=TODAY) + assert result.status == expected + assert "CANARY_SECRET" not in repr(result) + + +@pytest.mark.parametrize("raw", ["", "invalid", "{}", '{"dependencies":[]}']) +def test_invalid_or_zero_package_audit_is_tool_error(raw: str) -> None: + # Given unusable audit output; When classified; Then it is never a clean scan. + assert classify_audit(raw, 0, policy_raw=POLICY, today=TODAY).status == Status.TOOL_ERROR + + +def test_skipped_package_is_incomplete() -> None: + # Given a package the tool skipped; When classified; Then coverage is incomplete. + raw = '{"dependencies":[{"name":"x","version":"1","skip_reason":"CANARY_SECRET"}]}' + assert classify_audit(raw, 0, policy_raw=POLICY, today=TODAY).status == Status.INCOMPLETE + + +@pytest.mark.parametrize( + "expires,expected", [("2026-10-21", Status.FAIL), ("2026-11-21", Status.PASS)] +) +def test_exception_expiry_and_affectedness(expires: str, expected: Status) -> None: + # Given an owned exception; When evaluated; Then only current/no-fix exceptions apply. + policy = json.loads(POLICY) + policy["exceptions"] = [ + { + "id": "CVE-2026-12345", + "package": "example", + "kind": "fix-not-available", + "reason": "no patched release", + "owner": "maintainer", + "added": "2026-10-01", + "expires": expires, + } + ] + result = classify_audit(payload(True), 1, policy_raw=json.dumps(policy), today=TODAY) + assert result.status == expected + assert bool(result.exceptions_applied) == (expected == Status.PASS) + raw = json.loads(payload(True)) + raw["dependencies"][0]["vulns"][0]["fix_versions"] = ["2"] + assert ( + classify_audit(json.dumps(raw), 1, policy_raw=json.dumps(policy), today=TODAY).status + == Status.FAIL + ) + + +def test_stale_exception_is_reported() -> None: + # Given a current unused exception; When evaluated; Then it is visible for removal. + policy = json.loads(POLICY) + policy["exceptions"] = [ + { + "id": "CVE-2026-12345", + "package": "example", + "kind": "not-affected", + "reason": "reviewed unreachable path", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + } + ] + result = classify_audit(payload(), 0, policy_raw=json.dumps(policy), today=TODAY) + assert result.status == Status.PASS + assert result.counts["stale_exceptions"] == 1 + + +@pytest.mark.parametrize( + "patch", + [ + {"expires": "2027-10-22"}, + {"added": "2026-11-01"}, + {"kind": "ignore-all"}, + {"line": -1}, + {"line": True}, + {"owner": ""}, + {"reason": ""}, + ], +) +def test_invalid_exception_policy_fails(patch: dict[str, str | int]) -> None: + # Given malformed policy; When parsed; Then no exception can hide a finding. + rule = { + "id": "CVE-1", + "package": "example", + "kind": "not-affected", + "reason": "reviewed", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + **patch, + } + with pytest.raises((EvidenceError, ValueError)): + parse_exceptions(load(json.dumps({"exceptions": [rule]}))["exceptions"], TODAY) + + +@pytest.mark.parametrize("code", [0, 2]) +def test_inconsistent_exit_status_fails(code: int) -> None: + # Given findings inconsistent with exit status; When classified; Then tooling failed. + assert ( + classify_audit(payload(True), code, policy_raw=POLICY, today=TODAY).status + == Status.TOOL_ERROR + ) diff --git a/tests/test_security_evidence.py b/tests/test_security_evidence.py new file mode 100644 index 0000000..a663d46 --- /dev/null +++ b/tests/test_security_evidence.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from datetime import date +import json +from pathlib import Path + +import pytest + +from tools.security_evidence import apply_bandit_policy, classify_bandit, semgrep_evidence +from tools.security_report import Check, Status, write_report + + +def payload() -> str: + return json.dumps( + { + "errors": [], + "metrics": {"_totals": {"loc": 10}, "src/x.py": {"loc": 10}}, + "results": [], + } + ) + + +@pytest.mark.parametrize( + "raw,code,status", + [ + ("", 2, Status.TOOL_ERROR), + ("invalid", 0, Status.TOOL_ERROR), + ('{"errors":[],"metrics":{"_totals":{}},"results":[]}', 0, Status.INCOMPLETE), + ], +) +def test_missing_evidence_fails_closed(raw: str, code: int, status: Status) -> None: + # Given missing evidence; When classified; Then it cannot pass. + assert classify_bandit(raw, code, floor=1).status == status + + +def test_complete_scan_has_real_counts() -> None: + # Given one scanned file; When classified; Then its inventory is reported. + result = classify_bandit(payload(), 0, floor=1) + assert result.status == Status.PASS + assert result.counts == {"files": 1, "loc": 10, "errors": 0} + + +@pytest.mark.parametrize("severity", ["LOW", "MEDIUM", "HIGH"]) +def test_findings_block_without_raw_diagnostics(severity: str, tmp_path: Path) -> None: + # Given secret-bearing diagnostics; When rendered; Then only metadata survives. + data = json.loads(payload()) + data["results"] = [ + { + "test_id": "B105", + "filename": "src/x.py", + "line_number": 3, + "issue_severity": severity, + "issue_confidence": "HIGH", + "code": "Authorization: Bearer CANARY_SECRET", + "issue_text": "AccountKey=CANARY_SECRET", + } + ] + result = classify_bandit(json.dumps(data), 1, floor=1) + summary = tmp_path / "summary.md" + assert write_report([result], tmp_path, summary) == 1 + saved = (tmp_path / "summary.json").read_text() + assert "CANARY_SECRET" not in saved + summary.read_text() + assert result.counts[f"severity.{severity}"] == 1 + parsed = json.loads(saved) + assert parsed["schema_version"] == 1 + assert set(parsed["checks"][0]) == { + "name", + "scope", + "status", + "exit_status", + "counts", + "suppressions", + "exceptions_applied", + "tool_versions", + "findings", + "message", + } + + +@pytest.mark.parametrize("code", [0, 2]) +def test_error_files_are_tool_errors(code: int) -> None: + # Given a scanner error; When classified; Then it is not a vulnerability result. + data = json.loads(payload()) + data["errors"] = [{"reason": "CANARY_SECRET"}] + assert classify_bandit(json.dumps(data), code, floor=1).status == Status.TOOL_ERROR + + +@pytest.mark.parametrize("field", ["nosec", "skipped_tests"]) +def test_suppression_overflow_is_incomplete(field: str) -> None: + # Given an unexpected scanner skip; When classified; Then completeness fails. + data = json.loads(payload()) + data["metrics"]["_totals"][field] = 1 + assert classify_bandit(json.dumps(data), 0, floor=1).status == Status.INCOMPLETE + + +def test_static_policy_matches_exact_location_and_expires() -> None: + # Given an exact reviewed exception; When evaluated; Then expiry/location matter. + data = json.loads(payload()) + data["results"] = [ + { + "test_id": "B404", + "filename": "src/x.py", + "line_number": 1, + "issue_severity": "LOW", + "issue_confidence": "HIGH", + } + ] + check = classify_bandit(json.dumps(data), 1, floor=1) + rule = { + "id": "B404", + "file": "src/x.py", + "line": 1, + "kind": "reviewed-static", + "reason": "fixed shell-free command", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + } + policy = json.dumps({"schema_version": 1, "exceptions": [rule]}) + applied = apply_bandit_policy(check, policy, date(2026, 10, 22)) + assert applied.status == Status.PASS + assert applied.exceptions_applied == ("B404:src/x.py:1",) + assert apply_bandit_policy(check, policy, date(2026, 11, 2)).status == Status.FAIL + rule["line"] = 2 + stale = apply_bandit_policy( + check, json.dumps({"schema_version": 1, "exceptions": [rule]}), date(2026, 10, 22) + ) + assert stale.status == Status.FAIL + assert stale.counts["stale_exceptions"] == 1 + assert apply_bandit_policy(check, "invalid", date(2026, 10, 22)).status == Status.TOOL_ERROR + + +@pytest.mark.parametrize( + "outcome,status", + [ + ("success", Status.INCOMPLETE), + ("failure", Status.TOOL_ERROR), + ("skipped", Status.INCOMPLETE), + ], +) +def test_semgrep_outcome_is_not_a_vulnerability(outcome: str, status: Status) -> None: + # Given action outcome only; When recorded; Then finding counts stay unknown. + assert semgrep_evidence(outcome).status == status + + +@pytest.mark.parametrize( + "status,code", + [(Status.PASS, 0), (Status.FAIL, 1), (Status.TOOL_ERROR, 2), (Status.INCOMPLETE, 3)], +) +def test_status_exit_codes_are_distinct(status: Status, code: int) -> None: + # Given a status; When returned to CI; Then its code is unambiguous. + assert Check("test", (), status, 0).returncode == code diff --git a/tests/test_security_report.py b/tests/test_security_report.py new file mode 100644 index 0000000..f90db3a --- /dev/null +++ b/tests/test_security_report.py @@ -0,0 +1,87 @@ +from datetime import date +import json +from pathlib import Path + +import pytest + +from tools.security_exceptions import parse_exceptions +from tools.security_report import ( + Check, + EvidenceError, + Status, + identifier, + integer, + load, + sequence, + text, + write_report, +) + + +@pytest.mark.parametrize("raw", ["null", "[]", '"text"', "true"]) +def test_json_boundary_requires_object(raw: str) -> None: + # Given non-object input; When parsed; Then the evidence shape is rejected. + with pytest.raises(EvidenceError): + load(raw) + + +@pytest.mark.parametrize("value", [None, 0, True, ""]) +def test_text_requires_nonempty_string(value: None | int | str) -> None: + # Given invalid text; When parsed; Then it cannot enter an identifier. + with pytest.raises(EvidenceError): + text(value) + + +@pytest.mark.parametrize("value", [None, True, -1, "1"]) +def test_count_requires_nonnegative_integer(value: None | int | str) -> None: + # Given invalid counts; When parsed; Then malformed evidence cannot pass. + with pytest.raises(EvidenceError): + integer(value) + + +@pytest.mark.parametrize("value", [None, "[]", 0]) +def test_array_requires_sequence(value: None | str | int) -> None: + # Given a nonarray; When parsed; Then the boundary rejects it. + with pytest.raises(EvidenceError): + sequence(value) + + +@pytest.mark.parametrize( + "value", + [ + "/tmp/file.py", + "../secret", + "https://example.com", + "Authorization: Bearer CANARY_SECRET", + "x?code=CANARY_SECRET", + ], +) +def test_sensitive_metadata_is_rejected(value: str) -> None: + # Given unsafe metadata; When parsed; Then no original credential enters evidence. + with pytest.raises(EvidenceError): + identifier(value) + + +def test_duplicate_exception_is_rejected() -> None: + # Given duplicate policy entries; When parsed; Then exception accounting is unambiguous. + rule = { + "id": "CVE-1", + "package": "example", + "kind": "not-affected", + "reason": "reviewed", + "owner": "maintainer", + "added": "2026-10-01", + "expires": "2026-11-01", + } + parsed = load(json.dumps({"exceptions": [rule, rule]})) + with pytest.raises(EvidenceError): + parse_exceptions(parsed["exceptions"], date(2026, 10, 22)) + + +def test_summary_appends_to_github_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given an existing job summary; When evidence is written; Then prior evidence is preserved. + summary = tmp_path / "summary.md" + summary.write_text("existing\n") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + assert write_report([Check("scanner", (), Status.PASS, 0)], tmp_path / "out") == 0 + assert summary.read_text().startswith("existing\n") diff --git a/tests/test_security_tool_execution.py b/tests/test_security_tool_execution.py new file mode 100644 index 0000000..4cc3b0f --- /dev/null +++ b/tests/test_security_tool_execution.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from datetime import date +import json +from pathlib import Path +import subprocess +import sys +from zipfile import ZipFile + +import pytest + +from tools import dependency_audit_policy, security_evidence +from tools.security_report import Check, Status + + +def test_bandit_inventory_matches_disk(monkeypatch: pytest.MonkeyPatch) -> None: + # Given a complete fake scan; When inventory loses a file; Then coverage fails closed. + metrics = { + p.as_posix(): {"loc": 1} + for surface in security_evidence.SCOPE + for p in Path(surface).rglob("*.py") + } + metrics["_totals"] = {"loc": len(metrics)} + raw = json.dumps({"errors": [], "metrics": metrics, "results": []}) + + def complete(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: + assert "--ignore-nosec" in command + return subprocess.CompletedProcess(command, 0, stdout=raw, stderr="CANARY_SECRET") + + monkeypatch.setattr(subprocess, "run", complete) + monkeypatch.setattr(security_evidence, "version", lambda name: "1.9.4") + assert security_evidence.run_bandit().status == Status.PASS + del metrics[next(iter(metrics))] + raw = json.dumps({"errors": [], "metrics": metrics, "results": []}) + assert security_evidence.run_bandit().status == Status.INCOMPLETE + + +@pytest.mark.parametrize("marker", ["# nosec", "# nosec B999", "# nosec B110"]) +def test_unlisted_suppression_is_incomplete( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, marker: str +) -> None: + # Given a new inline marker; When scanned; Then its exact location needs review. + (tmp_path / "src").mkdir() + (tmp_path / "src/x.py").write_text(f"value = 1 {marker}\n") + (tmp_path / "security").mkdir() + (tmp_path / "security/bandit-policy.json").write_text( + '{"schema_version":1,"exceptions":[],"inline_suppressions":[]}' + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(security_evidence, "SCOPE", ("src",)) + raw = json.dumps( + { + "errors": [], + "metrics": {"_totals": {"loc": 40}, **{f"src/{i}.py": {} for i in range(40)}}, + "results": [], + } + ) + + def complete(*args: str, **kwargs: str) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess([], 0, stdout=raw, stderr="") + + monkeypatch.setattr(subprocess, "run", complete) + monkeypatch.setattr(security_evidence, "version", lambda name: "1.9.4") + assert security_evidence.run_bandit().status == Status.INCOMPLETE + + +@pytest.mark.parametrize("failure", [OSError(), subprocess.TimeoutExpired("scanner", 1)]) +def test_execution_errors_never_pass( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: OSError | subprocess.TimeoutExpired +) -> None: + # Given unavailable tools; When invoked; Then no raw diagnostic is retained. + (tmp_path / "example.dist-info").mkdir() + + def fail(*args: str, **kwargs: str) -> subprocess.CompletedProcess[str]: + raise failure + + monkeypatch.setattr(subprocess, "run", fail) + assert security_evidence.run_bandit().status == Status.TOOL_ERROR + assert dependency_audit_policy.run_audit(tmp_path, tmp_path).status == Status.INCOMPLETE + + +def test_audit_uses_only_candidate_installation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given an isolated inventory; When audited; Then tooling cannot contaminate its closure. + (tmp_path / "example.dist-info").mkdir() + (tmp_path / "example.dist-info/METADATA").write_text("Name: example\nVersion: 1\n") + (tmp_path / "candidate.dist-info").mkdir() + (tmp_path / "candidate.dist-info/METADATA").write_text("Name: candidate\nVersion: 99\n") + wheel = tmp_path / "candidate.whl" + with ZipFile(wheel, "w") as archive: + archive.writestr("candidate.dist-info/METADATA", "Name: candidate\nVersion: 99\n") + policy = tmp_path / "policy.json" + policy.write_text('{"schema_version":1,"blocking":"all-known-runtime","exceptions":[]}') + + def complete(command: list[str], **kwargs: str) -> subprocess.CompletedProcess[str]: + assert Path(command[command.index("--requirement") + 1]).read_text() == "example==1\n" + return subprocess.CompletedProcess( + command, + 0, + stdout='{"dependencies":[{"name":"example","version":"1","vulns":[]}]}', + stderr="CANARY_SECRET", + ) + + monkeypatch.setattr(subprocess, "run", complete) + monkeypatch.setattr(dependency_audit_policy, "version", lambda name: "2.10.1") + assert dependency_audit_policy.run_audit(tmp_path, policy, wheel).status == Status.PASS + + +@pytest.mark.parametrize("development", [False, True]) +@pytest.mark.parametrize( + "status,code", [(Status.FAIL, 1), (Status.TOOL_ERROR, 2), (Status.INCOMPLETE, 3)] +) +def test_audit_cli_preserves_failure_codes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, development: bool, status: Status, code: int +) -> None: + # Given a classified failure; When CLI writes evidence; Then the same failure reaches CI. + args = [ + "audit", + "--site-packages", + str(tmp_path), + "--output", + str(tmp_path), + "--candidate-wheel", + "candidate.whl", + ] + if development: + args.append("--development") + monkeypatch.setattr(sys, "argv", args) + monkeypatch.setattr( + dependency_audit_policy, + "run_audit", + lambda path, policy, wheel: Check("pip-audit", (), status, 1), + ) + assert dependency_audit_policy.main() == code + assert json.loads((tmp_path / "summary.json").read_text())["checks"][0]["status"] == status + + +def test_real_cli_writes_stdlib_only_evidence(tmp_path: Path) -> None: + # Given the real CLI; When action evidence is missing; Then it writes INCOMPLETE and exits 3. + result = subprocess.run( + [ + sys.executable, + "-m", + "tools.security_evidence", + "semgrep", + "--outcome", + "cancelled", + "--output", + str(tmp_path), + ], + capture_output=True, + check=False, + ) + assert result.returncode == 3 + assert json.loads((tmp_path / "summary.json").read_text())["schema_version"] == 1 + + +def test_incomplete_scan_cannot_be_excepted() -> None: + # Given missing coverage; When exceptions are applied; Then they cannot waive completeness. + check = Check("bandit", (), Status.INCOMPLETE, 0) + assert security_evidence.apply_bandit_policy(check, "{}", date(2026, 10, 22)) == check diff --git a/tests/test_security_workflows.py b/tests/test_security_workflows.py new file mode 100644 index 0000000..a7afae5 --- /dev/null +++ b/tests/test_security_workflows.py @@ -0,0 +1,93 @@ +from pathlib import Path +import tomllib + +import yaml + +from tools.lint_release_workflows import check_pins, check_publish_needs +from tools.security_evidence import SCOPE + +ROOT = Path(__file__).resolve().parents[1] + + +def test_release_gate_preserves_existing_tiers_and_candidate() -> None: + # Given release configuration; When inspected; Then the audit adds to every existing gate. + text = (ROOT / ".github/workflows/publish-pypi.yml").read_text() + jobs = yaml.safe_load(text)["jobs"] + assert check_publish_needs(text, "publish-pypi.yml") == [] + assert set(jobs["publish"]["needs"]) == { + "build", + "lib-tests", + "cookbook-smoke", + "cookbook-host-smoke", + "azure-e2e", + "runtime-dependency-audit", + } + gate = jobs["runtime-dependency-audit"] + assert gate["permissions"] == {"contents": "read"} + assert gate["needs"] == "build" + assert gate["strategy"]["matrix"]["interpreter"] == ["3.11", "3.12", "3.13", "3.14"] + assert gate["strategy"]["matrix"]["closure"] == ["base", "otel"] + assert not gate.get("continue-on-error") + assert all(not s.get("continue-on-error") for s in gate["steps"]) + download = next(s for s in gate["steps"] if "download-artifact@" in s.get("uses", "")) + assert download["with"]["name"] == "dist" + commands = "\n".join(s.get("run", "") for s in gate["steps"]) + assert "pip-audit==2.10.1" in commands + assert "--without-pip" in commands + assert '"${wheels[0]}[otel]"' in commands + assert "make security-runtime" in commands + assert "--candidate-wheel" in commands + assert "build --wheel" not in commands + + +def test_evidence_uploads_always_run_with_canonical_pins() -> None: + # Given every evidence workflow; When inspected; Then failed scans retain redacted evidence. + for filename in ("security.yml", "dependency-audit.yml", "publish-pypi.yml"): + text = (ROOT / ".github/workflows" / filename).read_text() + workflow = yaml.safe_load(text) + assert workflow["permissions"] == {"contents": "read"} + assert check_pins(text, filename) == [] + uploads = [ + s + for job in workflow["jobs"].values() + for s in job.get("steps", []) + if "upload-artifact@" in s.get("uses", "") + and s.get("with", {}).get("path", "").startswith("security-results") + ] + assert uploads + assert all( + s["if"] == "always()" and s["with"]["if-no-files-found"] == "error" for s in uploads + ) + + +def test_security_scans_declared_executable_surfaces() -> None: + # Given declared surfaces; When workflow runs; Then neither source nor examples are omitted. + workflow = yaml.safe_load((ROOT / ".github/workflows/security.yml").read_text()) + assert SCOPE == ("src", "examples", "scripts", "tools", "benchmarks") + steps = workflow["jobs"]["bandit"]["steps"] + assert any( + s.get("if") == "always()" and "make security-bandit" in s.get("run", "") for s in steps + ) + assert any("security-typecheck" in s.get("run", "") for s in steps) + semgrep = workflow["jobs"]["semgrep"]["steps"] + action = next(s for s in semgrep if "semgrep/semgrep-action@" in s.get("uses", "")) + assert action["uses"] == "semgrep/semgrep-action@713efdd345f3035192eaa63f56867b88e63e4e5d" + assert action["with"]["config"] == "p/python" + assert not action.get("continue-on-error") + assert any( + s.get("if") == "always()" and "make security-semgrep" in s.get("run", "") for s in semgrep + ) + scripts = tomllib.loads((ROOT / "pyproject.toml").read_text())["tool"]["hatch"]["envs"][ + "default" + ]["scripts"] + assert "--strict" in scripts["security-typecheck"] + + +def test_scheduled_audits_reuse_policy_as_reporting_only() -> None: + # Given scheduled audit lanes; When inspected; Then runtime/OTel/dev use one classifier. + workflow = yaml.safe_load((ROOT / ".github/workflows/dependency-audit.yml").read_text()) + steps = workflow["jobs"]["pip-audit"]["steps"] + audits = [s for s in steps if "make security-runtime-report" in s.get("run", "")] + assert len(audits) == 3 + assert all(s["continue-on-error"] for s in audits) + assert any("--development" in s["run"] for s in audits) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/candidate_inventory.py b/tools/candidate_inventory.py new file mode 100644 index 0000000..82a84ab --- /dev/null +++ b/tools/candidate_inventory.py @@ -0,0 +1,39 @@ +"""Verify wheel identity against isolated installed metadata before excluding it.""" + +from email.parser import BytesParser +from importlib.metadata import distributions +from pathlib import Path +import re +from zipfile import ZipFile + + +def normalize(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def inventory(site_packages: Path, wheel: Path) -> tuple[tuple[str, str], dict[str, str]]: + with ZipFile(wheel) as archive: + names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] + if len(names) != 1: + raise ValueError("Ambiguous wheel metadata") + metadata = BytesParser().parsebytes(archive.read(names[0])) + name, version = metadata.get("Name", ""), metadata.get("Version", "") + if not name or not version: + raise ValueError("Missing candidate identity") + candidate = (normalize(name), version) + installed: dict[str, str] = {} + for dist in distributions(path=[str(site_packages)]): + package = normalize(dist.metadata["Name"] or "") + release = dist.version + if ( + not re.fullmatch(r"[a-z0-9][a-z0-9-]*", package) + or not release + or not re.fullmatch(r"[a-zA-Z0-9.!+_-]+", release) + ): + raise ValueError("Invalid installed identity") + if package in installed: + raise ValueError("Duplicate installed identity") + installed[package] = release + if installed.pop(candidate[0], None) != candidate[1] or not installed: + raise ValueError("Candidate mismatch or empty third-party inventory") + return candidate, installed diff --git a/tools/dependency_audit_policy.py b/tools/dependency_audit_policy.py new file mode 100644 index 0000000..16b5357 --- /dev/null +++ b/tools/dependency_audit_policy.py @@ -0,0 +1,192 @@ +"""Audit only an isolated wheel environment, with fail-closed redacted evidence.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +from datetime import date +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +import re +import subprocess +import sys +import tempfile +from zipfile import BadZipFile + +from tools.candidate_inventory import inventory, normalize +from tools.security_exceptions import parse_exceptions +from tools.security_report import ( + Check, + EvidenceError, + Finding, + Status, + identifier, + load, + mapping, + sequence, + text, + write_report, +) + + +def classify_audit(raw: str, exit_status: int, *, policy_raw: str, today: date) -> Check: + result = Check("pip-audit", ("resolved-runtime-wheel",), Status.TOOL_ERROR, exit_status) + try: + policy = load(policy_raw) + if policy["schema_version"] != 1 or policy["blocking"] != "all-known-runtime": + raise EvidenceError("unsupported policy") + rules = parse_exceptions(policy["exceptions"], today) + if any(r.file or r.line or not r.package or r.kind == "reviewed-static" for r in rules): + raise EvidenceError("invalid dependency exception") + dependencies = sequence(load(raw)["dependencies"]) + if exit_status not in {0, 1} or not dependencies: + raise EvidenceError("tool crash or zero packages") + if any("skip_reason" in mapping(d) for d in dependencies): + return replace(result, status=Status.INCOMPLETE, message="Skipped audit scope.") + findings: list[Finding] = [] + applied: set[str] = set() + blocking = 0 + for dependency in dependencies: + item = mapping(dependency) + package = re.sub(r"[-_.]+", "-", identifier(item["name"]).lower()) + text(item["version"]) + for advisory in sequence(item["vulns"]): + vuln = mapping(advisory) + advisory_id = identifier(vuln["id"]) + severity = identifier(vuln.get("severity", "UNKNOWN")) + findings.append(Finding(advisory_id, severity=severity, package=package)) + matches = [ + r + for r in rules + if r.id == advisory_id + and re.sub(r"[-_.]+", "-", r.package.lower()) == package + and r.expires >= today + ] + if matches and ( + matches[0].kind != "fix-not-available" or not sequence(vuln["fix_versions"]) + ): + applied.add(f"{package}:{advisory_id}") + else: + blocking += 1 + if bool(findings) != (exit_status == 1): + raise EvidenceError("exit status disagrees with findings") + expired = sum(r.expires < today for r in rules) + return replace( + result, + status=Status.FAIL if blocking or expired else Status.PASS, + counts={ + "packages": len(dependencies), + "findings": len(findings), + "blocking": blocking, + "expired_exceptions": expired, + "stale_exceptions": len(rules) - len(applied), + }, + findings=tuple(findings), + exceptions_applied=tuple(sorted(applied)), + message="All known runtime advisories block without a current applicable exception.", + ) + except (ValueError, KeyError, TypeError): + return replace(result, message="Invalid audit/tool/policy evidence; release blocked.") + + +def run_audit(site_packages: Path, policy: Path, candidate_wheel: Path | None = None) -> Check: + if candidate_wheel is None: + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.INCOMPLETE, + None, + message="Verified candidate wheel required.", + ) + try: + candidate, installed = inventory(site_packages, candidate_wheel) + except (OSError, ValueError, BadZipFile, KeyError): + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.INCOMPLETE, + None, + message="Candidate identity or third-party inventory invalid.", + ) + try: + with tempfile.TemporaryDirectory(prefix="runtime-audit-") as temporary: + requirements = Path(temporary) / "requirements.txt" + requirements.write_text( + "".join(f"{name}=={release}\n" for name, release in sorted(installed.items())) + ) + process = subprocess.run( + [ + sys.executable, + "-m", + "pip_audit", + "--requirement", + str(requirements), + "--no-deps", + "--disable-pip", + "--format", + "json", + "--desc", + "off", + "--progress-spinner", + "off", + "--timeout", + "30", + ], + capture_output=True, + text=True, + check=False, + timeout=300, + ) + result = classify_audit( + process.stdout, + process.returncode, + policy_raw=policy.read_text(encoding="utf-8"), + today=date.today(), + ) + if result.status in {Status.PASS, Status.FAIL}: + observed: dict[str, str] = {} + for dependency in sequence(load(process.stdout)["dependencies"]): + item = mapping(dependency) + name = normalize(text(item["name"])) + release = text(item["version"]) + if name in observed or (name == candidate[0] and release != candidate[1]): + return replace( + result, status=Status.INCOMPLETE, message="Audit identity mismatch." + ) + observed[name] = release + observed.pop(candidate[0], None) + if observed != installed: + return replace( + result, + status=Status.INCOMPLETE, + message="Audit omitted installed dependencies.", + ) + return replace(result, tool_versions={"pip-audit": version("pip-audit")}) + except (OSError, subprocess.TimeoutExpired, PackageNotFoundError): + return Check( + "pip-audit", + ("resolved-runtime-wheel",), + Status.TOOL_ERROR, + None, + message="Audit execution failed; release blocked.", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--site-packages", type=Path, required=True) + parser.add_argument( + "--policy", type=Path, default=Path("security/dependency-audit-policy.json") + ) + parser.add_argument("--output", type=Path, default=Path("security-results")) + parser.add_argument("--development", action="store_true") + parser.add_argument("--candidate-wheel", type=Path, required=True) + args = parser.parse_args() + check = run_audit(args.site_packages, args.policy, args.candidate_wheel) + if args.development: + check = replace(check, name="pip-audit-development", scope=("runtime-and-development",)) + return write_report([check], args.output) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/security_evidence.py b/tools/security_evidence.py new file mode 100644 index 0000000..5c8cabf --- /dev/null +++ b/tools/security_evidence.py @@ -0,0 +1,233 @@ +"""Execute Bandit's explicit product inventory and record Semgrep action evidence.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +from datetime import date +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +import re +import subprocess +import sys +import tokenize + +from tools.security_exceptions import parse_exceptions +from tools.security_report import ( + Check, + EvidenceError, + Finding, + Status, + identifier, + integer, + load, + mapping, + sequence, + text, + write_report, +) + +SCOPE = ("src", "examples", "scripts", "tools", "benchmarks") +FILE_FLOOR = 38 + + +def classify_bandit(raw: str, exit_status: int, *, floor: int = FILE_FLOOR) -> Check: + result = Check("bandit", SCOPE, Status.TOOL_ERROR, exit_status) + try: + data = load(raw) + metrics = mapping(data["metrics"]) + totals = mapping(metrics["_totals"]) + files = tuple(identifier(f.removeprefix("./")) for f in metrics if f != "_totals") + findings: list[Finding] = [] + counts = { + "files": len(files), + "loc": integer(totals.get("loc", 0)), + "errors": len(sequence(data["errors"])), + } + for entry in sequence(data["results"]): + item = mapping(entry) + severity, confidence = text(item["issue_severity"]), text(item["issue_confidence"]) + if severity not in {"LOW", "MEDIUM", "HIGH", "UNDEFINED"} or confidence not in { + "LOW", + "MEDIUM", + "HIGH", + "UNDEFINED", + }: + raise EvidenceError("invalid finding classification") + findings.append( + Finding( + identifier(item["test_id"]), + identifier(text(item["filename"]).removeprefix("./")), + integer(item["line_number"]), + severity, + confidence, + ) + ) + for key in (f"severity.{severity}", f"confidence.{confidence}"): + counts[key] = counts.get(key, 0) + 1 + suppressions = integer(totals.get("nosec", 0)) + integer(totals.get("skipped_tests", 0)) + result = replace(result, counts=counts, suppressions=suppressions, findings=tuple(findings)) + if exit_status not in {0, 1} or counts["errors"]: + return replace(result, message="Scanner failure/error files; scan unusable.") + if len(files) < floor or counts["loc"] == 0 or suppressions: + return replace( + result, status=Status.INCOMPLETE, message="Scan floor/skip policy failed." + ) + if bool(findings) != (exit_status == 1): + return replace(result, message="Scanner exit status disagrees with findings.") + return replace(result, status=Status.FAIL if findings else Status.PASS) + except (ValueError, KeyError, TypeError): + return replace(result, message="Invalid Bandit JSON; raw diagnostics discarded.") + + +def apply_bandit_policy(check: Check, policy_raw: str, today: date) -> Check: + if check.status not in {Status.PASS, Status.FAIL}: + return check + try: + policy = load(policy_raw) + if policy["schema_version"] != 1: + raise EvidenceError("unsupported policy") + rules = parse_exceptions( + sequence(policy["exceptions"]) + sequence(policy.get("inline_suppressions", [])), today + ) + if any( + r.expires < today + or not r.file + or not r.line + or r.package + or r.kind != "reviewed-static" + for r in rules + ): + return replace( + check, status=Status.FAIL, message="Expired or invalid static exception." + ) + applied: list[str] = [] + blocking = 0 + for finding in check.findings: + if any( + (r.id, r.file, r.line) == (finding.id, finding.file, finding.line) for r in rules + ): + applied.append(f"{finding.id}:{finding.file}:{finding.line}") + else: + blocking += 1 + return replace( + check, + status=Status.FAIL if blocking else Status.PASS, + exceptions_applied=tuple(applied), + counts={ + **check.counts, + "blocking": blocking, + "stale_exceptions": len(rules) - len(set(applied)), + }, + ) + except (ValueError, KeyError, TypeError): + return replace(check, status=Status.TOOL_ERROR, message="Invalid Bandit exception policy.") + + +def run_bandit() -> Check: + try: + process = subprocess.run( + [sys.executable, "-m", "bandit", "-r", *SCOPE, "-f", "json", "--ignore-nosec"], + capture_output=True, + text=True, + check=False, + timeout=180, + ) + result = classify_bandit(process.stdout, process.returncode) + expected = {p.as_posix() for surface in SCOPE for p in Path(surface).rglob("*.py")} + scanned = set(mapping(load(process.stdout)["metrics"])) - {"_totals"} + if result.status != Status.TOOL_ERROR and ( + {p.removeprefix("./") for p in scanned} != expected + or any(not Path(surface).is_dir() for surface in SCOPE) + ): + result = replace( + result, + status=Status.INCOMPLETE, + message="Scanned inventory differs from declared surfaces.", + ) + policy = Path("security/bandit-policy.json").read_text(encoding="utf-8") + result = apply_bandit_policy(result, policy, date.today()) + rules = parse_exceptions(load(policy)["inline_suppressions"], date.today()) + observed: set[tuple[str, str, int]] = set() + suppressions = 0 + for filename in expected: + with tokenize.open(filename) as stream: + for token in tokenize.generate_tokens(stream.readline): + if token.type != tokenize.COMMENT or "nosec" not in token.string: + continue + ids = re.findall(r"B\d{3}", token.string) + suppressions += 1 + observed.update((rule_id, filename, token.start[0]) for rule_id in ids) + if not ids: + observed.add(("blanket", filename, token.start[0])) + allowed = {(r.id, r.file, r.line) for r in rules} + if observed != allowed and result.status != Status.TOOL_ERROR: + result = replace( + result, + status=Status.INCOMPLETE, + message="Inline suppression inventory differs from reviewed allow-list.", + ) + if any(r.expires < date.today() for r in rules) and result.status in { + Status.PASS, + Status.FAIL, + }: + result = replace( + result, status=Status.FAIL, message="Expired inline suppression review." + ) + return replace( + result, + suppressions=suppressions, + tool_versions={"bandit": version("bandit")}, + counts={**result.counts, "inline_suppressions_reviewed": len(allowed)}, + message=result.message or "Existing inline suppressions audited with ignore-nosec.", + ) + except ( + OSError, + subprocess.TimeoutExpired, + ValueError, + KeyError, + TypeError, + PackageNotFoundError, + tokenize.TokenError, + SyntaxError, + ): + return Check( + "bandit", + SCOPE, + Status.TOOL_ERROR, + None, + message="Bandit execution/evidence collection failed.", + ) + + +def semgrep_evidence(outcome: str) -> Check: + code: int | None + match outcome: + case "success": + status, code = Status.INCOMPLETE, 0 + case "failure": + status, code = Status.TOOL_ERROR, 1 + case _: + status, code = Status.INCOMPLETE, None + return Check( + "semgrep", + ("p/python", *SCOPE), + status, + code, + tool_versions={"action": "713efdd345f3035192eaa63f56867b88e63e4e5d"}, + message="action outcome only; finding count unknown; failures may include findings.", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("check", choices=("bandit", "semgrep")) + parser.add_argument("--outcome", default="") + parser.add_argument("--output", type=Path, default=Path("security-results")) + args = parser.parse_args() + check = run_bandit() if args.check == "bandit" else semgrep_evidence(args.outcome) + return write_report([check], args.output) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/security_exceptions.py b/tools/security_exceptions.py new file mode 100644 index 0000000..2162a63 --- /dev/null +++ b/tools/security_exceptions.py @@ -0,0 +1,49 @@ +"""Owned, exact exceptions with a maximum 90-day validity window.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date + +from tools.security_report import JSON, EvidenceError, identifier, integer, mapping, sequence, text + + +@dataclass(frozen=True, slots=True) +class ExceptionRule: + id: str + package: str + file: str + line: int + kind: str + expires: date + added: date + + +def parse_exceptions(data: JSON, today: date) -> tuple[ExceptionRule, ...]: + rules: list[ExceptionRule] = [] + for entry in sequence(data): + item = mapping(entry) + text(item["reason"]) + text(item["owner"]) + added = date.fromisoformat(text(item["added"])) + expires = date.fromisoformat(text(item["expires"])) + if not 0 < (expires - added).days <= 90 or added > today: + raise EvidenceError("invalid exception validity window") + rule = ExceptionRule( + identifier(item["id"]), + identifier(item["package"]) if "package" in item else "", + identifier(item["file"]) if "file" in item else "", + integer(item.get("line", 0)), + text(item["kind"]), + expires, + added, + ) + if rule.kind not in {"fix-not-available", "not-affected", "reviewed-static"}: + raise EvidenceError("unknown exception kind") + if any( + (r.id, r.package, r.file, r.line) == (rule.id, rule.package, rule.file, rule.line) + for r in rules + ): + raise EvidenceError("duplicate exception") + rules.append(rule) + return tuple(rules) diff --git a/tools/security_report.py b/tools/security_report.py new file mode 100644 index 0000000..0692c5f --- /dev/null +++ b/tools/security_report.py @@ -0,0 +1,120 @@ +"""Allow-listed security evidence; raw scanner diagnostics are never retained.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from enum import StrEnum +import json +import os +from pathlib import Path +import re +from typing import TypeAlias, assert_never + +JSON: TypeAlias = "None | bool | int | float | str | list[JSON] | dict[str, JSON]" + + +class EvidenceError(ValueError): + """Invalid evidence; messages contain no external input.""" + + +def mapping(value: JSON) -> dict[str, JSON]: + if not isinstance(value, dict): + raise EvidenceError("expected JSON object") + return value + + +def sequence(value: JSON) -> list[JSON]: + if not isinstance(value, list): + raise EvidenceError("expected JSON array") + return value + + +def text(value: JSON) -> str: + if not isinstance(value, str) or not value: + raise EvidenceError("expected nonempty string") + return value + + +def integer(value: JSON) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise EvidenceError("expected nonnegative integer") + return value + + +def load(raw: str) -> dict[str, JSON]: + parsed: JSON = json.loads(raw) + return mapping(parsed) + + +def identifier(value: JSON) -> str: + candidate = text(value) + if not re.fullmatch(r"[A-Za-z0-9_.:/@+-]{1,240}", candidate): + raise EvidenceError("unsafe evidence identifier") + if candidate.startswith(("/", "http:", "https:")) or ".." in candidate: + raise EvidenceError("unsafe evidence path") + return candidate + + +class Status(StrEnum): + PASS = "PASS" + FAIL = "FAIL" + TOOL_ERROR = "TOOL_ERROR" + INCOMPLETE = "INCOMPLETE" + + +@dataclass(frozen=True, slots=True) +class Finding: + id: str + file: str = "" + line: int = 0 + severity: str = "UNKNOWN" + confidence: str = "UNKNOWN" + package: str = "" + + +@dataclass(frozen=True, slots=True) +class Check: + name: str + scope: tuple[str, ...] + status: Status + exit_status: int | None + counts: dict[str, int] = field(default_factory=dict) + suppressions: int = 0 + exceptions_applied: tuple[str, ...] = () + tool_versions: dict[str, str] = field(default_factory=dict) + findings: tuple[Finding, ...] = () + message: str = "" + + @property + def returncode(self) -> int: + match self.status: + case Status.PASS: + return 0 + case Status.FAIL: + return 1 + case Status.TOOL_ERROR: + return 2 + case Status.INCOMPLETE: + return 3 + case unreachable: + assert_never(unreachable) + + +def write_report(checks: list[Check], output: Path, summary: Path | None = None) -> int: + output.mkdir(parents=True, exist_ok=True) + (output / "summary.json").write_text( + json.dumps({"schema_version": 1, "checks": [asdict(c) for c in checks]}, indent=2) + "\n", + encoding="utf-8", + ) + markdown = "## Security evidence\n\n| Check | Status | Counts |\n| --- | --- | --- |\n" + for check in checks: + markdown += f"| {check.name} | {check.status} | {json.dumps(check.counts)} |\n" + markdown += f"\n{check.message}\n\n" + if summary is None and os.environ.get("GITHUB_STEP_SUMMARY"): + summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) + if summary is not None: + with summary.open("a", encoding="utf-8") as stream: + stream.write(markdown) + for check in checks: + print(f"{check.name}: {check.status} {json.dumps(check.counts)} {check.message}") + return max((c.returncode for c in checks), default=3)