From cc5a6ebd5ca1c0d47fd94a32cc73b4fff296f5d1 Mon Sep 17 00:00:00 2001 From: Josui Date: Wed, 18 Nov 2026 03:44:25 +0000 Subject: [PATCH 1/4] fix: pip-audit says when the network failed instead of passing or failing pip-audit 2.9.0 exits 1 for a PyPI read timeout, the same status as a vulnerability, and an egress path answering 404 makes it skip every package and exit 0. Raise --timeout to 60, retry a network failure twice with 10s/30s backoff, and report one that outlasts the retries as exit 2 (could not run). "Not found" for a package uv.lock takes from PyPI counts as a failed lookup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R7Y1jckWHhAgXg2WBwkWbb --- .github/workflows/container.yml | 2 +- CHANGELOG.md | 13 ++ sdlc | 111 ++++++++++++++--- tests/test_pip_audit_resilience.py | 187 +++++++++++++++++++++++++++++ 4 files changed, 297 insertions(+), 16 deletions(-) create mode 100755 tests/test_pip_audit_resilience.py diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index da9c9d5..86751dd 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -127,7 +127,7 @@ jobs: for t in tests/test_plan.py tests/test_codeql_globs.py \ tests/test_pycache_prefix.py tests/test_gourmand_sidecar.py \ tests/test_codeql_resilience.py tests/test_convert_repo.py \ - tests/test_fleet_wave.py; do + tests/test_fleet_wave.py tests/test_pip_audit_resilience.py; do podman run --rm -v "$PWD:/src:ro,z" -w /src \ --entrypoint /opt/constitution-venv/bin/python3 \ "localhost/python-sdlc:$GITHUB_SHA" "$t" diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f60c0..f24979a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to this project are documented here, following ## [Unreleased] +### Fixed + +- **pip-audit reports what happened on the network, not a verdict it never + reached.** pip-audit 2.9.0 exits 1 for a PyPI read timeout, which `sdlc ci` + printed as a failed gate on the busy self-hosted runners; and an egress path + that answers 404 makes it skip every package and exit 0, which read as a + pass. pip-audit now runs with `--timeout 60` (was 15), retries a network + failure twice with 10s then 30s backoff, and reports one that outlasts the + retries as exit 2, COULD NOT RUN. A package that uv.lock takes from PyPI and + pip-audit calls "not found" counts as a failed lookup, not as unauditable. + A real vulnerability is still a 1, at once, with no retry. + `tests/test_pip_audit_resilience.py` covers each case. + ## [0.9.1] - 2026-11-17 ### Changed diff --git a/sdlc b/sdlc index 233c8cc..2bb93a4 100755 --- a/sdlc +++ b/sdlc @@ -41,6 +41,7 @@ import re import shutil import subprocess import sys +import time import tomllib from collections.abc import Callable from datetime import UTC, datetime @@ -1571,17 +1572,85 @@ def t_semgrep(s: dict) -> int: )[0] +# pip-audit's own --timeout is pip's 15s socket default. On a runner whose cores +# are all busy that is too short for a read that PyPI answers in well under a +# second when idle, so the timeout is raised and a network failure is retried +# before it is reported. The delays are the pauses BETWEEN attempts. +PIP_AUDIT_TIMEOUT = 60 +PIP_AUDIT_BACKOFF = (10, 30) + +# What pip-audit, or the `uv run --with pip-audit` that fetches it, prints when +# the network failed rather than the scan. pip-audit 2.9.0 turns a connect +# timeout into "Could not connect ..." and lets a READ timeout, a reset or a +# 5xx escape as a raw requests traceback -- every one of them exit 1, which is +# the same status as "found a vulnerability". None of these is a verdict. +_PIP_AUDIT_NETWORK = re.compile( + r"ReadTimeout|ConnectTimeout|ConnectionError|ProxyError|SSLError" + r"|Max retries exceeded|Could not connect to PyPI|PyPI is not redirecting" + r"|requests\.exceptions\.HTTPError|Failed to fetch|error sending request" +) + + +def pypi_locked(lock: Path) -> set[str]: + """Normalised names of the packages uv.lock resolves from PyPI itself. + + These all have a PyPI record by construction, so pip-audit reporting one of + them "not found on PyPI" means the lookup failed, not that it is unauditable. + """ + try: + data = tomllib.loads(lock.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + cannot(f"cannot read {lock}: {exc}") + return { + re.sub(r"[-_.]+", "-", str(pkg.get("name", ""))).lower() + for pkg in data.get("package", []) + if "pypi.org" in str((pkg.get("source") or {}).get("registry", "")) + } + + +def pip_audit_once(env: dict[str, str]) -> tuple[int, str, list[str]]: + """One pip-audit run: (its status, its output, what it skipped as not found).""" + rc, out = run( + [ + "uv", + "run", + "--locked", + "--all-extras", + "--with", + "pip-audit", + "pip-audit", + "--timeout", + str(PIP_AUDIT_TIMEOUT), + "--progress-spinner", + "off", + ], + env=env, + cwd=project_dir(), + tee=True, + ) + unaudited = sorted( + { + line.split("could not be audited:")[-1].strip() + for line in out.splitlines() + if "could not be audited" in line + } + ) + return rc, out, unaudited + + def t_pip_audit(s: dict) -> int: """Audit the project's locked dependencies, and GATE on it. 17 repos in the fleet run this as `pip-audit … || true`, which means the scan cannot fail and has never once blocked anything. This one returns the - tool's verdict. + tool's verdict -- and only the tool's verdict. A network failure is retried, + and one that outlasts the retries is a 2, never a 1 that reads as a finding + and never a 0 that reads as a pass. """ need_project() env = project_env() sync_project(env) - step("pip-audit") + pypi = pypi_locked(project_dir() / "uv.lock") # NOT --strict. A local path dependency is legitimately absent from PyPI -- # trentina vendors `petit-plus` from tests/fixtures -- and --strict turns # "could not audit this one" into a failed gate. The fleet's own @@ -1589,23 +1658,35 @@ def t_pip_audit(s: dict) -> int: # than CI trains people to skip the local gate. # # The cost is that an unauditable package passes quietly, so it is reported - # explicitly below rather than left in the log for nobody. - rc, out = run( - ["uv", "run", "--locked", "--all-extras", "--with", "pip-audit", "pip-audit"], - env=env, - cwd=project_dir(), - tee=True, - ) - unaudited = [ - line.split("could not be audited:")[-1].strip() - for line in out.splitlines() - if "could not be audited" in line - ] + # explicitly below rather than left in the log for nobody. A package the + # lock takes from PyPI cannot be unauditable, though: "not found" for one of + # those is a lookup that failed -- an egress proxy answering 404 skips every + # package and exits 0 -- so it is retried with the network failures. + attempts = len(PIP_AUDIT_BACKOFF) + 1 + for attempt in range(1, attempts + 1): + step(f"pip-audit (attempt {attempt} of {attempts})" if attempt > 1 else "pip-audit") + rc, out, unaudited = pip_audit_once(env) + lost = [u for u in unaudited if re.sub(r"[-_.]+", "-", u.split(" (")[0]).lower() in pypi] + if lost: + why = f"PyPI answered 'not found' for {len(lost)} package(s) uv.lock takes from PyPI" + elif rc != CLEAN and _PIP_AUDIT_NETWORK.search(out): + why = "the PyPI lookup failed on the network" + else: + break + if attempt == attempts: + cannot( + f"pip-audit could not run: {why}, on all {attempts} attempts " + f"(timeout {PIP_AUDIT_TIMEOUT}s). Nothing was audited, so this is " + f"neither a pass nor a finding." + ) + delay = PIP_AUDIT_BACKOFF[attempt - 1] + print(f"\npip-audit: {why}; retrying in {delay}s.", flush=True) + time.sleep(delay) if unaudited: print( "\nNOT AUDITED -- no PyPI record, so these were skipped rather than " "cleared:\n " - + "\n ".join(sorted(set(unaudited))) + + "\n ".join(unaudited) + "\nA local path or vendored dependency is expected here. Anything " "else in this list is a real gap in the scan." ) diff --git a/tests/test_pip_audit_resilience.py b/tests/test_pip_audit_resilience.py new file mode 100755 index 0000000..f8a6194 --- /dev/null +++ b/tests/test_pip_audit_resilience.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""A pip-audit that never reached PyPI must say so: not a finding, not a pass. + +On the self-hosted runners a busy host made pip-audit's 15s read timeout trip +against a PyPI that answers in under a second, and pip-audit exits 1 for that -- +the status of a real vulnerability. The opposite also happened: lookups that all +came back "not found" exited 0 and read as a pass. These pin the fix: a network +failure is retried with backoff, and one that outlasts the retries is a 2. + +Run: tests/test_pip_audit_resilience.py (needs the image's python3 and PyYAML) +""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path + +# Loaded by path, as in test_codeql_globs.py: `sdlc` has no .py extension. +_SPEC = importlib.util.spec_from_loader( + "sdlc_under_test", + importlib.machinery.SourceFileLoader( + "sdlc_under_test", str(Path(__file__).resolve().parent.parent / "sdlc") + ), +) +assert _SPEC is not None and _SPEC.loader is not None +_SDLC = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_SDLC) + +LOCK = """\ +version = 1 + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "petit-plus" +version = "0.1.0" +source = { directory = "tests/fixtures/petit-plus" } +""" + +# What pip-audit 2.9.0 actually prints for each case. +READ_TIMEOUT = ( + "Traceback (most recent call last):\n" + "urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='pypi.org', port=443): " + "Read timed out. (read timeout=15)\n" + "requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='pypi.org', port=443): " + "Read timed out. (read timeout=15)\n" +) +CLEAN_RUN = "No known vulnerabilities found, 1 dependency was skipped\n" + ( + "Name Skip Reason\n" + "---------- -----------\n" + "petit-plus Dependency not found on PyPI and could not be audited: petit-plus (0.1.0)\n" +) +VULNERABLE = ( + "Found 1 known vulnerability in 1 package\n" + "Name Version ID Fix Versions\n" + "requests 2.32.3 GHSA-xxxx-xxxx-xxxx 2.32.4\n" +) +# The other ways the network fails, one representative line each: pip-audit's own +# connect-timeout message, the requests exceptions it lets escape, and uv failing +# to fetch pip-audit itself. +OTHER_NETWORK = { + "connect timeout": "ERROR:pip_audit._cli:Could not connect to PyPI's vulnerability feed\n", + "redirect loop": "ERROR:pip_audit._cli:PyPI is not redirecting properly\n", + "connection reset": "requests.exceptions.ConnectionError: ('Connection aborted.')\n", + "proxy": "requests.exceptions.ProxyError: HTTPSConnectionPool(host='pypi.org')\n", + "tls": "requests.exceptions.SSLError: HTTPSConnectionPool(host='pypi.org')\n", + "5xx": "requests.exceptions.HTTPError: 503 Server Error: Service Unavailable\n", + "retries": "HTTPSConnectionPool(host='pypi.org', port=443): Max retries exceeded\n", + "uv fetch": "error: Failed to fetch: `https://pypi.org/simple/pip-audit/`\n", + "uv request": "Caused by: error sending request for url (https://pypi.org/simple/)\n", + "connect timeout, raw": "requests.exceptions.ConnectTimeout: HTTPSConnectionPool\n", +} + +# pip-audit's own --timeout default, which is pip's. +PIP_AUDIT_DEFAULT_TIMEOUT = 15 +ONE_RETRY = 2 + +PROXY_404 = "No known vulnerabilities found, 1 dependency was skipped\n" + ( + "requests Dependency not found on PyPI and could not be audited: requests (2.32.3)\n" +) + + +def check(ok: bool, what: str) -> bool: + print(f"{'ok ' if ok else 'FAIL'} {what}") + return ok + + +class Harness: + """Stand in for uv, the project and the clock; replay pip-audit's answers.""" + + def __init__(self, project: Path, answers: list[tuple[int, str]]) -> None: + self.answers = list(answers) + self.calls: list[list[str]] = [] + self.slept: list[float] = [] + _SDLC.need_project = lambda: None + _SDLC.project_env = dict + _SDLC.sync_project = lambda _env: None + _SDLC.project_dir = lambda: project + _SDLC.run = self.run + _SDLC.time.sleep = self.slept.append + + def run(self, cmd: list[str], **_: object) -> tuple[int, str]: + self.calls.append(cmd) + return self.answers.pop(0) + + +def outcome(fn: Callable[[], int]) -> int | str: + """The status `fn` returns, or "cannot" when it raised CannotRunError.""" + try: + return fn() + except _SDLC.CannotRunError: + return "cannot" + + +def cases() -> bool: + ok = True + real_sleep = _SDLC.time.sleep + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) + (project / "uv.lock").write_text(LOCK, encoding="utf-8") + try: + h = Harness(project, [(1, READ_TIMEOUT), (0, CLEAN_RUN)]) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check(got == 0, "a read timeout, then a clean run: retried, and a pass") + ok &= check(len(h.calls) == ONE_RETRY, "exactly one retry after one timeout") + ok &= check(h.slept == [_SDLC.PIP_AUDIT_BACKOFF[0]], "it backed off before retrying") + ok &= check( + h.calls[0][-4:-2] == ["--timeout", str(_SDLC.PIP_AUDIT_TIMEOUT)] + and _SDLC.PIP_AUDIT_TIMEOUT > PIP_AUDIT_DEFAULT_TIMEOUT, + "pip-audit runs with a longer timeout than its 15s default", + ) + + attempts = len(_SDLC.PIP_AUDIT_BACKOFF) + 1 + h = Harness(project, [(1, READ_TIMEOUT)] * attempts) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check(got == "cannot", "a timeout on every attempt is COULD NOT RUN, not FOUND") + ok &= check(len(h.calls) == attempts, f"it gave up after {attempts} attempts") + ok &= check(h.slept == list(_SDLC.PIP_AUDIT_BACKOFF), "the backoff grows") + + for what, text in OTHER_NETWORK.items(): + h = Harness(project, [(1, text)] * attempts) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == "cannot" and len(h.calls) == attempts, + f"{what}: retried, then COULD NOT RUN", + ) + + h = Harness(project, [(0, PROXY_404)] * attempts) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == "cannot", + "'not found' for a package the lock takes from PyPI is not a pass", + ) + + h = Harness(project, [(0, CLEAN_RUN)]) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == 0 and len(h.calls) == 1, + "a local path dependency skipped as not found is still a pass, first time", + ) + + h = Harness(project, [(1, VULNERABLE)]) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == 1 and len(h.calls) == 1 and not h.slept, + "a real vulnerability is a 1, at once, with no retry", + ) + finally: + _SDLC.time.sleep = real_sleep + return ok + + +def main() -> int: + ok = cases() + print("pip-audit resilience PASSED" if ok else "pip-audit resilience FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 4ac95c2433c35ec8e65725af9cf2dcbd5e34b6dc Mon Sep 17 00:00:00 2001 From: Josui Date: Wed, 18 Nov 2026 03:44:35 +0000 Subject: [PATCH 2/4] test: a missing or malformed uv.lock is a 2 for pip-audit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R7Y1jckWHhAgXg2WBwkWbb --- tests/test_pip_audit_resilience.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_pip_audit_resilience.py b/tests/test_pip_audit_resilience.py index f8a6194..6348630 100755 --- a/tests/test_pip_audit_resilience.py +++ b/tests/test_pip_audit_resilience.py @@ -177,8 +177,24 @@ def cases() -> bool: return ok +def bad_lock() -> bool: + """A lock that cannot be read is a 2 before pip-audit runs, never a guess.""" + ok = True + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) + h = Harness(project, []) + ok &= check(outcome(lambda: _SDLC.t_pip_audit({})) == "cannot", "no uv.lock is a 2") + (project / "uv.lock").write_text("[[package]\nname = ", encoding="utf-8") + ok &= check( + outcome(lambda: _SDLC.t_pip_audit({})) == "cannot" and not h.calls, + "a malformed uv.lock is a 2, and pip-audit never ran", + ) + return ok + + def main() -> int: ok = cases() + ok &= bad_lock() print("pip-audit resilience PASSED" if ok else "pip-audit resilience FAILED") return 0 if ok else 1 From 406f7a46924acdac7337922906023446f69d2d3c Mon Sep 17 00:00:00 2001 From: Josui Date: Wed, 18 Nov 2026 04:05:32 +0000 Subject: [PATCH 3/4] fix: pin pip-audit to the image's baked version in the project venv Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R7Y1jckWHhAgXg2WBwkWbb --- CHANGELOG.md | 4 +++- sdlc | 12 ++++++++++-- tests/test_pip_audit_resilience.py | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f24979a..f8650d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,9 @@ All notable changes to this project are documented here, following retries as exit 2, COULD NOT RUN. A package that uv.lock takes from PyPI and pip-audit calls "not found" counts as a failed lookup, not as unauditable. A real vulnerability is still a 1, at once, with no retry. - `tests/test_pip_audit_resilience.py` covers each case. + pip-audit is now pinned to the image's own version (`--with pip-audit==2.9.0`) + instead of whatever PyPI's newest is. `tests/test_pip_audit_resilience.py` + covers each case. ## [0.9.1] - 2026-11-17 diff --git a/sdlc b/sdlc index 2bb93a4..e6ab999 100755 --- a/sdlc +++ b/sdlc @@ -1609,7 +1609,15 @@ def pypi_locked(lock: Path) -> set[str]: def pip_audit_once(env: dict[str, str]) -> tuple[int, str, list[str]]: - """One pip-audit run: (its status, its output, what it skipped as not found).""" + """One pip-audit run: (its status, its output, what it skipped as not found). + + It runs inside the project venv, so it comes from `--with` rather than the + image's own tool install -- pinned to that install's version, because the + network-failure patterns above are read from that version's code. + """ + version = BAKED["pip-audit"] + if version == "unknown": + cannot("SDLC_PIP_AUDIT_VERSION is not set, so pip-audit has no version to pin") rc, out = run( [ "uv", @@ -1617,7 +1625,7 @@ def pip_audit_once(env: dict[str, str]) -> tuple[int, str, list[str]]: "--locked", "--all-extras", "--with", - "pip-audit", + f"pip-audit=={version}", "pip-audit", "--timeout", str(PIP_AUDIT_TIMEOUT), diff --git a/tests/test_pip_audit_resilience.py b/tests/test_pip_audit_resilience.py index 6348630..bee0529 100755 --- a/tests/test_pip_audit_resilience.py +++ b/tests/test_pip_audit_resilience.py @@ -105,6 +105,7 @@ def __init__(self, project: Path, answers: list[tuple[int, str]]) -> None: _SDLC.project_dir = lambda: project _SDLC.run = self.run _SDLC.time.sleep = self.slept.append + _SDLC.BAKED["pip-audit"] = "2.9.0" def run(self, cmd: list[str], **_: object) -> tuple[int, str]: self.calls.append(cmd) @@ -136,6 +137,10 @@ def cases() -> bool: and _SDLC.PIP_AUDIT_TIMEOUT > PIP_AUDIT_DEFAULT_TIMEOUT, "pip-audit runs with a longer timeout than its 15s default", ) + ok &= check( + "pip-audit==2.9.0" in h.calls[0], + "pip-audit is pinned to the version the image bakes", + ) attempts = len(_SDLC.PIP_AUDIT_BACKOFF) + 1 h = Harness(project, [(1, READ_TIMEOUT)] * attempts) @@ -192,9 +197,21 @@ def bad_lock() -> bool: return ok +def unpinned() -> bool: + """With no baked version there is nothing to pin, and that is a 2.""" + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) + (project / "uv.lock").write_text(LOCK, encoding="utf-8") + h = Harness(project, []) + _SDLC.BAKED["pip-audit"] = "unknown" + got = outcome(lambda: _SDLC.t_pip_audit({})) + return check(got == "cannot" and not h.calls, "no baked pip-audit version is a 2") + + def main() -> int: ok = cases() ok &= bad_lock() + ok &= unpinned() print("pip-audit resilience PASSED" if ok else "pip-audit resilience FAILED") return 0 if ok else 1 From 9db37a74532481bfd5bf08cbbe772daa36fadca3 Mon Sep 17 00:00:00 2001 From: Josui Date: Wed, 18 Nov 2026 04:47:39 +0000 Subject: [PATCH 4/4] fix: a pip-audit finding is never retried, and an all-skipped run is a 2 Check pip-audit's "Found N known vulnerabilit" summary before classifying anything: it prints only after every lookup returned, so a finding beside a version PyPI no longer has, or beside uv's own fetch noise, stays a 1. A run that skipped every registry package installed in the venv is a failed lookup, which closes the exit-0 path behind a non-PyPI mirror. Test fixtures now use pip-audit 2.9.0's real columns layout. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01R7Y1jckWHhAgXg2WBwkWbb --- CHANGELOG.md | 6 +- sdlc | 55 ++++++++++++--- tests/test_pip_audit_resilience.py | 107 +++++++++++++++++++++++++---- 3 files changed, 142 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8650d9..1e7c3db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,11 @@ All notable changes to this project are documented here, following failure twice with 10s then 30s backoff, and reports one that outlasts the retries as exit 2, COULD NOT RUN. A package that uv.lock takes from PyPI and pip-audit calls "not found" counts as a failed lookup, not as unauditable. - A real vulnerability is still a 1, at once, with no retry. + So is a run that skipped every registry package installed in the venv, + which covers a mirror whose packages are not from PyPI at all. A real + vulnerability is still a 1, at once, with no retry: pip-audit's + `Found N known vulnerabilities` line is checked first, so a finding beside + a skipped package or uv's own network noise is never retried into a 2. pip-audit is now pinned to the image's own version (`--with pip-audit==2.9.0`) instead of whatever PyPI's newest is. `tests/test_pip_audit_resilience.py` covers each case. diff --git a/sdlc b/sdlc index e6ab999..99e4106 100755 --- a/sdlc +++ b/sdlc @@ -1591,20 +1591,42 @@ _PIP_AUDIT_NETWORK = re.compile( ) -def pypi_locked(lock: Path) -> set[str]: - """Normalised names of the packages uv.lock resolves from PyPI itself. +# pip-audit's summary when it finished auditing and found something +# (_cli.py:607 in 2.9.0). It is printed only after every lookup came back, so a +# run that printed it reached a verdict, whatever else is in its output. +_PIP_AUDIT_FOUND = re.compile(r"^Found \d+ known vulnerabilit", re.MULTILINE) - These all have a PyPI record by construction, so pip-audit reporting one of - them "not found on PyPI" means the lookup failed, not that it is unauditable. + +def _dist(name: str) -> str: + """A distribution name normalised as PEP 503 does, so lock, venv and pip-audit agree.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def locked_registry(lock: Path) -> tuple[set[str], set[str]]: + """Packages uv.lock resolves from a registry: (from PyPI itself, from any registry). + + The PyPI ones all have a PyPI record by construction, so pip-audit reporting + one of them "not found on PyPI" means the lookup failed, not that it is + unauditable. """ try: data = tomllib.loads(lock.read_text(encoding="utf-8")) except (OSError, tomllib.TOMLDecodeError) as exc: cannot(f"cannot read {lock}: {exc}") - return { - re.sub(r"[-_.]+", "-", str(pkg.get("name", ""))).lower() + registry = { + _dist(str(pkg.get("name", ""))): str((pkg.get("source") or {}).get("registry", "")) for pkg in data.get("package", []) - if "pypi.org" in str((pkg.get("source") or {}).get("registry", "")) + if (pkg.get("source") or {}).get("registry") + } + return {n for n, url in registry.items() if "pypi.org" in url}, set(registry) + + +def installed(env: dict[str, str]) -> set[str]: + """Distributions installed in the project venv, which is what pip-audit audits.""" + venv = Path(env.get("UV_PROJECT_ENVIRONMENT", "")) + return { + _dist(d.name.removesuffix(".dist-info").rsplit("-", 1)[0]) + for d in venv.glob("lib/python*/site-packages/*.dist-info") } @@ -1658,7 +1680,7 @@ def t_pip_audit(s: dict) -> int: need_project() env = project_env() sync_project(env) - pypi = pypi_locked(project_dir() / "uv.lock") + pypi, registry = locked_registry(project_dir() / "uv.lock") # NOT --strict. A local path dependency is legitimately absent from PyPI -- # trentina vendors `petit-plus` from tests/fixtures -- and --strict turns # "could not audit this one" into a failed gate. The fleet's own @@ -1669,14 +1691,25 @@ def t_pip_audit(s: dict) -> int: # explicitly below rather than left in the log for nobody. A package the # lock takes from PyPI cannot be unauditable, though: "not found" for one of # those is a lookup that failed -- an egress proxy answering 404 skips every - # package and exits 0 -- so it is retried with the network failures. + # package and exits 0 -- so it is retried with the network failures. So is + # a run that skipped every registry package it should have audited, which + # is the same failure behind a mirror, where none of them is from PyPI. attempts = len(PIP_AUDIT_BACKOFF) + 1 for attempt in range(1, attempts + 1): step(f"pip-audit (attempt {attempt} of {attempts})" if attempt > 1 else "pip-audit") rc, out, unaudited = pip_audit_once(env) - lost = [u for u in unaudited if re.sub(r"[-_.]+", "-", u.split(" (")[0]).lower() in pypi] + # A verdict is a verdict: a finding next to a skipped package or uv's + # own network noise is still a finding, and retrying it would turn a 1 + # into a 2 on every run. + if _PIP_AUDIT_FOUND.search(out): + break + skipped = {_dist(u.split(" (")[0]) for u in unaudited} + auditable = installed(env) & registry + lost = skipped & pypi if lost: why = f"PyPI answered 'not found' for {len(lost)} package(s) uv.lock takes from PyPI" + elif auditable and auditable <= skipped: + why = f"every one of the {len(auditable)} registry packages was skipped" elif rc != CLEAN and _PIP_AUDIT_NETWORK.search(out): why = "the PyPI lookup failed on the network" else: @@ -1684,7 +1717,7 @@ def t_pip_audit(s: dict) -> int: if attempt == attempts: cannot( f"pip-audit could not run: {why}, on all {attempts} attempts " - f"(timeout {PIP_AUDIT_TIMEOUT}s). Nothing was audited, so this is " + f"(timeout {PIP_AUDIT_TIMEOUT}s). The audit never completed, so this is " f"neither a pass nor a finding." ) delay = PIP_AUDIT_BACKOFF[attempt - 1] diff --git a/tests/test_pip_audit_resilience.py b/tests/test_pip_audit_resilience.py index bee0529..00def4e 100755 --- a/tests/test_pip_audit_resilience.py +++ b/tests/test_pip_audit_resilience.py @@ -38,35 +38,88 @@ version = "2.32.3" source = { registry = "https://pypi.org/simple" } +[[package]] +name = "urllib3" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } + [[package]] name = "petit-plus" version = "0.1.0" source = { directory = "tests/fixtures/petit-plus" } """ -# What pip-audit 2.9.0 actually prints for each case. +LOCK_MIRROR = """\ +version = 1 + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://mirror.example/simple" } + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://mirror.example/simple" } +""" + + +def skip_table(*rows: tuple[str, str]) -> str: + """pip-audit's skipped-dependency table, as ColumnsFormat lays it out.""" + lines = [ + (n, f"Dependency not found on PyPI and could not be audited: {n} ({v})") for n, v in rows + ] + width = max(len("Name"), *(len(n) for n, _ in lines)) + reason = max(len(r) for _, r in lines) + return "".join( + [ + f"{'Name'.ljust(width)} Skip Reason\n", + f"{'-' * width} {'-' * reason}\n", + *(f"{n.ljust(width)} {r}\n" for n, r in lines), + ] + ) + + +# What pip-audit 2.9.0 prints for each case, stderr and stdout merged as tee=True +# captures them: the summary on stderr first, then the tables on stdout. READ_TIMEOUT = ( "Traceback (most recent call last):\n" + ' File "/tmp/venv/lib/python3.14/site-packages/requests/adapters.py", line 713, in send\n' "urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='pypi.org', port=443): " "Read timed out. (read timeout=15)\n" + "\nDuring handling of the above exception, another exception occurred:\n\n" "requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='pypi.org', port=443): " "Read timed out. (read timeout=15)\n" ) -CLEAN_RUN = "No known vulnerabilities found, 1 dependency was skipped\n" + ( - "Name Skip Reason\n" - "---------- -----------\n" - "petit-plus Dependency not found on PyPI and could not be audited: petit-plus (0.1.0)\n" -) +CLEAN_RUN = "No known vulnerabilities found\n" + skip_table(("petit-plus", "0.1.0")) VULNERABLE = ( "Found 1 known vulnerability in 1 package\n" "Name Version ID Fix Versions\n" - "requests 2.32.3 GHSA-xxxx-xxxx-xxxx 2.32.4\n" + "-------- ------- ------------------- ------------\n" + "requests 2.32.3 GHSA-9hjg-9r4m-mvj7 2.32.4\n" +) +# A real finding beside a version PyPI no longer has, and beside uv's own noise. +VULNERABLE_AND_404 = VULNERABLE + skip_table(("urllib3", "1.26.0")) +VULNERABLE_AND_UV_NOISE = ( + "warning: Failed to fetch: `https://pypi.org/simple/pip-audit/`, retrying\n" + VULNERABLE +) +PROXY_404 = "No known vulnerabilities found\n" + skip_table( + ("petit-plus", "0.1.0"), ("requests", "2.32.3") +) +MIRROR_ALL_SKIPPED = "No known vulnerabilities found\n" + skip_table( + ("idna", "3.10"), ("requests", "2.32.3") ) +MIRROR_ONE_SKIPPED = "No known vulnerabilities found\n" + skip_table(("idna", "3.10")) + # The other ways the network fails, one representative line each: pip-audit's own # connect-timeout message, the requests exceptions it lets escape, and uv failing # to fetch pip-audit itself. OTHER_NETWORK = { - "connect timeout": "ERROR:pip_audit._cli:Could not connect to PyPI's vulnerability feed\n", + "connect timeout": ( + "ERROR:pip_audit._cli:Could not connect to PyPI's vulnerability feed\n" + "ERROR:pip_audit._cli:Tip: your network may be blocking this service. " + "Try another service with `-s SERVICE`\n" + ), "redirect loop": "ERROR:pip_audit._cli:PyPI is not redirecting properly\n", "connection reset": "requests.exceptions.ConnectionError: ('Connection aborted.')\n", "proxy": "requests.exceptions.ProxyError: HTTPSConnectionPool(host='pypi.org')\n", @@ -82,10 +135,6 @@ PIP_AUDIT_DEFAULT_TIMEOUT = 15 ONE_RETRY = 2 -PROXY_404 = "No known vulnerabilities found, 1 dependency was skipped\n" + ( - "requests Dependency not found on PyPI and could not be audited: requests (2.32.3)\n" -) - def check(ok: bool, what: str) -> bool: print(f"{'ok ' if ok else 'FAIL'} {what}") @@ -100,7 +149,11 @@ def __init__(self, project: Path, answers: list[tuple[int, str]]) -> None: self.calls: list[list[str]] = [] self.slept: list[float] = [] _SDLC.need_project = lambda: None - _SDLC.project_env = dict + venv = project / "venv" + site = venv / "lib" / "python3.14" / "site-packages" + for dist in ("requests-2.32.3", "idna-3.10", "petit_plus-0.1.0"): + (site / f"{dist}.dist-info").mkdir(parents=True, exist_ok=True) + _SDLC.project_env = lambda: {"UV_PROJECT_ENVIRONMENT": str(venv)} _SDLC.sync_project = lambda _env: None _SDLC.project_dir = lambda: project _SDLC.run = self.run @@ -109,7 +162,8 @@ def __init__(self, project: Path, answers: list[tuple[int, str]]) -> None: def run(self, cmd: list[str], **_: object) -> tuple[int, str]: self.calls.append(cmd) - return self.answers.pop(0) + # The last answer repeats, so an extra call shows up as a count, not a crash. + return self.answers.pop(0) if len(self.answers) > 1 else self.answers[0] def outcome(fn: Callable[[], int]) -> int | str: @@ -177,6 +231,31 @@ def cases() -> bool: got == 1 and len(h.calls) == 1 and not h.slept, "a real vulnerability is a 1, at once, with no retry", ) + + for what, text in { + "a version PyPI no longer has": VULNERABLE_AND_404, + "uv's own network noise": VULNERABLE_AND_UV_NOISE, + }.items(): + h = Harness(project, [(1, text)]) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == 1 and len(h.calls) == 1 and not h.slept, + f"a real finding beside {what} is a 1, with no retry", + ) + + (project / "uv.lock").write_text(LOCK_MIRROR, encoding="utf-8") + h = Harness(project, [(0, MIRROR_ALL_SKIPPED)] * attempts) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == "cannot" and len(h.calls) == attempts, + "behind a mirror, every registry package skipped is a 2, not a pass", + ) + h = Harness(project, [(0, MIRROR_ONE_SKIPPED)]) + got = outcome(lambda: _SDLC.t_pip_audit({})) + ok &= check( + got == 0 and len(h.calls) == 1, + "behind a mirror, one package skipped while others were audited is a pass", + ) finally: _SDLC.time.sleep = real_sleep return ok