Refresh exact builder header and pip vendor evidence

Author: chen21019

.github/dapper-headers.json

@@ -27,6 +27,23 @@ def violations(report, ledger, boundary, headers):
     if (headers.get("package") != "linux-libc-dev"
             or headers.get("version") != HEADER_VERSION or headers.get("purl") != HEADER_PURL):
         raise ValueError("Changed exact header review")
+    refresh = headers.get("mediumRefresh", {})
+    refreshed = {}
+    for entry in refresh.get("entries", []):
+        identifier = entry.get("id")
+        official = ("https://raw.githubusercontent.com/canonical/ubuntu-security-notices/"
+                    + str(refresh.get("officialSnapshot")) + "/osv/cve/2026/UBUNTU-"
+                    + str(identifier) + ".json")
+        if (identifier in refreshed or identifier not in headers.get("ids", [])
+                or entry.get("package") != "linux-libc-dev" or entry.get("version") != HEADER_VERSION
+                or entry.get("purl") != HEADER_PURL or entry.get("severity") != "MEDIUM"
+                or entry.get("decision") != "UNFIXED_MEDIUM_HEADER_ONLY"
+                or entry.get("officialFixedVersions") != []
+                or entry.get("officialRangeEvents") != [{"introduced": "0"}]
+                or entry.get("officialUrl") != official
+                or not entry.get("rawSha256") or not entry.get("target")):
+            raise ValueError("Invalid exact medium header refresh")
+        refreshed[identifier] = entry
     reviewed = {}
     for entry in ledger.get("unfixed", []):
         key = (entry["id"], entry["package"], entry["version"], entry["purl"])
@@ -54,7 +71,8 @@ def violations(report, ledger, boundary, headers):
                 if boundary != expected:
                     raise ValueError("Missing exact installed header-only proof")
                 if (not fixed and key[0] in headers.get("ids", [])
-                        and key[2] == HEADER_VERSION and key[3] == HEADER_PURL):
+                        and key[2] == HEADER_VERSION and key[3] == HEADER_PURL
+                        and (key[0] not in refreshed or finding.get("Severity") == "MEDIUM")):
                     continue
             severity = finding.get("Severity", "UNKNOWN")
             entry = reviewed.get(key)

.github/scripts/enforce-builder-vulnerabilities.py

@@ -1,4 +1,6 @@
 import importlib.util
+import json
+from copy import deepcopy
 import unittest
 from pathlib import Path
 spec = importlib.util.spec_from_file_location("gate", Path(__file__).with_name("enforce-builder-vulnerabilities.py"))
@@ -69,6 +71,28 @@ def test_fixed_header_rejected(self):
         self.header()["FixedVersion"] = "7.0.0-99"; self.assertTrue(self.check())
     def test_new_header_advisory_rejected(self):
         self.header()["VulnerabilityID"] = "NEW"; self.assertTrue(self.check())
+    def refreshed_header(self):
+        self.headers = json.loads((Path(__file__).resolve().parents[1] / "dapper-headers.json").read_text())
+        f = self.header()
+        f.update(VulnerabilityID=self.headers["mediumRefresh"]["entries"][0]["id"], Severity="MEDIUM")
+        return f
+    def test_refreshed_exact_medium_header(self):
+        self.refreshed_header()
+        self.assertEqual(len(self.headers["mediumRefresh"]["entries"]), 222)
+        self.assertEqual(self.check(), [])
+    def test_refreshed_headers_cannot_allow_high_unknown_or_available_fix(self):
+        for severity in ("LOW", "HIGH", "CRITICAL", "UNKNOWN"):
+            self.refreshed_header()["Severity"] = severity
+            with self.subTest(severity=severity): self.assertTrue(self.check())
+        self.refreshed_header()["FixedVersion"] = "7.0.0-99"
+        self.assertTrue(self.check())
+    def test_refreshed_review_fixed_or_package_drift_rejected(self):
+        self.refreshed_header()
+        original = deepcopy(self.headers)
+        for key, value in (("officialFixedVersions", ["7.0.0-99"]), ("purl", "pkg:generic/other@1")):
+            self.headers = deepcopy(original)
+            self.headers["mediumRefresh"]["entries"][0][key] = value
+            with self.subTest(key=key), self.assertRaises(ValueError): self.check()
     def test_header_version_or_purl_drift_rejected(self):
         for field in ("InstalledVersion", "PkgIdentifier"):
             f = self.header()

.github/scripts/test_builder_vulnerability_policy.py

@@ -30,7 +30,7 @@ else
 fi
 "$scratch/venv/bin/python" /opt/python-security/scripts/test-pip-overlay.py
 if [ -n "${PYTHON_SECURITY_VIRTUALENV_PYTHON:-}" ]; then
-    "$PYTHON_SECURITY_VIRTUALENV_PYTHON" -m virtualenv --no-download --no-periodic-update --pip="$expected" --extra-search-dir=/usr/share/python-wheels "$scratch/virtualenv"
+    "$PYTHON_SECURITY_VIRTUALENV_PYTHON" -m virtualenv --app-data "$scratch/virtualenv-app-data" --no-download --no-periodic-update --pip="$expected" --extra-search-dir=/usr/share/python-wheels "$scratch/virtualenv"
     "$scratch/virtualenv/bin/python" /opt/python-security/scripts/test-pip-overlay.py
 fi
 printf 'PYTHON_SECURITY_TOOLS_OK pip=%s certifi=2026.10.19 ensurepip=%s\n' "$expected" "$ensurepip_status"

scripts/install-python-security-tools

@@ -67,9 +67,23 @@ bash verify-pip-runtime.sh --virtualenv /usr/share/python-wheels
 
 `check-pip-vendor-slice.py` 會比對實際載入的全部 299 個 vendor 檔案與
 固定 wheel，拒絕額外檔案、其他同版本完整 distribution。只有取得此證據後，
-`filter-pip-vendor-findings.py` 才可對上述三個 CVE（含 msgpack GHSA 別名）
+`filter-pip-vendor-findings.py` 才可對已驗證的 CVE（含 msgpack GHSA 別名）
 產生綁定該映像的 OpenVEX 與有效掃描檔。原始掃描必須另行保存；未知 CVE、
 其他版本、非 SBOM 套件來源與 urllib3 finding 全部保留，不做泛用版本排除。
-執行 `python3 test-pip-vex.py` 可驗證以上六組正反控制。
+執行 `python3 test-pip-vex.py` 可驗證以上正反控制。
+
+新增公告仍限同一個 wheel 與實際 vendor slice：
+
+- [CVE-2026-114613／GHSA-j586-36cw-2gc2](https://github.com/msgpack/msgpack-python/security/advisories/GHSA-j586-36cw-2gc2)
+  的 C extension 不在固定 wheel，實際載入 pure-Python fallback。
+- [GHSA-xmj3-9gf7-h52w](https://github.com/pypa/setuptools/security/advisories/GHSA-xmj3-9gf7-h52w)
+  所需 `setuptools/wheel.py` 不在此 slice。
+
+每個 Compose builder 都須重新驗證 299 個 live 檔案、固定 seed wheel、
+SHA-256、脆弱模組缺席及整個檔案系統中相關 wheel／SBOM／distribution 來源。
+只有已列明的 Compose 路徑且內容完全一致才可承接此證據；未知位置的同版
+SBOM、額外完整套件或 seed copy 會拒絕。負向 fixtures 包含 sys.path 外完整
+distribution、外來 CycloneDX/SPDX、不同 bytes 與未知 wheel。虛擬環境驗證的
+app-data 放在其自有暫存目錄，不留下另一份未追蹤 seed。
 
 官方政策：https://pip.pypa.io/en/stable/development/vendoring-policy/

scripts/pip-security/README.md

@@ -9,16 +9,27 @@
 import zipfile
 import pip
 from pip._vendor import msgpack, urllib3
+from vendor_source_inventory import inventory
 
 EXPECTED_SHA = "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a"
+BOM_SHA = "3581cb8f0a6128c19ccec4e9aeab0d3ac8212fd19bd19d485b2e24fd4dc10f2a"
 FINDINGS = {
     "GHSA-6v7p-g79w-8964": ("msgpack", "1.1.2", "C/Cython unpacker is absent; runtime selects pure Python fallback"),
     "CVE-2026-57585": ("msgpack", "1.1.2", "C/Cython unpacker is absent; runtime selects pure Python fallback"),
+    "CVE-2026-114613": ("msgpack", "1.1.2", "_unpacker.pyx/unpack.h strict-map-key C extension is absent; fallback.py is not affected"),
+    "GHSA-j586-36cw-2gc2": ("msgpack", "1.1.2", "_unpacker.pyx/unpack.h strict-map-key C extension is absent; fallback.py is not affected"),
     "CVE-2025-47273": ("setuptools", "70.3.0", "setuptools/package_index.py is absent; only pkg_resources is vendored"),
     "CVE-2026-59890": ("setuptools", "70.3.0", "setuptools/command/egg_info.py and FileList are absent"),
+    "GHSA-xmj3-9gf7-h52w": ("setuptools", "70.3.0", "setuptools/wheel.py Wheel._fix_namespace_packages is absent; only pkg_resources is vendored"),
 }
 
 
+def require_absent_vulnerable_modules(names):
+    names = tuple(names)
+    assert not any(name.startswith("pip/_vendor/msgpack/_cmsgpack") for name in names), "Native msgpack module present"
+    assert not any(name.startswith("pip/_vendor/setuptools/") for name in names), "Full setuptools module present"
+
+
 def verify(wheel):
     assert hashlib.sha256(wheel.read_bytes()).hexdigest() == EXPECTED_SHA
     assert pip.__version__ == "26.2.2+pasturestack.1"
@@ -36,8 +47,7 @@ def verify(wheel):
             expected_files.add(name.removeprefix("pip/_vendor/"))
             checked += 1
         assert not any(name.startswith("pip/_vendor/") and name.endswith((".so", ".pyd")) for name in names)
-        assert not any(name.startswith("pip/_vendor/setuptools/") for name in names)
-        assert not any(name.startswith("pip/_vendor/msgpack/_cmsgpack") for name in names)
+        require_absent_vulnerable_modules(names)
     def live_files(directory, prefix=""):
         for entry in directory.iterdir():
             if entry.name == "__pycache__":
@@ -47,17 +57,20 @@ def live_files(directory, prefix=""):
                 yield from live_files(entry, relative + "/")
             else:
                 yield relative
-    assert set(live_files(live.joinpath("_vendor"))) == expected_files, "Unexpected live vendor files"
+    observed_files = set(live_files(live.joinpath("_vendor")))
+    require_absent_vulnerable_modules("pip/_vendor/" + name for name in observed_files)
+    assert observed_files == expected_files, "Unexpected live vendor files"
     # A separate full setuptools installation cannot inherit the vendor-slice VEX.
     for distribution in importlib.metadata.distributions():
         if distribution.metadata.get("Name", "").lower() == "setuptools":
             assert distribution.version != "70.3.0", "A full setuptools 70.3.0 distribution is installed"
         if distribution.metadata.get("Name", "").lower() == "msgpack":
             assert distribution.version != "1.1.2", "A separate msgpack 1.1.2 distribution is installed"
-    return {"verifiedWheelSha256": EXPECTED_SHA, "pip": pip.__version__,
+    return {"verifiedWheelSha256": EXPECTED_SHA, "verifiedWheelPath": str(wheel.resolve()), "pip": pip.__version__,
             "urllib3": urllib3.__version__, "vendorFilesByteCompared": checked,
             "msgpackImplementation": msgpack.Unpacker.__module__,
-            "setuptoolsPackageAbsent": True, "scope": "this exact pip vendor slice only",
+            "setuptoolsPackageAbsent": True, "vulnerableModulesAbsent": ["msgpack._cmsgpack", "setuptools.wheel"],
+            "scope": "this exact pip vendor slice only",
             "notAffected": [{"id": key, "package": value[0], "version": value[1],
                              "justification": "vulnerable_code_not_present", "detail": value[2]}
                             for key, value in FINDINGS.items()]}
@@ -81,6 +94,7 @@ def live_files(directory, prefix=""):
     else:
         with ensurepip._get_pip_whl_path_ctx() as wheel:
             result = verify(Path(wheel))
+    result["vendorSourceInventory"] = inventory(Path("/"), EXPECTED_SHA, BOM_SHA)
     data = json.dumps(result, indent=2) + "\n"
     if args.output:
         args.output.write_text(data)

scripts/pip-security/check-pip-vendor-slice.py

@@ -5,21 +5,39 @@
 from datetime import datetime, timezone
 import json
 from pathlib import Path
+from vendor_source_inventory import SBOM_PATHS, WHEEL_PATHS
 
 WHEEL_SHA = "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a"
+WHEEL_NAME = "pip-26.2.2+pasturestack.1-py3-none-any.whl"
+BOM_SHA = "3581cb8f0a6128c19ccec4e9aeab0d3ac8212fd19bd19d485b2e24fd4dc10f2a"
 RULES = {
     "GHSA-6v7p-g79w-8964": ("msgpack", "1.1.2"),
     "CVE-2026-57585": ("msgpack", "1.1.2"),
+    "CVE-2026-114613": ("msgpack", "1.1.2"),
+    "GHSA-j586-36cw-2gc2": ("msgpack", "1.1.2"),
     "CVE-2025-47273": ("setuptools", "70.3.0"),
     "CVE-2026-59890": ("setuptools", "70.3.0"),
+    "GHSA-xmj3-9gf7-h52w": ("setuptools", "70.3.0"),
 }
 
 
 def apply(raw, proof, subject):
+    inventory = proof.get("vendorSourceInventory", {})
+    sources = inventory.get("sources", [])
+    valid_sources = (sources and len({item.get("path") for item in sources}) == len(sources) and
+                     any(item.get("kind") == "sbom" for item in sources) and
+                     any(item.get("kind") == "wheel" and item.get("path") == proof.get("verifiedWheelPath") for item in sources) and
+                     all((item.get("kind") == "wheel" and item.get("path") in WHEEL_PATHS and item.get("sha256") == WHEEL_SHA) or
+                         (item.get("kind") == "sbom" and item.get("path") in SBOM_PATHS and item.get("sha256") == BOM_SHA)
+                         for item in sources))
     valid = (proof["verifiedWheelSha256"] == WHEEL_SHA and
+             proof.get("verifiedWheelPath") in WHEEL_PATHS and
              proof["pip"] == "26.2.2+pasturestack.1" and proof["urllib3"] == "2.8.0" and
              proof["msgpackImplementation"] == "pip._vendor.msgpack.fallback" and
-             proof["setuptoolsPackageAbsent"] is True and proof["vendorFilesByteCompared"] == 299)
+             proof["setuptoolsPackageAbsent"] is True and proof["vendorFilesByteCompared"] == 299 and
+             proof.get("vulnerableModulesAbsent") == ["msgpack._cmsgpack", "setuptools.wheel"] and
+             inventory.get("complete") is True and inventory.get("noForeignMatchingDistribution") is True and
+             inventory.get("expectedBomSha256") == BOM_SHA and valid_sources)
     if not valid:
         raise ValueError("Exact installed pip vendor proof is required")
     reasons = {item["id"]: item for item in proof["notAffected"]}
@@ -31,17 +49,20 @@ def apply(raw, proof, subject):
             key = item.get("VulnerabilityID")
             expected = RULES.get(key)
             actual = (item.get("PkgName"), item.get("InstalledVersion"))
+            purl = "pkg:pypi/" + str(actual[0]) + "@" + str(actual[1])
             packages = [pkg for pkg in section.get("Packages", []) if
                         (pkg.get("Name"), pkg.get("Version")) == actual]
             reason = reasons.get(key, {})
             eligible = (expected == actual and key in reasons and
                         (reason.get("package"), reason.get("version")) == actual and
                         section.get("Type") == "python-pkg" and packages and
-                        all(pkg.get("AnalyzedBy") == "sbom" for pkg in packages))
+                        all(pkg.get("AnalyzedBy") == "sbom" for pkg in packages) and
+                        (item.get("PkgIdentifier") or {}).get("PURL") == purl and
+                        all(not pkg.get("FilePath") and not pkg.get("PkgPath") for pkg in packages) and
+                        not item.get("PkgPath"))
             if not eligible:
                 retained.append(item)
                 continue
-            purl = "pkg:pypi/" + actual[0] + "@" + actual[1]
             statements.append({
                 "vulnerability": {"name": key},
                 "products": [{"@id": subject, "subcomponents": [{"@id": purl}]}],