From 74b7126204685847784dbb1147d6183a466f645e Mon Sep 17 00:00:00 2001 From: chen21019 <19357113+chen21019@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:19:26 +0800 Subject: [PATCH] Bind namespace-wheel VEX to the verified pip slice and native SBOM --- scripts/build-agent-release.sh | 18 ++++- tools/pip-security-overlay/README.md | 9 ++- .../check-pip-vendor-slice.py | 19 ++++- .../filter-pip-vendor-findings.py | 41 +++++++++- tools/pip-security-overlay/test-pip-vex.py | 74 ++++++++++++++++--- 5 files changed, 140 insertions(+), 21 deletions(-) diff --git a/scripts/build-agent-release.sh b/scripts/build-agent-release.sh index e7791f1228..bf2ab62f3c 100644 --- a/scripts/build-agent-release.sh +++ b/scripts/build-agent-release.sh @@ -23,13 +23,25 @@ candidate="local/pasturestack-node-agent-${COMPONENT}:${GITHUB_RUN_ID}" helper_container="pasturestack-agent-helper-${GITHUB_RUN_ID}" backports_container= bundle_container= -scratch=$(mktemp -d) +[[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]] +[[ "${RUNNER_TEMP:-/tmp}" = /* ]] +scratch_root=$(realpath -- "${RUNNER_TEMP:-/tmp}") +test "$scratch_root" != / +scratch=$(mktemp -d "$scratch_root/pasturestack-agent-${GITHUB_RUN_ID}.XXXXXX") cleanup() { + [[ "$(realpath -e -- "$scratch")" = "$scratch" && + "$scratch" = "$scratch_root"/pasturestack-agent-"$GITHUB_RUN_ID".* ]] || return 1 if [[ -n "$backports_container" ]]; then docker rm -v "$backports_container" >/dev/null 2>&1 || true; fi if [[ -n "$bundle_container" ]]; then docker rm -v "$bundle_container" >/dev/null 2>&1 || true; fi docker rm -v "$helper_container" >/dev/null 2>&1 || true docker logout ghcr.io >/dev/null 2>&1 || true - rm -rf "$scratch" + if [[ -d "$scratch/trivy-cache" ]]; then + [[ -n "${trivy:-}" && "$(realpath -e -- "$scratch/trivy-cache")" = "$scratch/trivy-cache" ]] || return 1 + docker run --rm --network none --user 0:0 \ + --mount "type=bind,source=$scratch/trivy-cache,target=/cache" \ + --entrypoint chown "$trivy" -R "$(id -u):$(id -g)" /cache + fi + rm -rf -- "$scratch" } trap cleanup EXIT @@ -167,6 +179,7 @@ pip_scan_input="$evidence/image-security-os-effective.json" fi python3 tools/pip-security-overlay/filter-pip-vendor-findings.py \ --raw "$pip_scan_input" --proof "$evidence/pip-vendor-slice.json" \ + --sbom "$evidence/image.cdx.json" \ --output "$evidence/image-security-effective.json" --vex "$evidence/image.vex.json" \ --subject "urn:sha256:${local_id#sha256:}" python3 - "$evidence" <<'PY' @@ -203,6 +216,7 @@ if [[ "$PUBLISH" = true ]]; then test "$(docker image inspect "$repository@$digest" --format '{{.Id}}')" = "$local_id" python3 tools/pip-security-overlay/filter-pip-vendor-findings.py \ --raw "$pip_scan_input" --proof "$evidence/pip-vendor-slice.json" \ + --sbom "$evidence/image.cdx.json" \ --output "$evidence/image-security-effective.json" --vex "$evidence/image.vex.json" \ --subject "https://ghcr.io/v2/${repository#ghcr.io/}/manifests/$digest" printf 'repository=%s\nversion=%s\nsource=%s\ndigest=%s\nimage_id=%s\n' \ diff --git a/tools/pip-security-overlay/README.md b/tools/pip-security-overlay/README.md index e354536a0c..97740e0d34 100644 --- a/tools/pip-security-overlay/README.md +++ b/tools/pip-security-overlay/README.md @@ -64,12 +64,17 @@ bash verify-pip-runtime.sh --virtualenv /usr/share/python-wheels - setuptools 70.3.0 僅保留 `pkg_resources`；CVE-2025-47273 所需的 `setuptools/package_index.py` 與 CVE-2026-59890 所需的 `setuptools/command/egg_info.py` 均不存在。 +- [GHSA-xmj3-9gf7-h52w](https://github.com/pypa/setuptools/security/advisories/GHSA-xmj3-9gf7-h52w) + 的 `setuptools/wheel.py`／`Wheel.install_as_egg` 同樣未包含於此 slice。 + 此 ID 額外要求實際 299 檔路徑與 SHA-256 inventory、明確的 wheel/live 模組缺席證據， + 以及同次原生 CycloneDX 中的完整 setuptools PURL；VEX 保留 inventory 與原 SBOM 的 SHA-256。 `check-pip-vendor-slice.py` 會比對實際載入的全部 299 個 vendor 檔案與 固定 wheel，拒絕額外檔案、其他同版本完整 distribution。只有取得此證據後， -`filter-pip-vendor-findings.py` 才可對上述三個 CVE（含 msgpack GHSA 別名） +`filter-pip-vendor-findings.py` 才可對上述已登記 ID（含 msgpack GHSA 別名） 產生綁定該映像的 OpenVEX 與有效掃描檔。原始掃描必須另行保存；未知 CVE、 其他版本、非 SBOM 套件來源與 urllib3 finding 全部保留，不做泛用版本排除。 -執行 `python3 test-pip-vex.py` 可驗證以上六組正反控制。 +執行 `python3 test-pip-vex.py` 可驗證以上八組正反控制，包含真正 wheel/live +`setuptools/wheel.py` 出現時拒絕，以及缺少 inventory／原生 SBOM 時保留 finding。 官方政策：https://pip.pypa.io/en/stable/development/vendoring-policy/ diff --git a/tools/pip-security-overlay/check-pip-vendor-slice.py b/tools/pip-security-overlay/check-pip-vendor-slice.py index 789bc28007..3fddeb776f 100644 --- a/tools/pip-security-overlay/check-pip-vendor-slice.py +++ b/tools/pip-security-overlay/check-pip-vendor-slice.py @@ -7,8 +7,6 @@ import json from pathlib import Path import zipfile -import pip -from pip._vendor import msgpack, urllib3 EXPECTED_SHA = "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a" FINDINGS = { @@ -16,28 +14,40 @@ "CVE-2026-57585": ("msgpack", "1.1.2", "C/Cython unpacker is absent; runtime selects pure Python fallback"), "CVE-2025-47273": ("setuptools", "70.3.0", "setuptools/package_index.py is absent; only pkg_resources is vendored"), "CVE-2026-59890": ("setuptools", "70.3.0", "setuptools/command/egg_info.py and FileList are absent"), + "GHSA-xmj3-9gf7-h52w": ("setuptools", "70.3.0", "setuptools/wheel.py and Wheel.install_as_egg are absent; only pkg_resources is vendored"), } +def require_setuptools_wheel_absent(wheel_names, live_vendor): + assert "pip/_vendor/setuptools/wheel.py" not in wheel_names, "Vulnerable setuptools wheel module is in the selected wheel" + assert not live_vendor.joinpath("setuptools/wheel.py").is_file(), "Vulnerable setuptools wheel module is installed" + + def verify(wheel): + import pip + from pip._vendor import msgpack, urllib3 assert hashlib.sha256(wheel.read_bytes()).hexdigest() == EXPECTED_SHA assert pip.__version__ == "26.2.2+pasturestack.1" assert urllib3.__version__ == "2.8.0" assert msgpack.Unpacker.__module__ == "pip._vendor.msgpack.fallback" live = importlib.resources.files("pip") checked = 0 + inventory = [] with zipfile.ZipFile(wheel) as archive: names = archive.namelist() expected_files = set() for name in names: if not name.startswith("pip/_vendor/") or name.endswith("/"): continue - assert live.joinpath(name.removeprefix("pip/")).read_bytes() == archive.read(name), name + content = archive.read(name) + assert live.joinpath(name.removeprefix("pip/")).read_bytes() == content, name expected_files.add(name.removeprefix("pip/_vendor/")) + inventory.append({"path": name.removeprefix("pip/_vendor/"), "sha256": hashlib.sha256(content).hexdigest()}) checked += 1 assert not any(name.startswith("pip/_vendor/") and name.endswith((".so", ".pyd")) for name in names) assert not any(name.startswith("pip/_vendor/setuptools/") for name in names) assert not any(name.startswith("pip/_vendor/msgpack/_cmsgpack") for name in names) + require_setuptools_wheel_absent(names, live.joinpath("_vendor")) def live_files(directory, prefix=""): for entry in directory.iterdir(): if entry.name == "__pycache__": @@ -54,10 +64,13 @@ def live_files(directory, prefix=""): assert distribution.version != "70.3.0", "A full setuptools 70.3.0 distribution is installed" if distribution.metadata.get("Name", "").lower() == "msgpack": assert distribution.version != "1.1.2", "A separate msgpack 1.1.2 distribution is installed" + inventory.sort(key=lambda item: item["path"]) + inventory_sha = hashlib.sha256(json.dumps(inventory, sort_keys=True, separators=(",", ":")).encode()).hexdigest() return {"verifiedWheelSha256": EXPECTED_SHA, "pip": pip.__version__, "urllib3": urllib3.__version__, "vendorFilesByteCompared": checked, "msgpackImplementation": msgpack.Unpacker.__module__, "setuptoolsPackageAbsent": True, "scope": "this exact pip vendor slice only", + "setuptoolsWheelModuleAbsent": True, "vendorInventory": inventory, "vendorInventorySha256": inventory_sha, "notAffected": [{"id": key, "package": value[0], "version": value[1], "justification": "vulnerable_code_not_present", "detail": value[2]} for key, value in FINDINGS.items()]} diff --git a/tools/pip-security-overlay/filter-pip-vendor-findings.py b/tools/pip-security-overlay/filter-pip-vendor-findings.py index 5a7c190794..1176e8dd40 100644 --- a/tools/pip-security-overlay/filter-pip-vendor-findings.py +++ b/tools/pip-security-overlay/filter-pip-vendor-findings.py @@ -4,18 +4,45 @@ from copy import deepcopy from datetime import datetime, timezone import json +import hashlib +import re from pathlib import Path WHEEL_SHA = "be5f63d537bbe611dc422fabb6740a44b5997888b592888df5091727f023802a" +VENDOR_INVENTORY_SHA = "144483e54419f59a5186652b932f7690d7b8bfaaab61ccc790c2881a89ab6212" RULES = { "GHSA-6v7p-g79w-8964": ("msgpack", "1.1.2"), "CVE-2026-57585": ("msgpack", "1.1.2"), "CVE-2025-47273": ("setuptools", "70.3.0"), "CVE-2026-59890": ("setuptools", "70.3.0"), + "GHSA-xmj3-9gf7-h52w": ("setuptools", "70.3.0"), } -def apply(raw, proof, subject): +def namespace_wheel_absent(proof, sbom): + rows = proof.get("vendorInventory", []) + if proof.get("setuptoolsWheelModuleAbsent") is not True or len(rows) != 299: + return False + if not all(isinstance(row, dict) and set(row) == {"path", "sha256"} + and isinstance(row["path"], str) and row["path"] and "\\" not in row["path"] + and not row["path"].startswith("/") and ".." not in row["path"].split("/") + and "setuptools" not in row["path"].split("/") + and isinstance(row["sha256"], str) and re.fullmatch("[0-9a-f]{64}", row["sha256"]) + for row in rows): + return False + if len({row["path"] for row in rows}) != 299 or rows != sorted(rows, key=lambda row: row["path"]): + return False + digest = hashlib.sha256(json.dumps(rows, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + if digest != proof.get("vendorInventorySha256") or digest != VENDOR_INVENTORY_SHA or sbom is None: + return False + document = json.loads(sbom) + packages = [c for c in document.get("components", []) if c.get("purl") == "pkg:pypi/setuptools@70.3.0"] + return document.get("bomFormat") == "CycloneDX" and packages and all( + c.get("name") == "setuptools" and c.get("version") == "70.3.0" and c.get("type") == "library" + for c in packages) + + +def apply(raw, proof, subject, sbom=None): valid = (proof["verifiedWheelSha256"] == WHEEL_SHA and proof["pip"] == "26.2.2+pasturestack.1" and proof["urllib3"] == "2.8.0" and proof["msgpackImplementation"] == "pip._vendor.msgpack.fallback" and @@ -38,6 +65,11 @@ def apply(raw, proof, subject): (reason.get("package"), reason.get("version")) == actual and section.get("Type") == "python-pkg" and packages and all(pkg.get("AnalyzedBy") == "sbom" for pkg in packages)) + if key == "GHSA-xmj3-9gf7-h52w": + exact_purl = "pkg:pypi/setuptools@70.3.0" + eligible = (eligible and item.get("PkgIdentifier", {}).get("PURL") == exact_purl + and all(pkg.get("Identifier", {}).get("PURL") == exact_purl for pkg in packages) + and namespace_wheel_absent(proof, sbom)) if not eligible: retained.append(item) continue @@ -48,6 +80,9 @@ def apply(raw, proof, subject): "status": "not_affected", "justification": "vulnerable_code_not_present", "impact_statement": reasons[key]["detail"] + "; exact pip wheel SHA-256 " + WHEEL_SHA, }) + if key == "GHSA-xmj3-9gf7-h52w": + statements[-1]["impact_statement"] += ("; live/selected-wheel inventory SHA-256 " + + proof["vendorInventorySha256"] + "; native CycloneDX SHA-256 " + hashlib.sha256(sbom).hexdigest()) section["Vulnerabilities"] = retained vex = {"@context": "https://openvex.dev/ns/v0.2.0", "@id": subject + "/pip-vendor-vex", "author": "PastureStack", "timestamp": datetime.now(timezone.utc).isoformat(), @@ -62,8 +97,10 @@ def apply(raw, proof, subject): parser.add_argument("--output", type=Path, required=True) parser.add_argument("--vex", type=Path, required=True) parser.add_argument("--subject", required=True) + parser.add_argument("--sbom", type=Path, help="Actual native CycloneDX bytes required for the namespace-wheel advisory") args = parser.parse_args() - report, vex = apply(json.loads(args.raw.read_text()), json.loads(args.proof.read_text()), args.subject) + report, vex = apply(json.loads(args.raw.read_text()), json.loads(args.proof.read_text()), args.subject, + args.sbom.read_bytes() if args.sbom else None) args.output.write_text(json.dumps(report, indent=2) + "\n") args.vex.write_text(json.dumps(vex, indent=2) + "\n") print(f"PROVEN_PIP_VENDOR_VEX statements={len(vex['statements'])}") diff --git a/tools/pip-security-overlay/test-pip-vex.py b/tools/pip-security-overlay/test-pip-vex.py index 9baa3e1c8c..18bbfad3ca 100644 --- a/tools/pip-security-overlay/test-pip-vex.py +++ b/tools/pip-security-overlay/test-pip-vex.py @@ -3,11 +3,18 @@ from copy import deepcopy import importlib.util from pathlib import Path +import hashlib +import json +import tempfile import unittest +from unittest.mock import patch spec = importlib.util.spec_from_file_location("pip_vex", Path(__file__).with_name("filter-pip-vendor-findings.py")) vex = importlib.util.module_from_spec(spec) spec.loader.exec_module(vex) +proof_spec = importlib.util.spec_from_file_location("pip_vendor_proof", Path(__file__).with_name("check-pip-vendor-slice.py")) +vendor_proof = importlib.util.module_from_spec(proof_spec) +proof_spec.loader.exec_module(vendor_proof) class VexBoundary(unittest.TestCase): @@ -17,17 +24,26 @@ def setUp(self): "setuptoolsPackageAbsent": True, "vendorFilesByteCompared": 299, "notAffected": [{"id": key, "package": value[0], "version": value[1], "detail": "Test fixture only"} for key, value in vex.RULES.items()]} + self.proof["setuptoolsWheelModuleAbsent"] = True + self.proof["vendorInventory"] = [{"path": f"fixture/{n:03d}.py", "sha256": "a" * 64} for n in range(299)] + self.rehash_inventory(self.proof) + # This synthetic metadata fixture is not a claim about installed files. + anchor = patch.object(vex, "VENDOR_INVENTORY_SHA", self.proof["vendorInventorySha256"]) + anchor.start(); self.addCleanup(anchor.stop) + self.sbom = json.dumps({"bomFormat": "CycloneDX", "components": [{"type": "library", "name": "setuptools", + "version": "70.3.0", "purl": "pkg:pypi/setuptools@70.3.0"}]}).encode() self.raw = {"Results": [{"Type": "python-pkg", "Packages": [ - {"Name": name, "Version": version, "AnalyzedBy": "sbom"} + {"Name": name, "Version": version, "AnalyzedBy": "sbom", "Identifier": {"PURL": "pkg:pypi/" + name + "@" + version}} for name, version in set(vex.RULES.values())], "Vulnerabilities": [ - {"VulnerabilityID": key, "PkgName": value[0], "InstalledVersion": value[1]} + {"VulnerabilityID": key, "PkgName": value[0], "InstalledVersion": value[1], + "PkgIdentifier": {"PURL": "pkg:pypi/" + value[0] + "@" + value[1]}} for key, value in vex.RULES.items()]}]} def test_exact_slice_only(self): original = deepcopy(self.raw) - result, document = vex.apply(self.raw, self.proof, "urn:test:exact-image") + result, document = vex.apply(self.raw, self.proof, "urn:test:exact-image", self.sbom) self.assertEqual(result["Results"][0]["Vulnerabilities"], []) - self.assertEqual(len(document["statements"]), 4) + self.assertEqual(len(document["statements"]), 5) self.assertEqual(self.raw, original, "raw evidence must not be mutated") def test_invalid_proof_fails_closed(self): @@ -35,34 +51,68 @@ def test_invalid_proof_fails_closed(self): ("vendorFilesByteCompared", 300), ("setuptoolsPackageAbsent", False)]: proof = dict(self.proof, **{key: value}) with self.subTest(key=key), self.assertRaises(ValueError): - vex.apply(self.raw, proof, "urn:test:image") + vex.apply(self.raw, proof, "urn:test:image", self.sbom) def test_full_distribution_is_not_excluded(self): for package in self.raw["Results"][0]["Packages"]: package["AnalyzedBy"] = "python-pkg" - result, document = vex.apply(self.raw, self.proof, "urn:test:image") - self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 4) + result, document = vex.apply(self.raw, self.proof, "urn:test:image", self.sbom) + self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 5) self.assertEqual(document["statements"], []) def test_missing_packages_is_not_excluded(self): self.raw["Results"][0]["Packages"] = [] - result, _ = vex.apply(self.raw, self.proof, "urn:test:image") - self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 4) + result, _ = vex.apply(self.raw, self.proof, "urn:test:image", self.sbom) + self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 5) def test_wrong_version_is_not_excluded(self): for finding in self.raw["Results"][0]["Vulnerabilities"]: finding["InstalledVersion"] = "999.0" - result, document = vex.apply(self.raw, self.proof, "urn:test:image") - self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 4) + result, document = vex.apply(self.raw, self.proof, "urn:test:image", self.sbom) + self.assertEqual(len(result["Results"][0]["Vulnerabilities"]), 5) self.assertEqual(document["statements"], []) def test_unknown_advisory_and_urllib3_are_retained(self): unknown = {"VulnerabilityID": "CVE-TEST-UNKNOWN", "PkgName": "setuptools", "InstalledVersion": "70.3.0"} urllib = {"VulnerabilityID": "CVE-2026-97689", "PkgName": "urllib3", "InstalledVersion": "2.7.0"} self.raw["Results"][0]["Vulnerabilities"] += [unknown, urllib] - result, _ = vex.apply(self.raw, self.proof, "urn:test:image") + result, _ = vex.apply(self.raw, self.proof, "urn:test:image", self.sbom) self.assertEqual(result["Results"][0]["Vulnerabilities"], [unknown, urllib]) + @staticmethod + def rehash_inventory(proof): + proof["vendorInventorySha256"] = hashlib.sha256(json.dumps( + proof["vendorInventory"], sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + def test_actual_wheel_module_presence_fails_source_proof(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + vendor_proof.require_setuptools_wheel_absent([], root) + with self.assertRaisesRegex(AssertionError, "selected wheel"): + vendor_proof.require_setuptools_wheel_absent(["pip/_vendor/setuptools/wheel.py"], root) + (root / "setuptools").mkdir(); (root / "setuptools/wheel.py").write_text("class Wheel: pass\n") + with self.assertRaisesRegex(AssertionError, "installed"): + vendor_proof.require_setuptools_wheel_absent([], root) + + def test_new_advisory_requires_inventory_module_absence_and_native_sbom(self): + for kind in ("missing-proof", "module-present", "inventory-tamper", "missing-sbom", "wrong-sbom", "finding-purl", "package-purl"): + proof = deepcopy(self.proof); sbom = self.sbom; raw = deepcopy(self.raw) + if kind == "missing-proof": proof.pop("setuptoolsWheelModuleAbsent") + if kind == "module-present": + proof["vendorInventory"][-1]["path"] = "setuptools/wheel.py" + proof["vendorInventory"].sort(key=lambda row: row["path"]); self.rehash_inventory(proof) + if kind == "inventory-tamper": proof["vendorInventory"][0]["sha256"] = "b" * 64 + if kind == "missing-sbom": sbom = None + if kind == "wrong-sbom": sbom = b'{"bomFormat":"CycloneDX","components":[]}' + if kind == "finding-purl": + next(v for v in raw["Results"][0]["Vulnerabilities"] if v["VulnerabilityID"] == "GHSA-xmj3-9gf7-h52w")["PkgIdentifier"]["PURL"] = "pkg:pypi/setuptools@999" + if kind == "package-purl": + next(p for p in raw["Results"][0]["Packages"] if p["Name"] == "setuptools")["Identifier"]["PURL"] = "pkg:pypi/setuptools@999" + result, document = vex.apply(raw, proof, "urn:test:image", sbom) + with self.subTest(kind=kind): + self.assertEqual([v["VulnerabilityID"] for v in result["Results"][0]["Vulnerabilities"]], ["GHSA-xmj3-9gf7-h52w"]) + self.assertEqual(len(document["statements"]), 4) + if __name__ == "__main__": unittest.main()