From 7d9997a31661fb69900ad420f2fea9bc589e03ad Mon Sep 17 00:00:00 2001 From: chaksaray Date: Wed, 4 Nov 2026 20:55:56 +0700 Subject: [PATCH] feat: ship the SARIF taxonomy as a verified catalog release asset (#442) Signed-off-by: chaksaray --- .github/workflows/catalog-release.yml | 20 ++- CHANGELOG.md | 7 + README.md | 27 +++- docs/specs/ave-implementer-guide.md | 21 ++- docs/specs/ave-in-sarif.md | 2 +- docs/specs/scaling-and-governance.md | 22 ++- scripts/check_release_assets.py | 90 ++++++++++-- tests/test_catalog_release_workflow.py | 60 ++++++++ tests/test_check_release_assets.py | 10 +- tests/test_release_taxonomy.py | 182 +++++++++++++++++++++++++ 10 files changed, 411 insertions(+), 30 deletions(-) create mode 100644 tests/test_catalog_release_workflow.py create mode 100644 tests/test_release_taxonomy.py diff --git a/.github/workflows/catalog-release.yml b/.github/workflows/catalog-release.yml index c7f5653..fd453be 100644 --- a/.github/workflows/catalog-release.yml +++ b/.github/workflows/catalog-release.yml @@ -1,8 +1,9 @@ name: catalog-release # Publishes a catalog release: a GitHub Release tagged v carrying -# ave-records.json, ave-records.manifest.json and SHA256SUMS, with a build -# provenance attestation for the two data files. Run by hand from +# ave-records.json, ave-records.manifest.json, ave-taxonomy.json (the SARIF +# taxonomy of exactly those records) and SHA256SUMS, with a build provenance +# attestation for the three data files. Run by hand from # Actions > catalog-release; it builds a commit already on main and never # pushes to the repository. Policy: docs/specs/scaling-and-governance.md, # section 6. @@ -126,19 +127,26 @@ jobs: --manifest-out release/ave-records.manifest.json jq --arg release "v$VERSION" '{release: $release} + .' release/ave-records.manifest.json > manifest.tmp mv manifest.tmp release/ave-records.manifest.json - (cd release && sha256sum ave-records.json ave-records.manifest.json > SHA256SUMS) + python3 scripts/build_taxonomy.py \ + --records release/ave-records.json \ + --release "v$VERSION" \ + --release-date "$(jq -r '.generated_at[:10]' release/ave-records.manifest.json)" \ + --download-uri "https://github.com/$GITHUB_REPOSITORY/releases/download/v$VERSION/ave-taxonomy.json" \ + --out release/ave-taxonomy.json + (cd release && sha256sum ave-records.json ave-records.manifest.json ave-taxonomy.json > SHA256SUMS) cat release/ave-records.manifest.json release/SHA256SUMS # the tag does not exist yet, so source_commit is checked against this run's commit - name: Verify the release files - run: python scripts/check_release_assets.py release --tag "v$VERSION" --commit "$(git rev-parse HEAD)" + run: python scripts/check_release_assets.py release --tag "v$VERSION" --commit "$(git rev-parse HEAD)" --repo "$GITHUB_REPOSITORY" - - name: Attest build provenance for both data files + - name: Attest build provenance for the three data files uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: | release/ave-records.json release/ave-records.manifest.json + release/ave-taxonomy.json - name: Create the release env: @@ -157,4 +165,4 @@ jobs: --title "AVE v$VERSION" \ --notes-file release-notes.md \ "${draft_flag[@]}" \ - release/ave-records.json release/ave-records.manifest.json release/SHA256SUMS + release/ave-records.json release/ave-records.manifest.json release/ave-taxonomy.json release/SHA256SUMS diff --git a/CHANGELOG.md b/CHANGELOG.md index ecb9fd0..31803ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ Format: [Semantic Versioning](https://semver.org). Schema versions and record se ## [Unreleased] ### SARIF +- **The taxonomy as a verified catalog-release asset** (#442). The `catalog-release` workflow builds + `ave-taxonomy.json` with `scripts/build_taxonomy.py` from the release's own `ave-records.json`, stamped with the + release's version, the manifest's date and its own download URL, lists it in `SHA256SUMS` and attests it with the + other data files. `scripts/check_release_assets.py` regenerates it from the release's records and demands a byte + match, naming the field or taxon that differs first, and takes a new `--repo owner/name` (default `aveproject/ave`) + so a fork's release is verified against the fork. Releases from v1.3.0 must carry the taxonomy; v1.0.0 to v1.2.0 + verify as before. That makes the order mechanical: this has to merge before v1.3.0 is cut. - **The AVE taxonomy as a canonical file, form C and taxonomy checks** (#420). `dist/ave-taxonomy-latest.json` is the AVE taxonomy as a SARIF 2.1.0 `toolComponent` with a frozen `guid` (`5110b285-34d7-5334-94db-0308b24cfb49`, the UUIDv5 of `https://aveproject.org/taxonomy/AVE`) and one taxon per record, including records whose status is not `active`. diff --git a/README.md b/README.md index e15576e..d78ca04 100644 --- a/README.md +++ b/README.md @@ -512,12 +512,16 @@ watching and the API. A catalog release is a [GitHub Release](https://github.com/aveproject/ave/releases) tagged `vX.Y.Z`: a fixed copy of the record set that does not change after it is published. Each one carries three files and a build provenance attestation for -the first two: +the first two; from v1.3.0 it carries a fourth, the SARIF taxonomy, which the +attestation covers too: - `ave-records.json`: every published record. - `ave-records.manifest.json`: the release tag, `schema_version`, `record_count`, the SHA-256 of `ave-records.json` and `source_commit`, the commit it was built from. -- `SHA256SUMS`: checksums of both files. +- `ave-taxonomy.json` (from v1.3.0): the [SARIF taxonomy](docs/specs/ave-in-sarif.md) + of exactly these records, stamped with the release's own version, date and + download link, ready to embed in a tool's SARIF output. +- `SHA256SUMS`: checksums of the data files (three of them from v1.3.0). Pin a tag, then check what you downloaded: @@ -535,7 +539,24 @@ gh attestation verify ave-records.manifest.json --repo aveproject/ave ``` `check_release_assets.py` confirms the checksums, that the manifest matches the -records file, and that `source_commit` is the commit the tag points at. +records file, and that `source_commit` is the commit the tag points at. From +v1.3.0 it also regenerates `ave-taxonomy.json` from the records file and requires +a byte for byte match, and refuses a release without it. + +From v1.3.0 on, add the taxonomy to the same steps. `--repo` is the repository the +release is published under, so verify a fork's release with `--repo owner/name`: + +```bash +TAG=v1.3.0 +curl -fsSLO "https://github.com/aveproject/ave/releases/download/$TAG/ave-taxonomy.json" +sha256sum -c SHA256SUMS # now covers three files +python ave-src/scripts/check_release_assets.py . --tag "$TAG" --git-dir ave-src --repo aveproject/ave +gh attestation verify ave-taxonomy.json --repo aveproject/ave +``` + +The taxonomy has stable URLs: `https://github.com/aveproject/ave/releases/download/vX.Y.Z/ave-taxonomy.json` +for a release, and `https://github.com/aveproject/ave/releases/latest/download/ave-taxonomy.json` +for the newest. Three kinds of record file, and which is which: diff --git a/docs/specs/ave-implementer-guide.md b/docs/specs/ave-implementer-guide.md index 98237ac..edc833a 100644 --- a/docs/specs/ave-implementer-guide.md +++ b/docs/specs/ave-implementer-guide.md @@ -35,13 +35,15 @@ comes first. ## Pin and verify a release A catalog release is a [GitHub Release](https://github.com/aveproject/ave/releases) -tagged `vX.Y.Z`. It never changes after it is published, and carries three files: +tagged `vX.Y.Z`. It never changes after it is published, and carries three files, +and from v1.3.0 a fourth: - `ave-records.json`: every published record, whatever its status, except drafts. - `ave-records.manifest.json`: what the release is (below). -- `SHA256SUMS`: checksums of both files. +- `ave-taxonomy.json` (from v1.3.0): the SARIF taxonomy of exactly those records. +- `SHA256SUMS`: checksums of the data files. -The first two also carry a build provenance attestation, signed by the workflow +The data files also carry a build provenance attestation, signed by the workflow that built them. `v1.2.0` is the first release with these files. Download a release and verify it. This needs network access, `curl`, `git`, @@ -78,6 +80,19 @@ What each check proves: `aveproject/ave`. This is the check that does not trust where you downloaded from. A file that differs by a single byte has no attestation and fails. +From v1.3.0 the release also carries `ave-taxonomy.json`, the AVE taxonomy for the SARIF you +emit ([AVE-in-SARIF](ave-in-sarif.md)), already stamped with the release's version, date and +download link. Download it with the other files; `sha256sum -c SHA256SUMS` then covers three +files, `check_release_assets.py` regenerates the taxonomy from the release's records and refuses a +release whose copy differs by a byte (pass `--repo owner/name` to verify a fork's release), and +the attestation check takes it too: + +```bash +curl -fsSL -o "ave-$TAG/ave-taxonomy.json" "https://github.com/aveproject/ave/releases/download/$TAG/ave-taxonomy.json" +gh attestation verify "ave-$TAG/ave-taxonomy.json" --repo aveproject/ave \ + --signer-workflow aveproject/ave/.github/workflows/catalog-release.yml +``` + The manifest of `v1.2.0`: ```json diff --git a/docs/specs/ave-in-sarif.md b/docs/specs/ave-in-sarif.md index 98f8918..c664c45 100644 --- a/docs/specs/ave-in-sarif.md +++ b/docs/specs/ave-in-sarif.md @@ -179,7 +179,7 @@ Add one taxon per AVE record that appears in the scan results. Scanners may incl python3 scripts/build_taxonomy.py --only AVE-2026-00001,AVE-2026-00075 --out - ``` -The release-stamped form, for a scanner pinned to a catalog release, takes that release's records file and its facts: `python3 scripts/build_taxonomy.py --records ave-records.json --release v1.2.0 --release-date 2026-10-29 --download-uri --out -`. The file is not yet a catalog-release asset. A taxon's `id` and `name` are the record's `ave_id` and `title` exactly, and the checker compares them with the catalog (`taxon_unknown`, `taxon_name_differs`). +The release-stamped form, for a scanner pinned to a catalog release, takes that release's records file and its facts: `python3 scripts/build_taxonomy.py --records ave-records.json --release v1.2.0 --release-date 2026-10-29 --download-uri --out -`. From v1.3.0 every catalog release carries the same file as an asset, `ave-taxonomy.json`: the pinned copy a tool should embed, already stamped with the release's `version`, `semanticVersion`, `releaseDateUtc` and `downloadUri` (`https://github.com/aveproject/ave/releases/download/vX.Y.Z/ave-taxonomy.json`, or `https://github.com/aveproject/ave/releases/latest/download/ave-taxonomy.json` for the newest), listed in the release's `SHA256SUMS`, attested, and checked byte for byte against the release's own records by `scripts/check_release_assets.py`. `dist/ave-taxonomy-latest.json` is the moving copy, and `latest` has no `version`. A taxon's `id` and `name` are the record's `ave_id` and `title` exactly, and the checker compares them with the catalog (`taxon_unknown`, `taxon_name_differs`). --- diff --git a/docs/specs/scaling-and-governance.md b/docs/specs/scaling-and-governance.md index 34cb089..9381980 100644 --- a/docs/specs/scaling-and-governance.md +++ b/docs/specs/scaling-and-governance.md @@ -293,8 +293,12 @@ The two are numbered separately. - `ave-records.manifest.json`: `release`, `schema_version`, `record_count`, `generated_at`, `source`, `sha256` (of `ave-records.json`) and `source_commit`, the tagged commit. -- `SHA256SUMS`: both files, in `sha256sum` format. -- A build provenance attestation for both files, checked with +- `ave-taxonomy.json` (from v1.3.0): the SARIF taxonomy (see + `docs/specs/ave-in-sarif.md`) of exactly this release's records, built by + `scripts/build_taxonomy.py` from `ave-records.json`, with the release's own + version, the manifest's `generated_at` date and its own download URL. +- `SHA256SUMS`: the data files, in `sha256sum` format (all three from v1.3.0). +- A build provenance attestation for each data file, checked with `gh attestation verify --repo aveproject/ave`. Asset names carry no version; the tag does. `scripts/check_release_assets.py` @@ -302,6 +306,20 @@ checks the files against each other, against the tag and against the tagged comm The release workflow runs it before publishing, and an implementer can run it after downloading (README, "Releases and pinning"). +**The taxonomy asset.** The verifier regenerates `ave-taxonomy.json` from the +release's own `ave-records.json` and demands a byte match, so no field, taxon or +character can differ from what the generator produces; a failure names the field +or taxon first. Releases from v1.3.0 must carry it and a release without it fails +verification; v1.0.0 to v1.2.0 shipped before it existed and verify without it. +That makes the order mechanical: a release cut by a workflow that does not build +the taxonomy fails its own verification, so the change that builds it has to merge +before v1.3.0 is cut. Its `downloadUri` must point at the repository it is +published under (`--repo owner/name`, default `aveproject/ave`), so a fork's +release is verified against the fork. Stable URLs: +`https://github.com/aveproject/ave/releases/download/vX.Y.Z/ave-taxonomy.json` +and `https://github.com/aveproject/ave/releases/latest/download/ave-taxonomy.json`. +`dist/ave-taxonomy-latest.json` is the moving copy and has no `version`. + **Cadence:** a release when records have changed, at most monthly unless a correction is urgent. This is the maintainer's policy, and the maintainer can change it. diff --git a/scripts/check_release_assets.py b/scripts/check_release_assets.py index acc09fa..fd97530 100644 --- a/scripts/check_release_assets.py +++ b/scripts/check_release_assets.py @@ -1,5 +1,5 @@ -# What: verifies the three files of an AVE catalog release (ave-records.json, -# ave-records.manifest.json, SHA256SUMS) against each other, against the +# What: verifies the files of an AVE catalog release (ave-records.json, +# ave-records.manifest.json, ave-taxonomy.json, SHA256SUMS) against each other, against the # release tag and against the commit the release was built from. # Why: a catalog release is what implementers pin to, so each file must # describe the others exactly: a checksum that no longer matches, a @@ -8,7 +8,10 @@ # catalog-release workflow before anything is published and on an # implementer's machine after download. # How: python scripts/check_release_assets.py --tag vX.Y.Z -# [--git-dir | --commit ] +# [--git-dir | --commit ] [--repo owner/name] +# ave-taxonomy.json is regenerated from ave-records.json with +# scripts/build_taxonomy.py and must match it byte for byte; releases from +# v1.3.0 on must carry it, earlier releases need not. # --git-dir checks source_commit against the commit points at in # that repository; --commit checks it against a given commit instead (the # workflow, which runs before the tag exists). Exit 0 when every check @@ -23,9 +26,19 @@ import sys from pathlib import Path +try: # imported as scripts.check_release_assets (tests) or run as a script from scripts/ + from scripts import build_taxonomy as bt +except ImportError: + import build_taxonomy as bt + RECORDS_FILE = "ave-records.json" MANIFEST_FILE = "ave-records.manifest.json" SUMS_FILE = "SHA256SUMS" +TAXONOMY_FILE = "ave-taxonomy.json" +# The first release that carries the SARIF taxonomy. A release from this tag on without it is refused; +# an earlier one (v1.0.0 to v1.2.0 shipped before it existed) may omit it. +TAXONOMY_FROM = (1, 3, 0) +DEFAULT_REPO = "aveproject/ave" TAG_PATTERN = re.compile(r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}") # sha256sum's own output: digest, a space, then " " (text) or "*" (binary), then the name @@ -44,7 +57,7 @@ def sha256_of(path: Path) -> str: # must cover both release files and match each byte for byte # How: parse each line in sha256sum format; refuse malformed lines and names # that leave the directory; compare each digest; require both files listed -def check_sums(directory: Path) -> list[str]: +def check_sums(directory: Path, extra_required=()) -> list[str]: sums = directory / SUMS_FILE if not sums.is_file(): return [f"{SUMS_FILE} is missing"] @@ -67,7 +80,7 @@ def check_sums(directory: Path) -> list[str]: errors.append(f"{SUMS_FILE} lists {name}, which is missing") elif sha256_of(target) != digest: errors.append(f"{SUMS_FILE}: {name} does not match its checksum") - for required in (RECORDS_FILE, MANIFEST_FILE): + for required in (RECORDS_FILE, MANIFEST_FILE, *extra_required): if required not in listed: errors.append(f"{SUMS_FILE} does not list {required}") return errors @@ -140,18 +153,69 @@ def check_manifest(directory: Path, tag: str, git_dir: Path | None, commit: str return errors +# What: whether a release must carry the taxonomy, from its tag +# Why: releases before the taxonomy existed stay valid; from TAXONOMY_FROM on it is part of the release +# How: compare the tag's (major, minor, patch) with TAXONOMY_FROM; a malformed tag requires nothing here (the tag check reports it) +def taxonomy_required(tag: str) -> bool: + match = TAG_PATTERN.fullmatch(tag) + return bool(match) and tuple(int(g) for g in match.groups()) >= TAXONOMY_FROM + + +# What: every way ave-taxonomy.json disagrees with the release it ships in +# Why: tools copy this file into their SARIF output, so a wrong guid, version, link or taxon would be copied +# with it; the only trustworthy file is the one the generator produces from the release's own records +# How: regenerate it from ave-records.json with the tag, the manifest's date and the asset's own URL, and +# compare byte for byte; when it differs, name the fields and taxa that differ before the generic message +def check_taxonomy(directory: Path, tag: str, repo: str = DEFAULT_REPO) -> list[str]: + path = directory / TAXONOMY_FILE + if not path.is_file(): + return [f"{TAXONOMY_FILE} is missing (releases from v{'.'.join(map(str, TAXONOMY_FROM))} carry it)"] if taxonomy_required(tag) else [] + try: + records = json.loads((directory / RECORDS_FILE).read_text(encoding="utf-8")) + manifest = json.loads((directory / MANIFEST_FILE).read_text(encoding="utf-8")) + actual_text = path.read_text(encoding="utf-8") + actual = json.loads(actual_text) + except (OSError, ValueError) as exc: + return [f"{TAXONOMY_FILE} cannot be checked, a release file is unreadable or not JSON: {exc}"] + if not (isinstance(records, list) and isinstance(manifest, dict) and isinstance(actual, dict) and TAG_PATTERN.fullmatch(tag)): + return [f"{TAXONOMY_FILE} cannot be checked against malformed records, manifest or tag"] + date = str(manifest.get("generated_at", ""))[:10] + uri = f"https://github.com/{repo}/releases/download/{tag}/{TAXONOMY_FILE}" + try: + expected = bt.build(records, release=tag, release_date=date, download_uri=uri) + except (ValueError, KeyError, TypeError) as exc: + return [f"{TAXONOMY_FILE} cannot be regenerated from {RECORDS_FILE}: {exc}"] + if actual_text == bt.dumps(expected): + return [] + errors = [] + for key in ("guid", "name", "version", "semanticVersion", "releaseDateUtc", "downloadUri", "isComprehensive"): + if actual.get(key) != expected.get(key): + errors.append(f"{TAXONOMY_FILE} {key} is {actual.get(key)!r}, expected {expected.get(key)!r}") + have = {t.get("id"): t for t in actual.get("taxa", []) if isinstance(t, dict)} + want = {t["id"]: t for t in expected["taxa"]} + for missing in sorted(set(want) - set(have)): + errors.append(f"{TAXONOMY_FILE} has no taxon for {missing}, which is in {RECORDS_FILE}") + for extra in sorted(set(have) - set(want)): + errors.append(f"{TAXONOMY_FILE} has a taxon for {extra}, which is not in {RECORDS_FILE}") + for common in sorted(set(have) & set(want)): + if have[common] != want[common]: + errors.append(f"{TAXONOMY_FILE} taxon {common} is not what {RECORDS_FILE} produces") + return errors or [f"{TAXONOMY_FILE} is not what scripts/build_taxonomy.py produces from {RECORDS_FILE} (same content, different bytes)"] + + # What: every check on one release directory, in the order they are reported # Why: main() and the tests both call this, so a check dropped from the # pipeline fails a test instead of passing silently -# How: the tag's form, then SHA256SUMS, then the manifest -def release_errors(directory: Path, tag: str, git_dir: Path | None = None, commit: str | None = None) -> list[str]: +# How: the tag's form, then SHA256SUMS, then the manifest, then the taxonomy +def release_errors(directory: Path, tag: str, git_dir: Path | None = None, commit: str | None = None, repo: str = DEFAULT_REPO) -> list[str]: errors = [] if not TAG_PATTERN.fullmatch(tag): errors.append(f"tag {tag!r} is not vX.Y.Z") if commit is not None and not COMMIT_PATTERN.fullmatch(commit): errors.append(f"--commit {commit!r} is not a 40-character commit SHA") commit = None - return errors + check_sums(directory) + check_manifest(directory, tag, git_dir, commit) + needs = (TAXONOMY_FILE,) if (directory / TAXONOMY_FILE).is_file() or taxonomy_required(tag) else () + return errors + check_sums(directory, needs) + check_manifest(directory, tag, git_dir, commit) + check_taxonomy(directory, tag, repo) def main(argv=None) -> int: @@ -163,9 +227,10 @@ def main(argv=None) -> int: source = parser.add_mutually_exclusive_group() source.add_argument("--git-dir", type=Path, help="a clone of aveproject/ave; source_commit must be the tag's commit there") source.add_argument("--commit", help="source_commit must be this full commit SHA (before the tag exists)") + parser.add_argument("--repo", default=DEFAULT_REPO, help="owner/name the release is published under; the taxonomy's downloadUri must point there") args = parser.parse_args(argv) - errors = release_errors(args.directory, args.tag, args.git_dir, args.commit) + errors = release_errors(args.directory, args.tag, args.git_dir, args.commit, args.repo) if errors: for error in errors: print(error) @@ -174,9 +239,14 @@ def main(argv=None) -> int: manifest = json.loads((args.directory / MANIFEST_FILE).read_text(encoding="utf-8")) checked = "the tag's commit" if args.git_dir else ("the given commit" if args.commit else "its form only") + taxonomy = ( + f" ave-taxonomy.json regenerated from the records and matches ({manifest['record_count']} taxa)." + if (args.directory / TAXONOMY_FILE).is_file() + else "" + ) print( f"AVE {args.tag}: {manifest['record_count']} records on schema {manifest['schema_version']}, " - f"checksums match, source_commit {manifest['source_commit']} (checked against {checked})." + f"checksums match, source_commit {manifest['source_commit']} (checked against {checked})." + taxonomy ) return 0 diff --git a/tests/test_catalog_release_workflow.py b/tests/test_catalog_release_workflow.py new file mode 100644 index 0000000..823dfe6 --- /dev/null +++ b/tests/test_catalog_release_workflow.py @@ -0,0 +1,60 @@ +"""The catalog-release workflow builds, verifies, attests and uploads the taxonomy with the other release files.""" + +import re +from pathlib import Path + +WORKFLOW = Path(".github/workflows/catalog-release.yml") + + +def steps(): + text = WORKFLOW.read_text(encoding="utf-8") + parts = re.split(r"(?m)^ - name: ", text)[1:] + return {p.split("\n", 1)[0].strip(): p for p in parts} + + +def step(prefix): + found = [body for name, body in steps().items() if name.startswith(prefix)] + assert len(found) == 1, (prefix, len(found)) + return found[0] + + +def test_the_header_says_the_release_carries_the_taxonomy(): + head = WORKFLOW.read_text(encoding="utf-8").split("\non:")[0] + assert "ave-taxonomy.json" in head and "three data files" in head + + +def test_the_taxonomy_is_built_from_the_release_records_with_the_tag_the_date_and_the_assets_own_url(): + body = step("Build the release files") + assert "python3 scripts/build_taxonomy.py" in body + for needle in ('--records release/ave-records.json', '--release "v$VERSION"', "jq -r '.generated_at[:10]' release/ave-records.manifest.json", + '--download-uri "https://github.com/$GITHUB_REPOSITORY/releases/download/v$VERSION/ave-taxonomy.json"', "--out release/ave-taxonomy.json"): + assert needle in body, needle + + +def test_the_taxonomy_is_built_before_the_checksums_and_the_checksums_cover_it(): + body = step("Build the release files") + assert body.index("build_taxonomy.py") < body.index("sha256sum") + assert "sha256sum ave-records.json ave-records.manifest.json ave-taxonomy.json > SHA256SUMS" in body + + +def test_the_verifier_is_told_which_repository_the_release_is_published_under(): + body = step("Verify the release files") + assert "scripts/check_release_assets.py release" in body and '--repo "$GITHUB_REPOSITORY"' in body and '--tag "v$VERSION"' in body + + +def test_the_taxonomy_is_attested_with_the_other_data_files(): + body = step("Attest build provenance") + for f in ("release/ave-records.json", "release/ave-records.manifest.json", "release/ave-taxonomy.json"): + assert f in body, f + + +def test_the_taxonomy_and_the_checksums_are_uploaded_with_the_release(): + body = step("Create the release") + assert "release/ave-taxonomy.json" in body and "release/SHA256SUMS" in body and "release/ave-records.json" in body + + +def test_no_run_step_interpolates_an_input_directly(): + # inputs reach the shell through env (VERSION, DRAFT) so a crafted input cannot inject commands + text = WORKFLOW.read_text(encoding="utf-8") + runs = re.findall(r"(?ms)^\s+run: (.*?)(?=^\s+- name:|\Z)", text) + assert runs and not any("${{ inputs." in r for r in runs) diff --git a/tests/test_check_release_assets.py b/tests/test_check_release_assets.py index 41ca37b..82f27aa 100644 --- a/tests/test_check_release_assets.py +++ b/tests/test_check_release_assets.py @@ -260,17 +260,17 @@ def test_source_commit_from_another_commit_is_refused_against_the_tag(tmp_path, def test_a_tag_missing_from_the_repository_is_refused(tmp_path, tagged_repo): repo, tagged, _, _ = tagged_repo - release = _release(tmp_path / "release", commit=tagged, release="v9.9.9") + release = _release(tmp_path / "release", commit=tagged, release="v1.2.9") - assert cra.release_errors(release, "v9.9.9", git_dir=repo) == [f"tag v9.9.9 was not found in {repo}"] + assert cra.release_errors(release, "v1.2.9", git_dir=repo) == [f"tag v1.2.9 was not found in {repo}"] def test_a_branch_named_like_the_tag_does_not_stand_in_for_it(tmp_path, tagged_repo): repo, tagged, later, _ = tagged_repo - _git(repo, "branch", "v9.9.9") # a branch, at the later commit, with a tag's name - release = _release(tmp_path / "release", commit=later, release="v9.9.9") + _git(repo, "branch", "v1.2.9") # a branch, at the later commit, with a tag's name + release = _release(tmp_path / "release", commit=later, release="v1.2.9") - assert cra.release_errors(release, "v9.9.9", git_dir=repo) == [f"tag v9.9.9 was not found in {repo}"] + assert cra.release_errors(release, "v1.2.9", git_dir=repo) == [f"tag v1.2.9 was not found in {repo}"] # --- the command line --------------------------------------------------------------- diff --git a/tests/test_release_taxonomy.py b/tests/test_release_taxonomy.py new file mode 100644 index 0000000..2983e25 --- /dev/null +++ b/tests/test_release_taxonomy.py @@ -0,0 +1,182 @@ +"""scripts/check_release_assets.py and the taxonomy asset: a release carries the file the generator produces from its own records.""" + +import hashlib +import json +from pathlib import Path + +import pytest + +from scripts import build_taxonomy as bt +from scripts import check_release_assets as cra + +COMMIT = "b" * 40 +REPO = "aveproject/ave" + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _records(): + return [ + {"ave_id": "AVE-2026-00001", "title": "Class one", "status": "active", "severity": "HIGH", "schema_version": "1.3.0", + "detection_stage": "static_detection", "attack_class": "A - one", "behavioral_fingerprint": "One thing.", "aivss": {"aivss_score": 7.5}}, + {"ave_id": "AVE-2026-00002", "title": "Class two", "status": "rejected", "severity": "MEDIUM", "schema_version": "1.3.0", + "detection_stage": "runtime_observed", "attack_class": "A - two", "behavioral_fingerprint": "Two things.", "aivss": {"aivss_score": 5.0}, + "rejection_reason": "https://example.org/decision"}, + ] + + +def release(directory: Path, tag="v1.3.0", taxonomy=True, sums=True, records=None, **override): + """A consistent release built the way the workflow builds one; `override` replaces fields of the taxonomy.""" + directory.mkdir(parents=True, exist_ok=True) + records = _records() if records is None else records + rpath = directory / "ave-records.json" + rpath.write_text(json.dumps(records, indent=2) + "\n", encoding="utf-8") + manifest = {"release": tag, "schema_version": "1.3.0", "record_count": len(records), "generated_at": "2026-11-05T09:30:00.000Z", + "source": "https://github.com/aveproject/ave", "sha256": _sha(rpath), "source_commit": COMMIT} + (directory / "ave-manifest.tmp").write_text("", encoding="utf-8") + (directory / "ave-manifest.tmp").unlink() + (directory / "ave-records.manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + names = ["ave-records.json", "ave-records.manifest.json"] + if taxonomy: + comp = bt.build(records, release=tag, release_date="2026-11-05", download_uri=f"https://github.com/{REPO}/releases/download/{tag}/ave-taxonomy.json") + comp.update(override) + (directory / "ave-taxonomy.json").write_text(bt.dumps(comp), encoding="utf-8") + names.append("ave-taxonomy.json") + if sums: + (directory / "SHA256SUMS").write_text("".join(f"{_sha(directory / n)} {n}\n" for n in names), encoding="utf-8") + return directory + + +def errors(directory, tag="v1.3.0", **kw): + return cra.release_errors(directory, tag, commit=COMMIT, **kw) + + +def test_a_consistent_release_with_its_taxonomy_passes(tmp_path): + assert errors(release(tmp_path)) == [] + + +def test_a_release_from_v1_3_0_without_the_taxonomy_is_refused(tmp_path): + out = errors(release(tmp_path, taxonomy=False)) + assert any("ave-taxonomy.json is missing" in e for e in out) + + +@pytest.mark.parametrize("tag,required", [("v1.2.9", False), ("v1.3.0", True), ("v1.3.1", True), ("v2.0.0", True), ("v1.10.0", True), ("v0.9.9", False)]) +def test_the_taxonomy_is_required_from_the_first_taxonomy_release_on(tmp_path, tag, required): + out = errors(release(tmp_path, tag=tag, taxonomy=False), tag=tag) + assert any("ave-taxonomy.json is missing" in e for e in out) is required + + +def test_an_earlier_release_with_no_taxonomy_still_passes(tmp_path): + assert errors(release(tmp_path, tag="v1.2.0", taxonomy=False), tag="v1.2.0") == [] + + +def test_an_earlier_release_that_has_a_taxonomy_is_checked_like_any_other(tmp_path): + assert errors(release(tmp_path, tag="v1.2.0"), tag="v1.2.0") == [] + out = errors(release(tmp_path / "bad", tag="v1.2.0", version="9.9.9"), tag="v1.2.0") + assert any("version" in e for e in out) + + +def test_a_taxonomy_missing_from_sha256sums_is_refused(tmp_path): + d = release(tmp_path) + (d / "SHA256SUMS").write_text("".join(f"{_sha(d / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json")), encoding="utf-8") + assert "SHA256SUMS does not list ave-taxonomy.json" in errors(d) + + +def test_a_taxonomy_changed_after_its_checksum_is_refused(tmp_path): + d = release(tmp_path) + (d / "ave-taxonomy.json").write_text((d / "ave-taxonomy.json").read_text(encoding="utf-8") + " ", encoding="utf-8") + assert "SHA256SUMS: ave-taxonomy.json does not match its checksum" in errors(d) + + +@pytest.mark.parametrize("field,value", [("version", "9.9.9"), ("semanticVersion", "9.9.9"), ("releaseDateUtc", "2020-01-01"), ("guid", "00000000-0000-4000-8000-000000000000"), + ("downloadUri", "https://example.org/ave-taxonomy.json"), ("isComprehensive", False), ("name", "Other")]) +def test_a_taxonomy_header_field_that_is_not_the_releases_is_refused(tmp_path, field, value): + out = errors(release(tmp_path, **{field: value})) + assert any(e.startswith(f"ave-taxonomy.json {field} is {value!r}") for e in out), out + + +def test_a_taxonomy_that_points_at_another_repository_is_refused(tmp_path): + out = errors(release(tmp_path), repo="someone/fork") + assert any("downloadUri" in e for e in out) + assert errors(release(tmp_path / "ok", ), repo=REPO) == [] + + +def test_a_missing_taxon_is_refused_and_named(tmp_path): + d = release(tmp_path) + comp = json.loads((d / "ave-taxonomy.json").read_text(encoding="utf-8")); comp["taxa"].pop() + (d / "ave-taxonomy.json").write_text(bt.dumps(comp), encoding="utf-8") + (d / "SHA256SUMS").write_text("".join(f"{_sha(d / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json", "ave-taxonomy.json")), encoding="utf-8") + assert "ave-taxonomy.json has no taxon for AVE-2026-00002, which is in ave-records.json" in errors(d) + + +def test_an_extra_taxon_is_refused_and_named(tmp_path): + d = release(tmp_path) + comp = json.loads((d / "ave-taxonomy.json").read_text(encoding="utf-8")); comp["taxa"].append({"id": "AVE-2026-00099", "name": "Invented"}) + (d / "ave-taxonomy.json").write_text(bt.dumps(comp), encoding="utf-8") + (d / "SHA256SUMS").write_text("".join(f"{_sha(d / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json", "ave-taxonomy.json")), encoding="utf-8") + assert "ave-taxonomy.json has a taxon for AVE-2026-00099, which is not in ave-records.json" in errors(d) + + +def test_a_taxon_that_is_not_what_its_record_produces_is_refused(tmp_path): + d = release(tmp_path) + comp = json.loads((d / "ave-taxonomy.json").read_text(encoding="utf-8")); comp["taxa"][0]["name"] = "Renamed" + (d / "ave-taxonomy.json").write_text(bt.dumps(comp), encoding="utf-8") + (d / "SHA256SUMS").write_text("".join(f"{_sha(d / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json", "ave-taxonomy.json")), encoding="utf-8") + assert "ave-taxonomy.json taxon AVE-2026-00001 is not what ave-records.json produces" in errors(d) + + +def test_the_same_content_in_other_bytes_is_refused(tmp_path): + d = release(tmp_path) + comp = json.loads((d / "ave-taxonomy.json").read_text(encoding="utf-8")) + (d / "ave-taxonomy.json").write_text(json.dumps(comp, indent=4), encoding="utf-8") + (d / "SHA256SUMS").write_text("".join(f"{_sha(d / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json", "ave-taxonomy.json")), encoding="utf-8") + assert any("same content, different bytes" in e for e in errors(d)) + + +def test_a_taxonomy_that_is_not_json_is_refused_not_crashed_on(tmp_path): + d = release(tmp_path) + (d / "ave-taxonomy.json").write_text("{not json", encoding="utf-8") + (d / "SHA256SUMS").write_text("".join(f"{_sha(d / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json", "ave-taxonomy.json")), encoding="utf-8") + assert any("cannot be checked" in e for e in errors(d)) + + +def _with_stand_in_taxonomy(directory: Path, records): + """A release whose taxonomy was written by some other tool, because the generator refuses these records.""" + release(directory, taxonomy=False, records=records) + (directory / "ave-taxonomy.json").write_text("{}\n", encoding="utf-8") + (directory / "SHA256SUMS").write_text("".join(f"{_sha(directory / n)} {n}\n" for n in ("ave-records.json", "ave-records.manifest.json", "ave-taxonomy.json")), encoding="utf-8") + return directory + + +def test_records_the_generator_cannot_use_are_reported_not_crashed_on(tmp_path): + out = errors(_with_stand_in_taxonomy(tmp_path / "dup", _records() + [_records()[0]])) + assert any("cannot be regenerated" in e and "duplicate" in e for e in out) + out = errors(_with_stand_in_taxonomy(tmp_path / "bad", [{"schema_version": "1.3.0"}])) + assert any("cannot be regenerated" in e for e in out) + + +def test_main_passes_a_consistent_release_and_names_what_it_checked(tmp_path, capsys): + d = release(tmp_path) + assert cra.main([str(d), "--tag", "v1.3.0", "--commit", COMMIT]) == 0 + out = capsys.readouterr().out + assert "AVE v1.3.0: 2 records" in out and "ave-taxonomy.json regenerated from the records and matches (2 taxa)" in out + + +def test_main_does_not_claim_a_taxonomy_check_for_a_release_without_one(tmp_path, capsys): + d = release(tmp_path, tag="v1.2.0", taxonomy=False) + assert cra.main([str(d), "--tag", "v1.2.0", "--commit", COMMIT]) == 0 + assert "ave-taxonomy.json" not in capsys.readouterr().out + + +def test_main_fails_a_release_without_its_taxonomy(tmp_path, capsys): + d = release(tmp_path, taxonomy=False) + assert cra.main([str(d), "--tag", "v1.3.0", "--commit", COMMIT]) == 1 + assert "ave-taxonomy.json is missing" in capsys.readouterr().out + + +def test_main_takes_the_repository_the_release_is_published_under(tmp_path, capsys): + d = release(tmp_path) + assert cra.main([str(d), "--tag", "v1.3.0", "--commit", COMMIT, "--repo", "someone/fork"]) == 1 + assert "downloadUri" in capsys.readouterr().out