From 069d07c35771b485715cd6b08ef42fb156a924a3 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sun, 16 Aug 2026 08:55:42 +0800 Subject: [PATCH] =?UTF-8?q?feat(imf):=20Interscript=20Model=20Format=20v1?= =?UTF-8?q?=20=E2=80=94=20spec,=20validator,=20packer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IMF v1 is the versioned portable artifact of interscript-ml: byte-level tokenizer only, ONNX opset 14 (Ruby onnxruntime gem compat), sha256 of every graph verified on load, metrics traceable to RESULTS.md anchors. - docs/imf-v1.md: full schema + validation-level documentation - src/imf: metadata schema, zip validator (base + strict release gate), packer that computes sha256 and self-validates, CLI (python -m imf) - tests: 20 specs over real zips with real ONNX graphs, no doubles - CI: pytest job installs export extras so graph checks run - khm-latn-1.0-fp16 upgraded from the PR #44 zip via 'imf pack' (base-valid; strict pending the WO03 parity gate) - license: BSD-3-Clause for code and model weights (LICENSE added; replaces the placeholder MIT) Found on the way: the fp32 khm zip on secryst-checkpoints:/khmer_byt5 has a CRC error in encoder.onnx — re-export needed (WO02); the fp16 zip is intact (validated). --- .github/workflows/test.yml | 4 +- .gitignore | 1 + LICENSE | 30 ++ README.md | 8 +- docs/imf-v1.md | 132 ++++++++ docs/model_card.md | 2 +- .../khm-latn/khm-latn-1.0-fp16.metadata.yaml | 26 ++ models/khm-latn/khm-latn-1.0.README.md | 42 +++ pyproject.toml | 4 +- src/imf/__init__.py | 25 ++ src/imf/__main__.py | 4 + src/imf/cli.py | 118 +++++++ src/imf/pack.py | 121 ++++++++ src/imf/schema.py | 171 ++++++++++ src/imf/validator.py | 236 ++++++++++++++ tests/test_imf.py | 293 ++++++++++++++++++ 16 files changed, 1210 insertions(+), 7 deletions(-) create mode 100644 LICENSE create mode 100644 docs/imf-v1.md create mode 100644 models/khm-latn/khm-latn-1.0-fp16.metadata.yaml create mode 100644 models/khm-latn/khm-latn-1.0.README.md create mode 100644 src/imf/__init__.py create mode 100644 src/imf/__main__.py create mode 100644 src/imf/cli.py create mode 100644 src/imf/pack.py create mode 100644 src/imf/schema.py create mode 100644 src/imf/validator.py create mode 100644 tests/test_imf.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 535af89..5ff0e36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - name: Install (dev extras only — train/export need GPU + system deps) run: | python -m pip install --upgrade pip - pip install -e ".[dev]" + pip install -e ".[dev,export]" - name: Run pytest run: PYTHONPATH=src python -m pytest tests/ -v - name: Lint (ruff) @@ -38,7 +38,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" - - run: pip install -e ".[dev]" + - run: pip install -e ".[dev,export]" - name: CLI smoke test run: | PYTHONPATH=src python -m src.cli list diff --git a/.gitignore b/.gitignore index ebb38d4..4b942f1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ data/augmented/ # Model artifacts (Git-LFS tracked if committed at all) models/*.onnx +models/**/*.zip models/*.safetensors models/*.ckpt *.tmp diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2c5eac2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2026, Ribose Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index acb4bb6..b76e663 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,11 @@ export tests are gated behind `@pytest.mark.gpu` and require the ## Distribution +Models ship as **IMF v1** zips (Interscript Model Format — spec in +[`docs/imf-v1.md`](./docs/imf-v1.md)): byte-level tokenizer only, ONNX +opset 14, sha256-verified graphs, metrics traceable to `RESULTS.md` +anchors. Build/validate with `PYTHONPATH=src python -m imf pack|validate`. + Models reach end users through three channels (full plan in [`TODO.distribution/`](./TODO.distribution/)): @@ -117,5 +122,4 @@ completes. ## License -MIT. Model weights are released under their own licenses (see -`docs/model_card.md` per task). +BSD-3-Clause, for code and model weights alike (see `LICENSE`). diff --git a/docs/imf-v1.md b/docs/imf-v1.md new file mode 100644 index 0000000..776ffa5 --- /dev/null +++ b/docs/imf-v1.md @@ -0,0 +1,132 @@ +# Interscript Model Format (IMF) v1 + +IMF v1 is the versioned, portable artifact of **interscript-ml** — the +phonological layer of Interscript. A model.zip is adoptable on its own +terms, like ONNX itself: any runtime that can open a zip, sha256 a file, +and run two ONNX sessions can serve the model. Adopting the artifact does +not require adopting our training code. + +The format exists to make three guarantees: + +1. **One runtime everywhere.** v1 supports exactly one tokenizer: raw + UTF-8 bytes (pad=0, EOS=1, ByT5 convention). No vocab files, no + sentencepiece, no per-model tokenization code in Ruby/TS/Python. + Non-byte models enter via distillation (TODO.runtime-arch/07), never + via a second tokenizer system. +2. **Old-runtimes load it.** Opset is pinned to 14 because the Ruby + `onnxruntime` gem bundles an old ORT that cannot load higher opsets + (verified the hard way — secryst PR #44). The validator enforces + graph opset == metadata opset <= 14. +3. **Every number is traceable.** Metrics in metadata must cite a + `RESULTS.md` anchor; the parity block records ONNX-vs-reference + agreement. Numbers that cannot be traced do not ship. + +## Zip layout + +``` +model.zip +├── metadata.yaml # manifest (schema below) — never hand-write the +│ # sha256 block; `imf pack` computes it +├── encoder.onnx # required. inputs: [input_ids], dynamic batch/seq +├── decoder.onnx # required (fallback path). inputs: [input_ids, +│ # encoder_hidden_states] -> [logits] +├── decoder-kv.onnx # optional (default artifact when present). +│ # inputs add past_*; outputs add present_* +└── README.md # required. Usage in all three APIs +``` + +## metadata.yaml schema + +| Field | Type | Constraint | +|---|---|---| +| `format` | str | must be `imf-v1` | +| `id` | str | `-.`, lowercase segments, e.g. `khm-latn-1.0` | +| `task` | enum | `g2p` \| `diacritization` \| `translit` | +| `source_script` | str | ISO 15924 script code (e.g. `Khmr`) | +| `target` | str | target script or scheme (e.g. `Latn`) | +| `tokenizer` | enum | `bytes` (the only v1 value) | +| `opset` | int | 7..14; must equal the graphs' opset | +| `decoder` | enum | `plain` \| `kv` (`kv` requires decoder-kv.onnx) | +| `precision` | enum | `fp32` \| `fp16` \| `int8` | +| `license` | str | non-empty (strict gate) | +| `trained_from` | str | repo + run/checkpoint id | +| `metrics` | list | `{name, value, protocol, source}`; `source` must be a `RESULTS.md#anchor` (strict gate) | +| `parity` | map? | `{samples, cer_delta}`; strict gate: samples >= 500, cer_delta <= 0.2pp | +| `sha256` | map | every `*.onnx` member -> hex digest; no dangling entries | + +The `id` does not encode precision: `khm-latn-1.0-fp16.zip` and +`khm-latn-1.0-int8.zip` share id `khm-latn-1.0`; the model index +(TODO.runtime-arch/08) resolves channel and precision. + +Metrics blocks are generated from `docs/RESULTS.md`, never hand-written +(TODO.runtime-arch/10). Parity is produced by the WO03 gate. + +## Example + +```yaml +format: imf-v1 +id: khm-latn-1.0 +task: translit +source_script: Khmr +target: Latn +tokenizer: bytes +opset: 14 +decoder: plain +precision: fp16 +license: BSD-3-Clause +trained_from: secryst train_khmer_byt5.py run-001 (secryst-checkpoints:/khmer_byt5/run-001/best) +metrics: + - name: cer + value: 27.42 + protocol: "greedy decode; 895 held-out pairs; split 16,120/895/895 seed 42" + source: secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14 + - name: em + value: 59.66 + protocol: "greedy decode; 895 held-out pairs; split 16,120/895/895 seed 42" + source: secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14 +parity: + samples: 500 + cer_delta: 0.03 +sha256: + encoder.onnx: a4a4eb... + decoder.onnx: b54b5c... +``` + +## Validation + +Two levels (`src/imf/validator.py`): + +- **Base** — what every runtime does on load: zip integrity (CRC), + required members, metadata parses, every `.onnx` sha256-verified, + graph opset matches metadata and stays <= 14, decoder contract + (`input_ids` / `encoder_hidden_states` / `past_*`-`present_*` names). +- **Strict** — the release gate: base + non-empty anchored metrics, + parity within thresholds, license present. No zip ships without it. + +CLI (also the CI entry point): + +``` +PYTHONPATH=src python -m imf validate models/khm-latn/khm-latn-1.0-fp16.zip +PYTHONPATH=src python -m imf validate --strict # release gate +PYTHONPATH=src python -m imf info # print manifest +PYTHONPATH=src python -m imf pack --source \ + --metadata [--readme ] --out # sha256 computed +``` + +Legacy notes: + +- Old secryst zips (`vocabs.yaml` + single `transformer.onnx`) and the + PR #44 byte-level zips (`metadata.yaml: name: byt5`) predate IMF. + They fail base validation with a pointer to re-export/upgrade. +- The fp32 Khmer zip on `secryst-checkpoints:/khmer_byt5/` has a CRC + error in `encoder.onnx` (found 2026-08-16, exactly the corrupt-download + class of failure the sha256-on-load rule exists for). It must be + re-exported by the WO02 pipeline; the fp16 zip is intact and was + upgraded to IMF v1 via `imf pack`. + +## Versioning + +Format changes bump the `format` field (`imf-v2`, ...). Model versioning +lives in `id` (`khm-latn-1.1`). Adding an optional member or metadata +field is v1-compatible; anything a v1 runtime would misinterpret is a +new format version. diff --git a/docs/model_card.md b/docs/model_card.md index 545ec67..8f9da07 100644 --- a/docs/model_card.md +++ b/docs/model_card.md @@ -5,7 +5,7 @@ - **Name:** `interscript/rababa_arabic` - **Type:** Character-level transformer (4 layers, 256 dim, 4 heads) - **Trained from:** distilled from Qwen3.5-4B-Instruct teacher -- **License:** MIT (code), CC-BY-4.0 (model weights) +- **License:** BSD-3-Clause (code and weights) ## Training data diff --git a/models/khm-latn/khm-latn-1.0-fp16.metadata.yaml b/models/khm-latn/khm-latn-1.0-fp16.metadata.yaml new file mode 100644 index 0000000..579ed39 --- /dev/null +++ b/models/khm-latn/khm-latn-1.0-fp16.metadata.yaml @@ -0,0 +1,26 @@ +format: imf-v1 +id: khm-latn-1.0 +task: translit +source_script: Khmr +target: Latn +tokenizer: bytes +opset: 14 +decoder: plain +precision: fp16 +license: BSD-3-Clause +trained_from: >- + secryst train_khmer_byt5.py run-001; checkpoint + secryst-checkpoints:/khmer_byt5/run-001/best +metrics: + - name: cer + value: 27.42 + protocol: >- + greedy decode; 895 held-out pairs; split 16,120/895/895 seed 42; + ByT5-small early stop @ep15 + source: secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14 + - name: em + value: 59.66 + protocol: >- + greedy decode; 895 held-out pairs; split 16,120/895/895 seed 42; + ByT5-small early stop @ep15 + source: secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14 diff --git a/models/khm-latn/khm-latn-1.0.README.md b/models/khm-latn/khm-latn-1.0.README.md new file mode 100644 index 0000000..f48713f --- /dev/null +++ b/models/khm-latn/khm-latn-1.0.README.md @@ -0,0 +1,42 @@ +# khm-latn-1.0 (fp16) + +Khmer → Latin transliteration. Byte-level seq2seq (ByT5-small): +the tokenizer is raw UTF-8 bytes (pad=0, EOS=1) — no vocab files. +IMF v1 artifact; format spec: interscript/ml-models docs/imf-v1.md. + +- precision: fp16 (mixed: LayerNorm parameters in fp32) +- decoder: plain greedy (KV-cache variant ships with the WO02 export) +- metrics: CER 27.42 / EM 59.66 on 895 held-out pairs — + secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14 +- trained from: secryst train_khmer_byt5.py run-001 + (secryst-checkpoints:/khmer_byt5/run-001/best) +- license: BSD-3-Clause + +## Usage + +Ruby (secryst gem, the Ruby binding of interscript-ml): + +```ruby +require "secryst" +translator = Secryst::Translator.new(model: "khm-latn-1.0") +translator.translate("ភាសា") +``` + +TypeScript (@interscript/ml): + +```ts +import { loadModel } from "@interscript/ml"; +const model = await loadModel("khm-latn-1.0"); +await model.translate("ភាសា"); +``` + +Python (interscript-ml): + +```python +from interscript_ml import Model +model = Model.load("khm-latn-1.0") +model.translate("ភាសា") +``` + +All three runtimes verify the sha256 of every ONNX member in this zip +against metadata.yaml before loading. diff --git a/pyproject.toml b/pyproject.toml index 3d1ac34..ed00818 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "interscript-ml-models" version = "0.1.0" description = "Unified training framework for Interscript ML models (rababa + secryst)" readme = "README.md" -license = { text = "MIT" } +license = { text = "BSD-3-Clause" } requires-python = ">=3.10" authors = [{ name = "Interscript Project" }] keywords = ["nlp", "transliteration", "diacritization", "onnx", "distillation"] @@ -48,7 +48,7 @@ interscript-ml = "src.cli:main" [tool.setuptools.packages.find] where = ["src"] -include = ["framework*", "tasks*"] +include = ["framework*", "tasks*", "imf*"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/imf/__init__.py b/src/imf/__init__.py new file mode 100644 index 0000000..2f9ae84 --- /dev/null +++ b/src/imf/__init__.py @@ -0,0 +1,25 @@ +"""Interscript Model Format (IMF) v1 — spec, validator, packer.""" + +from imf.pack import PackError, pack_zip +from imf.schema import ( + FORMAT, + MAX_OPSET, + MetadataError, + Metric, + ModelMetadata, + Parity, +) +from imf.validator import ValidationResult, validate_zip + +__all__ = [ + "FORMAT", + "MAX_OPSET", + "MetadataError", + "Metric", + "ModelMetadata", + "PackError", + "Parity", + "ValidationResult", + "pack_zip", + "validate_zip", +] diff --git a/src/imf/__main__.py b/src/imf/__main__.py new file mode 100644 index 0000000..22ef28b --- /dev/null +++ b/src/imf/__main__.py @@ -0,0 +1,4 @@ +from imf.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/imf/cli.py b/src/imf/cli.py new file mode 100644 index 0000000..82e96b6 --- /dev/null +++ b/src/imf/cli.py @@ -0,0 +1,118 @@ +"""``python -m imf`` — validate, inspect, and pack IMF v1 model zips. + +- ``validate [--strict]`` exit 0 iff the zip conforms + (``--strict`` adds the release gate: metrics, parity, thresholds) +- ``info `` print the parsed manifest +- ``pack`` build a conforming zip from a legacy + zip or a directory of graphs + a metadata YAML (sha256 computed here) +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from imf.pack import PackError, pack_zip +from imf.schema import ModelMetadata +from imf.validator import validate_zip + + +def _cmd_validate(args: argparse.Namespace) -> int: + result = validate_zip(args.zip, strict=args.strict) + for warning in result.warnings: + print(f"warn: {warning}") + for error in result.errors: + print(f"error: {error}", file=sys.stderr) + label = "strict " if args.strict else "" + print(f"{args.zip}: {'OK' if result.ok else 'FAILED'} ({label}validation)") + return 0 if result.ok else 1 + + +def _cmd_info(args: argparse.Namespace) -> int: + result = validate_zip(args.zip) + for error in result.errors: + print(f"error: {error}", file=sys.stderr) + if result.metadata is None: + return 1 + m = result.metadata + print(f"id: {m.id}") + print(f"task: {m.task} ({m.source_script} -> {m.target})") + print(f"tokenizer: {m.tokenizer}") + print(f"decoder: {m.decoder}") + print(f"precision: {m.precision}") + print(f"opset: {m.opset}") + print(f"license: {m.license}") + print(f"trained_from: {m.trained_from}") + for metric in m.metrics: + print(f"metric: {metric.name} = {metric.value} [{metric.source}]") + if m.parity is not None: + print(f"parity: cer_delta {m.parity.cer_delta}pp on {m.parity.samples} samples") + else: + print("parity: (not measured)") + for name, digest in sorted(m.sha256.items()): + print(f"sha256: {name} {digest}") + return 0 if result.ok else 1 + + +def _cmd_pack(args: argparse.Namespace) -> int: + metadata = ModelMetadata.from_yaml(Path(args.metadata).read_text(encoding="utf-8")) + readme = ( + Path(args.readme).read_text(encoding="utf-8") if args.readme else _default_readme(metadata) + ) + try: + out = pack_zip(args.source, metadata, readme, args.out) + except PackError as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(f"wrote {out}") + return 0 + + +def _default_readme(metadata: ModelMetadata) -> str: + return ( + f"# {metadata.id}\n\n" + f"{metadata.task} ({metadata.source_script} -> {metadata.target}), " + f"{metadata.precision} precision, byte-level tokenizer, " + f"decoder: {metadata.decoder}.\n\n" + f"Trained from: {metadata.trained_from}\n" + f"License: {metadata.license}\n\n" + "IMF v1 artifact — see the interscript/ml-models docs/imf-v1.md spec.\n" + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="imf", description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + p_validate = sub.add_parser("validate", help="validate a model.zip") + p_validate.add_argument("zip", type=Path) + p_validate.add_argument( + "--strict", action="store_true", help="release gate: metrics + parity thresholds" + ) + p_validate.set_defaults(func=_cmd_validate) + + p_info = sub.add_parser("info", help="print the manifest of a model.zip") + p_info.add_argument("zip", type=Path) + p_info.set_defaults(func=_cmd_info) + + p_pack = sub.add_parser("pack", help="build a conforming model.zip") + p_pack.add_argument( + "--source", required=True, type=Path, + help="directory of .onnx graphs, or a legacy zip to upgrade", + ) + p_pack.add_argument( + "--metadata", required=True, type=Path, + help="metadata YAML (sha256 block is computed and overwritten)", + ) + p_pack.add_argument("--readme", type=Path, help="README.md content for the zip") + p_pack.add_argument("--out", required=True, type=Path) + p_pack.set_defaults(func=_cmd_pack) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + return int(args.func(args)) diff --git a/src/imf/pack.py b/src/imf/pack.py new file mode 100644 index 0000000..1de926c --- /dev/null +++ b/src/imf/pack.py @@ -0,0 +1,121 @@ +"""Build (or upgrade) an IMF v1 model.zip. + +``pack`` takes ONNX graphs from a directory or a legacy zip, a metadata +mapping (sha256 computed here, never by hand), and a README, and writes +a conforming zip. The output is validated before the function returns — +a zip that leaves this function always passes base validation. +""" + +from __future__ import annotations + +import hashlib +import zipfile +from pathlib import Path +from typing import Any + +import yaml + +from imf.schema import ModelMetadata +from imf.validator import ValidationResult, validate_zip + + +class PackError(ValueError): + """Raised when the inputs cannot produce a conforming zip.""" + + +def _read_onnx_sources(source: Path) -> dict[str, bytes]: + """Collect .onnx payloads from a directory or an existing zip.""" + graphs: dict[str, bytes] = {} + if source.is_dir(): + for path in sorted(source.glob("*.onnx")): + graphs[path.name] = path.read_bytes() + elif zipfile.is_zipfile(source): + with zipfile.ZipFile(source) as zf: + for name in sorted(n for n in zf.namelist() if n.endswith(".onnx")): + graphs[name] = zf.read(name) + else: + raise PackError(f"source must be a directory or a zip: {source}") + if "encoder.onnx" not in graphs or "decoder.onnx" not in graphs: + raise PackError("source must contain encoder.onnx and decoder.onnx") + return graphs + + +def _to_dict(metadata: ModelMetadata) -> dict[str, Any]: + data: dict[str, Any] = { + "format": metadata.format, + "id": metadata.id, + "task": metadata.task, + "source_script": metadata.source_script, + "target": metadata.target, + "tokenizer": metadata.tokenizer, + "opset": metadata.opset, + "decoder": metadata.decoder, + "precision": metadata.precision, + "license": metadata.license, + "trained_from": metadata.trained_from, + } + if metadata.metrics: + data["metrics"] = [ + { + "name": m.name, + "value": m.value, + "protocol": m.protocol, + "source": m.source, + } + for m in metadata.metrics + ] + if metadata.parity is not None: + data["parity"] = { + "samples": metadata.parity.samples, + "cer_delta": metadata.parity.cer_delta, + } + data["sha256"] = dict(metadata.sha256) + return data + + +def pack_zip( + source: Path | str, + metadata: ModelMetadata, + readme: str, + out: Path | str, +) -> Path: + source = Path(source) + out = Path(out) + graphs = _read_onnx_sources(source) + + unknown = set(graphs) - {"encoder.onnx", "decoder.onnx", "decoder-kv.onnx"} + if unknown: + raise PackError(f"unexpected .onnx members in source: {sorted(unknown)}") + + final = ModelMetadata( + format=metadata.format, + id=metadata.id, + task=metadata.task, + source_script=metadata.source_script, + target=metadata.target, + tokenizer=metadata.tokenizer, + opset=metadata.opset, + decoder=metadata.decoder, + precision=metadata.precision, + license=metadata.license, + trained_from=metadata.trained_from, + metrics=metadata.metrics, + parity=metadata.parity, + sha256={name: hashlib.sha256(data).hexdigest() for name, data in graphs.items()}, + ) + + out.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr( + "metadata.yaml", + yaml.safe_dump(_to_dict(final), sort_keys=False, allow_unicode=True), + ) + for name in sorted(graphs): + zf.writestr(name, graphs[name]) + zf.writestr("README.md", readme) + + result: ValidationResult = validate_zip(out) + if not result.ok: + out.unlink(missing_ok=True) + raise PackError(f"packed zip failed validation: {result.errors}") + return out diff --git a/src/imf/schema.py b/src/imf/schema.py new file mode 100644 index 0000000..e3e6324 --- /dev/null +++ b/src/imf/schema.py @@ -0,0 +1,171 @@ +"""Interscript Model Format v1 — metadata schema. + +``metadata.yaml`` inside every model.zip. The zip is the portable, +adoptable artifact (like ONNX itself): any runtime that can read a zip, +sha256 a file, and run two ONNX sessions can serve the model — no +Interscript training code required. + +Field-by-field documentation lives in ``docs/imf-v1.md``. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +FORMAT = "imf-v1" + +TASKS = frozenset({"g2p", "diacritization", "translit"}) +DECODERS = frozenset({"plain", "kv"}) +PRECISIONS = frozenset({"fp32", "fp16", "int8"}) + +# The Ruby onnxruntime gem bundles an old ORT that cannot load opset > 14. +# Opset is pinned to 14 and validated against the actual graphs on load. +MAX_OPSET = 14 + +# v1 supports exactly one tokenizer: raw UTF-8 bytes (pad=0, EOS=1). +# Anything else (sentencepiece, BPE, char vocab) must be distilled or +# adapted before it can enter an IMF zip — see TODO.runtime-arch/00. +TOKENIZERS = frozenset({"bytes"}) + +REQUIRED_ONNX = ("encoder.onnx", "decoder.onnx") +OPTIONAL_ONNX = ("decoder-kv.onnx",) + +ID_PATTERN = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*-\d+\.\d+$") + + +class MetadataError(ValueError): + """Raised when metadata.yaml does not conform to IMF v1.""" + + +@dataclass(frozen=True) +class Metric: + """One evaluated number, always traceable to a documented protocol.""" + + name: str + value: float + protocol: str + source: str # e.g. "secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14" + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> Metric: + try: + return cls( + name=str(raw["name"]), + value=float(raw["value"]), + protocol=str(raw["protocol"]), + source=str(raw["source"]), + ) + except KeyError as e: + raise MetadataError(f"metric entry missing field {e}") from e + + +@dataclass(frozen=True) +class Parity: + """ONNX-vs-reference agreement, measured by the gate in WO03.""" + + samples: int + cer_delta: float # percentage points + + MAX_CER_DELTA = 0.2 + MIN_SAMPLES = 500 + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> Parity: + try: + return cls( + samples=int(raw["samples"]), + cer_delta=float(raw["cer_delta"]), + ) + except KeyError as e: + raise MetadataError(f"parity block missing field {e}") from e + + +@dataclass(frozen=True) +class ModelMetadata: + format: str + id: str + task: str + source_script: str + target: str + tokenizer: str + opset: int + decoder: str + precision: str + license: str + trained_from: str + metrics: tuple[Metric, ...] = () + parity: Parity | None = None + sha256: dict[str, str] = field(default_factory=dict) + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> ModelMetadata: + for key in ( + "format", + "id", + "task", + "source_script", + "target", + "tokenizer", + "opset", + "decoder", + "precision", + "license", + "trained_from", + ): + if key not in raw: + raise MetadataError(f"metadata.yaml missing required field: {key}") + + if raw["format"] != FORMAT: + raise MetadataError( + f"unsupported format {raw['format']!r} (expected {FORMAT!r})" + ) + if not ID_PATTERN.match(str(raw["id"])): + raise MetadataError( + f"invalid id {raw['id']!r}: expected e.g. 'khm-latn-1.0' " + "(lowercase segments, trailing major.minor version)" + ) + for field_name, allowed in ( + ("task", TASKS), + ("decoder", DECODERS), + ("precision", PRECISIONS), + ("tokenizer", TOKENIZERS), + ): + if raw[field_name] not in allowed: + raise MetadataError( + f"invalid {field_name} {raw[field_name]!r} (allowed: {sorted(allowed)})" + ) + opset = int(raw["opset"]) + if not 7 <= opset <= MAX_OPSET: + raise MetadataError(f"opset {opset} outside supported range 7..{MAX_OPSET}") + + metrics = tuple(Metric.from_dict(m) for m in raw.get("metrics", [])) + parity = Parity.from_dict(raw["parity"]) if raw.get("parity") is not None else None + sha256 = {str(k): str(v) for k, v in raw.get("sha256", {}).items()} + + return cls( + format=raw["format"], + id=str(raw["id"]), + task=raw["task"], + source_script=str(raw["source_script"]), + target=str(raw["target"]), + tokenizer=raw["tokenizer"], + opset=opset, + decoder=raw["decoder"], + precision=raw["precision"], + license=str(raw["license"]), + trained_from=str(raw["trained_from"]), + metrics=metrics, + parity=parity, + sha256=sha256, + ) + + @classmethod + def from_yaml(cls, text: str) -> ModelMetadata: + import yaml + + raw = yaml.safe_load(text) + if not isinstance(raw, dict): + raise MetadataError("metadata.yaml must be a mapping at top level") + return cls.from_dict(raw) diff --git a/src/imf/validator.py b/src/imf/validator.py new file mode 100644 index 0000000..e1e01cf --- /dev/null +++ b/src/imf/validator.py @@ -0,0 +1,236 @@ +"""IMF v1 zip validator. + +Two levels: + +- base: the zip is structurally a valid IMF v1 artifact — required files + present, metadata parses, every ``*.onnx`` member is sha256-verified, + recorded opset matches the graphs (and is <= 14 for Ruby gem compat). + This is what a runtime does on every load. + +- strict (release gate): base + the zip is shippable — metrics with + anchored sources, parity block present and within the WO03 thresholds. + No zip is released without passing this. +""" + +from __future__ import annotations + +import hashlib +import zipfile +from dataclasses import dataclass, field +from pathlib import Path + +from imf.schema import ( + MAX_OPSET, + MetadataError, + ModelMetadata, + Parity, +) + +_SHA256_BUF_SIZE = 1024 * 1024 + + +@dataclass +class ValidationResult: + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + metadata: ModelMetadata | None = None + + @property + def ok(self) -> bool: + return not self.errors + + def error(self, message: str) -> None: + self.errors.append(message) + + def warn(self, message: str) -> None: + self.warnings.append(message) + + +def _sha256_member(zf: zipfile.ZipFile, name: str) -> str: + digest = hashlib.sha256() + with zf.open(name) as fh: + while chunk := fh.read(_SHA256_BUF_SIZE): + digest.update(chunk) + return digest.hexdigest() + + +def _graph_opsets(data: bytes) -> dict[str, int]: + import onnx + + model = onnx.load_model_from_string(data) + return {op.domain or "ai.onnx": op.version for op in model.opset_import} + + +def _graph_io_names(data: bytes) -> tuple[list[str], list[str]]: + import onnx + + model = onnx.load_model_from_string(data) + inputs = [i.name for i in model.graph.input] + outputs = [o.name for o in model.graph.output] + return inputs, outputs + + +def _check_encoder_contract(result: ValidationResult, data: bytes, opset: int) -> None: + inputs, _ = _graph_io_names(data) + if inputs != ["input_ids"]: + result.error( + f"encoder.onnx inputs must be exactly ['input_ids'] (got {inputs})" + ) + graph_opsets = _graph_opsets(data) + graph_opset = graph_opsets.get("ai.onnx") + if graph_opset != opset: + result.error( + f"encoder.onnx graph opset {graph_opset} != metadata opset {opset}" + ) + if graph_opset is not None and graph_opset > MAX_OPSET: + result.error( + f"encoder.onnx graph opset {graph_opset} > {MAX_OPSET}: " + "the Ruby onnxruntime gem cannot load it" + ) + + +def _check_decoder_contract( + result: ValidationResult, data: bytes, opset: int, kv: bool +) -> None: + inputs, outputs = _graph_io_names(data) + expected = ["input_ids", "encoder_hidden_states"] + if kv: + has_past = any(i.startswith("past_") for i in inputs) + has_present = any(o.startswith("present_") for o in outputs) + if not has_past or not has_present: + result.error( + "decoder-kv.onnx must take past_* inputs and emit present_* outputs " + f"(got inputs={inputs}, outputs={outputs})" + ) + else: + if inputs != expected: + result.error( + f"decoder.onnx inputs must be exactly {expected} (got {inputs})" + ) + graph_opsets = _graph_opsets(data) + graph_opset = graph_opsets.get("ai.onnx") + if graph_opset != opset: + name = "decoder-kv.onnx" if kv else "decoder.onnx" + result.error(f"{name} graph opset {graph_opset} != metadata opset {opset}") + if graph_opset is not None and graph_opset > MAX_OPSET: + name = "decoder-kv.onnx" if kv else "decoder.onnx" + result.error(f"{name} graph opset {graph_opset} > {MAX_OPSET}") + + +def validate_zip(path: Path | str, strict: bool = False) -> ValidationResult: + """Validate a model.zip. Never raises on invalid content; collects errors.""" + result = ValidationResult() + path = Path(path) + + if not path.is_file(): + result.error(f"not a file: {path}") + return result + + try: + zf = zipfile.ZipFile(path) + except zipfile.BadZipFile as e: + result.error(f"not a valid zip: {e}") + return result + + with zf: + bad_member = zf.testzip() + if bad_member is not None: + result.error(f"corrupt member (CRC mismatch): {bad_member}") + + names = zf.namelist() + + if "vocabs.yaml" in names: + result.error( + "legacy secryst zip (vocabs.yaml): IMF v1 is byte-tokenizer only; " + "re-export from the training checkpoint" + ) + return result + + for required in ("metadata.yaml", "encoder.onnx", "decoder.onnx", "README.md"): + if required not in names: + result.error(f"missing required file: {required}") + if not result.ok: + return result + + try: + raw = zf.read("metadata.yaml").decode("utf-8") + except UnicodeDecodeError as e: + result.error(f"metadata.yaml is not valid UTF-8: {e}") + return result + + try: + metadata = ModelMetadata.from_yaml(raw) + except MetadataError as e: + result.error(f"metadata.yaml: {e}") + return result + result.metadata = metadata + + onnx_members = sorted(n for n in names if n.endswith(".onnx")) + for name in onnx_members: + recorded = metadata.sha256.get(name) + if recorded is None: + result.error(f"{name} is not covered by the metadata sha256 block") + continue + actual = _sha256_member(zf, name) + if actual != recorded: + result.error( + f"{name} sha256 mismatch: zip has {actual}, metadata says {recorded}" + ) + for name in metadata.sha256: + if name not in names: + result.error(f"sha256 block references missing file: {name}") + + has_kv = "decoder-kv.onnx" in names + if metadata.decoder == "kv" and not has_kv: + result.error("metadata declares decoder: kv but decoder-kv.onnx is missing") + if has_kv and metadata.decoder != "kv": + result.warn( + "decoder-kv.onnx present but metadata decoder is " + f"'{metadata.decoder}'; the kv graph will not be selected" + ) + + try: + import onnx # noqa: F401 + except ImportError: + result.warn("onnx package not installed: graph contracts not verified") + else: + try: + _check_encoder_contract( + result, zf.read("encoder.onnx"), metadata.opset + ) + _check_decoder_contract( + result, zf.read("decoder.onnx"), metadata.opset, kv=False + ) + if has_kv: + _check_decoder_contract( + result, zf.read("decoder-kv.onnx"), metadata.opset, kv=True + ) + except Exception as e: # onnx parse failure of a hashed member + result.error(f"failed to parse ONNX graph: {e}") + + if strict: + if not metadata.metrics: + result.error("strict: metrics block is empty") + for metric in metadata.metrics: + if "#" not in metric.source: + result.error( + f"strict: metric {metric.name!r} source lacks a " + "RESULTS.md anchor" + ) + if metadata.parity is None: + result.error("strict: parity block is missing (run the WO03 gate)") + else: + if metadata.parity.cer_delta > Parity.MAX_CER_DELTA: + result.error( + f"strict: parity cer_delta {metadata.parity.cer_delta}pp " + f"exceeds {Parity.MAX_CER_DELTA}pp" + ) + if metadata.parity.samples < Parity.MIN_SAMPLES: + result.error( + f"strict: parity measured on {metadata.parity.samples} " + f"samples, fewer than {Parity.MIN_SAMPLES}" + ) + if not metadata.license.strip(): + result.error("strict: license is empty") + + return result diff --git a/tests/test_imf.py b/tests/test_imf.py new file mode 100644 index 0000000..32d44f6 --- /dev/null +++ b/tests/test_imf.py @@ -0,0 +1,293 @@ +"""Tests for ``imf`` — schema, validator, packer. + +All fixtures are real zips containing real (tiny) ONNX graphs built with +the onnx package; no mocks. The onnx-dependent tests are skipped when +onnx is not installed (base validation still runs everywhere). +""" + +from __future__ import annotations + +import hashlib +import zipfile +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from imf.pack import PackError, pack_zip +from imf.schema import ModelMetadata +from imf.validator import validate_zip + +onnx = pytest.importorskip("onnx", reason="graph checks need onnx") +import numpy as np # noqa: E402 +from onnx import TensorProto, helper, numpy_helper # noqa: E402 + + +def _graph_bytes( + inputs: list[tuple[str, list[str]]], + outputs: list[tuple[str, list[str]]], + opset: int = 14, +) -> bytes: + graph = helper.make_graph( + nodes=[helper.make_node("Add", [inputs[0][0], "bias"], [outputs[0][0]])], + name="tiny", + inputs=[ + helper.make_tensor_value_info(n, TensorProto.INT64, d) for n, d in inputs + ], + outputs=[ + helper.make_tensor_value_info(n, TensorProto.INT64, d) for n, d in outputs + ], + initializer=[numpy_helper.from_array(np.zeros(1, dtype=np.int64), "bias")], + ) + model = helper.make_model( + graph, + opset_imports=[helper.make_opsetid("", opset)], + ir_version=7, + ) + return model.SerializeToString() + + +def _encoder_bytes(opset: int = 14) -> bytes: + return _graph_bytes( + inputs=[("input_ids", ["batch", "seq"])], + outputs=[("last_hidden_state", ["batch", "seq", "d"])], + opset=opset, + ) + + +def _decoder_bytes(opset: int = 14, kv: bool = False) -> bytes: + inputs = [("input_ids", ["batch", "seq"]), ("encoder_hidden_states", ["batch", "seq", "d"])] + outputs = [("logits", ["batch", "seq", "vocab"])] + if kv: + inputs.append(("past_key_0", ["batch", "heads", "seq", "dk"])) + outputs.append(("present_key_0", ["batch", "heads", "seq", "dk"])) + return _graph_bytes(inputs=inputs, outputs=outputs, opset=opset) + + +METADATA: dict[str, Any] = { + "format": "imf-v1", + "id": "khm-latn-1.0", + "task": "translit", + "source_script": "Khmr", + "target": "Latn", + "tokenizer": "bytes", + "opset": 14, + "decoder": "plain", + "precision": "fp16", + "license": "BSD-3-Clause", + "trained_from": "secryst train_khmer_byt5.py run-001", + "metrics": [ + { + "name": "cer", + "value": 27.42, + "protocol": "greedy decode, 895 held-out pairs, split seed 42", + "source": "secryst/docs/RESULTS.md#khmer-transliteration-2026-08-14", + } + ], + "parity": {"samples": 500, "cer_delta": 0.03}, +} + + +def _write_zip( + path: Path, + metadata: dict[str, Any] | None = None, + encoder: bytes | None = None, + decoder: bytes | None = None, + extra: dict[str, bytes] | None = None, + include_readme: bool = True, +) -> Path: + meta = dict(metadata if metadata is not None else METADATA) + members: dict[str, bytes] = { + "encoder.onnx": encoder if encoder is not None else _encoder_bytes(), + "decoder.onnx": decoder if decoder is not None else _decoder_bytes(), + } + if extra: + members.update(extra) + if "sha256" not in meta: + meta["sha256"] = { + name: hashlib.sha256(data).hexdigest() for name, data in members.items() + } + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("metadata.yaml", yaml.safe_dump(meta, sort_keys=False)) + for name, data in members.items(): + zf.writestr(name, data) + if include_readme: + zf.writestr("README.md", "# model\n") + return path + + +def test_valid_zip_passes_base_and_strict(tmp_path: Path) -> None: + z = _write_zip(tmp_path / "m.zip") + assert validate_zip(z).ok + assert validate_zip(z, strict=True).ok + + +def test_missing_encoder_fails(tmp_path: Path) -> None: + path = tmp_path / "m.zip" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("metadata.yaml", yaml.safe_dump(METADATA)) + zf.writestr("decoder.onnx", _decoder_bytes()) + zf.writestr("README.md", "# model\n") + result = validate_zip(path) + assert not result.ok + assert any("encoder.onnx" in e for e in result.errors) + + +def test_sha256_mismatch_fails(tmp_path: Path) -> None: + z = _write_zip(tmp_path / "m.zip") + tampered = tmp_path / "tampered.zip" + with zipfile.ZipFile(z) as src, zipfile.ZipFile(tampered, "w") as dst: + for name in src.namelist(): + data = src.read(name) + if name == "encoder.onnx": + data = data[:-1] + bytes([data[-1] ^ 0xFF]) + dst.writestr(name, data) + result = validate_zip(tampered) + assert not result.ok + assert any("sha256 mismatch" in e for e in result.errors) + + +def test_unhashed_onnx_member_fails(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["sha256"] = {} + z = _write_zip(tmp_path / "m.zip", metadata=meta) + result = validate_zip(z) + assert not result.ok + assert any("not covered" in e for e in result.errors) + + +def test_dangling_sha256_entry_fails(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["sha256"] = {"decoder-kv.onnx": "0" * 64} + z = _write_zip(tmp_path / "m.zip", metadata=meta) + result = validate_zip(z) + assert any("missing file" in e for e in result.errors) + + +def test_opset_mismatch_between_metadata_and_graph_fails(tmp_path: Path) -> None: + z = _write_zip(tmp_path / "m.zip", encoder=_encoder_bytes(opset=13)) + result = validate_zip(z) + assert any("graph opset 13" in e for e in result.errors) + + +def test_opset_above_14_fails_ruby_compat(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["opset"] = 15 + z = _write_zip(tmp_path / "m.zip", metadata=meta, encoder=_encoder_bytes(opset=15)) + result = validate_zip(z) + assert any("opset" in e and "14" in e for e in result.errors) + + +def test_non_bytes_tokenizer_rejected_by_schema() -> None: + meta = dict(METADATA) + meta["tokenizer"] = "sentencepiece" + with pytest.raises(Exception, match="tokenizer"): + ModelMetadata.from_dict(meta) + + +def test_legacy_vocab_zip_fails_with_clear_error(tmp_path: Path) -> None: + path = tmp_path / "legacy.zip" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("vocabs.yaml", "input: [a, b]\n") + zf.writestr("transformer.onnx", b"not-really") + result = validate_zip(path) + assert any("legacy" in e for e in result.errors) + + +def test_decoder_kv_declared_but_missing_fails(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["decoder"] = "kv" + z = _write_zip(tmp_path / "m.zip", metadata=meta) + result = validate_zip(z) + assert any("decoder-kv.onnx" in e for e in result.errors) + + +def test_kv_graph_without_past_present_fails(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["decoder"] = "kv" + z = _write_zip( + tmp_path / "m.zip", + metadata=meta, + extra={"decoder-kv.onnx": _decoder_bytes(kv=False)}, + ) + result = validate_zip(z) + assert any("past_" in e for e in result.errors) + + +def test_strict_gate_requires_parity_and_metrics(tmp_path: Path) -> None: + meta = dict(METADATA) + meta.pop("parity") + meta.pop("metrics") + z = _write_zip(tmp_path / "m.zip", metadata=meta) + assert validate_zip(z).ok + strict = validate_zip(z, strict=True) + assert any("parity" in e for e in strict.errors) + assert any("metrics" in e for e in strict.errors) + + +def test_strict_gate_rejects_high_cer_delta(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["parity"] = {"samples": 500, "cer_delta": 0.5} + z = _write_zip(tmp_path / "m.zip", metadata=meta) + strict = validate_zip(z, strict=True) + assert any("cer_delta" in e for e in strict.errors) + + +def test_strict_gate_rejects_small_parity_sample(tmp_path: Path) -> None: + meta = dict(METADATA) + meta["parity"] = {"samples": 100, "cer_delta": 0.01} + z = _write_zip(tmp_path / "m.zip", metadata=meta) + strict = validate_zip(z, strict=True) + assert any("samples" in e for e in strict.errors) + + +def test_pack_from_directory_and_roundtrip(tmp_path: Path) -> None: + graphs = tmp_path / "graphs" + graphs.mkdir() + (graphs / "encoder.onnx").write_bytes(_encoder_bytes()) + (graphs / "decoder.onnx").write_bytes(_decoder_bytes()) + metadata = ModelMetadata.from_dict(METADATA) + out = pack_zip(graphs, metadata, "# packed\n", tmp_path / "packed.zip") + assert validate_zip(out, strict=True).ok + + +def test_pack_from_legacy_zip_upgrades_conformance(tmp_path: Path) -> None: + legacy = tmp_path / "legacy.zip" + with zipfile.ZipFile(legacy, "w") as zf: + zf.writestr("metadata.yaml", "name: byt5\n") + zf.writestr("encoder.onnx", _encoder_bytes()) + zf.writestr("decoder.onnx", _decoder_bytes()) + metadata = ModelMetadata.from_dict(METADATA) + out = pack_zip(legacy, metadata, "# upgraded\n", tmp_path / "upgraded.zip") + result = validate_zip(out) + assert result.ok + assert result.metadata is not None + assert set(result.metadata.sha256) == {"encoder.onnx", "decoder.onnx"} + + +def test_pack_rejects_unknown_onnx_members(tmp_path: Path) -> None: + legacy = tmp_path / "legacy.zip" + with zipfile.ZipFile(legacy, "w") as zf: + zf.writestr("encoder.onnx", _encoder_bytes()) + zf.writestr("decoder.onnx", _decoder_bytes()) + zf.writestr("bonus.onnx", _encoder_bytes()) + with pytest.raises(PackError, match="bonus.onnx"): + pack_zip(legacy, ModelMetadata.from_dict(METADATA), "# x\n", tmp_path / "o.zip") + + +def test_cli_validate_and_info(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + from imf.cli import main + + z = _write_zip(tmp_path / "m.zip") + assert main(["validate", str(z)]) == 0 + assert main(["validate", str(z), "--strict"]) == 0 + assert main(["info", str(z)]) == 0 + out = capsys.readouterr().out + assert "khm-latn-1.0" in out + assert "cer" in out + + bad_meta = dict(METADATA) + bad_meta["sha256"] = {} + bad = _write_zip(tmp_path / "bad.zip", metadata=bad_meta) + assert main(["validate", str(bad)]) == 1