From 1d69334735f366cae6c1ceb7b50514212cdff9eb Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 18:23:12 -0800 Subject: [PATCH 1/5] pins: goldens move with their pins; per-site gaps that kept bot PRs red Golden substitution skipped any old pin string that a file outside tests/ still held: a docstring example in hack/pinver.py, a prose note in integrations/eas/pins.json, a bench script. So the k3s tag in the audit goldens, the compute-runtime version in host_setup_test.sh and the Hugging Face revisions in the model optimizer fixtures never moved, and those bumps went red. A test now keeps an old string only when another pin's site still holds it, when the test names a file outside tests/ that still holds it (it reads that record), or when it is a bare version in test code, where it is as likely the test's own input. Prose and records no test names hold nothing back. Per-site fixes: - tests/audit_test.sh reads the k3s tag from the Dockerfile and moves to versions no upstream will publish (v9.9.9-k3s1, cuda 99.9.9); 13.5.0 is a real release now. - tests/bench_vs_test.sh reads each stack's image from bench/competitors.lock. - tests/host_setup_test.sh reads compute-runtime, IGC, level-zero and the NPU archive from the launcher, as it already did for the PPA builds. - hf-meta hook (group hf-console-fallback): each moved repo's record in the saved metadata bundles under tests/fixtures/hf is read again at the new revision. - pins/registry.toml derives the launcher's vver= for vllm-xpu, llm-scaler and gaudi-vllm; tests/pins_test.py checks every canary package constant has a site. - packaging/check.sh verifies the unsigned rpm with --nosignature: RPM 6.1 in Fedora 45 enforces signatures on -K and -i. --- hack/bump | 66 ++++++++++++++-- hack/pinlib/goldens.py | 128 ++++++++++++++++++++++--------- hack/substitute-goldens | 9 ++- packaging/check.sh | 6 +- pins/README.md | 27 ++++++- pins/registry.toml | 8 ++ tests/audit_test.sh | 11 ++- tests/bench_vs_test.sh | 15 +++- tests/bump_test.sh | 157 ++++++++++++++++++++++++++++++++++++++- tests/host_setup_test.sh | 41 ++++++---- tests/pins_test.py | 26 +++++++ 11 files changed, 420 insertions(+), 74 deletions(-) diff --git a/hack/bump b/hack/bump index 0b4635fa..ca02faf3 100755 --- a/hack/bump +++ b/hack/bump @@ -12,9 +12,11 @@ pin is moved by its group's hook and checked after it). 3. Hooks, in the group's order: lock (images.lock), readme-history, render-latest, ppa-write, convert-pins, ledger-rules, constraints, docs-lock, flake-update, - repin (the digests group); ledger-rules also runs whenever console/ changed. + repin (the digests group), hf-meta (each moved Hugging Face repo's record in + the saved metadata bundles, tests/fixtures/hf, read again at its new revision); + ledger-rules also runs whenever console/ changed. 4. hack/substitute-goldens: each exact old pin string becomes its new one in - tests/** only. + tests/** only (hack/pinlib/goldens.py says which strings a test keeps). 5. The group's tests (skipped by --dry-run and --no-tests). 6. The policy verdict (hack/pins policy) of the resulting diff, and a PR body. @@ -47,10 +49,10 @@ import tomllib sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from pinlib import goldens, ledgerinfo, lockfile, oci, policy, registry, sources # noqa: E402 +from pinlib import goldens, http, ledgerinfo, lockfile, oci, policy, registry, sources # noqa: E402 HOOKS = ("lock", "readme-history", "render-latest", "ppa-write", "convert-pins", "ledger-rules", "constraints", - "docs-lock", "flake-update", "repin") + "docs-lock", "flake-update", "repin", "hf-meta") class BumpError(Exception): @@ -296,10 +298,61 @@ def hook_repin(root, ctx): ctx["touched"].update(path for path, *_ in changes) +HF_FIXTURES = "tests/fixtures/hf" + + +def _hf_get(url, rng=None): + """console/model_opt.py's get (bytes, None on 401 or 404) through pinlib.http, so + PINLIB_FIXTURES serves it in tests.""" + try: + return http.request(url, {"Range": "bytes=%d-%d" % rng} if rng else None).body + except http.HTTPError as e: + if e.status in (401, 404): + return None + raise BumpError("hf-meta: %s" % e) from None + + +def hook_hf_meta(root, ctx): + """Each saved metadata bundle (tests/fixtures/hf) that records a moved Hugging Face + repo at its old revision gets that repo's record as Hugging Face serves it at the + new one, read by console/model_opt.py's own repo_record; the other records stay.""" + moved = {p["source"]["model"]: (old, new) for p, old, new in ctx["moves"] + if p.get("source", {}).get("kind") == "hf" and old and new and old != new} + d = os.path.join(root, HF_FIXTURES) + if not moved or not os.path.isdir(d): + return + keep, sys.dont_write_bytecode = sys.dont_write_bytecode, True # no __pycache__ in the bumped tree + try: + mo = oci.load_path("model_opt_bump", os.path.join(root, "console", "model_opt.py")) + finally: + sys.dont_write_bytecode = keep + for name in sorted(os.listdir(d)): + if not name.endswith(".json"): + continue + path = os.path.join(d, name) + with open(path, encoding="utf-8") as f: + bundle = json.load(f) + repos = bundle.get("repos") or {} + hit = False + for rid, (old, new) in sorted(moved.items()): + if (repos.get(rid) or {}).get("sha") != old: + continue + rec = mo.repo_record(_hf_get, rid, new, target=rid in (bundle.get("requested"), bundle.get("target"))) + if rec is None or rec.get("sha") != new: + raise BumpError("hf-meta: Hugging Face has no %s at %s" % (rid, new)) + repos[rid], hit = rec, True + ctx["notes"].append("hf-meta: %s/%s %s %s -> %s" % (HF_FIXTURES, name, rid, old[:8], new[:8])) + if hit: + with open(path + ".tmp", "w", encoding="utf-8") as f: + f.write(json.dumps(bundle, indent=1, sort_keys=True) + "\n") + os.replace(path + ".tmp", path) + ctx["touched"].add("%s/%s" % (HF_FIXTURES, name)) + + HOOK_FNS = {"lock": hook_lock, "readme-history": hook_readme_history, "render-latest": hook_render_latest, "ppa-write": hook_ppa_write, "convert-pins": hook_convert_pins, "ledger-rules": hook_ledger_rules, "constraints": hook_constraints, "docs-lock": hook_docs_lock, "flake-update": hook_flake_update, - "repin": hook_repin} + "repin": hook_repin, "hf-meta": hook_hf_meta} # ---- known issues ------------------------------------------------------------------- @@ -509,7 +562,8 @@ def bump(root, group, reason=None, dry_run=False, resolved=None, run_tests=True, continue edits += edits_for(p, spans, cur, r) pairs = apply_edits(root, tree, edits) - ctx = {"touched": {s.file for s, _ in edits}, "edits": edits, "pins": [m[0] for m in moves], "notes": notes} + ctx = {"touched": {s.file for s, _ in edits}, "edits": edits, "pins": [m[0] for m in moves], "notes": notes, + "moves": [(p, cur.get("v"), r.v) for p, _, cur, r in moves]} for h in hooks: if h not in HOOK_FNS: raise BumpError("group %s: unknown hook %r" % (group, h)) diff --git a/hack/pinlib/goldens.py b/hack/pinlib/goldens.py index 7f565702..c661c6e0 100644 --- a/hack/pinlib/goldens.py +++ b/hack/pinlib/goldens.py @@ -2,16 +2,29 @@ A test that names a pinned value (a golden transcript, a recorded image list) moves with the pin. Only tests/** is written; never a ledger, historical, arm or -eat-stock path, nor .github/. Idempotent: a second run with the same pairs -changes nothing, even when an old string is part of its new one (1.37 -> 1.37.1). +eat-stock path, nor .github/. A pair leaves a test alone when + - another pin's site outside tests/ still holds the old string (two images at + one digest: the test may check the other one); + - the test names a file outside tests/ that still holds it (it reads that + record: a ledger arm's or a historical run's pin); + - the old string is a bare version (a site's value alone, no image name, no + key) and the test is code (.py, .sh or a #! script): there it is as likely + the test's own input, as in a version-compare case. In a recorded file (a + golden transcript, a fixture) it moves. +Prose and records that no test names hold nothing back. Idempotent: a second run +with the same pairs changes nothing, even when an old string is part of its new +one (1.37 -> 1.37.1). """ import os +import re import subprocess from . import classes MASK = "\x00{%d}\x00" +BARE = re.compile(r"[A-Za-z0-9._+~-]+") +HEX = re.compile(r"(sha256:)?[0-9a-f]{40,64}") def substitute_text(text, pairs): @@ -28,51 +41,98 @@ def substitute_text(text, pairs): return masked -def test_files(root): +def _tracked(root): try: - out = subprocess.run(["git", "-C", root, "ls-files", "-z", "tests"], capture_output=True, check=True).stdout - paths = [p for p in out.decode().split("\0") if p] + out = subprocess.run(["git", "-C", root, "ls-files", "-z"], capture_output=True, check=True).stdout + return sorted(p for p in out.decode().split("\0") if p) except (OSError, subprocess.CalledProcessError): paths = [] - for d, _, fs in os.walk(os.path.join(root, "tests")): - paths += [os.path.relpath(os.path.join(d, f), root) for f in fs] - return sorted(p for p in paths if p.startswith("tests/") and classes.writable(root, p)) + for d, ds, fs in os.walk(root): + ds[:] = [x for x in ds if x != ".git"] + paths += [os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/") for f in fs] + return sorted(paths) + + +def _read(root, path): + try: + with open(os.path.join(root, path), encoding="utf-8") as f: + return f.read() + except (UnicodeDecodeError, FileNotFoundError, IsADirectoryError, NotADirectoryError): + return None + + +def test_files(root): + return sorted(p for p in _tracked(root) if p.startswith("tests/") and classes.writable(root, p)) + + +def is_code(path, text): + """A test script, as opposed to a recorded file (a golden, a fixture).""" + return path.endswith((".py", ".sh")) or text.startswith("#!") + + +def is_bare(s): + """A version or tag on its own (v1.37.1-k3s1, release-1.1.0), not a digest, + a revision or a string with an image name or key around it.""" + return bool(BARE.fullmatch(s)) and not HEX.fullmatch(s) + + +def site_texts(root): + """Every string a registry pin's site outside tests/ holds now: each site's + whole match and each of its values.""" + from . import registry + path = os.path.join(root, "pins", "registry.toml") + if not os.path.exists(path): + return set() + tree = registry.Tree(root) + _, _, spans_by = registry.all_pins(tree, registry.load(path)) + out = set() + for spans in spans_by.values(): + for s in spans: + if s.file.startswith("tests/"): + continue + out.add(s.text) + if s.match: + out.add(tree.read(s.file)[s.match[0]:s.match[1]]) + return out def live_pairs(root, pairs): - """The pairs whose old string no file outside tests/ holds any more (prose and - recorded data aside). An old string still pinned elsewhere (a ledger arm, a - historical run) may be what a test checks, so it stays.""" - from . import oci - texts = [] - for p in oci.images_lock().scanned_files(root): - if p.startswith("tests/"): - continue - try: - with open(os.path.join(root, p), encoding="utf-8") as f: - texts.append(f.read()) - except (UnicodeDecodeError, FileNotFoundError, IsADirectoryError): - continue - blob = "\x00".join(texts) - return [(o, n) for o, n in pairs if o and o != n and o not in blob] + """The pairs whose old string no pin site outside tests/ holds any more.""" + held = site_texts(root) + return [(o, n) for o, n in pairs if o and o != n and o not in held] def substitute(root, pairs, paths=None): """Apply pairs to tests/** (or the given paths, still only under tests/); the - paths changed. Only pairs whose old string is gone from the rest of the repo.""" + paths changed. See the module docstring for the pairs a test keeps.""" pairs = live_pairs(root, pairs) - changed = [] + tests = {} for p in paths if paths is not None else test_files(root): - if not p.startswith("tests/") or not classes.writable(root, p): - continue - full = os.path.join(root, p) - try: - with open(full, encoding="utf-8") as f: - text = f.read() - except (UnicodeDecodeError, FileNotFoundError, IsADirectoryError): - continue - new = substitute_text(text, pairs) + if p.startswith("tests/") and classes.writable(root, p): + text = _read(root, p) + if text is not None: + tests[p] = text + # the files outside tests/ that still hold each old string some test holds + wanted = {o for o, _ in pairs if any(o in t for t in tests.values())} + holders = {o: [] for o in wanted} + if wanted: + for p in _tracked(root): + if p.startswith("tests/"): + continue + text = _read(root, p) + if text is None: + continue + for o in wanted: + if o in text: + holders[o].append(p) + changed = [] + for p, text in sorted(tests.items()): + code = is_code(p, text) + mine = [(o, n) for o, n in pairs if o in text and not (code and is_bare(o)) + and not any(h in text for h in holders.get(o, ()))] + new = substitute_text(text, mine) if new != text: + full = os.path.join(root, p) with open(full + ".tmp", "w", encoding="utf-8") as f: f.write(new) os.chmod(full + ".tmp", os.stat(full).st_mode) diff --git a/hack/substitute-goldens b/hack/substitute-goldens index af683b6e..7d6cc2bc 100755 --- a/hack/substitute-goldens +++ b/hack/substitute-goldens @@ -5,10 +5,11 @@ A test that names a pinned value (a golden transcript, a recorded image list) moves with the pin. Only tests/** is written: never a ledger, historical, arm or -eat-stock path, nor .github/. A pair applies only once no file outside tests/ -holds its old string any more (one still pinned elsewhere may be what a test -checks). Idempotent: a second run changes nothing, even when an old string is -part of its new one (1.37 -> 1.37.1). hack/bump runs it after every bump. +eat-stock path, nor .github/. A test keeps an old string that another pin's site +still holds, that a file outside tests/ it names still holds (a ledger run it +reads), or that is a bare version in test code (hack/pinlib/goldens.py, pins/README.md). +Idempotent: a second run changes nothing, even when an old string is part of its +new one (1.37 -> 1.37.1). hack/bump runs it after every bump. --pairs FILE: a JSON list of [old, new]. Prints the paths changed. """ diff --git a/packaging/check.sh b/packaging/check.sh index d9806c55..5199c352 100755 --- a/packaging/check.sh +++ b/packaging/check.sh @@ -74,8 +74,10 @@ nfpm_verify() { rpm -qpi /dist/${rpm} rpm -qpR /dist/${rpm} rpm -qplv /dist/${rpm} - rpm -K /dist/${rpm} - rpm -i /dist/${rpm} + # The package is unsigned: check its digests only. RPM 6 enforces signatures by + # default (Fedora 45's 6.1 fails a plain -K or -i on an unsigned package). + rpm -K --nosignature /dist/${rpm} + rpm -i --nosignature /dist/${rpm} rpm -V k3d-gpu ${installed_checks} rpm -e k3d-gpu diff --git a/pins/README.md b/pins/README.md index d0f32c6c..4d297703 100644 --- a/pins/README.md +++ b/pins/README.md @@ -29,7 +29,7 @@ Until `pins/classes` exists, `hack/pinlib/classes.py` gives every path its class | `hack/pins propose-amendment ` | a PENDING row's runner image pins under a dated amendment stub | | `hack/bump ` | resolve, rewrite the sites, run the hooks, substitute goldens, run the group's tests, write the PR body and verdict | | `hack/pin-only-diff ` | is the diff only pin moves (else: changes-defaults) | -| `hack/substitute-goldens OLD=NEW...` | exact-string substitution in `tests/**` only | +| `hack/substitute-goldens OLD=NEW...` | exact-string substitution in `tests/**` only (see Goldens below) | `hack/bump` writes the working tree it runs in; try a group in a scratch clone: @@ -84,9 +84,28 @@ git clone -q . /tmp/k && /tmp/k/hack/bump telemetry --root /tmp/k --dry-run A `[groups.]` table: `tests` (suites `hack/bump` runs after the rewrite; CI runs everything again on the PR), `hooks` (built-in steps after the rewrite, in order: `lock`, `readme-history`, `render-latest`, `ppa-write`, `convert-pins`, -`ledger-rules`, `constraints`, `docs-lock`, `flake-update`, `repin`) and `outputs` -(files those hooks write; `hack/pin-only-diff` accepts any change there). -Coupled components go in one group, so they land in one PR. +`ledger-rules`, `constraints`, `docs-lock`, `flake-update`, `repin`, `hf-meta`) and +`outputs` (files those hooks write; `hack/pin-only-diff` accepts any change there). +Coupled components go in one group, so they land in one PR. `hf-meta` reads each +moved Hugging Face repo's record again at its new revision, in every saved metadata +bundle under `tests/fixtures/hf` that recorded it at the old one. + +## Goldens + +After the rewrite and the hooks, `hack/bump` replaces each exact old pin string +(a whole site match, a digest, a revision, a version or tag) with its new one in +`tests/**`. A test keeps the old string when + +- another pin's site outside `tests/` still holds it (two images at one digest); +- the test names a file outside `tests/` that still holds it: it reads that + record (a ledger arm's or a historical run's pin), which never moves; +- the old string is a bare version or tag (`v1.37.1-k3s1`, `release-1.1.0`) and + the test is code (`.py`, `.sh` or a `#!` script), where it is as likely the + test's own input. In a recorded file (a golden transcript, a fixture) it moves. + +Prose and records that no test names hold nothing back. A test script that checks +a pinned value reads it from the file that pins it (the launcher, the Dockerfile, +`bench/competitors.lock`), so a bump never has to edit the script. ## Hold a pin diff --git a/pins/registry.toml b/pins/registry.toml index d8999ed5..c024a493 100644 --- a/pins/registry.toml +++ b/pins/registry.toml @@ -104,6 +104,8 @@ tests = ["tests/detect_test.sh"] [groups.hf-console-fallback] tests = ["tests/model_opt_test.sh"] +hooks = ["hf-meta"] +outputs = ["tests/fixtures/hf/*.json"] [groups.convert-pins] tests = ["tests/console_test.sh", "tests/model_opt_test.sh"] @@ -504,6 +506,8 @@ derived = [ { file = "console/model_opt.py", regex = 'o\.get\("vllm_version"\) or "(?P[0-9.]+)"', from = "{v_bare}" }, { file = "console/model_opt.py", regex = '^VLLM_[A-Z_]+ = "https://github\.com/vllm-project/vllm/blob/(?Pv[0-9.]+)/', from = "{v}", count = 3 }, { file = "console/model_opt.py", regex = '"--language-model-only [^\n]*"https://github\.com/vllm-project/vllm/blob/(?Pv[0-9.]+)/vllm/engine/arg_utils\.py', from = "{v}" }, + # the vLLM version the launcher hands the optimizer (agent_optimize) + { file = "scripts/k3d-gpu", regex = 'device=\$\{AGENT_SEL%% \*\} vver=(?P[0-9.]+) ', from = "{v_bare}" }, ] [[pin]] @@ -522,6 +526,9 @@ sites = [ { file = "console/moe_predict.py", ref = "intel/llm-scaler-vllm" }, { file = "console/plan.py", regex = '"(?P[0-9.]+)" if backend == "llm-scaler"' }, ] +derived = [ + { file = "scripts/k3d-gpu", regex = 'llm-scaler\) vver=(?P[0-9.]+) ;;', from = "{vllm}" }, +] [[pin]] id = "gaudi-vllm" @@ -539,6 +546,7 @@ sites = [ ] derived = [ { file = "console/plan.py", regex = '^AGENT_VLLM_VERSION = \{"HPU": "(?P[0-9.]+)"', from = "{vllm}" }, + { file = "scripts/k3d-gpu", regex = '\[ -z "\$\{AGENT_HPU:-\}" \] \|\| vver=(?P[0-9.]+)', from = "{vllm}" }, ] [[pin]] diff --git a/tests/audit_test.sh b/tests/audit_test.sh index 48697099..f9271dab 100755 --- a/tests/audit_test.sh +++ b/tests/audit_test.sh @@ -87,10 +87,13 @@ lock_drift "a moved backend digest in the launcher" "vllm/vllm-openai-xpu:${XPU_ "sed -i 's|\(vllm/vllm-openai-xpu:[^@]*\)@sha256:[0-9a-f]*|\1@sha256:$(printf 'e%.0s' {1..64})|' scripts/k3d-gpu" lock_drift "a changed pinned digest" "the repo pins sha256:$(printf 'f%.0s' {1..64})" \ "sed -i 's/node-feature-discovery:v0.20.1@sha256:[0-9a-f]*/node-feature-discovery:v0.20.1@sha256:$(printf 'f%.0s' {1..64})/' share/node-feature-discovery.yml" -lock_drift "a k3s bump" "images.lock is for k3s v1.37.1-k3s1, the Dockerfile pins v1.38.0-k3s1" \ - "sed -i 's/^ARG K3S_TAG=.*/ARG K3S_TAG=\"v1.38.0-k3s1\"/' Dockerfile" -lock_drift "a bumped test image" "not in images.lock: docker.io/nvidia/cuda:13.5.0-base-ubuntu26.04" \ - "sed -i 's|K3D_GPU_TEST_IMAGE:-nvidia/cuda:[^}]*|K3D_GPU_TEST_IMAGE:-nvidia/cuda:13.5.0-base-ubuntu26.04|' scripts/k3d-gpu" +# The pinned k3s tag is read from the Dockerfile and the edits move to releases no +# upstream will publish (v9.9.9, 99.9.9), so a bump of either pin leaves these cases as they are. +K3S_TAG=$(sed -n 's/^ARG K3S_TAG="\{0,1\}\([^"]*\)"\{0,1\}$/\1/p' "${REPO}/Dockerfile") +lock_drift "a k3s bump" "images.lock is for k3s ${K3S_TAG}, the Dockerfile pins v9.9.9-k3s1" \ + "sed -i 's/^ARG K3S_TAG=.*/ARG K3S_TAG=\"v9.9.9-k3s1\"/' Dockerfile" +lock_drift "a bumped test image" "not in images.lock: docker.io/nvidia/cuda:99.9.9-base-ubuntu26.04" \ + "sed -i 's|K3D_GPU_TEST_IMAGE:-nvidia/cuda:[^}]*|K3D_GPU_TEST_IMAGE:-nvidia/cuda:99.9.9-base-ubuntu26.04|' scripts/k3d-gpu" lock_drift "a dropped entry" "not in images.lock: docker.io/library/ubuntu:26.04 (from Dockerfile)" \ "python3 -c \"import json; d=json.load(open('images.lock')); d['images']=[e for e in d['images'] if e['name']!='docker.io/library/ubuntu']; open('images.lock','w').write(json.dumps(d))\"" lock_drift "a lock entry with no digest" "schema violations" \ diff --git a/tests/bench_vs_test.sh b/tests/bench_vs_test.sh index 379ed3af..c4a1339e 100755 --- a/tests/bench_vs_test.sh +++ b/tests/bench_vs_test.sh @@ -41,11 +41,19 @@ refuses() { dry() { "${K}" bench --dry-run "$@"; } +# lock_image : the stack's image in bench/competitors.lock, cut to +# name:tag@sha256: as the plans print it (a bump moves the lock, not this test). +lock_image() { + python3 -c 'import json, sys +i = json.load(open(sys.argv[1]))["stacks"][sys.argv[2]]["image"] +print(i[:i.index("@sha256:") + 16])' "${ROOT}/bench/competitors.lock" "$1" +} + dry_ollama() { local o o=$(dry --vs ollama --id H10 --cpuset 0-15) || return 1 grep -q '^docker run -d --name k3dgpu-g-vs-ollama -p 127.0.0.1:[0-9]*:11434 ' <<< "${o}" \ - && grep -q -- '--cpuset-cpus 0-15 ollama/ollama:0.40.13@sha256:b2f0122c' <<< "${o}" \ + && grep -qF -- "--cpuset-cpus 0-15 $(lock_image ollama)" <<< "${o}" \ && grep -q -- '--id H10 ' <<< "${o}" \ && grep -q -- '--arm ollama=http://127.0.0.1:[0-9]*/v1,qwen3:4b-q4_K_M$' <<< "${o}" \ && grep -q '^verify: sha256 /root/.ollama/models/blobs/sha256-163553aea1b1de62de7c5eb2ef5afb756b4b3133308d9ae7e42e951d8d696ef5 = 163553ae' <<< "${o}" @@ -54,7 +62,8 @@ dry_ollama() { dry_llamacpp() { local o o=$(dry --vs llamacpp) || return 1 - grep -q 'llama.cpp:server-b11960@sha256:8052c13e.* -mu https://huggingface.co/Qwen/Qwen3-4B-GGUF/resolve/bc640142c66e1fdd12af0bd68f40445458f3869b/Qwen3-4B-Q4_K_M.gguf -m /root/.cache/llama.cpp/Qwen3-4B-Q4_K_M.gguf --alias Qwen3-4B --host 0.0.0.0 --port 8080$' <<< "${o}" \ + grep -qF "$(lock_image llamacpp)" <<< "${o}" \ + && grep -q ' -mu https://huggingface.co/Qwen/Qwen3-4B-GGUF/resolve/bc640142c66e1fdd12af0bd68f40445458f3869b/Qwen3-4B-Q4_K_M.gguf -m /root/.cache/llama.cpp/Qwen3-4B-Q4_K_M.gguf --alias Qwen3-4B --host 0.0.0.0 --port 8080$' <<< "${o}" \ && ! grep -q -- ' -hf ' <<< "${o}" \ && grep -q -- '--arm llamacpp=http://127.0.0.1:[0-9]*/v1,Qwen3-4B$' <<< "${o}" \ && grep -q -- '--id H0 --out bench-H0-llamacpp/result.json' <<< "${o}" @@ -63,7 +72,7 @@ dry_llamacpp() { dry_ovms() { local o o=$(dry --vs ovms) || return 1 - grep -q 'model_server:2026.4.1-gpu@sha256:a928903f.* --source_model OpenVINO/Qwen3-4B-int4-ov ' <<< "${o}" \ + grep -qF "$(lock_image ovms)" <<< "${o}" && grep -q ' --source_model OpenVINO/Qwen3-4B-int4-ov ' <<< "${o}" \ && ! grep -q -- '--target_device' <<< "${o}" \ && grep -q -- ' --user 0 ' <<< "${o}" \ && grep -q -- '--arm ovms=http://127.0.0.1:[0-9]*/v3,OpenVINO/Qwen3-4B-int4-ov$' <<< "${o}" diff --git a/tests/bump_test.sh b/tests/bump_test.sh index fda05aa4..66c40183 100755 --- a/tests/bump_test.sh +++ b/tests/bump_test.sh @@ -4,8 +4,12 @@ # digests): no network. Checks that a bump # - changes only declared sites, derived values and hook outputs # (hack/pin-only-diff passes on the result); -# - substitutes goldens (exact old -> new strings) in tests/** only, and only -# for strings no other file still pins; +# - substitutes goldens (exact old -> new strings) in tests/** only: not a string +# another pin's site still holds, not in a test that names a file still holding +# it, and a bare version only in recorded files (prose and records no test reads +# hold nothing back); +# - with the hf-meta hook, re-reads the saved Hugging Face metadata of each moved +# repo at its new revision; # - leaves ledger (bench/), historical, eat-stock files and .github/ alone; # - is a no-op when run a second time; # - moves no held pin to or past its hold, from --resolved values and from a @@ -124,7 +128,8 @@ check "the tool site moved" grep -qx 'TOOL_VERSION=v1.3.0' "${R}/share/tool.env" check "the derived value followed" grep -qx 'built with tool 1.3.0' "${R}/share/notes.txt" check "the image moved with its new digest" grep -qx "image: x/img:1.1.0@${D_NEW}" "${R}/share/app.yml" check "the golden of the moved image was substituted" grep -qx "want x/img:1.1.0@${D_NEW}" "${R}/tests/golden.txt" -check "a golden of a string the ledger still pins stays" grep -qx 'want TOOL_VERSION=v1.2.0' "${R}/tests/golden.txt" +check "a golden the ledger run also holds moves (the test does not read that run)" \ + grep -qx 'want TOOL_VERSION=v1.3.0' "${R}/tests/golden.txt" check "the eat-stock golden stays" grep -qx "want x/eat:2.0.0@${D_EAT}" "${R}/tests/golden.txt" for f in bench/h1/run.sh .github/workflows/x.yml EXPERIMENTS.md integrations/eat/values.yaml; do check "${f} untouched" git -C "${R}" diff --quiet -- "${f}" @@ -329,6 +334,152 @@ check "bump from the sources with held pins: nothing to move (rc ${rc}: ${out##* test "${rc}" -eq 0 -a "${out%%$'\n'*}" = "tools: nothing to move" check "bump from the sources with held pins: the tree is unchanged" test -z "$(git -C "${R5}" status --porcelain)" +# goldens: which strings in tests/ move with a bump. A golden moves unless another +# pin's site still holds its old string (two images at one digest), the test names a +# file outside tests/ that still holds it (it reads that record: a ledger run's +# image), or it is a bare version (a site's value alone) in test code, where it is as +# likely the test's own input. A prose line or a kept record no test names holds +# nothing back. +R6="${WORK}/repo6" +mkdir -p "${R6}/pins" "${R6}/share" "${R6}/tests" "${R6}/bench/h1" +cat > "${R6}/pins/registry.toml" <<'EOF' +[groups.tools] +tests = [] + +[groups.other] +tests = [] + +[[pin]] +id = "k3s" +class = "ours" +group = "tools" +family = "tools" +scheme = "k8s" +source = { kind = "github-release", repo = "o/k3s", tag = "{v}" } +sites = [{ file = "share/pins.env", var = "K3S_TAG" }] + +[[pin]] +id = "img" +class = "ours" +group = "tools" +family = "tools" +scheme = "semver" +source = { kind = "oci", image = "x/img", tag = "{v}" } +sites = [{ file = "share/app.yml", ref = "x/img" }] + +[[pin]] +id = "mirror" +class = "ours" +group = "other" +family = "tools" +scheme = "semver" +source = { kind = "oci", image = "y/img", tag = "{v}" } +sites = [{ file = "share/mirror.yml", ref = "y/img" }] +EOF +echo 'K3S_TAG=v1.2.0-k3s1' > "${R6}/share/pins.env" +printf 'image: x/img:1.0.0@%s\n' "${D_OLD}" > "${R6}/share/app.yml" +printf 'image: y/img:3.0.0@%s\n' "${D_OLD}" > "${R6}/share/mirror.yml" +# prose in an ours file (a pinned name:tag@digest there would be a site of its own) +echo '# before this release: K3S_TAG=v1.2.0-k3s1' > "${R6}/share/CHANGES" +printf 'IMAGE=x/img:1.0.0@%s # H1 measured this\n' "${D_OLD}" > "${R6}/bench/h1/run.sh" +printf 'k3s v1.2.0-k3s1\nimage x/img:1.0.0@%s\ndigest %s\n' "${D_OLD}" "${D_OLD}" > "${R6}/tests/audit.golden" +printf '#!/usr/bin/env bash\ngrep -q "x/img:1.0.0@%s" share/app.yml\n' "${D_OLD}" > "${R6}/tests/app_test.sh" +printf '#!/usr/bin/env bash\ngrep -q "IMAGE=x/img:1.0.0@%s" bench/h1/run.sh\n' "${D_OLD}" > "${R6}/tests/h1_test.sh" +printf 'assert parse("v1.2.0-k3s1") == (1, 2, 0)\n' > "${R6}/tests/version_test.py" +: > "${R6}/EXPERIMENTS.md" +: > "${R6}/images.unpinned" +git -C "${R6}" init -q -b main && git -C "${R6}" add -A && git -C "${R6}" commit -qm base +echo '{"k3s": {"v": "v1.2.1-k3s1"}, "img": {"v": "1.1.0", "tag": "1.1.0"}}' > "${WORK}/resolved6.json" +out=$(PINLIB_FIXTURES="${FIX}" python3 "${ROOT}/hack/bump" --root "${R6}" --resolved "${WORK}/resolved6.json" \ + --no-tests tools 2>&1); rc=$? +check "goldens: bump exits 0 (rc ${rc}: ${out##*$'\n'})" test "${rc}" -eq 0 +check "goldens: a recorded golden's bare version moves past a prose line that keeps it" \ + grep -qx 'k3s v1.2.1-k3s1' "${R6}/tests/audit.golden" +check "goldens: a recorded golden's image moves past a ledger run that keeps it" \ + grep -qx "image x/img:1.1.0@${D_NEW}" "${R6}/tests/audit.golden" +check "goldens: a digest another pin's site still holds stays" grep -qx "digest ${D_OLD}" "${R6}/tests/audit.golden" +check "goldens: a test's image moves when the test does not read the ledger run" \ + grep -q "x/img:1.1.0@${D_NEW}" "${R6}/tests/app_test.sh" +check "goldens: a test that names the ledger run keeps the run's image" \ + git -C "${R6}" diff --quiet -- tests/h1_test.sh +check "goldens: a bare version in test code stays (a test input)" git -C "${R6}" diff --quiet -- tests/version_test.py +for f in share/CHANGES share/mirror.yml bench/h1/run.sh; do + check "goldens: ${f} untouched" git -C "${R6}" diff --quiet -- "${f}" +done + +# the hf-meta hook: a moved Hugging Face pin's record in each saved metadata bundle +# (tests/fixtures/hf) is read again at the new revision; other repos' records and +# bundles without the repo stay as they are. +R7="${WORK}/repo7" +HF_OLD=$(printf 'a%.0s' {1..40}) HF_NEW=$(printf 'b%.0s' {1..40}) HF_BASE=$(printf 'c%.0s' {1..40}) +mkdir -p "${R7}/pins" "${R7}/console" "${R7}/tests/fixtures/hf" +cp "${ROOT}/console/model_opt.py" "${ROOT}/console/convert-pins.json" "${R7}/console/" +cat > "${R7}/pins/registry.toml" <<'EOF' +[groups.hf] +tests = [] +hooks = ["hf-meta"] +outputs = ["tests/fixtures/hf/*.json"] + +[[pin]] +id = "hf-m" +class = "ours" +group = "hf" +family = "hf" +scheme = "hf-sha" +source = { kind = "hf", model = "o/m-int4" } +sites = [{ file = "console/fallback.txt", regex = '"o/m-int4": \{"revision": "(?P[0-9a-f]{40})"' }] +EOF +printf '"o/m-int4": {"revision": "%s"}\n' "${HF_OLD}" > "${R7}/console/fallback.txt" +python3 - "${R7}" "${HF_OLD}" "${HF_BASE}" <<'PY' +import json, sys +root, old, base = sys.argv[1:] +def rec(rid, sha, lines): + return {"id": rid, "sha": sha, "url": "https://huggingface.co/%s/tree/%s" % (rid, sha), "files": ["README.md"], + "card": {}, "card_lines": lines, "library_name": None, "pipeline_tag": None, "params_total": None, "tags": []} +b = {"schema": "k3d-gpu.hfmeta/v1", "requested": "o/m", "target": "o/m", "revision": "main", "source": "huggingface", + "searches": [], "repos": {"o/m": rec("o/m", base, []), "o/m-int4": rec("o/m-int4", old, [{"line": 1, "text": "* mode: **INT4**"}])}} +other = dict(b, requested="o/n", target="o/n", repos={"o/n": rec("o/n", base, [])}) +for name, d in (("o__m.json", b), ("o__n.json", other)): + with open("%s/tests/fixtures/hf/%s" % (root, name), "w") as f: + f.write(json.dumps(d, indent=1, sort_keys=True) + "\n") +PY +: > "${R7}/EXPERIMENTS.md" +: > "${R7}/images.unpinned" +git -C "${R7}" init -q -b main && git -C "${R7}" add -A && git -C "${R7}" commit -qm base +FIX7="${WORK}/fixtures7.json" +cat > "${FIX7}" < "${WORK}/resolved7.json" +out=$(PINLIB_FIXTURES="${FIX7}" python3 "${ROOT}/hack/bump" --root "${R7}" --resolved "${WORK}/resolved7.json" \ + --no-tests hf --summary "${WORK}/hf.json" 2>&1); rc=$? +check "hf-meta: bump exits 0 (rc ${rc}: ${out##*$'\n'})" test "${rc}" -eq 0 +check "hf-meta: the moved repo's record is the one Hugging Face serves at the new revision" \ + python3 - "${R7}" "${HF_NEW}" "${HF_BASE}" "${ROOT}" <<'PY' +import json, sys +root, new, base, repo = sys.argv[1:] +sys.dont_write_bytecode = True +sys.path.insert(0, repo + "/console") +import model_opt +readme = "# m-int4\n\n* mode: **INT4**\n* group_size: **128**\n" +text = open(root + "/tests/fixtures/hf/o__m.json").read() +assert text == json.dumps(json.loads(text), indent=1, sort_keys=True) + "\n", "bundle not written as saved" +r = json.loads(text)["repos"] +m = r["o/m-int4"] +assert m["sha"] == new and m["url"].endswith("/tree/" + new), m +assert m["tags"] == ["openvino"] and m["card_lines"] == model_opt.card_lines(readme) != [], m +assert r["o/m"]["sha"] == base, r["o/m"] +PY +check "hf-meta: a bundle without the repo is untouched" git -C "${R7}" diff --quiet -- tests/fixtures/hf/o__n.json +check "hf-meta: the summary notes the refreshed bundle" \ + grep -q "hf-meta: tests/fixtures/hf/o__m.json o/m-int4 ${HF_OLD:0:8} -> ${HF_NEW:0:8}" "${WORK}/hf.json" +git -C "${R7}" add -A && git -C "${R7}" commit -qm bump +check "hack/pin-only-diff passes on the hf-meta bump" \ + python3 "${ROOT}/hack/pin-only-diff" --root "${R7}" --group hf HEAD~1 HEAD + # the registry: one group (one branch) for both NRI components check "nri-balloons and nri-balloons-guard bump in group nri (one branch)" python3 - "${ROOT}" <<'PY' import sys, tomllib diff --git a/tests/host_setup_test.sh b/tests/host_setup_test.sh index f0e5f9b5..23a835bf 100755 --- a/tests/host_setup_test.sh +++ b/tests/host_setup_test.sh @@ -112,6 +112,19 @@ says "the nvctk pin derives HS_NVCTK from it" 'derived scripts/k3d-gpu ^HS_NVCTK # name for that release's build (26.3.5-1~ppa1 -> 26.3.5-1~24.04~ppa1). pin() { sed -n "s/^$1=//p" "${K3D_GPU}"; } ppa() { local v; v=$(pin "$1"); echo "${v%%~*}~$2~${v#*~}"; } +# The GitHub .debs and the NPU archive move with group host-intel (hack/bump): the +# plans below are checked against what the launcher pins, read from it here. +CR=$(pin HS_CR) +NPU_REL=$(pin HS_NPU_REL) +NPU_BUILD=$(pin HS_NPU_BUILD) +ZE_DEB=$(sed -n 's/^HS_ZE_DEB="\([^ ]*\) .*/\1/p' "${K3D_GPU}") +IGC_OCL_DEB=$(sed -n 's/^\(intel-igc-opencl-2_[^ ]*\.deb\) .*/\1/p' "${K3D_GPU}") +OCL_ICD_SHA=$(sed -n 's/^intel-opencl-icd_[$]{HS_CR}-0_amd64\.deb \([0-9a-f]\{64\}\) .*/\1/p' "${K3D_GPU}") +NPU_2604_SHA=$(sed -n 's/.*-ubuntu2604\.tar\.gz \([0-9a-f]\{64\}\)".*/\1/p' "${K3D_GPU}") +for v in CR NPU_REL NPU_BUILD ZE_DEB IGC_OCL_DEB OCL_ICD_SHA NPU_2604_SHA; do + OUT="${v}=${!v}"; ok=1; [ -n "${!v}" ] && ok=0 + check "the launcher's ${v} pin is read" "${ok}" "${v} is empty" +done for id_v in ppa-media:HS_PPA_MEDIA ppa-gmm:HS_PPA_GMM ppa-libva:HS_PPA_LIBVA; do v=${id_v#*:} OUT="${v}=$(pin "${v}")" @@ -260,10 +273,10 @@ says "ubuntu pins libva-x11-2/libva-wayland2 with libva (vainfo needs both)" "li says "ubuntu keeps apt from swapping the GitHub compute .debs for PPA builds" "Pin: release o=LP-PPA-kobuk-team-intel-graphics" says "the apt pin covers the runtime packages" "Package: intel-opencl-icd libze-intel-gpu1 libze1" says "downloads go to a private dir that must be the user's own" "mkdir -p '${WORK}/dl' && [ ! -L '${WORK}/dl' ] && [ -O '${WORK}/dl' ] && chmod 0700 '${WORK}/dl'" -says "ubuntu downloads compute-runtime 26.35 .debs" "releases/download/26.35.39758.10/libze-intel-gpu1_26.35.39758.10-0_amd64.deb" -says "ubuntu verifies the opencl-icd sha256" "61712caaddeba3d38e4f79e2a0fb23fea25596ca2d72c3144c6eea2331ec4301 ${WORK}/dl/intel-opencl-icd_26.35.39758.10-0_amd64.deb' | sha256sum -c -" -says "ubuntu installs IGC 2.41.5 with the runtime" "intel-igc-opencl-2_2.41.5+22716_amd64.deb" -says "ubuntu installs the level-zero loader .deb" "libze1_1.34.0+u24.04_amd64.deb" +says "ubuntu downloads the pinned compute-runtime .debs" "releases/download/${CR}/libze-intel-gpu1_${CR}-0_amd64.deb" +says "ubuntu verifies the opencl-icd sha256" "${OCL_ICD_SHA} ${WORK}/dl/intel-opencl-icd_${CR}-0_amd64.deb' | sha256sum -c -" +says "ubuntu installs the pinned IGC with the runtime" "${IGC_OCL_DEB}" +says "ubuntu installs the level-zero loader .deb" "${ZE_DEB}" not_says "no NVIDIA plan without an NVIDIA GPU" "nvidia-container-toolkit" not_says "no NPU plan without an NPU" "linux-npu-driver" says "dry run says nothing changed" "Dry run: nothing was changed" @@ -276,13 +289,13 @@ osrel "${r}" ubuntu 26.04 kernel "${r}" 6.17.0-5-generic pci "${r}" 0000:00:0b.0 0x8086 0xb03e 0x120000 intel_vpu run "${r}" "" -says "ubuntu 26.04 NPU uses the 26.04 driver archive" "linux-npu-driver-v1.38.0.20260910-34487311128-ubuntu2604.tar.gz" -says "NPU archive sha256 is checked" "83685e6d92f4db1ca65e442846d487c7feb1335634b33994292ceb38f933bcc7" -says "NPU install pulls libtbb12 and the archive .debs by name" "libtbb12 '${WORK}/dl/npu-v1.38.0/intel-level-zero-npu_1.38.0.20260910-34487311128~ubuntu26.04_amd64.deb' '${WORK}/dl/npu-v1.38.0/intel-driver-compiler-npu_1.38.0.20260910-34487311128~ubuntu26.04_amd64.deb' '${WORK}/dl/npu-v1.38.0/intel-fw-npu_1.38.0.20260910-34487311128~ubuntu26.04_amd64.deb'" +says "ubuntu 26.04 NPU uses the 26.04 driver archive" "linux-npu-driver-v${NPU_BUILD}-ubuntu2604.tar.gz" +says "NPU archive sha256 is checked" "${NPU_2604_SHA}" +says "NPU install pulls libtbb12 and the archive .debs by name" "libtbb12 '${WORK}/dl/npu-${NPU_REL}/intel-level-zero-npu_${NPU_BUILD}~ubuntu26.04_amd64.deb' '${WORK}/dl/npu-${NPU_REL}/intel-driver-compiler-npu_${NPU_BUILD}~ubuntu26.04_amd64.deb' '${WORK}/dl/npu-${NPU_REL}/intel-fw-npu_${NPU_BUILD}~ubuntu26.04_amd64.deb'" not_says "NPU install never globs the download dir" "*.deb" -says "only the named .debs are unpacked from the checked archive" "-C '${WORK}/dl/npu-v1.38.0' './intel-level-zero-npu_1.38.0.20260910-34487311128~ubuntu26.04_amd64.deb'" +says "only the named .debs are unpacked from the checked archive" "-C '${WORK}/dl/npu-${NPU_REL}' './intel-level-zero-npu_${NPU_BUILD}~ubuntu26.04_amd64.deb'" says "NPU needs the loader, and 26.04 has no pinned one: said, not guessed" "no pinned Level Zero loader for ubuntu-26.04" -not_says "26.04 NPU never gets the 24.04 loader .deb" "libze1_1.34.0+u24.04_amd64.deb" +not_says "26.04 NPU never gets the 24.04 loader .deb" "${ZE_DEB}" not_says "NPU-only host skips the media PPA" "add-apt-repository" says "missing NPU firmware is reported" "intel/vpu/vpu_50xx_* firmware missing" @@ -514,9 +527,9 @@ osrel "${r}" ubuntu 26.04 pci "${r}" 0000:00:0b.0 0x8086 0x7d1d 0x120000 intel_vpu run "${r}" "y" --apply rc_is "ubuntu --apply succeeds" 0 -logged "ubuntu --apply downloads the NPU archive" "curl -fL --retry 3 -o ${WORK}/dl/linux-npu-driver-v1.38.0.20260910-34487311128-ubuntu2604.tar.gz" +logged "ubuntu --apply downloads the NPU archive" "curl -fL --retry 3 -o ${WORK}/dl/linux-npu-driver-v${NPU_BUILD}-ubuntu2604.tar.gz" logged "ubuntu --apply checks the sha256" "sha256sum -c -" -logged "ubuntu --apply unpacks the archive" "tar -xzf ${WORK}/dl/linux-npu-driver-v1.38.0.20260910-34487311128-ubuntu2604.tar.gz -C ${WORK}/dl/npu-v1.38.0" +logged "ubuntu --apply unpacks the archive" "tar -xzf ${WORK}/dl/linux-npu-driver-v${NPU_BUILD}-ubuntu2604.tar.gz -C ${WORK}/dl/npu-${NPU_REL}" r=$(newroot apply-arch-units) osrel "${r}" manjaro @@ -761,8 +774,8 @@ says "crescent without --plan: nothing to install" "Every check passed: nothing # One failing check: only its own package set. rm "${r}/usr/lib/x86_64-linux-gnu/libze_intel_gpu.so.1" run "${r}" "" "${CR_OK[@]}" -says "missing Level Zero GPU driver plans the 26.35 compute runtime" "releases/download/26.35.39758.10/libze-intel-gpu1_26.35.39758.10-0_amd64.deb" -# The 26.35 .debs depend on libigdgmm12 (>= 22.10.0); the 24.04 and 26.04 +says "missing Level Zero GPU driver plans the pinned compute runtime" "releases/download/${CR}/libze-intel-gpu1_${CR}-0_amd64.deb" +# The compute-runtime .debs depend on libigdgmm12 (>= 22.10.0); the 24.04 and 26.04 # archives carry 22.3.17 and 22.9.0, so the PPA's gmmlib comes first. says "missing Level Zero GPU driver adds the PPA for its gmmlib" " sudo add-apt-repository -y ppa:kobuk-team/intel-graphics" says "missing Level Zero GPU driver: the PPA's tool is a prerequisite" "software-properties-common" @@ -790,7 +803,7 @@ not_says "media failure plans no 24.04 package" "24.04" rm "${r}/usr/lib/x86_64-linux-gnu/libze_loader.so.1" run "${r}" "" "${CR_OK[@]}" says "26.04 has no pinned loader .deb and says so" "no pinned Level Zero loader for ubuntu-26.04" -not_says "26.04 never gets the u24.04 loader" "libze1_1.34.0+u24.04" +not_says "26.04 never gets the u24.04 loader" "${ZE_DEB%_amd64.deb}" # accel-config: planned only when a planned DSA/IAA step needs it and it is missing. r=$(crescent crescent-dsa) diff --git a/tests/pins_test.py b/tests/pins_test.py index 7c9d18aa..32a28516 100644 --- a/tests/pins_test.py +++ b/tests/pins_test.py @@ -122,6 +122,32 @@ def test_every_group_names_existing_tests_and_known_hooks(self): used = {p["group"] for p in reg["pin"]} | {x["group"] for x in reg["images"]["default"]["groups"]} self.assertEqual(set(reg["groups"]) - used - {"digests"}, set(), "groups with no pin") + def test_every_canary_derived_constant_moves_with_its_pin(self): + """A package version canary/engines.toml checks against an engine's image (the + vLLM version the launcher hands the optimizer, say) lies inside a site or + derived value of the pin whose canary names that engine, so a bump moves it + too. A deb or firmware inside the image is a fact only the canary reads.""" + import re + import tomllib + sys.path.insert(0, str(ROOT / "hack")) + from pinlib import registry + reg = registry.load(str(ROOT / "pins" / "registry.toml")) + tree = registry.Tree(str(ROOT)) + with open(ROOT / "canary" / "engines.toml", "rb") as f: + engines = tomllib.load(f)["engine"] + missing = [] + for e in engines: + pins = [p for p in reg["pin"] if e["id"] in p.get("canary", [])] + spans = [s for p in pins for s in registry.locate(tree, p)] + for d in (d for d in e.get("derived", []) if d.get("of") == "package"): + m = re.search(d["regex"], tree.read(d["file"]) or "", re.M) + self.assertIsNotNone(m, "%s: %s matches nothing in %s" % (e["id"], d["regex"], d["file"])) + a, b = m.span("v") + if not any(s.file == d["file"] and s.start <= a and b <= s.end for s in spans): + missing.append("%s: %s %r (pins %s)" % (e["id"], d["file"], m.group(0), + ", ".join(p["id"] for p in pins) or "none")) + self.assertEqual(missing, [], "canary derived constants no registry site or derived value covers") + class Check(unittest.TestCase): def run_check(self, registry=REGISTRY, files=None): From fa6e9c6e4c00e57633f0bd7ef26a54f64780830b Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 18:30:08 -0800 Subject: [PATCH 2/5] pins: goldens move cut digests and revisions; only kept records hold a test back A plan or an audit prints a revision or digest cut to 12 hex digits, so each digest or revision pair also moves in that form. The names rule now reads only kept records (files no bumper writes, prose aside): a test whose comment mentions EXPERIMENTS.md or README.md no longer keeps every string those files hold. --- hack/pinlib/goldens.py | 33 ++++++++++++++++++++++++++------- tests/bump_test.sh | 26 +++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/hack/pinlib/goldens.py b/hack/pinlib/goldens.py index c661c6e0..57efb866 100644 --- a/hack/pinlib/goldens.py +++ b/hack/pinlib/goldens.py @@ -2,11 +2,13 @@ A test that names a pinned value (a golden transcript, a recorded image list) moves with the pin. Only tests/** is written; never a ledger, historical, arm or -eat-stock path, nor .github/. A pair leaves a test alone when +eat-stock path, nor .github/. A digest or revision also moves cut to its first 12 +hex digits, as plans and audits print it. A pair leaves a test alone when - another pin's site outside tests/ still holds the old string (two images at one digest: the test may check the other one); - - the test names a file outside tests/ that still holds it (it reads that - record: a ledger arm's or a historical run's pin); + - the test names a kept record that still holds it (a ledger arm's or a + historical run's file: one no bumper writes, prose (.md) aside) and so + reads that record; - the old string is a bare version (a site's value alone, no image name, no key) and the test is code (.py, .sh or a #! script): there it is as likely the test's own input, as in a version-compare case. In a recorded file (a @@ -25,6 +27,7 @@ MASK = "\x00{%d}\x00" BARE = re.compile(r"[A-Za-z0-9._+~-]+") HEX = re.compile(r"(sha256:)?[0-9a-f]{40,64}") +SHORT = 12 # hex digits of a cut digest or revision def substitute_text(text, pairs): @@ -97,9 +100,25 @@ def site_texts(root): def live_pairs(root, pairs): - """The pairs whose old string no pin site outside tests/ holds any more.""" + """The pairs whose old string no pin site outside tests/ holds any more, each + digest or revision also cut to its first 12 hex digits.""" held = site_texts(root) - return [(o, n) for o, n in pairs if o and o != n and o not in held] + out = [] + for o, n in pairs: + if not o or o == n or o in held: + continue + out.append((o, n)) + if HEX.fullmatch(o) and HEX.fullmatch(n): + k = len("sha256:") + SHORT if o.startswith("sha256:") else SHORT + if o[:k] != n[:k]: + out.append((o[:k], n[:k])) + return out + + +def kept_record(root, path): + """A file outside tests/ that no bumper writes (a ledger arm, a historical run), + prose (.md) aside.""" + return not path.startswith("tests/") and not path.endswith(".md") and not classes.writable(root, path) def substitute(root, pairs, paths=None): @@ -112,12 +131,12 @@ def substitute(root, pairs, paths=None): text = _read(root, p) if text is not None: tests[p] = text - # the files outside tests/ that still hold each old string some test holds + # the kept records that still hold each old string some test holds wanted = {o for o, _ in pairs if any(o in t for t in tests.values())} holders = {o: [] for o in wanted} if wanted: for p in _tracked(root): - if p.startswith("tests/"): + if not kept_record(root, p): continue text = _read(root, p) if text is None: diff --git a/tests/bump_test.sh b/tests/bump_test.sh index 66c40183..2503dde0 100755 --- a/tests/bump_test.sh +++ b/tests/bump_test.sh @@ -367,6 +367,15 @@ scheme = "semver" source = { kind = "oci", image = "x/img", tag = "{v}" } sites = [{ file = "share/app.yml", ref = "x/img" }] +[[pin]] +id = "rev" +class = "ours" +group = "tools" +family = "hf" +scheme = "hf-sha" +source = { kind = "hf", model = "o/m" } +sites = [{ file = "share/model.env", regex = '^MODEL_REV=(?P[0-9a-f]{40})$' }] + [[pin]] id = "mirror" class = "ours" @@ -386,10 +395,18 @@ printf 'k3s v1.2.0-k3s1\nimage x/img:1.0.0@%s\ndigest %s\n' "${D_OLD}" "${D_OLD} printf '#!/usr/bin/env bash\ngrep -q "x/img:1.0.0@%s" share/app.yml\n' "${D_OLD}" > "${R6}/tests/app_test.sh" printf '#!/usr/bin/env bash\ngrep -q "IMAGE=x/img:1.0.0@%s" bench/h1/run.sh\n' "${D_OLD}" > "${R6}/tests/h1_test.sh" printf 'assert parse("v1.2.0-k3s1") == (1, 2, 0)\n' > "${R6}/tests/version_test.py" -: > "${R6}/EXPERIMENTS.md" +# a revision: the ledger and the README keep the old one as history; a plan prints it cut to 12 +REV_OLD=$(printf 'd%.0s' {1..40}) REV_NEW=$(printf 'e%.0s' {1..40}) +echo "MODEL_REV=${REV_OLD}" > "${R6}/share/model.env" +printf 'serve o/m@%s\n' "${REV_OLD:0:12}" > "${R6}/tests/plan.golden" +printf '#!/usr/bin/env bash\n# reads no ledger (EXPERIMENTS.md) and no README.md\ngrep -q -- "--revision=%s" out\n' \ + "${REV_OLD}" > "${R6}/tests/model_test.sh" +echo "H1 measured o/m at ${REV_OLD}" > "${R6}/EXPERIMENTS.md" +echo "o/m at ${REV_OLD}" > "${R6}/README.md" : > "${R6}/images.unpinned" git -C "${R6}" init -q -b main && git -C "${R6}" add -A && git -C "${R6}" commit -qm base -echo '{"k3s": {"v": "v1.2.1-k3s1"}, "img": {"v": "1.1.0", "tag": "1.1.0"}}' > "${WORK}/resolved6.json" +echo "{\"k3s\": {\"v\": \"v1.2.1-k3s1\"}, \"img\": {\"v\": \"1.1.0\", \"tag\": \"1.1.0\"}, \"rev\": {\"v\": \"${REV_NEW}\"}}" \ + > "${WORK}/resolved6.json" out=$(PINLIB_FIXTURES="${FIX}" python3 "${ROOT}/hack/bump" --root "${R6}" --resolved "${WORK}/resolved6.json" \ --no-tests tools 2>&1); rc=$? check "goldens: bump exits 0 (rc ${rc}: ${out##*$'\n'})" test "${rc}" -eq 0 @@ -403,7 +420,10 @@ check "goldens: a test's image moves when the test does not read the ledger run" check "goldens: a test that names the ledger run keeps the run's image" \ git -C "${R6}" diff --quiet -- tests/h1_test.sh check "goldens: a bare version in test code stays (a test input)" git -C "${R6}" diff --quiet -- tests/version_test.py -for f in share/CHANGES share/mirror.yml bench/h1/run.sh; do +check "goldens: a revision cut to 12 digits moves with the revision" grep -qx "serve o/m@${REV_NEW:0:12}" "${R6}/tests/plan.golden" +check "goldens: naming the ledger or the README in a comment holds nothing back" \ + grep -q -- "--revision=${REV_NEW}" "${R6}/tests/model_test.sh" +for f in share/CHANGES share/mirror.yml bench/h1/run.sh EXPERIMENTS.md README.md; do check "goldens: ${f} untouched" git -C "${R6}" diff --quiet -- "${f}" done From a8c6407f8d55804b113dec96893275a3ba2ef981 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 18:50:20 -0800 Subject: [PATCH 3/5] tests: chat-ui, agent-sandbox and the EAS sandbox fixture follow the pins they check tests/chat_ui_test.sh reads the Open WebUI version from the manifest's comment and the digest from images.lock for that tag, so the three still have to agree but a bump no longer edits the test. tests/detect_test.sh checks both agent-sandbox images against the launcher's SANDBOX_VERSION. The k3d-gpu-sandbox EAS fixture names the controller by tag, as the other sandbox fixtures do: the refusal it tests reads the namespace, and a digest-pinned copy of the live image moved with every bump while the expected refusal kept the old tag. --- tests/chat_ui_test.sh | 11 +++++++---- tests/detect_test.sh | 6 ++++-- tests/fixtures/eas/config/k3d-gpu-sandbox/facts.json | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/chat_ui_test.sh b/tests/chat_ui_test.sh index 45fb286f..0852956b 100755 --- a/tests/chat_ui_test.sh +++ b/tests/chat_ui_test.sh @@ -40,10 +40,13 @@ expect_env() { # Image pinned by digest under its version tag, never by a moving tag alone. has "image pinned by digest" '^ +image: ghcr\.io/open-webui/open-webui:v[0-9.]+@sha256:[0-9a-f]{64}$' lacks "no tag-only image" '^ +image: [^@]+:[^@]+$' -# The exact index digest of ghcr.io/open-webui/open-webui:v0.12.2, and the -# version the header comment names. A bump changes all three together. -OWUI_VERSION=v0.12.2 -OWUI_DIGEST=sha256:10e6a78ba2d172493460a9e8a92c496aaa7a64dc4d26d622a2e95e0c736081c9 +# The version the comment above the image names, the image's tag and digest, and +# the version the header names agree, and the digest is images.lock's for that tag. +# A bump changes all of them together (hack/bump opt-in-manifests). +OWUI_VERSION=$(sed -n 's|^ *# ghcr\.io/open-webui/open-webui:\(v[0-9.]*\)$|\1|p' "${MANIFEST}") +OWUI_DIGEST=$(python3 -c 'import json, sys +print("".join(e["digest"] for e in json.load(open(sys.argv[1]))["images"] + if e["name"] == "ghcr.io/open-webui/open-webui" and e["tag"] == sys.argv[2]))' "${ROOT}/images.lock" "${OWUI_VERSION}") has "image is ${OWUI_VERSION} digest" "^ +image: ghcr\.io/open-webui/open-webui:${OWUI_VERSION//./\\.}@${OWUI_DIGEST}$" has "header names ${OWUI_VERSION}" "^# Source: https://github\.com/open-webui/open-webui ${OWUI_VERSION//./\\.}\." diff --git a/tests/detect_test.sh b/tests/detect_test.sh index 485c2732..de16b23e 100755 --- a/tests/detect_test.sh +++ b/tests/detect_test.sh @@ -4864,8 +4864,10 @@ if grep -A1 -- '^ - name: HOME$' "${SHARE}/agent-sandbox-k3d.yml" | grep else FAILS=$((FAILS + 1)); echo "FAIL sandbox profile sets a writable HOME for pip (no 'name: HOME' followed by 'value: /tmp')" fi -manifest_has "sandbox runtime image pinned by digest" 'image: registry.k8s.io/agent-sandbox/python-runtime-sandbox:v1.0.8@sha256:[0-9a-f]{64}$' agent-sandbox-k3d.yml -manifest_has "agent-sandbox controller image pinned by digest" 'image: registry.k8s.io/agent-sandbox/agent-sandbox-controller:v1.0.8@sha256:[0-9a-f]{64}$' agent-sandbox.yml +# Both images are the release the launcher installs (SANDBOX_VERSION; group opt-in-manifests moves them together). +SANDBOX_TAG=$(sed -n 's/^SANDBOX_VERSION="\(v[0-9.]*\)"$/\1/p' "${K3D_GPU}") +manifest_has "sandbox runtime image pinned by digest" "image: registry.k8s.io/agent-sandbox/python-runtime-sandbox:${SANDBOX_TAG:-unset}@sha256:[0-9a-f]{64}\$" agent-sandbox-k3d.yml +manifest_has "agent-sandbox controller image pinned by digest" "image: registry.k8s.io/agent-sandbox/agent-sandbox-controller:${SANDBOX_TAG:-unset}@sha256:[0-9a-f]{64}\$" agent-sandbox.yml if grep -E 'image: ' "${SHARE}/agent-sandbox.yml" "${SHARE}/agent-sandbox-k3d.yml" | grep -v '@sha256:' | grep -v ':[[:space:]]*$' >/dev/null; then FAILS=$((FAILS + 1)); echo "FAIL every agent-sandbox image is pinned by digest" else diff --git a/tests/fixtures/eas/config/k3d-gpu-sandbox/facts.json b/tests/fixtures/eas/config/k3d-gpu-sandbox/facts.json index 39a4b09e..b5c9267d 100644 --- a/tests/fixtures/eas/config/k3d-gpu-sandbox/facts.json +++ b/tests/fixtures/eas/config/k3d-gpu-sandbox/facts.json @@ -1,6 +1,6 @@ { "agent_sandbox": { - "controller_image": "registry.k8s.io/agent-sandbox/agent-sandbox-controller:v1.0.8@sha256:e1d96312487eba7d8f001e848a9d4e644b7deea8c2a4564ccc318abb729be731" + "controller_image": "registry.k8s.io/agent-sandbox/agent-sandbox-controller:v1.0.8" }, "cluster": "k3d-gpu", "containerd_cni_bin_dirs": [ From 5ed9cac50d070cfb45e69033abf64a60161d2839 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 18:54:23 -0800 Subject: [PATCH 4/5] pins: document the cut digests and the kept-record rule of golden substitution --- hack/substitute-goldens | 7 ++++--- pins/README.md | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hack/substitute-goldens b/hack/substitute-goldens index 7d6cc2bc..5a28d6bb 100755 --- a/hack/substitute-goldens +++ b/hack/substitute-goldens @@ -5,9 +5,10 @@ A test that names a pinned value (a golden transcript, a recorded image list) moves with the pin. Only tests/** is written: never a ledger, historical, arm or -eat-stock path, nor .github/. A test keeps an old string that another pin's site -still holds, that a file outside tests/ it names still holds (a ledger run it -reads), or that is a bare version in test code (hack/pinlib/goldens.py, pins/README.md). +eat-stock path, nor .github/. A digest or revision also moves cut to 12 hex +digits. A test keeps an old string that another pin's site still holds, that a +kept record it names still holds (a ledger run it reads), or that is a bare +version in test code (hack/pinlib/goldens.py, pins/README.md). Idempotent: a second run changes nothing, even when an old string is part of its new one (1.37 -> 1.37.1). hack/bump runs it after every bump. --pairs FILE: a JSON list of [old, new]. Prints the paths changed. diff --git a/pins/README.md b/pins/README.md index 4d297703..9a4cdb25 100644 --- a/pins/README.md +++ b/pins/README.md @@ -94,11 +94,12 @@ bundle under `tests/fixtures/hf` that recorded it at the old one. After the rewrite and the hooks, `hack/bump` replaces each exact old pin string (a whole site match, a digest, a revision, a version or tag) with its new one in -`tests/**`. A test keeps the old string when +`tests/**`; a digest or revision also moves cut to its first 12 hex digits, as +plans and audits print it. A test keeps the old string when - another pin's site outside `tests/` still holds it (two images at one digest); -- the test names a file outside `tests/` that still holds it: it reads that - record (a ledger arm's or a historical run's pin), which never moves; +- the test names a kept record that still holds it (a file no bumper writes, + prose aside: a ledger arm's or a historical run's pin), so it reads that record; - the old string is a bare version or tag (`v1.37.1-k3s1`, `release-1.1.0`) and the test is code (`.py`, `.sh` or a `#!` script), where it is as likely the test's own input. In a recorded file (a golden transcript, a fixture) it moves. From 008a1cfca2b1986fcf8948fbf26b80eb5948546a Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:36:31 -0800 Subject: [PATCH 5/5] pins: pin-only-diff counts the cut digests and revisions goldens now move Golden substitution moves a digest or revision cut to its first 12 hex digits (sha256:<12> in audits, name@<12> in plans), but hack/pin-only-diff only knew a cut digest followed by an ellipsis. A bump whose goldens carry the cut form was then labelled changes-defaults and held for review: the hf-console-fallback bump (the explain goldens' @<12> revisions) and the opt-in-manifests bump (the audit goldens' sha256:<12> digests). Both forms are now pin tokens; a twelve-digit number elsewhere still is not. --- hack/pinlib/pindiff.py | 2 ++ pins/README.md | 3 ++- tests/bump_test.sh | 3 +++ tests/pin_only_diff_test.py | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/hack/pinlib/pindiff.py b/hack/pinlib/pindiff.py index 99e6d642..f570e596 100644 --- a/hack/pinlib/pindiff.py +++ b/hack/pinlib/pindiff.py @@ -19,6 +19,8 @@ r"(?`, `@<12>`: `hack/pin-only-diff` counts +both as pin tokens). A test keeps the old string when - another pin's site outside `tests/` still holds it (two images at one digest); - the test names a kept record that still holds it (a file no bumper writes, diff --git a/tests/bump_test.sh b/tests/bump_test.sh index 2503dde0..073b505a 100755 --- a/tests/bump_test.sh +++ b/tests/bump_test.sh @@ -426,6 +426,9 @@ check "goldens: naming the ledger or the README in a comment holds nothing back" for f in share/CHANGES share/mirror.yml bench/h1/run.sh EXPERIMENTS.md README.md; do check "goldens: ${f} untouched" git -C "${R6}" diff --quiet -- "${f}" done +git -C "${R6}" add -A && git -C "${R6}" commit -qm bump +check "hack/pin-only-diff passes on the goldens bump (the cut revision too)" \ + python3 "${ROOT}/hack/pin-only-diff" --root "${R6}" --group tools HEAD~1 HEAD # the hf-meta hook: a moved Hugging Face pin's record in each saved metadata bundle # (tests/fixtures/hf) is read again at the new revision; other repos' records and diff --git a/tests/pin_only_diff_test.py b/tests/pin_only_diff_test.py index f2f21b73..02ac529c 100644 --- a/tests/pin_only_diff_test.py +++ b/tests/pin_only_diff_test.py @@ -44,6 +44,8 @@ class = "ours" "share/tool.sh": 'TOOL_URL="https://example.com/v1.2.0/tool-amd64.tar"\nARGS="--a"\n', "share/notes.txt": "tool v1.2.0 (2026-10-01), digest %s, rev %s\n" % (D1, "a" * 40), "share/lock.json": '{"x": 1}\n', + "share/plan.txt": "ok image x/y@%s: %s in images.lock\nserve o/m@%s\nbytes 123456789012\n" + % (D1, D1[:19], "a" * 12), "README.md": "# x\n\n## Release History\n\n| Date | K3s | NV | Intel | Gaudi |\n" "|------------|-----|----|-------|-------|\n| 2026-10-01 | a | b | v1 | c |\n", } @@ -97,6 +99,18 @@ def test_version_digest_revision_and_date_tokens_pass(self): rc, out = self.run_case({"share/notes.txt": "tool v1.3.1 (2026-10-29), digest %s, rev %s\n" % (D2, "b" * 40)}) self.assertEqual(rc, 0, out) + def test_digests_and_revisions_cut_to_12_pass(self): + # hack/pinlib/goldens.py moves the cut form audits (sha256:<12>) and plans (@<12>) print + rc, out = self.run_case({"share/plan.txt": "ok image x/y@%s: %s in images.lock\nserve o/m@%s\n" + "bytes 123456789012\n" % (D2, D2[:19], "b" * 12)}) + self.assertEqual(rc, 0, out) + + def test_a_twelve_digit_number_is_not_a_cut_revision(self): + rc, out = self.run_case({"share/plan.txt": "ok image x/y@%s: %s in images.lock\nserve o/m@%s\n" + "bytes 123456789013\n" % (D1, D1[:19], "a" * 12)}) + self.assertEqual(rc, 1, out) + self.assertIn("share/plan.txt:3: +bytes 123456789013", out) + def test_a_site_may_change_beyond_its_tokens(self): # the site regex covers the whole URL, so a reshaped asset name is the pin's own move rc, out = self.run_case({"share/tool.sh": 'TOOL_URL="https://example.com/v1.3.0/tool-linux.tar"\nARGS="--a"\n'})