From 4d7796fe242fd2dd1ad68b6467a399e8dd8bb883 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:12:23 -0800 Subject: [PATCH 1/8] ledger(H62r): pre-register H62 at the GA tags, with its runner H62's re-run trigger fired: EAT release-2.0.0 was tagged (7b827e2d) and EAS v1.2.0 was released (a051f00f), naming enterprise-inference v1.1.0 (d035b1ef). H62r is the new row H62 named for that case: H62's hypothesis and predictions copied verbatim (B5's release identifiers aside), A1 unchanged, and H62's design, statistic, gates, caps, budget and verdict rule, with its own L3 seed. Between H62's commits and the GA tags only docs, VERSION, the layer manifests and enterprise-inference's LiteLLM changed, so the sizing that starved EAT's sandbox router in H62 is the same code. The pre-registration traces it before any measurement: EAS's topology probe counts the comma fields of cpu0's thread_siblings_list, reads the guest's adjacent pair "0-1" as one thread per core and falls back to its standard reservation (12 CPUs), so cpu: auto gave gemma-4-26b 76 CPUs. On crescent's host ("0,48") the probe reads two and EAS reserves the 48 siblings. A spike in a KVM guest of H62's shape reproduces the read with EAS's own probe (bench/h62r/topology-spike.sh). bench/h62r/ starts as a copy of bench/h62/, which stays untouched. Changes: the GA pins (Docker CE in VM a at its current patch release), H62r's own guests, ports and state, D1 as a check of the GA manifest, D3's sibling read (nri_reserved_cpu_list in global_config.yaml, EAS's documented setting, only when the probe reads one thread per core where the kernel lists two), sizing records in VM b whether the install passes or fails, a check that A1's render carries H50's OVMS image, main's EAS check in the smoke, a bound of 3 setup attempts, and the trigger's thresholds after the GA releases. pairs.py and live.sh learn H62r; pins/classes classifies bench/h62r and adds H62r to the harness lines that list H62; the technical report counts 83 rows and 10 PENDING. Nothing is measured here. --- .github/workflows/ci.yml | 2 + EXPERIMENTS.md | 252 +++++++++ bench/eatflow/live.sh | 2 + bench/eatflow/pairs.py | 9 +- bench/eatflow/test_pairs.py | 39 ++ bench/h62r/arm-A1 | 3 + bench/h62r/arm-B5 | 3 + bench/h62r/arm.sh | 89 ++++ bench/h62r/campaign.sh | 69 +++ bench/h62r/guest-a.sh | 84 +++ bench/h62r/guest-b.sh | 110 ++++ bench/h62r/h62r.py | 677 ++++++++++++++++++++++++ bench/h62r/pairs-hold.sh | 27 + bench/h62r/qemu.Dockerfile | 12 + bench/h62r/setup.sh | 103 ++++ bench/h62r/smoke.sh | 51 ++ bench/h62r/teardown.sh | 35 ++ bench/h62r/test_h62r.py | 907 +++++++++++++++++++++++++++++++++ bench/h62r/topology-spike.sh | 41 ++ bench/h62r/upstream.sh | 12 + bench/h62r/vm.sh | 104 ++++ docs/intel/technical-report.md | 4 +- pins/classes | 7 +- 23 files changed, 2633 insertions(+), 9 deletions(-) create mode 100755 bench/h62r/arm-A1 create mode 100755 bench/h62r/arm-B5 create mode 100755 bench/h62r/arm.sh create mode 100755 bench/h62r/campaign.sh create mode 100755 bench/h62r/guest-a.sh create mode 100755 bench/h62r/guest-b.sh create mode 100644 bench/h62r/h62r.py create mode 100755 bench/h62r/pairs-hold.sh create mode 100644 bench/h62r/qemu.Dockerfile create mode 100755 bench/h62r/setup.sh create mode 100755 bench/h62r/smoke.sh create mode 100755 bench/h62r/teardown.sh create mode 100644 bench/h62r/test_h62r.py create mode 100755 bench/h62r/topology-spike.sh create mode 100755 bench/h62r/upstream.sh create mode 100755 bench/h62r/vm.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02ef55e6..42b2e5bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -301,6 +301,8 @@ jobs: # fixtures, arm.sh against a local stub server, its records scored by h59.py. - run: python3 bench/h59/test_run.py - run: python3 bench/h62/test_h62.py + # H62r's runner: H62's at the GA tags, with D1 as a check and D3's sibling read. + - run: python3 bench/h62r/test_h62r.py # H58's runner (amendment R1): h58.py's interlock, placement and clause scorers, and run.sh # end to end against stand-ins for k3d-gpu, kubectl, docker and the Hub API. - run: python3 bench/h58/test_h58.py diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index df90e2f0..855d4a82 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -69,6 +69,7 @@ setup too weak to decide). Only `SUPPORTED` backs a claim. | H50o | (Model, arms and sampling superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20): Gemma 4 26B-A4B QAT.) H50's comparison with arm B3, EAT's own OVMS chart (`core/helm-charts/ovms` at release-1.0.3-hotfix, stock values, `openvino/model_server` 2025.4, the closest OpenVINO/ model it pulls, `OpenVINO/Qwen3-4B-int4-ov`) instead of B1: A1 delivers far more B24 tasks within budget than B3. With H50 it splits B1 -> A1 into EAT switching its own engine (B1 -> B3) and what k3d-gpu adds on top (B3 -> A1). | Superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20), which restates the predictions; as first written: Written 2026-10-20, before any measurement. B3's stock graph has no tool parser (the chart's `ovms --pull` writes none) and its model reasons by default, so its tool calls arrive as text: B3 within-budget success <= 0.10 (0.7), paired A1 - B3 >= +60 points (0.65). Pair yield: a B3 attempt should end within 30-90 s (a failed tool call or a reasoned text answer), so a 40-min hold should run 15-25 pairs and 24 pairs fit 2-3 holds; L2 passes (0.6), so SUPPORTED (0.5), INCONCLUSIVE (0.45). | As H50 with arm B = B3 (same pairs, gates, thresholds, statistic, cluster per hold `k3dgpu-e-h50o`); holds of at most 40 min until 24 pairs or 6 holds. The B1 -> B3 and B3 -> A1 splits are described from the two rows' matched pairs, never gated. `bench/eatflow/crescent/pairs-hold.sh H50o `, `agentbench pairs-report --hyp H50o --arm-a A1 --arm-b B3`. Model and predictions from [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20). Analysis rule L3 from [amendment A3](#crescent-amendment-a3-analysis-rule-l3-for-the-crescent-pairs-rows-2026-10-20). B3 is unchanged under [amendment E1](#crescent-amendment-e1-eat-stock-is-release-110-for-h50-h50o-and-h51-2026-10-29) (EAT release-1.1.0's OVMS chart renders the same objects); node image, deploy commit and order per [amendment E2](#crescent-amendment-e2-pins-deploy-commits-order-and-the-restore-check-for-the-pending-crescent-rows-2026-10-29); H50o's deploy commit, B3's records and its wiring smoke per [H50o amendment O1](#h50o-amendment-o1-deploy-commit-b3s-records-and-a-b3-wiring-smoke-2026-10-29). Result ([H50o run](#h50o-run-xeon-6741p-2026-10-29)): B3 was not ready in hold 1, so under M1's rule H50o runs no further hold and no pair ran (1 of the 6 permitted holds). On k3d, B3's PVC bound, its pod was scheduled and its init container pulled `OpenVINO/gemma-4-26b-a4b-it-int4-ov` at `4e5885c3` (every file equal to the Hub's), but OVMS 2025.4's server failed on each of its 7 starts, going into CrashLoopBackOff; the last start's recorded exit is 139 (SIGSEGV), a second into loading the model. The same server in a plain container, outside k3d, on the same revision, exited 139 the same way with and without the chart's limits. Not deliverable: EAT's OVMS chart as shipped does not serve this model. | INCONCLUSIVE | | H51 | (Model, arms and sampling superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20): Gemma 4 26B-A4B QAT.) Graded agents per box on the Xeon 6741P: H45's ramp, gates and scoring (B15-RO, typical load, 300 s steps, pooled p95 turn <= 10 s, success >= the arm's N=1 success - 10 points, errors <= 1%, >= 60 window requests), extended to N=16, for A1 and B1. | Superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20), which restates the predictions; as first written: Written 2026-10-20, before any measurement. A1 in [1, 8] (0.5). B1 in [0, 4] (0.5). B1 <= A1 (0.55). | H45's `apb` runner and scorer (`apb-score --hyp H51`, ramp 1, 2, 4, 8, 16), one fresh cluster per hold, at most 4 holds of at most 40 min. A prediction holds when the arm's bracket lies inside its range and is refuted when it lies outside; B1 <= A1 is decided only on matched load. REFUTED when any prediction is refuted, SUPPORTED when all three hold, else INCONCLUSIVE. Runs only if H50's holds end within their cap. Model and predictions from [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20). B1 as H50, from [amendment E1](#crescent-amendment-e1-eat-stock-is-release-110-for-h50-h50o-and-h51-2026-10-29); node image, deploy commit and order per [amendment E2](#crescent-amendment-e2-pins-deploy-commits-order-and-the-restore-check-for-the-pending-crescent-rows-2026-10-29); deploy commit, M1's ranges in the scorer, one hold per arm and the wiring smokes per [H51 amendment S1](#h51-amendment-s1-deploy-commit-scorer-ranges-ramp-per-hold-and-wiring-smokes-2026-10-29). Result ([H51 run](#h51-run-xeon-6741p-2026-10-29)): one hold per arm, 2 of the 4 permitted, none VOID. Both arms failed N=1. A1: 65 window requests, turn p95 10.41 s against the 10 s SLO, success 30/32. B1: 58 window requests (under 60), p95 11.52 s, success 27/29. Agents per box A1 0, outside its predicted [1, 4], so REFUTED. B1's 0 lies inside [0, 4], and B1 <= A1 holds on matched load (0 <= 0). | REFUTED | | H62 | Against Intel's new default. On the Xeon 6741P, inside EAT's banking flow (B24) with Gemma 4 26B-A4B on its QAT weights, the k3d-gpu CPU stack A1 (H50's A1 unchanged: `agent manifest`, OVMS 2026.4.1 int4 on the snapped QAT IR, CPU, no draft) delivers far more tasks within the 120 s turn budget than B5, Intel's documented single-node default for its agent toolkit: EAS v1.2.0-rc.1 (`5c27fb3`) installing EAT 2.0 (`release-2.0.0` `036b662`) and enterprise-inference `release-1.1` (`170eb3a`) with Kubespray onto one node, serving its catalog entry `gemma-4-26b` (vLLM CPU v0.28.0 on KServe, NRI balloons) on the same QAT weights. Each stack runs in its own KVM guest of one shape on the same host; both are live in one hold, interleaved. Deviations from stock are named (D1-D4; D4: the harness reaches B5's workload Service, not EAS's LiteLLM gateway and guardrail). A stack-vs-stack claim (amendment 16). | Written 2026-10-29, before any H62 step and after H50 (A1 19/24, B1 21/24) and H51 were known. A1 within-budget success >= 0.70 (0.65). B5 >= 0.70 (0.6). Paired A1 - B5 in [-0.20, +0.15] (0.6). Balance gate passes (0.9). REFUTED (0.6), INCONCLUSIVE (0.35: mostly B5 not deliverable in its guest within 120 min, a failed wiring smoke or a gate), SUPPORTED (0.05). Not gated: median attempt A1/B5 <= 0.85 (0.5); A1's median attempt in its guest over H50's 8.0 s on the host in [1.0, 1.3] (0.6). | [H62 pre-registration](#h62-pre-registration-k3d-gpu-against-intels-new-default-eas-with-eat-20-on-the-xeon-6741p-2026-10-29): H50's design, statistic, gates and verdict rule (rule L3, amendment A3, seed `H62-crescent-2026-10-29`), two KVM guests on crescent (VM a 96 vCPU and 96 GiB for A1, VM b 96 vCPU and 192 GiB for B5), the harness's LiteLLM per arm on the host, holds of at most 40 min until 24 pairs or 8 holds, each hold restarting both serving pods. Setup and wiring-smoke gates (memory, disk, AMX in each guest, NFD label, B5's weights equal to the QAT checkpoint, EAS's own `validate` and gateway check) come before hold 1. SUPPORTED: paired difference CI lower bound >= +30 points and A1 >= 0.6, with every gate passed. REFUTED: CI upper bound < +30 points. Otherwise INCONCLUSIVE. `bench/h62/campaign.sh`; `agentbench pairs-report --hyp H62 --arm-a A1 --arm-b B5`. Re-run as a new row when an EAS release at v1.2.0 or later or an EAT release at 2.0.0 or later is tagged (`bench/h62/upstream.sh`). Mechanics per [H62 amendment V1](#h62-amendment-v1-the-wiring-smokes-eas-check-without-the-rest-of-260-2026-11-01) (the smoke's EAS check), [V2](#h62-amendment-v2-eass-configure-answered-on-a-terminal-2026-11-01) (configure on a terminal), [V3](#h62-amendment-v3-d2-reads-the-catalogs-commented-name-lines-2026-11-01) (D2's catalog parse) and [V4](#h62-amendment-v4-the-clusters-state-is-recorded-when-b5s-install-fails-2026-11-01) (records on an install failure; the last setup attempt). Result ([H62 run](#h62-run-xeon-6741p-2026-11-01)): no hold ran, 0 of the 8 permitted, and no smoke ran. A1 served in its guest within its 45 min ceiling (`guest-a.sh` done at 35.7 and 34.7 min). Intel's install of B5 failed twice at the same step, inside its 120 min ceiling: EAT's sandbox-router could not be scheduled ("Insufficient cpu") because enterprise-inference's `cpu: auto` had sized gemma-4-26b to 76 of the node's 95.4 allocatable CPUs, beside Qwen3Guard's 8 and the platform's requests, which left 0.25 CPU for the router's 0.5. B5 is not deliverable in its guest, so H62 is INCONCLUSIVE by amendment V4's rule. | INCONCLUSIVE | +| H62r | H62's hypothesis, verbatim but for B5's release identifiers (the GA tags): Against Intel's new default. On the Xeon 6741P, inside EAT's banking flow (B24) with Gemma 4 26B-A4B on its QAT weights, the k3d-gpu CPU stack A1 (H50's A1 unchanged: `agent manifest`, OVMS 2026.4.1 int4 on the snapped QAT IR, CPU, no draft) delivers far more tasks within the 120 s turn budget than B5, Intel's documented single-node default for its agent toolkit: EAS v1.2.0 (`a051f00`) installing EAT 2.0 (`release-2.0.0` `7b827e2`) and enterprise-inference `v1.1.0` (`d035b1e`) with Kubespray onto one node, serving its catalog entry `gemma-4-26b` (vLLM CPU v0.28.0 on KServe, NRI balloons) on the same QAT weights. Each stack runs in its own KVM guest of one shape on the same host; both are live in one hold, interleaved. Deviations from stock are named (D1-D4; D4: the harness reaches B5's workload Service, not EAS's LiteLLM gateway and guardrail). A stack-vs-stack claim (amendment 16). | Written 2026-11-06, before any H62r step, after H62 (INCONCLUSIVE: no hold ran, B5's install not deliverable in its guest) was known. H62's predictions, copied verbatim and not tuned: A1 within-budget success >= 0.70 (0.65). B5 >= 0.70 (0.6). Paired A1 - B5 in [-0.20, +0.15] (0.6). Balance gate passes (0.9). REFUTED (0.6), INCONCLUSIVE (0.35: mostly B5 not deliverable in its guest within 120 min, a failed wiring smoke or a gate), SUPPORTED (0.05). Not gated: median attempt A1/B5 <= 0.85 (0.5); A1's median attempt in its guest over H50's 8.0 s on the host in [1.0, 1.3] (0.6). | [H62r pre-registration](#h62r-pre-registration-h62-at-the-ga-tags-eas-v120-with-eat-20-on-the-xeon-6741p-2026-11-06): H62's design, statistic, gates, caps, budget and verdict rule, unchanged (rule L3, amendment A3, its own seed `H62r-crescent-2026-11-06`; VM a 96 vCPU and 96 GiB for A1, VM b 96 vCPU and 192 GiB for B5; the harness's LiteLLM per arm on the host; holds of at most 40 min until 24 pairs or 8 holds, each restarting both serving pods; VM a's setup 45 min and VM b's install 120 min). B5 at the GA tags: EAS `v1.2.0`, whose layer manifest names enterprise-inference `v1.1.0` and EAT `release-2.0.0` by commit itself, so D1 is a check. From H62's commits to these, only docs, `VERSION`, the layer manifests and enterprise-inference's LiteLLM (v1.84.0) changed. Mechanics for H62's failure, registered before any measurement: EAS's topology probe counts the comma fields of cpu0's `thread_siblings_list`, reads the guest's adjacent pair (`0-1`) as one thread per core and falls back to its `standard` reservation (12 CPUs), so `cpu: auto` gave gemma-4-26b 96 - 12 - 8 = 76 CPUs and starved EAT's sandbox router; on crescent's host (`0,48`) the probe reads two and reserves the 48 siblings. D3's sibling read: when EAS's probe reads one thread per core where the guest kernel lists two, `nri_reserved_cpu_list` (EAS's documented setting in `global_config.yaml`) names the guest's 48 sibling vCPUs, the reservation EAS's default `siblings` profile makes when it reads SMT; expected, not gated, gemma-4-26b at 40 CPUs. A1's render must carry H50's OVMS image. At most 3 setup attempts. SUPPORTED: paired difference CI lower bound >= +30 points and A1 >= 0.6, with every gate passed. REFUTED: CI upper bound < +30 points. Otherwise INCONCLUSIVE. `bench/h62r/campaign.sh`; `agentbench pairs-report --hyp H62r --arm-a A1 --arm-b B5`. Re-run as a new row when an EAS release after v1.2.0 or an EAT release after `release-2.0.0` is tagged (`bench/h62r/upstream.sh`). | PENDING | | H52 | A plain `k3d-gpu up` (no flags, the auto-adapt default) on the Xeon 6741P host as found detects and enables every engine actually present and advertises each usable one to the scheduler, with nothing missing and nothing phantom; engines present but not configured on the host are named, not advertised. | Written 2026-10-20 from an inventory read from sysfs, lspci and CPUID, not from k3d-gpu. 4 x Arc Pro B70 (xe) as `gpu.intel.com/xe: 4`; AMX TILE, BF16, INT8, FP16 and 14 AVX-512/VNNI labels; no hybrid, NUMA, NVIDIA, Gaudi or NPU objects; DSA, IAA and QAT named as present but unconfigured (0.4: `up` may not name them). | The table in [H50-H52 pre-registration](#h50-h52-pre-registration-the-xeon-6741p-2026-10-20): `k3d-gpu doctor`, `K3D_GPU_CLUSTER=k3dgpu-h52 k3d-gpu up`, `kubectl get node -o json`, `k3d-gpu test`, and one enumeration pod per B70. SUPPORTED: every row matches. REFUTED: any row does not. Run 2026-10-20 ([H52 run](#h52-run-xeon-6741p-2026-10-20)): the four B70s (`gpu.intel.com/xe: 4`, `xmx=true`), AMX (TILE, BF16, INT8, FP16) and all 14 AVX-512/VNNI labels detected and reached pods, nothing phantom; but no media engine label on `xe`, and `up` does not name the unconfigured DSA, IAA and QAT (`doctor` does). | REFUTED | | H52b | After `k3d-gpu host-setup --apply` configures the host's DSA, IAA and QAT (the fix that adds that path, merged first), a plain `k3d-gpu up` also advertises them, and each one reaches a pod; the host is then restored exactly. | Written 2026-10-20, before the fix exists. DSA, IAA and QAT allocatable > 0, each at the count host-setup reports configuring; their test pods see `/dev/dsa`, `/dev/iax`, `/dev/vfio` (0.6). The other rows read as H52's expected table, media labels included ([amendment D1](#h52-amendment-d1-the-fixes-and-h52b-2026-10-20)). | Runs after H52, once the host-setup fix is merged; same procedure plus `host-setup --apply` before `up` and its printed revert after. SUPPORTED: the three engines advertised and reached, the H52 rows unchanged, and the host inventory read back equal to the one taken before. REFUTED: any of these fails. The host-setup command (`--apply --with-kernel-wqs`), the plan read first, the counts, the plain `up`'s node image and the revert per [amendment D2](#h52-amendment-d2-h52bs-host-setup-step-on-crescent-2026-10-29); the runner and deploy commit per [amendment D3](#h52-amendment-d3-h52bs-runner-and-deploy-commit-2026-10-29). Result ([H52b run](#h52b-run-xeon-6741p-2026-10-29)): one hold at k3d-gpu 0.1.79; the plan passed D2's gate (8 commands) and applied. DSA advertised 3 of the 3 configured work queues and its pod saw `/dev/dsa`; IAA got no work queue and no resource (as D2 item 4 disclosed); QAT got 48 VFs, but its plugin crash-looped and advertised none; `gpu.intel.com/xe` read 2, not 4: the serving pairs hold about half the VRAM of all four cards, and `up`'s busy-card filter (half the VRAM) left out the two just over half (16.1, 16.2 GiB) and kept the two just under it (15.4, 15.5 GiB), which the pairs also hold, so two enumeration pods ran, not four. Media labels (`vcs` 2, `vecs` 2), AMX, the 14 AVX-512/VNNI labels, CPU 96 and no other vendor read as expected. The third inventory equals the first. Three clauses fail, so REFUTED. | REFUTED | | H53 | Inside EAT's banking flow on the Arc Pro B70s with Qwen3.8-27B (thinking off), k3d-gpu's own GPU choice AG (OVMS int4 on one card, `agent manifest`, auto device) delivers far more B24 tasks within the 120 s turn budget than B2, "EAT PR #21 nearest profile (Qwen3.6-27B) on Qwen3.8 weights, fp16" (vLLM XPU v0.28.0, TP 4). A stack-vs-stack claim. | Written 2026-10-20, before any H53 measurement. AG >= 0.70 (0.6); B2 boots by ladder rung (b) (0.6); B2 >= 0.50 (0.5); block mean AG - B2 in [-0.15, +0.30] (0.5). INCONCLUSIVE (0.6), REFUTED (0.3), SUPPORTED (0.1). | Solo holds in the order AG, B2, B2, AG, AG, B2 (up to 8), blocks = hold pairs (2k-1, 2k), block-level inference: mean of per-block d over >= 3 counted blocks, t-interval. SUPPORTED: lower bound >= +30 points and AG >= 0.6, L3 gates passed. REFUTED: upper bound < +30 points. Else INCONCLUSIVE. `bench/eatflow/crescent/solo-hold.sh N ARM`, `agentbench solo-report --hyp H53 --arm-a AG --arm-b B2`. See [H53 pre-registration](#h53-pre-registration-eats-banking-flow-on-arc-pro-b70-2026-10-20). Node image, deploy commit, order and the restore check per [amendment E2](#crescent-amendment-e2-pins-deploy-commits-order-and-the-restore-check-for-the-pending-crescent-rows-2026-10-29); the solo runner fix, setup, the wiring smokes, the window and its futility stop per [H53 amendment R1](#h53-amendment-r1-the-solo-runner-fix-setup-wiring-smokes-and-the-gpu-window-2026-10-30); tools per [amendment E4](#crescent-amendment-e4-kubectl-v1372-helm-v431-and-uv-0136-in-the-crescent-workspace-2026-10-29). Result ([H53 run](#h53-run-arc-pro-b70-2026-10-30)): no hold ran, 0 of the 8 permitted. AG's wiring smoke passed (OVMS on one card, warm-up answered). B2 needed rung (b) (`setvars.sh` is not in the v0.28.0 image), then failed in its first cross-card collective: oneCCL opens `/dev/dri/by-path`, which the k3d node and pod lack. No further rung applies, so B2 is not deliverable and H53 is INCONCLUSIVE by its pre-registered rule. | INCONCLUSIVE | @@ -5145,6 +5146,257 @@ node-config-hash labels, which are hashes. default inside EAT's banking flow. On this guest shape that default does not finish installing, so the row is not decided. No new row is pre-registered here. +### H62r pre-registration: H62 at the GA tags, EAS v1.2.0 with EAT 2.0, on the Xeon 6741P (2026-11-06) + +Written 2026-11-06 and committed with its runner (`bench/h62r/`) before any H62r step. No guest +exists on crescent, nothing has been installed there for H62r, and nothing below is a measurement +of either arm. Crescent was only read, over read-only ssh, at 2026-11-07T03:36Z (its CPU sibling +lists). The hypothesis and predictions in the row are H62's, copied verbatim (B5's release +identifiers aside) and not tuned; they are never edited. A protocol change is a dated amendment, +merged before the data it governs. + +**Why.** + +- H62's pre-registration names this row: when an EAS release at v1.2.0 or later or an EAT + release at 2.0.0 or later is tagged, "a new row, H62r, is pre-registered: the same design with + B5 at the tagged release (its release notes' component table)". H62 ended at setup with no hold + (INCONCLUSIVE, not deliverable), so it keeps its verdict and its records stay as they are. +- The trigger has fired. `bench/h62/upstream.sh`, run 2026-11-07T04:06Z: `fired: eas v1.2.0, eat + release-2.0.0`. +- H62's question stands at the release Intel now ships: does k3d-gpu's CPU stack deliver far more + banking tasks within budget than Intel's documented single-node default for its agent toolkit? + +**Heads, read 2026-11-06 (`gh api`, `git ls-remote`, the release pages, the registries).** + +- EAS: `v1.2.0` is a lightweight tag on `a051f00fcb2dd473fe763610471cd509db82c433` ("Update rag's + revs (#108)", 2026-11-06T11:15:59Z). Its release (2026-11-06T13:53:39Z) is not a pre-release. + `release-1.2` is at the same commit; main is `35be51f8`. `VERSION` reads 1.2.0. A + `v1.2.0-rc.2` tag (`328b2322`) also exists and is not used. +- The EAS release notes' solution table: enterprise-agent-toolkit v2.0.0 (`release-2.0.0`), + ai-inference v1.1.0 and ai-erag v3.0.1. They give the install as `init agent-toolkit`, then + `install agent-toolkit`. +- EAT: `release-2.0.0` is an annotated tag on `7b827e2d95cec826a20c7e3f5a30eb4818ed2992` + (release published 2026-11-06T06:05:17Z). Its release notes say to clone enterprise-ai-solutions + and follow `docs/single-node-deployment.md`, then init, install and validate. That guide still + clones `-b release-1.2` (line 19), which is `a051f00`. +- enterprise-inference (EI): `v1.1.0` is an annotated tag on + `d035b1ef1981fbf5084ff05c9a337e620478bb87`, also the head of `release-1.1`. +- The v1.2.0 layer manifest (`configs/repos/repos.agent-toolkit.yaml` lines 16 and 25) names both + commits itself: `rev: "d035b1ef..." # v1.1.0` and `rev: "7b827e2d..." # release-2.0.0`. EAS main's + manifest names the branches `main` and `agentic-toolkit-migration`, so an unpinned clone is not + the release. +- What changed since H62's commits (`gh api repos/intel//compare/...`): + - EAS `5c27fb3`..`a051f00` (3 commits, 23 files): docs, `VERSION` and the layer manifests. The + installer and every role are unchanged. `configure` still refuses `--force` and still needs a + terminal (H62 amendment V2), and the topology probe and the reservation code are rc.1's. + - EAT `036b662`..`7b827e2` (4 commits, 17 files): docs only (the rename to "Intel® Agent + Toolkit"). `config.yaml` and `roles/agent_sandbox` are unchanged, so the sandbox router is + still the upstream manifest's 2 replicas at 250m CPU each, on the floating + `sandbox-router:latest-main` tag. + - EI `170eb3a`..`d035b1e` (2 commits, 21 files): docs, and LiteLLM moved to v1.84.0 (#53, a CVE + fix). `model_manager/` is unchanged: the `gemma-4-26b` entry (`cpu: auto`, 128Gi, vLLM CPU + `server_version` 0.28.0, the same args and env) and `nri_policy.py` are 170eb3a's byte for byte. +- `google/gemma-4-26B-A4B-it-qat-q4_0-unquantized` is still at `f1e06dc5` and not gated. + `vllm/vllm-openai-cpu:v0.28.0` still serves `sha256:197354e6...`. + +**H62's failure, traced to its mechanism, from source at the GA tags.** These are the mechanics +for enterprise-inference's `cpu: auto` starving EAT's sandbox router, registered before any +measurement. + +1. EAS's NRI role reads each node's topology with `roles/nri_cpu_balloons/files/probe-node-topology.sh`. + Its threads per core is the number of comma fields in `cpu0/topology/thread_siblings_list` + (line 31: `awk -F, '{print NF}'`). +2. Linux writes that list as a cpulist, with adjacent CPUs as a range. On crescent's host a core's + threads are `n` and `n+48`: cpu0 reads `0,48` and cpu1 `1,49` (read 2026-11-07T03:36Z), two + fields. QEMU numbers a guest's threads of one core next to each other, so in a guest of H62's + shape cpu0 and cpu1 are one core and the list reads `0-1`, one field. +3. Reproduced on the desktop, not on crescent: `bench/h62r/topology-spike.sh` boots a KVM guest + with `-smp 96,sockets=1,cores=48,threads=2` (QEMU 10.2.1, `-cpu host`, kernel 6.6, no disk and no + network) and runs the probe fetched from EAS at `a051f00`. It printed `cpu1 + thread_siblings_list=0-1`, `NRI_NUMA=1`, `NRI_LOGICAL=96`, `NRI_TPC=1` and `NRI_PHYSICAL=96`. + That matches H62's read of VM b (H62 run: `numa=1 physical_cpus=96 threads_per_core=1`). +4. With one thread per core, the default `siblings` profile of + `roles/nri_cpu_balloons/files/compute_reserved_cpus_advanced.sh` logs "SMT/Hyperthreading + disabled — no siblings to reserve" and falls back to `standard` (line 264): 12 reserved CPUs. + With two, it reserves every sibling, the higher thread of each core: on crescent's host + `48-95`, 48 CPUs. `gen-balloon-types.py` (line 71) documents that reservation as how the + balloons keep vLLM on physical cores. +5. EI's `auto_cpu` (`model_manager/core/nri_policy.py` line 569) gives a TP 1 model one NUMA + node's capacity (`status.capacity.cpu` over the NUMA nodes), minus the reserved CPUs per NUMA + node, minus the CPU requests of pods already in balloons. It does not subtract pods outside the + balloons. In H62's guest that was 96 - 12 - 8 (Qwen3Guard) = 76. With the rest of the stack's + 11.2 CPUs, 95.154 of 95.4 allocatable were requested, and 0.246 was left for the router's 0.5. +6. With the reservation the host's numbering gives, the same arithmetic is 96 - 48 - 8 = 40. About + 59.2 of 95.4 is then requested, and the router fits. This is Intel's own arithmetic on the + recorded requests, not a measurement. B5 cannot run on crescent's host to test it, because + Kubespray would remove Docker (H62 pre-registration). +7. So H62's "not deliverable" came from the guest's numbering meeting the probe's comma count + (D3), with EAS's sizing code doing what it does on any node it reads that way. The GA tags + change neither the probe nor the sizing, so a stock install in the same guest is expected to + fail at the same step. + +**B5 at the GA tags.** Intel's documented single-node install in VM b, with deviations D1-D4 +only. The steps, in order (changes from H62 marked): + +1. The node kernel settings of EAT's `docs/prerequisites.md`, verbatim. +2. `git clone -b v1.2.0`; the head must be `a051f00` (changed: H62 cloned `-b v1.2.0-rc.1`). +3. D1, now a check (changed). `h62r.py eas-check` requires each layer's one rev line to name the + pinned commit and, in its comment, the pinned release. Nothing is rewritten. +4. `./es_auto_installer.sh configure` on a new terminal that is sent `y` (H62 amendment V2). +5. `init agent-toolkit`. The two cloned layers must be at `d035b1e` and `7b827e2`. +6. The one documented edit (`llm_services_autodeploy_models: ["gemma-4-26b"]`), then D2. +7. D3's sibling read (new, below). +8. `install agent-toolkit`, with `HF_TOKEN` unset (recorded as set or unset). +9. Wait for the gemma-4-26b workload pod to be Ready. + +Left at EAS's and EAT's defaults, as in H62: Kubespray onto localhost, NRI balloons with the +`siblings` profile, Istio ambient, MetalLB, Envoy's AI gateway, observability, LiteLLM (now +v1.84.0) with the content-safety guardrail and the hybrid router, Langfuse, Redis Stack, pgvector +and agent-sandbox v0.5.0. `base_domain_name` stays `solutions.ai`. + +**Deviations from stock, named.** H62's D1-D4, with these changes. + +- **D1, commits: a check.** The GA manifest names the release commits itself (above), so D1 + changes nothing in the file. It stays named because the runner refuses any other manifest. +- **D2, weights:** unchanged. +- **D3, the guest:** H62's guest shape, image and QEMU, unchanged (below), plus the sibling read. + - In VM b, after the edit and before the install, `guest-b.sh` runs EAS's own probe from the + clone (`probe.txt`) and reads every CPU's `thread_siblings_list` (`thread-siblings.txt`). + - `h62r.py sibling-read` compares them. When the probe and the kernel agree, nothing is set + and B5 is stock. When the probe reads one thread per core and the kernel lists two on every + core, the result is the higher thread of each core: in this guest the 48 odd vCPUs, `1,3,...,95`. + Anything else stops the setup for an amendment. + - `h62r.py reserve` writes that cpuset as `nri_reserved_cpu_list: "..."` into + `env/local/global_config.yaml`. It refuses the file when the key already holds another value. + - That key is EAS's documented operator setting (`docs/customize/nri_cpu_balloons.md`, section 2, + "Edit global_config.yaml": "Optional: pin reserved CPUs to a fixed cpuset"). The installer + passes `global_config.yaml` to every playbook as extra vars (`es_auto_installer.sh` line 1490). + The NRI role then skips its own computation and writes the cpuset into the node's + BalloonsPolicy (`roles/nri_cpu_balloons/tasks/install_advanced.yaml` line 50, "Use + operator-supplied reserved CPU list"). `nri_deployment_profile` stays `siblings`. + - The lean. The setting restores the reservation EAS makes on crescent's own numbering, so the + guest stops changing Intel's sizing. It does not tune B5 to a number. Expected, not gated: + `cpu: auto` renders gemma-4-26b at 40 CPUs, where H62's stock read rendered 76 and left the + stack undeliverable. A1 serves on 48 CPUs in its guest (`agent manifest`'s count in H62's + setup). The guests' vCPUs float over the host's CPUs, so the guest's sibling split is not the + host's; that is part of the VM tax, described for both arms. +- **D4, the request path:** unchanged. + +**A1: H50's A1 unchanged.** The same guest steps and the same `agent manifest` command as H62 (no +backend named), at the deploy commit. New check: the render must carry H50's OVMS image, +`openvino/model_server:2026.4.1-gpu@sha256:a928903f...`, or the setup stops. Main has moved from +k3d-gpu 0.1.83 (H62's deploy) to 0.1.89, and since H59 the fit's engine serves a model the fit +sizes on this CPU class. The local IR is not sized (H62's records: "fit: no metadata"), so OVMS +is expected to serve it. + +**Guests (D3).** H62's shape: KVM, `-cpu host`, 96 vCPUs as 1 socket of 48 cores with 2 +threads, unpinned, no hugepages; VM a 96 GiB, VM b 192 GiB; user-mode networking with each +guest's ports on 127.0.0.1 only. Every guest pin was read again on 2026-11-06 and is still the +current one: the Ubuntu 24.04 cloud image `release-20261017` (the newest noble release, sha256 +`04e8f246...6c94`, 621284352 bytes), `ubuntu:26.04@sha256:6a06111c...` (the tag's digest is +unchanged), and `qemu-system-x86` 1:10.2.1+ds-1ubuntu3.3 and genisoimage 9:1.1.11-5 (the +candidates in that image). The one exception is Docker CE in VM a, which moves to 5:29.9.3, its +current patch release (a security release, 2026-11-03); containerd.io 2.4.2-3 is unchanged. +Docker's apt key still carries `9DC85822...0EBFCD88`. The guests, ports and state are H62r's own: +`h62r-vm-a` and `h62r-vm-b`, ssh 18842 and 18843, serve 18851 and 18852, `~/k3dgpu-bench/h62r`. + +**Harness, model, sampling, workload, design, statistic and verdict: H62's.** + +- The harness per arm on the host, the port-forwards to A1's `svc/ovms` and B5's + `gemma-4-26b-kserve-workload-svc:8000`, amendment M1's model and sampling (profile + `agent_gemma4`), B24 at N=1 under `typical` load, both arms live in one hold, interleaved pairs. +- Rule L3 (amendment A3) with seed `H62r-crescent-2026-11-06`. +- The settle gap and the 10 s pre-window, the balance gate, at least 16 complete pairs, at most 2 + void pairs and the idle-arm check (at or under 0.5 CPU, read in the other arm's guest). +- 120 s turn budget, 240 s task cap, H49b's outcome classes and grader. +- Scorer: `agentbench pairs-report --hyp H62r --arm-a A1 --arm-b B5`, H50's banking scorer + unchanged (`pairs.DESIGNS["H62r"]` is `("H49", "L3")`). +- **SUPPORTED** when the paired difference's 95% CI lower bound is at least +30 points and A1's + within-budget success is at least 0.6, with every gate passed. **REFUTED** when the CI's upper + bound is under +30 points. Otherwise **INCONCLUSIVE**, including a row that is not deliverable. +- Reported, never gated: H62's list (median attempt and turn p95, paired J per attempt, the + period-balanced estimate and L2's matched subset, the serving covariate and each guest's + CPU-s/s, B5's rendered CPU request, cpuset, `nproc` and ISA per hold, and the VM tax by + `h62r.py vmtax`), plus the sizing records below. + +**Sizing records**, reported, never gated. In VM b: `probe.txt`, `thread-siblings.txt`, +`d3-siblings.txt` and its diff, and the env files after the edit. After the install, whether it +passed (`ok-*`) or failed (`fail-*`): the BalloonsPolicy CRs, every pod's CPU request and limit, +`kubectl describe nodes` and the sandbox router's Deployment. A failed install also keeps H62 +amendment V4's records. + +**Holds, caps and gates.** H62's items 1-8, with these changes. + +- Deploy commit: one clean, detached checkout at `~/k3dgpu-bench/k3d-gpu-h62r`, at an + `origin/main` commit at or after this PR's merge. PR #260 merged on 2026-11-01, so there is no + run-branch condition. VM a gets the same commit (`git archive`), and every record names it. +- Setup: at most 3 attempts in all. `campaign.sh` counts its starts in `attempts.txt` and + refuses a fourth before anything starts. Between attempts, a dated amendment names the cause and + the change. If the records trace a failed install to stock B5 itself (EAS's or EAT's own sizing, + images or defaults in this guest with D3's sibling read), H62r is INCONCLUSIVE (not + deliverable) with no further attempt, as H62 amendment V4 ruled. A router that is still not + placed with the sibling read is that case. +- Wiring smoke (`bench/h62r/smoke.sh`): `./es_auto_installer.sh validate agent-toolkit`, then + `integrations/eat/eat-check.sh` with `EAT_MODE=eas` through EAS's own gateway on `gemma-4-26b` + (#260's check, now on main, with `integrations/eas/eas-ready.py` beside it), both from the deploy + commit and recorded by sha256. Then one hold, `H62r-smoke`, with the pair-start guard past its + end. All of it must pass before hold 1. +- Holds (`bench/h62r/pairs-hold.sh N` under `bench/eatflow/crescent/hold.sh`): at most 40 min each, + until 24 pairs or 8 holds, each restarting both serving pods with H62's 900 s readiness rule, + two not-deliverable holds in a row ending the row, the 600 s pair-start guard and the hold-start + rule. +- One hold at a time on crescent (amendment E2 item 4; H56 forbids two unsliced holds at once). The + guests stay up from setup to teardown, and other rows run no hold in that window. + +**Risks, named before measuring.** H62's list, plus these. + +- The operator cpuset path has not run here. The NRI role still generates its balloon types from + the probe's read (96 CPUs, one thread per core), now beside a reserved cpuset of 48. A failure + there goes to the setup bound above. +- B5 runs vLLM on about 40 vCPUs, not 76. If B5 then misses the 120 s turn budget more often, + that is the result of Intel's sizing on this host's numbering, and it is reported with the + rendered request. +- Another limit may wait after the router step, which H62 never passed. EAS's gateway now runs + LiteLLM v1.84.0; the smoke's check reads the guardrail's answers of v1.83.14 and v1.105.1. +- The router image is EAT's floating staging tag; each hold records the pod's image ID. +- The trigger can fire again during the run (below). + +**Re-run trigger.** `bench/h62r/upstream.sh` (`h62r.py trigger`) reads both repositories' tags. +It fires on an EAS release tag after v1.2.0 (`vX.Y.Z`, not a release candidate) or an EAT release +tag after `release-2.0.0` (`release-X.Y.Z`, `vX.Y.Z`, or the `-hotfix` form, including +`release-2.0.0-hotfix`). On 2026-11-07T04:06Z it read `fired: none`. It runs before setup and +before each hold, and its output goes into the records. + +- Fired before hold 1: a dated amendment re-pins B5 to the tagged release, by amendment E1's + logic. If the tag's components equal the pinned commits, the amendment only records that. +- Fired after hold 1: H62r's holds finish at these pins, and H62r keeps its verdict. A new row + is pre-registered for the tagged release. + +**Runner (this PR), with regression tests.** + +- `bench/h62r/` started as a copy of `bench/h62/`, which stays as it is (historical). The + differences: the pins (`h62r.py PINS`), the guests' names, ports, label, share tag and state + directory, D1 as a check (`eas-check`), D3's sibling read (`sibling-read`, `reserve`) and the + sizing records in `guest-b.sh`, A1's image check in `guest-a.sh`, main's EAS check in + `smoke.sh`, the setup bound in `campaign.sh`, and the trigger's thresholds. `topology-spike.sh` + is the spike of item 3 above. +- `bench/eatflow/pairs.py`: `DESIGNS["H62r"] = ("H49", "L3")` and its order seed. + `bench/eatflow/live.sh`: `ARM=H62r` runs H62's remote-arm branch. +- `pins/classes`: `bench/h62r/**` is H62r's ledger line, and H62r joins the `bench/eatflow` lines + that list H62. +- Tests: `bench/h62r/test_h62r.py` (in CI) and `H62rTest` in `bench/eatflow/test_pairs.py`. + +**Reproduce (planned).** On crescent, from `~/k3dgpu-bench/k3d-gpu-h62r` at the deploy commit: +`bench/h62r/upstream.sh`, then `bench/h62r/campaign.sh` (setup, smoke, holds, score, teardown). +Records go to `~/k3dgpu-bench/results/H62r/`, with the hold records in +`~/k3dgpu-bench/runs/H62r-hold/` and the smoke in `~/k3dgpu-bench/smoke/H62r/`. They come back +under `bench/eatflow/results/2026-11-06/H62r/`. + +**Also found, offered upstream.** EAS v1.2.0's probe reads one thread per core on any Linux node +that lists a core's threads as a range, as this KVM guest does. Counting the CPUs of the cpulist, +with ranges expanded, instead of its comma fields would read two. It is an upstreamable finding +for Intel's EAS team; nothing is filed from this row. + ### H53 pre-registration: EAT's banking flow on Arc Pro B70 (2026-10-20) Written 2026-10-20, before any H53 measurement and before either H53 arm has run on a GPU. The diff --git a/bench/eatflow/live.sh b/bench/eatflow/live.sh index bc3bda1f..061741d5 100755 --- a/bench/eatflow/live.sh +++ b/bench/eatflow/live.sh @@ -47,6 +47,7 @@ # pairs: ARM = H47, H49, H47b, H49b, H50 or H50o, both arms live in cluster k3dgpu-e-, one hold # of interleaved pairs into LIVE_PAIRS_DIR (LIVE_HOLD = this hold's number); # ARM = H62: both servers in KVM guests, driven by LIVE_REMOTE_A and LIVE_REMOTE_B (bench/h62); +# ARM = H62r: the same, with H62r's guests (bench/h62r); # code: H43 (amendment W1), the coding agent on C12 with `k3d-gpu sandbox up`, one # block per level in LIVE_CODE_LEVELS (default "typical heavy heavy typical") into # LIVE_CODE_DIR/h43-- (shared by an arm's holds, so a level's halves resume); @@ -498,6 +499,7 @@ if [ "${MODE}" = pairs ]; then DESIGN=${FAMILY} [ "${ARM}" = H50o ] && DESIGN=H50o [ "${ARM}" = H62 ] && DESIGN=H62 + [ "${ARM}" = H62r ] && DESIGN=H62 # H62 re-run at the GA tags: the same remote arms (bench/h62r) [ "${ARM}" = H50 ] && KV_GIB=${LIVE_EAT_KV_GIB:-40} # EAT's shipped KV cache (40 GiB) mkdir -p "${PD}" PD=$(cd "${PD}" && pwd) diff --git a/bench/eatflow/pairs.py b/bench/eatflow/pairs.py index b9f1222d..cd36b1b4 100644 --- a/bench/eatflow/pairs.py +++ b/bench/eatflow/pairs.py @@ -17,7 +17,7 @@ pairs' pre-windows. bg_cpu, package W and throttle during the attempt are reported, never gated. -Load rule L3 (H50, H50o, H53, H62; EXPERIMENTS.md "Crescent amendment A3"): the same +Load rule L3 (H50, H50o, H53, H62, H62r; EXPERIMENTS.md "Crescent amendment A3"): the same pre-windows, no match filter. Every complete pair enters the statistic; a balance gate (a sign-flip test of arm on each pre covariate, p < 0.01 gates) replaces it; a two-regressor adjustment and L2's matched subset are reported, never gated. The arm order of pair j is drawn @@ -538,16 +538,17 @@ def boot_share(d, pred, boot=stats.BOOT, seed=stats.SEED, stat=None): # Rule L3 rows: the seed of their arm order, fixed in the pre-registration (amendment A3). ORDER_SEEDS = {"H50": "H50-crescent-2026-10-20", "H50o": "H50o-crescent-2026-10-20", "H53": "H53-crescent-2026-10-20", "H62": "H62-crescent-2026-10-29", - "H58": "H58-crescent-2026-11-01"} + "H58": "H58-crescent-2026-11-01", "H62r": "H62r-crescent-2026-11-06"} # Hypothesis id -> (design, load rule). H47b and H49b are H47 and H49 under rule L2; H50 is # H49b's design on the Xeon 6741P, H50o the same with arm B3 (EXPERIMENTS.md "H50-H52 pre-registration"); # H53 the H49 statistic over solo-hold block pairs on the Arc Pro B70s (solo.py; block-level verdict, "H53 pre-registration"); # H62 H50's design with arm B5, Intel's EAS default, each server in a KVM guest ("H62 pre-registration"); -# H58 one solo hold of arm Z, the zero-touch agent, for its clause (c) ("H58 amendment R1"). +# H58 one solo hold of arm Z, the zero-touch agent, for its clause (c) ("H58 amendment R1"); +# H62r H62's design with B5 at the GA tags ("H62r pre-registration"). DESIGNS = {"H47": ("H47", "L1"), "H49": ("H49", "L1"), "H47b": ("H47", "L2"), "H49b": ("H49", "L2"), "H50": ("H49", "L3"), "H50o": ("H49", "L3"), "H53": ("H49", "L3"), "H62": ("H49", "L3"), - "H58": ("H49", "L3")} + "H58": ("H49", "L3"), "H62r": ("H49", "L3")} # The ledger a rule-L3 row must be pre-registered in before any of its holds runs. diff --git a/bench/eatflow/test_pairs.py b/bench/eatflow/test_pairs.py index c796d317..91a70871 100644 --- a/bench/eatflow/test_pairs.py +++ b/bench/eatflow/test_pairs.py @@ -1092,6 +1092,45 @@ def test_live_sh_runs_h62_against_remote_arms(self): self.assertIn("REMOTE=()", live[:live.index("cleanup() {")]) +class H62rTest(unittest.TestCase): + """H62r (EXPERIMENTS.md "H62r pre-registration"): H62 re-run with B5 at the GA tags. Its own + row, seed and pairs registration; live.sh runs it through H62's remote-arm branch.""" + + REG = {"hyp": "H62r", "rule": "L3", "seed": "H62r-crescent-2026-11-06"} + + def test_h62r_is_h62s_design_and_rule_with_its_own_seed(self): + self.assertEqual(PR.design("H62r"), ("H49", "L3")) + self.assertEqual(PR.ORDER_SEEDS["H62r"], self.REG["seed"]) + self.assertIn("H62r", AB.PAIRS_HYPS) + at = [dict(r, arm="B5") if r["arm"] == "B1" else r for r in L2Test.h49_like(L2Test())] + res = PR.analyze(at, pairs_of(24), "H62r", "A1", "B5", registration=self.REG) + ref = PR.analyze(at, pairs_of(24), "H62", "A1", "B5", registration=H62Test.REG) + self.assertEqual((res["rule"], res["arms"]), ("L3", ["A1", "B5"])) + self.assertEqual((res["verdict"], res["paired_diff"]), (ref["verdict"], ref["paired_diff"])) # H62's statistic + with self.assertRaises(ValueError): # H62's registration is not H62r's + PR.analyze(at, pairs_of(24), "H62r", "A1", "B5", registration=H62Test.REG) + + def test_h62r_draws_its_own_arm_order(self): + h62 = [PR.order(j, "A1", "B5", PR.ORDER_SEEDS["H62"]) for j in range(24)] + h62r = [PR.order(j, "A1", "B5", PR.ORDER_SEEDS["H62r"]) for j in range(24)] + self.assertNotEqual(h62, h62r) + self.assertEqual(h62r, [PR.order(j, "A1", "B5", "H62r-crescent-2026-11-06") for j in range(24)]) + + def test_registration_names_h62r(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + self.assertEqual(PR.register(tmp, "H62r"), self.REG) + with self.assertRaises(PR.HoldAborted): # an H62 hold cannot reuse the directory + PR.register(tmp, "H62") + + def test_live_sh_runs_h62r_through_the_remote_arm_branch(self): + with open(os.path.join(HERE, "live.sh")) as f: + live = f.read() + self.assertIn('[ "${ARM}" = H62r ] && DESIGN=H62', live) + pairs = live[live.index('if [ "${MODE}" = pairs ]; then'):] + self.assertLess(pairs.index('[ "${ARM}" = H62r ] && DESIGN=H62'), pairs.index("case ${DESIGN} in")) + + class B3RecordsTest(unittest.TestCase): """H50o amendment O1 item 2: the records of B3's pulled model that the pre-registration names (the init container's log, every file's sha256, the graph each hold serves), read from the diff --git a/bench/h62r/arm-A1 b/bench/h62r/arm-A1 new file mode 100755 index 00000000..c54a54f4 --- /dev/null +++ b/bench/h62r/arm-A1 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# arm-A1: bench/h62r/arm.sh for arm A1, as one word (live.sh's LIVE_REMOTE_A). +exec "$(dirname "$0")/arm.sh" A1 "$@" diff --git a/bench/h62r/arm-B5 b/bench/h62r/arm-B5 new file mode 100755 index 00000000..cc0fe57a --- /dev/null +++ b/bench/h62r/arm-B5 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# arm-B5: bench/h62r/arm.sh for arm B5, as one word (live.sh's LIVE_REMOTE_B). +exec "$(dirname "$0")/arm.sh" B5 "$@" diff --git a/bench/h62r/arm.sh b/bench/h62r/arm.sh new file mode 100755 index 00000000..6fe2d43c --- /dev/null +++ b/bench/h62r/arm.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# arm.sh ARM CMD [ARG]: one H62r arm's server in its KVM guest (EXPERIMENTS.md "H62r +# pre-registration"). bench/eatflow/live.sh's remote-arm mode calls it through arm-A1 and arm-B5. +# port the host port its Service is forwarded to (the guest's hostfwd) +# restart SECONDS delete its serving pod, then wait until a new pod is Ready; exit 3 when none is +# Ready within SECONDS (the hold's 900 s readiness rule passes the time left) +# records DIR the pod, the CPU ISA it sees (amx_tile lines), its cpuset and nproc, the nodes, +# the guest's load and memory, and B5's LLMInferenceService +# logs DIR every pod, events, the pod description and the server log (a hold that fails) +# forward (re)start `kubectl port-forward` of its Service in the guest, on the serve port +# unforward stop that port-forward +# cgroup "exec: ARM cpustat ": its pod's cgroup in the guest, as the +# idle-arm check reads it (bench/eatflow/pairs.py) +# cpustat DIR that cgroup's cpu.stat +# ARM: A1 (VM a: k3d-gpu's agent OVMS) or B5 (VM b: EAS's gemma-4-26b LLMInferenceService). +# H62R_STATE (default ~/k3dgpu-bench/h62r) holds ssh_config; H62R_SSH is the ssh client (tests stub +# it); H62R_POLL_S is the restart poll period (default 5 s). Each guest's ~/h62r/env sets PATH and +# KUBECONFIG for kubectl there. +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SELF=${HERE}/arm.sh +P() { python3 "${HERE}/h62r.py" "$@"; } +ARM=${1:?usage: arm.sh A1|B5 CMD [ARG]} CMD=${2:?usage: arm.sh A1|B5 CMD [ARG]} +fields=$(P arm-fields "${ARM}" 2>/dev/null) || { echo "arm.sh: no arm ${ARM} (A1 or B5)" >&2; exit 2; } +read -r _ HOST PORT NS SEL SVC SPORT CTR EXTRA <<< "${fields}" +STATE=${H62R_STATE:-${HOME}/k3dgpu-bench/h62r} +# g CMD: one command in the guest, after its ~/h62r/env. +g() { ${H62R_SSH:-ssh} -F "${STATE}/ssh_config" "${HOST}" ". ~/h62r/env && $*"; } +pods() { g "kubectl -n ${NS} get pods -l ${SEL} -o json"; } +ready_pod() { pods | P pick-pod "${1:-}"; } # " " of a Ready pod whose uid is not in $1 +case ${CMD} in +port) + echo "${PORT}" ;; +restart) + left=${3:-} + [ -n "${left}" ] || { echo "usage: arm.sh ${ARM} restart SECONDS" >&2; exit 2; } + [ "${left}" -gt 0 ] 2>/dev/null || { echo "no time left to restart"; exit 3; } + end=$(( SECONDS + left )) + old=$(pods | P uids) || { echo "cannot list the pods"; exit 1; } + g "kubectl -n ${NS} delete pod -l ${SEL} --wait=false" || { echo "cannot delete the pods"; exit 1; } + while [ "${SECONDS}" -lt "${end}" ]; do + if p=$(ready_pod "${old}"); then echo "ready: ${p}"; exit 0; fi + sleep "${H62R_POLL_S:-5}" + done + echo "not ready within ${left} s" + exit 3 ;; +records) + d=${3:?usage: arm.sh ARM records DIR} + mkdir -p "${d}" + read -r name uid <<< "$(ready_pod "")" + [ -n "${name:-}" ] || { echo "no Ready pod"; exit 1; } + echo "${name} ${uid}" > "${d}/pod.txt" + g "kubectl -n ${NS} get pod ${name} -o json" > "${d}/pod.json" + g "kubectl -n ${NS} exec ${name} -c ${CTR} -- grep -c amx_tile /proc/cpuinfo" > "${d}/isa.txt" 2>&1 + g "kubectl -n ${NS} exec ${name} -c ${CTR} -- sh -c 'cat /sys/fs/cgroup/cpuset.cpus.effective; nproc'" \ + > "${d}/cpuset.txt" 2>&1 + g "kubectl get nodes -o json" > "${d}/nodes.json" + g "uptime; free -g; grep -c amx_tile /proc/cpuinfo; nproc" > "${d}/guest.txt" 2>&1 + [ -z "${EXTRA}" ] || g "kubectl -n ${NS} get ${EXTRA} -o yaml" > "${d}/extra.yaml" 2>&1 + g "kubectl get pods -A -o wide" > "${d}/pods.txt" 2>&1 + echo "records: ${d}" ;; +logs) + d=${3:?usage: arm.sh ARM logs DIR} + mkdir -p "${d}" + g "kubectl get pods -A -o wide" > "${d}/pods.txt" 2>&1 + g "kubectl get events -A --sort-by=.lastTimestamp" > "${d}/events.txt" 2>&1 + g "kubectl -n ${NS} describe pods -l ${SEL}" > "${d}/describe.txt" 2>&1 + g "kubectl -n ${NS} logs -l ${SEL} -c ${CTR} --tail=1000" > "${d}/server.log" 2>&1 + echo "logs: ${d}" ;; +forward) + g "pid=\$(cat ~/h62r/pf-${ARM}.pid 2>/dev/null) && kill \${pid} 2>/dev/null;" \ + "setsid nohup kubectl -n ${NS} port-forward --address 0.0.0.0 ${SVC} ${PORT}:${SPORT}" \ + "> ~/h62r/pf-${ARM}.log 2>&1 < /dev/null & echo \$! > ~/h62r/pf-${ARM}.pid" ;; +unforward) + g "pid=\$(cat ~/h62r/pf-${ARM}.pid 2>/dev/null) && kill \${pid} 2>/dev/null; rm -f ~/h62r/pf-${ARM}.pid" ;; +cgroup) + read -r _ uid <<< "$(ready_pod "")" + [ -n "${uid:-}" ] || { echo "no Ready pod" >&2; exit 1; } + # cgroupfs driver (k3s in VM a's k3d node): pod; systemd driver (Kubespray's containerd + # in VM b): ...-pod.slice. + dir=$(g "find /sys/fs/cgroup -maxdepth 9 -type d \\( -name pod${uid} -o -name '*pod${uid//-/_}.slice' \\) -print -quit") + [ -n "${dir}" ] || { echo "no cgroup for pod ${uid}" >&2; exit 1; } + echo "exec:${SELF} ${ARM} cpustat ${dir}" ;; +cpustat) + g "cat ${3:?usage: arm.sh ARM cpustat DIR}/cpu.stat" ;; +*) + echo "usage: arm.sh A1|B5 port|restart SECONDS|records DIR|logs DIR|forward|unforward|cgroup|cpustat DIR" >&2 + exit 2 ;; +esac diff --git a/bench/h62r/campaign.sh b/bench/h62r/campaign.sh new file mode 100755 index 00000000..8d3d659c --- /dev/null +++ b/bench/h62r/campaign.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# campaign.sh: H62r end to end on crescent (EXPERIMENTS.md "H62r pre-registration"), from the H62r +# checkout at the deploy commit: setup.sh, smoke.sh, then holds (pairs-hold.sh 1, 2, ...) until +# B24's 24 pairs are recorded or 8 holds have run, then the score. teardown.sh runs at the end, +# also after an abort. +# - at most 3 setup attempts in all (attempts.txt, one line per start); a fourth start exits 10 +# before anything is started +# - setup exit 6 (the host gate still failed after 60 min): the two qwen38 serving pairs are +# stopped under the box owner's grant (gpu-serving.sh stop, never rm) and setup runs once more; +# teardown.sh starts them again +# - a hold that exits 3 (a server not ready with 900 s left) counts as one of the 8 and runs no +# pair; after 2 such holds in a row no further hold runs (INCONCLUSIVE, not deliverable) +# Records: ~/k3dgpu-bench/results/H62r (campaign.log, attempts.txt, setup/, pairs/, hold/, +# report.txt, vmtax.json). +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(cd "${HERE}/../.." && pwd) +B=${HOME}/k3dgpu-bench +R=${B}/results/H62r +MAX_HOLDS=8 NOT_DELIVERABLE_MAX=2 ATTEMPTS_MAX=3 +mkdir -p "${R}" +log() { echo "== campaign $* $(date -u +%FT%TZ)" | tee -a "${R}/campaign.log"; } +tries=$(grep -c . "${R}/attempts.txt" 2>/dev/null) +if [ "${tries:-0}" -ge "${ATTEMPTS_MAX}" ]; then + log "setup attempt bound reached (${tries:-0} of ${ATTEMPTS_MAX}): no further attempt" + exit 10 +fi +echo "$(date -u +%FT%TZ) $(git -C "${REPO}" rev-parse HEAD)" >> "${R}/attempts.txt" +teardown() { bash "${HERE}/teardown.sh" >> "${R}/teardown.log" 2>&1; log "teardown exit $?"; } +trap teardown EXIT +log "start (attempt $(( ${tries:-0} + 1 )) of ${ATTEMPTS_MAX}) at $(git -C "${REPO}" rev-parse HEAD)" +bash "${HERE}/setup.sh" +rc=$? +if [ "${rc}" = 6 ]; then + log "host gate failed: stopping the qwen38 serving pairs (owner's grant; docker stop, never rm)" + for c in qwen38-a-hr qwen38-b-hr; do + if bash "${REPO}/bench/eatflow/crescent/gpu-serving.sh" stop "${c}" >> "${R}/serving.log" 2>&1; then + echo "${c}" >> "${R}/serving-stopped.txt" + fi + done + bash "${HERE}/setup.sh" + rc=$? +fi +[ "${rc}" = 0 ] || { log "setup exit ${rc}: no hold"; exit "${rc}"; } +bash "${HERE}/smoke.sh" +rc=$? +[ "${rc}" = 0 ] || { log "wiring smoke failed (exit ${rc}): no hold"; exit 9; } +nd=0 +for n in $(seq 1 "${MAX_HOLDS}"); do + left=$(python3 "${HERE}/h62r.py" pairs-left "${R}/pairs") + [ "${left}" -gt 0 ] || break + log "hold ${n}: ${left} pair(s) left" + bash "${HERE}/pairs-hold.sh" "${n}" + rc=$? + log "hold ${n} exit ${rc}" + if [ "${rc}" = 3 ]; then + nd=$(( nd + 1 )) + [ "${nd}" -lt "${NOT_DELIVERABLE_MAX}" ] || { log "${nd} holds in a row not deliverable: no further hold"; break; } + else + nd=0 + fi +done +python3 "${REPO}/bench/eatflow/agentbench.py" pairs-report "${R}/pairs" --hyp H62r --arm-a A1 --arm-b B5 \ + > "${R}/report.txt" 2>&1 +log "pairs-report exit $?" +h50=${REPO}/bench/eatflow/results/2026-10-20/H50/pairs/analysis.json +if [ -f "${R}/pairs/analysis.json" ]; then + python3 "${HERE}/h62r.py" vmtax "${h50}" "${R}/pairs/analysis.json" > "${R}/vmtax.json" +fi diff --git a/bench/h62r/guest-a.sh b/bench/h62r/guest-a.sh new file mode 100755 index 00000000..df1c3c31 --- /dev/null +++ b/bench/h62r/guest-a.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# guest-a.sh [deploy]: VM a's setup for H62r's A1 (EXPERIMENTS.md "H62r pre-registration"), run +# in the guest by setup.sh as user ubuntu, under its 45 min ceiling. A1 is H50's A1 unchanged, as +# in H62: +# 1. Docker CE from Docker's apt repository at the pinned version (its key's fingerprint +# checked first) and python3-yaml for bench/apb/stage.py; then this script runs again in a +# shell with the docker group, as `deploy` +# 2. A1's IR from the host's read-only share (virtfs tag h62rshare), checked against its +# manifest.json +# 3. `k3d-gpu up` at the deploy commit (~/h62r/k3d-gpu, copied by setup.sh), with H50's node +# image and crescent amendment C1's kubelet eviction thresholds, cluster k3dgpu-e-h62r +# 4. `k3d-gpu agent manifest` with live.sh's gemma4 settings for A1 (CPU, the snapped QAT IR, +# tool parser gemma4, no reasoning parser, no draft, no autoquant); the render must carry +# H50's OVMS 2026.4.1 image, or the setup stops here; then staged and applied as live.sh +# does, and rolled out +# 5. records: amendment A4's `k3d-gpu plan` line, versions, the node, the pod +# k3d and kubectl are setup.sh's checked binaries in ~/h62r/bin. Records: ~/h62r/records. +set -euo pipefail +G=${HOME}/h62r +P() { python3 "${G}/h62r.py" "$@"; } +R=${G}/records +REPO=${G}/k3d-gpu +mkdir -p "${R}" +export PATH=${G}/bin:${PATH} KUBECONFIG=${G}/kubeconfig XDG_STATE_HOME=${G}/state K3D_GPU_CLUSTER=k3dgpu-e-h62r +step() { echo "== $1 $(date -u +%FT%TZ)" | tee -a "${R}/steps.txt"; } +if [ "${1:-}" != deploy ]; then + step docker + # ${PATH} expands when a command in the guest sources ~/h62r/env (bench/h62r/arm.sh). + # shellcheck disable=SC2016 + printf 'export PATH=%s/bin:${PATH} KUBECONFIG=%s/kubeconfig\n' "${G}" "${G}" > "${G}/env" + curl -fsSLo "${G}/docker.asc" https://download.docker.com/linux/ubuntu/gpg + gpg --show-keys --with-colons "${G}/docker.asc" | awk -F: '/^fpr/ {print $10}' | grep -x "$(P pin docker_gpg_fpr)" > /dev/null \ + || { echo "Docker's apt key is not $(P pin docker_gpg_fpr)"; exit 1; } + sudo install -m 0755 -d /etc/apt/keyrings + sudo install -m 0644 "${G}/docker.asc" /etc/apt/keyrings/docker.asc + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update -q 2>&1 | tee "${R}/apt-update.log" > /dev/null + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -q \ + docker-ce="$(P pin docker_ce)" docker-ce-cli="$(P pin docker_ce)" containerd.io="$(P pin containerd)" \ + python3-yaml="$(P pin python3_yaml)" 2>&1 | tee "${R}/apt-install.log" > /dev/null + sudo usermod -aG docker "$(id -un)" + exec sg docker -c "bash ${G}/guest-a.sh deploy" +fi +step ir +sudo mkdir -p /mnt/h62r-ir +mountpoint -q /mnt/h62r-ir || sudo mount -t 9p -o trans=virtio,version=9p2000.L,ro h62rshare /mnt/h62r-ir +P ir-verify /mnt/h62r-ir > "${R}/ir-verify.txt" +step up +K3D_GPU_IMAGE="$(P pin node_image)" \ + K3D_GPU_K3S_ARGS="--kubelet-arg=eviction-hard=memory.available<100Mi,nodefs.available<1%,imagefs.available<1%,nodefs.inodesFree<1%" \ + timeout 900 "${REPO}/scripts/k3d-gpu" up > "${R}/up.log" 2>&1 +step manifest +# H50's and H62's command. Since H59 a model the fit sizes is served by the fit's engine on this +# CPU class; the local IR is not sized (H62's records), so OVMS serves it. The render is checked +# for H50's OVMS image, so a deploy commit that serves A1 otherwise stops here. +K3D_GPU_AGENT_DEVICE=CPU K3D_GPU_AGENT_MODEL=k3dgpu/gemma-4-26B-A4B-it-qat-int4-ov K3D_GPU_AGENT_MODEL_NAME=gemma-4-26b-a4b-it \ + K3D_GPU_AGENT_DRAFT=off K3D_GPU_AGENT_TOOL_PARSER=gemma4 K3D_GPU_AGENT_REASONING_PARSER='' K3D_GPU_AUTOQUANT=off \ + "${REPO}/scripts/k3d-gpu" agent manifest > "${R}/agent.yaml" 2> "${R}/agent.err" +grep -qF "image: $(P pin ovms_image)" "${R}/agent.yaml" \ + || { echo "agent manifest does not render $(P pin ovms_image): A1 would not be H50's A1"; exit 1; } +python3 "${REPO}/bench/apb/stage.py" ovms "${R}/agent.yaml" "${R}/deploy-a.yaml" +step stage +# As live.sh's stage_path: the IR copied into the node at the path the staged manifest mounts. +node=$(docker ps --filter "label=k3d.cluster=${K3D_GPU_CLUSTER}" --filter label=k3d.role=server --format '{{.Names}}' | head -1) +[ -n "${node}" ] || { echo "no k3d server node"; exit 1; } +t=/var/apb-models/k3dgpu/gemma-4-26B-A4B-it-qat-int4-ov +docker exec "${node}" mkdir -p "${t}" /var/apb-models/.ov_cache +docker cp -L /mnt/h62r-ir/. "${node}:${t}/" +docker exec "${node}" rm -rf "${t}/.cache" "${t}/.git" +docker exec "${node}" chown -R 5000:5000 "${t}" +docker exec "${node}" chown 5000:5000 /var/apb-models/.ov_cache +docker exec "${node}" chmod 1777 /var/apb-models +step apply +kubectl apply -f "${R}/deploy-a.yaml" > "${R}/apply.log" +kubectl -n k3d-gpu-agent rollout status deploy/ovms --timeout=1500s > "${R}/rollout.log" 2>&1 +step records +{ timeout 600 "${REPO}/scripts/k3d-gpu" plan --device cpu --model google/gemma-4-26B-A4B-it --explain; echo "rc=$?"; } \ + > "${R}/plan.txt" 2>&1 || true +{ docker version; k3d version; kubectl version --client; uname -r; } > "${R}/versions.txt" 2>&1 +kubectl get nodes -o json > "${R}/nodes.json" +kubectl -n k3d-gpu-agent get pods -o json > "${R}/pods.json" +docker exec "${node}" sh -c "cd ${t} && sha256sum ./*.xml ./*.bin" > "${R}/staged-ir.txt" +step finished diff --git a/bench/h62r/guest-b.sh b/bench/h62r/guest-b.sh new file mode 100755 index 00000000..924a5cfc --- /dev/null +++ b/bench/h62r/guest-b.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# guest-b.sh: VM b's setup for H62r's B5 (EXPERIMENTS.md "H62r pre-registration"), run in the +# guest by setup.sh as user ubuntu, under its 120 min ceiling. B5 is Intel's documented +# single-node install of the Agent Toolkit at the GA tags: EAT release-2.0.0 +# docs/single-node-deployment.md from Step 1, with EAS cloned at v1.2.0 (the commit the guide's +# `-b release-1.2` names, read 2026-11-06), on the inventory EAS ships (Option A, localhost: +# Kubespray installs Kubernetes here). +# - prerequisites.md "Node Kernel Settings", verbatim +# - clone; D1, now a check (the v1.2.0 layer manifest names the release commits itself); +# configure (its y/N prompt answered y on a terminal, H62 amendment V2); init agent-toolkit +# - the documented edit (gemma-4-26b in llm_services_autodeploy_models) and D2 (gemma-4-26b's +# model_id is the QAT checkpoint) +# - D3's sibling read: EAS's own topology probe (from this clone) and the kernel's +# thread_siblings_list; when the probe reads one thread per core where the kernel lists two, +# nri_reserved_cpu_list (EAS's global config, env/local) names the cpuset EAS's default +# 'siblings' profile reserves when it reads SMT (the higher thread of each core) +# - install agent-toolkit, wait for the gemma-4-26b workload pod, then the records; the sizing +# records (balloon policies, every pod's CPU request, the nodes, the sandbox router) pass or +# fail; on a failure also H62 amendment V4's records +# Nothing else is changed: guardrail, hybrid router, observability, NRI balloons, Istio, MetalLB +# and the agent sandbox are EAS's and EAT's defaults. Records: ~/h62r/records (no kubeconfig, no +# secret file). +set -euo pipefail +G=${HOME}/h62r +P() { python3 "${G}/h62r.py" "$@"; } +R=${G}/records +E=${HOME}/enterprise-ai-solutions +TAG=$(P pin eas_tag) SHA=$(P pin eas_sha) +mkdir -p "${R}" +step() { echo "== $1 $(date -u +%FT%TZ)" | tee -a "${R}/steps.txt"; } +# sizing PREFIX: what `cpu: auto` read and rendered, and whether the sandbox router was placed. +sizing() { + kubectl get balloonspolicies.config.nri -A -o yaml > "${R}/$1-balloonspolicies.yaml" 2>&1 || true + kubectl get pods -A -o custom-columns='NS:.metadata.namespace,POD:.metadata.name,PHASE:.status.phase,CPU:.spec.containers[*].resources.requests.cpu,LIMIT:.spec.containers[*].resources.limits.cpu' \ + > "${R}/$1-cpu-requests.txt" 2>&1 || true + kubectl describe nodes > "${R}/$1-nodes.txt" 2>&1 || true + kubectl -n agent-sandbox-system get deploy sandbox-router-deployment -o wide > "${R}/$1-router.txt" 2>&1 || true +} +echo "HF_TOKEN: $([ -n "${HF_TOKEN:-}" ] && echo set || echo unset)" > "${R}/hf-token.txt" +step sysctl +sudo tee /etc/sysctl.d/90-inotify.conf > /dev/null <<'EOF' +fs.inotify.max_user_instances = 8192 +fs.inotify.max_user_watches = 1048576 +EOF +sudo sysctl --system 2>&1 | tee "${R}/sysctl.txt" > /dev/null +step clone +[ -d "${E}" ] || git clone -q -b "${TAG}" https://github.com/intel/enterprise-ai-solutions.git "${E}" +cd "${E}" +[ "$(git rev-parse HEAD)" = "${SHA}" ] || { echo "EAS ${TAG} is not ${SHA}"; exit 1; } +step d1 +P eas-check configs/repos/repos.agent-toolkit.yaml > "${R}/d1.txt" +step configure +# configure takes no option and its one y/N prompt refuses a stdin that is not a terminal (v1.2.0's +# es_auto_installer.sh is rc.1's), so it runs on a terminal that answers y, as a person at one +# would (H62 amendment V2); configure's own steps are unchanged. +P on-tty "${R}/configure.log" y ./es_auto_installer.sh configure +step init +./es_auto_installer.sh init agent-toolkit > "${R}/init.log" 2>&1 +for pair in "enterprise.ai-inference $(P pin ei_sha)" "enterprise.agent-toolkit $(P pin eat_sha)"; do + read -r dest want <<< "${pair}" + have=$(git -C "ext/${dest}" rev-parse HEAD) + echo "${dest} ${have}" >> "${R}/ext-commits.txt" + [ "${have}" = "${want}" ] || { echo "ext/${dest} is at ${have}, not ${want}"; exit 1; } +done +step edit +P eas-edit env/local > "${R}/edit.diff" +step d3-siblings +sh roles/nri_cpu_balloons/files/probe-node-topology.sh > "${R}/probe.txt" +for c in /sys/devices/system/cpu/cpu[0-9]*; do + echo "${c##*/} $(cat "${c}/topology/thread_siblings_list")" +done > "${R}/thread-siblings.txt" +cpus=$(P sibling-read /sys/devices/system/cpu "${R}/probe.txt") +if [ -n "${cpus}" ]; then echo "${cpus}"; else echo "none: the probe reads the kernel threads per core"; fi > "${R}/d3-siblings.txt" +[ -z "${cpus}" ] || P reserve "${cpus}" env/local/global_config.yaml > "${R}/d3-siblings.diff" +for f in global_config.yaml config.agent-toolkit.yaml config.inference.yaml models.yaml inventory/hosts.yaml; do + [ ! -f "env/local/${f}" ] || cp "env/local/${f}" "${R}/env-${f//\//-}" +done +step install +if ! ./es_auto_installer.sh install agent-toolkit > "${R}/install.log" 2>&1; then + # H62 amendment V4: the cluster's state when Intel's install fails, read before teardown + # removes the guest, and the sizing records. Records only; the install still counts as failed. + step install-failed + export KUBECONFIG=${E}/env/local/kubeconfig.yaml + { kubectl get nodes -o wide; kubectl describe nodes; } > "${R}/fail-nodes.txt" 2>&1 || true + kubectl get pods -A -o wide > "${R}/fail-pods.txt" 2>&1 || true + kubectl get events -A --sort-by=.lastTimestamp > "${R}/fail-events.txt" 2>&1 || true + kubectl get pods -A --no-headers 2>/dev/null \ + | awk '{split($3, r, "/"); if (r[1] != r[2] && $4 != "Completed") print $1, $2}' \ + | while read -r ns pod; do + echo "=== ${ns}/${pod}" + kubectl -n "${ns}" describe pod "${pod}" + kubectl -n "${ns}" logs "${pod}" --all-containers --tail=50 + done > "${R}/fail-describe.txt" 2>&1 || true + sizing fail + exit 1 +fi +step ready +export KUBECONFIG=${E}/env/local/kubeconfig.yaml +printf 'export KUBECONFIG=%s\n' "${KUBECONFIG}" > "${G}/env" +NS=$(P arm B5 ns) SEL=$(P arm B5 selector) +until kubectl -n "${NS}" get pods -l "${SEL}" -o json | P pick-pod "" > "${R}/ready-pod.txt"; do sleep 15; done +step records +sizing ok +{ ./es_auto_installer.sh status; echo "rc=$?"; } > "${R}/status.txt" 2>&1 || true +{ python3 --version; yq --version; kubectl version; helm version; uname -r; } > "${R}/versions.txt" 2>&1 || true +kubectl get nodes -o json > "${R}/nodes.json" +kubectl get pods -A -o wide > "${R}/pods.txt" +kubectl -n "${NS}" get pods -o json > "${R}/llm-inference-pods.json" +kubectl -n "${NS}" get llminferenceservices -o yaml > "${R}/llminferenceservices.yaml" +step finished diff --git a/bench/h62r/h62r.py b/bench/h62r/h62r.py new file mode 100644 index 00000000..c16ff559 --- /dev/null +++ b/bench/h62r/h62r.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +"""h62r.py: the pure parts of the H62r runner (EXPERIMENTS.md "H62r pre-registration"): H62 +re-run with B5 at the GA tags (EAS v1.2.0 installing EAT release-2.0.0 and enterprise-inference +v1.1.0), k3d-gpu's CPU stack A1 unchanged, each in its own KVM guest on crescent. It started as a +copy of H62's runner (historical, never edited); the differences are the pins, the guests' +names and ports, D1 as a check, D3's sibling read and the re-run trigger. Python 3 standard +library only, so the same file runs on the host and in either guest. + + h62r.py pin NAME one pin of PINS (the shell scripts read every pin here) + h62r.py qemu-image the QEMU image's name + h62r.py vm VM FIELD a field of VMS (VM a or b) + h62r.py arm ARM FIELD a field of ARMS (A1 or B5) + h62r.py arm-fields ARM every field arm.sh reads, on one line + h62r.py ssh-config STATE the ssh_config of both guests + h62r.py user-data VM PUBKEY_FILE the guest's cloud-init user-data + h62r.py meta-data VM its cloud-init meta-data + h62r.py qemu-argv VM STATE IMAGE UID GID KVM_GID [SHARE] + the `docker run` of the guest's QEMU, one argument per line + h62r.py host-facts the host gate's facts (MemAvailable, / used, reachability), JSON + h62r.py gates PHASE FACTS_JSON PASS/FAIL lines of one gate phase (host, guest, nfd, + weights); exit 1 on any FAIL + h62r.py ir-verify DIR A1's IR against its manifest.json; exit 1 on any mismatch + h62r.py eas-check MANIFEST D1 at the GA tags: the agent-toolkit layer manifest names the + pinned commits and releases; prints them, writes nothing + h62r.py eas-edit ENV_DIR the documented catalog edit and D2, in place; prints the diffs + h62r.py sibling-read CPU_DIR PROBE D3's sibling read: the cpuset to reserve when EAS's topology + probe (its output in PROBE) reads one thread per core where the + kernel's thread_siblings_list under CPU_DIR has two; nothing + when they agree; exit 2 on anything else + h62r.py reserve CPUS GLOBAL_CONFIG nri_reserved_cpu_list in EAS's global_config.yaml, in place; + prints the diff + h62r.py uids < PODS_JSON every pod's uid, comma-separated + h62r.py pick-pod OLD_UIDS < PODS_JSON " " of a Ready pod not in OLD_UIDS; exit 1 if none + h62r.py model-mount CONTAINER < POD_JSON + where CONTAINER mounts the model-store PVC + h62r.py verify-script SIBLINGS_JSON a self-contained script that checks a weights directory + against the Hub's file list (run in B5's pod) + h62r.py trigger EAS_TAGS EAT_TAGS the re-run trigger, from two `git ls-remote --tags` outputs + h62r.py on-tty LOG REPLY CMD... CMD with stdin on a new terminal that is sent REPLY, a newline + and end of input; its output appended to LOG; its exit code + h62r.py pairs-left PAIRS_DIR B24 pairs still to run + h62r.py vmtax H50_ANALYSIS H62R_ANALYSIS + A1 on the host (H50) and in its guest, and H50's B1 on the + host against B5 in its guest, described +""" +import difflib +import glob +import json +import os +import re +import sys + +PINS = { + # B5: EAS's GA release and the solution versions its release notes list (read 2026-11-06). + # The v1.2.0 layer manifest names both commits itself; EAS release-1.2 is at the same commit. + "eas_repo": "https://github.com/intel/enterprise-ai-solutions.git", + "eas_tag": "v1.2.0", + "eas_sha": "a051f00fcb2dd473fe763610471cd509db82c433", + "ei_tag": "v1.1.0", + "ei_sha": "d035b1ef1981fbf5084ff05c9a337e620478bb87", + "eat_tag": "release-2.0.0", + "eat_sha": "7b827e2d95cec826a20c7e3f5a30eb4818ed2992", + "catalog_name": "gemma-4-26b", + "stock_model_id": "google/gemma-4-26B-A4B-it", + # D2: the weights both arms serve (crescent amendment M1). + "qat_repo": "google/gemma-4-26B-A4B-it-qat-q4_0-unquantized", + "qat_rev": "f1e06dc520982d9b9edd76859fdb7ab209449949", + # The image enterprise-inference v1.1.0's catalog names for gemma-4-26b (by tag; the digest it + # serves, unchanged since 2026-10-29). Recorded, not forced. + "vllm_cpu_image": "vllm/vllm-openai-cpu:v0.28.0@sha256:197354e6475e638eb5ccca09beb078488046a82de0e204e7429bf1a804df9647", + # The guests (D3), as H62 had them: every pin below is still the current one (read 2026-11-06) + # but Docker CE, at its current patch release. + "cloud_image_url": "https://cloud-images.ubuntu.com/releases/noble/release-20261017/ubuntu-24.04-server-cloudimg-amd64.img", + "cloud_image_sha256": "04e8f2461a16a44e78a370284471f29616cdd9baf1893df25807b71f985b6c94", + "cloud_image_bytes": 621284352, + "qemu_base": "ubuntu:26.04@sha256:6a06111c3b0a8600feadb270aec771dc239bf407b25625f93f44ab00e11f4d3c", + "qemu_pkg": "1:10.2.1+ds-1ubuntu3.3", + "genisoimage_pkg": "9:1.1.11-5", + # A1's guest: Docker CE from Docker's apt repository, H50's node image and H50's OVMS. + "docker_ce": "5:29.9.3-1~ubuntu.24.04~noble", + "containerd": "2.4.2-3~ubuntu.24.04~noble", + "docker_gpg_fpr": "9DC858229FC7DD38854AE2D88D81803C0EBFCD88", + "python3_yaml": "6.0.1-2build2", # bench/apb/stage.py in VM a + "jq": "1.7.1-3ubuntu0.24.04.2", # integrations/eat/eat-check.sh in VM b (the smoke) + "node_image": "cryptoandcoffee/k3d-gpu:0.1.61@sha256:8cd264aa68c77681d7fa474693944de0db8d55de5881253e4029bf72a67e175c", + "ovms_image": "openvino/model_server:2026.4.1-gpu@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58", +} +QEMU_IMAGE = "h62r-qemu:" + re.sub(r"[^A-Za-z0-9_.-]", "-", PINS["qemu_pkg"]) + +VCPUS = 96 +SMP = "96,sockets=1,cores=48,threads=2" # crescent's topology: 1 socket, 48 cores, 2 threads +HOST_RESERVE_GIB = 48 # MemAvailable left to the host's own tenants +ROOT_MAX_PCT = 90 +REACH = {"huggingface.co": "https://huggingface.co/api/models/google/gemma-4-26B-A4B-it-qat-q4_0-unquantized", + "registry-1.docker.io": "https://registry-1.docker.io/v2/", + "registry.k8s.io": "https://registry.k8s.io/v2/"} +VMS = { + "a": {"name": "h62r-vm-a", "host": "h62r-a", "arm": "A1", "mem_gib": 96, "disk_gib": 200, + "ssh_port": 18842, "serve_port": 18851, "share": True}, + "b": {"name": "h62r-vm-b", "host": "h62r-b", "arm": "B5", "mem_gib": 192, "disk_gib": 400, + "ssh_port": 18843, "serve_port": 18852, "share": False}, +} +ARMS = { + # k3d-gpu's agent OVMS (agent manifest: Deployment ovms, label app=ovms, Service ovms:8000). + "A1": {"vm": "a", "ns": "k3d-gpu-agent", "selector": "app=ovms", "svc": "svc/ovms", "svc_port": 8000, + "container": "ovms", "extra": ""}, + # EAS's gemma-4-26b: a KServe LLMInferenceService (enterprise-inference v1.1.0's model_manager + # and KServe role are 170eb3a's), behind -kserve-workload-svc:8000. + "B5": {"vm": "b", "ns": "llm-inference", + "selector": "app.kubernetes.io/part-of=llminferenceservice,app.kubernetes.io/name=gemma-4-26b," + "kserve.io/component=workload", + "svc": "svc/gemma-4-26b-kserve-workload-svc", "svc_port": 8000, "container": "main", + "extra": "llminferenceservice/gemma-4-26b"}, +} +TASKS = 24 # B24's task ids, one pair each + + +def arm_fields(arm): + """What bench/h62r/arm.sh needs, in one call: VM, guest host, serve port, namespace, selector, + Service, Service port, container, then the extra object.""" + a = ARMS[arm] + v = VMS[a["vm"]] + return [a["vm"], v["host"], v["serve_port"], a["ns"], a["selector"], a["svc"], a["svc_port"], a["container"], + a["extra"]] + + +def mem_needed_kib(): + return (sum(v["mem_gib"] for v in VMS.values()) + HOST_RESERVE_GIB) * 1024 * 1024 + + +def host_facts(meminfo="/proc/meminfo", df_text=None, probe=None): + """MemAvailable (KiB), how full / is (df's percentage) and, per REACH host, whether its URL + answered over HTTPS at all (any HTTP status counts: a registry answers 401).""" + import subprocess + with open(meminfo) as f: + mem = {k: v for k, _, v in (ln.partition(":") for ln in f)} + if df_text is None: + df_text = subprocess.run(["df", "-P", "/"], capture_output=True, text=True).stdout + pct = int(df_text.splitlines()[-1].split()[4].rstrip("%")) + if probe is None: + def probe(url): + return subprocess.run(["curl", "-sS", "-o", "/dev/null", "--max-time", "20", url], + capture_output=True).returncode == 0 + return {"mem_available_kib": int(mem["MemAvailable"].split()[0]), "root_used_pct": pct, + "reach": {h: bool(probe(u)) for h, u in REACH.items()}} + + +def ir_verify(d): + """(lines, bad) of A1's IR directory against its manifest.json: every listed file's sha256 + and size, and the files the manifest does not list.""" + import hashlib + files = _load(os.path.join(d, "manifest.json"))["files"] + out, bad = [], 0 + for n, want in sorted(files.items()): + p = os.path.join(d, n) + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + h.update(chunk) + ok = h.hexdigest() == want["sha256"] and os.path.getsize(p) == want["bytes"] + bad += not ok + out.append(f"{'ok ' if ok else 'BAD'} {n} {h.hexdigest()} {want['bytes']}") + extra = sorted(set(os.listdir(d)) - set(files) - {"manifest.json"}) + out.append(f"dir {d}; {len(files)} files, {bad} bad; not in the manifest: {extra}") + return out, bad + + +def ssh_config(state): + out = [] + for v in VMS.values(): + out += [f"Host {v['host']}", " HostName 127.0.0.1", f" Port {v['ssh_port']}", " User ubuntu", + f" IdentityFile {state}/id_ed25519", " IdentitiesOnly yes", f" UserKnownHostsFile {state}/known_hosts", + " StrictHostKeyChecking accept-new", " BatchMode yes", " ControlMaster auto", + f" ControlPath {state}/cm-%C", " ControlPersist 600", " ServerAliveInterval 30", ""] + return "\n".join(out) + + +def _key(pubkey): + k = pubkey.strip() + if not re.fullmatch(r"ssh-(ed25519|rsa) [A-Za-z0-9+/=]+( [^\n]*)?", k): + raise ValueError("not one ssh public key line") + return k + + +def user_data(vm, pubkey): + """cloud-init for one guest: the harness's key on the default user, no password login, and + the image's package state kept (D3): no upgrade at boot and no apt timer running alongside the + installs.""" + return "\n".join([ + "#cloud-config", + f"hostname: {VMS[vm]['host']}", + "ssh_pwauth: false", + "ssh_authorized_keys:", + f" - {_key(pubkey)}", + "package_update: false", + "package_upgrade: false", + "runcmd:", + " - [systemctl, disable, --now, unattended-upgrades.service, apt-daily.timer, apt-daily-upgrade.timer]", + ""]) + + +def meta_data(vm): + h = VMS[vm]["host"] + return f"instance-id: {h}\nlocal-hostname: {h}\n" + + +def qemu_argv(vm, state, image, share=None, uid=None, gid=None, kvm_gid=None): + """The guest's QEMU, in the pinned container: KVM, -cpu host (AMX passes through), the host's + topology, user-mode networking with the ssh and serve ports on 127.0.0.1 only, a qcow2 overlay + on the cloud image, and for VM a A1's IR shared read-only.""" + v = VMS[vm] + if v["share"] and not share: + raise ValueError(f"VM {vm} needs A1's IR directory to share") + if share and not v["share"]: + raise ValueError(f"VM {vm} shares nothing") + if None in (uid, gid, kvm_gid): + raise ValueError("uid, gid and the kvm group are required") + argv = ["docker", "run", "-d", "--name", v["name"], "--network", "host", "--device", "/dev/kvm", + "--user", f"{uid}:{gid}", "--group-add", str(kvm_gid), "--label", "io.k3d-gpu.h62r=vm", + "-v", f"{state}:/h62r"] + if share: + argv += ["-v", f"{share}:/share:ro"] + fwd = f"hostfwd=tcp:127.0.0.1:{v['ssh_port']}-:22,hostfwd=tcp:127.0.0.1:{v['serve_port']}-:{v['serve_port']}" + argv += [image, "qemu-system-x86_64", "-name", v["name"], "-machine", "q35,accel=kvm", "-cpu", "host", + "-smp", SMP, "-m", f"{v['mem_gib']}G", + "-drive", f"file=/h62r/{vm}/disk.qcow2,if=virtio,format=qcow2", + "-drive", f"file=/h62r/{vm}/seed.iso,if=virtio,format=raw,readonly=on", + "-netdev", f"user,id=n0,{fwd}", "-device", "virtio-net-pci,netdev=n0"] + if share: + argv += ["-virtfs", "local,path=/share,mount_tag=h62rshare,security_model=none,readonly=on"] + return argv + ["-display", "none", "-serial", "stdio", "-monitor", "none"] + + +def gates(phase, f): + """[(ok, message)] of one gate phase, from its facts.""" + out = [] + if phase == "host": + need = mem_needed_kib() + have = f.get("mem_available_kib") + out.append((have is not None and have >= need, + f"MemAvailable {have} KiB >= {need} KiB (both guests + {HOST_RESERVE_GIB} GiB)")) + pct = f.get("root_used_pct") + out.append((pct is not None and pct <= ROOT_MAX_PCT, f"/ used {pct}% <= {ROOT_MAX_PCT}%")) + reach = f.get("reach") or {} + for h in REACH: + out.append((reach.get(h) is True, f"{h} reachable")) + elif phase == "guest": + for vm in VMS: + amx = (f.get("amx_tile") or {}).get(vm) + out.append((amx == VCPUS, f"VM {vm}: amx_tile on {amx} of {VCPUS} vCPUs")) + n = (f.get("nproc") or {}).get(vm) + out.append((n == VCPUS, f"VM {vm}: nproc {n} == {VCPUS}")) + elif phase == "nfd": + out.append((f.get("amxbf16") == "true", f"VM b node label cpu-cpuid.AMXBF16={f.get('amxbf16')}")) + elif phase == "weights": + ok = f.get("bad") == 0 and f.get("missing") == 0 and (f.get("checked") or 0) > 0 + out.append((ok, f"B5's weights: {f.get('checked')} LFS files checked, {f.get('bad')} differ, " + f"{f.get('missing')} missing, against {PINS['qat_repo']}@{PINS['qat_rev']}")) + else: + raise ValueError(f"no gate phase {phase}") + return out + + +def _diff(old, new, name): + return "".join(difflib.unified_diff(old.splitlines(True), new.splitlines(True), name, name)) + + +def eas_check(text): + """D1 at the GA tags. EAS v1.2.0's agent-toolkit layer manifest names each layer's commit + itself, with its release in the line's comment (the release notes' solution table). Checked, + never rewritten: each layer must carry one rev line, the pinned commit and the pinned release. + Returns one line per layer.""" + want = {"inference": (PINS["ei_sha"], PINS["ei_tag"]), "agent-toolkit": (PINS["eat_sha"], PINS["eat_tag"])} + lines = text.splitlines() + starts = [i for i, ln in enumerate(lines) if re.match(r'\s*- layer: "', ln)] + seen = {} + for k, i in enumerate(starts): + layer = re.match(r'\s*- layer: "([^"]+)"', lines[i]).group(1) + if layer not in want: + continue + end = starts[k + 1] if k + 1 < len(starts) else len(lines) + revs = [lines[j] for j in range(i, end) if re.match(r"\s*rev:", lines[j])] + if len(revs) != 1: + raise ValueError(f"layer {layer}: {len(revs)} rev lines, want 1") + sha, tag = want[layer] + m = re.fullmatch(r'\s*rev: "([0-9a-f]{40})" # (\S+)\s*', revs[0]) + if not m or (m.group(1), m.group(2)) != (sha, tag): + raise ValueError(f"layer {layer}: rev line is not {sha} ({tag}): {revs[0].strip()}") + seen[layer] = f"{layer} {sha} ({tag})" + if set(seen) != set(want): + raise ValueError(f"layers missing from the manifest: {sorted(set(want) - set(seen))}") + return [seen[k] for k in want] + + +def set_autodeploy(text, name): + """The documented edit (single-node-deployment.md step 3): the one live + llm_services_autodeploy_models line names one catalog value.""" + pat = re.compile(r"^llm_services_autodeploy_models:.*$", re.M) + if len(pat.findall(text)) != 1: + raise ValueError("want exactly one live llm_services_autodeploy_models line") + return pat.sub(f'llm_services_autodeploy_models: ["{name}"]', text) + + +def set_model_id(text, name, old_id, new_id): + """D2: the model_id of catalog entry `name` (and only that entry) moves from the stock id to + new_id. Refuses when the entry or its stock id is not there.""" + lines = text.splitlines(True) + heads = [i for i, ln in enumerate(lines) if re.match(r"\s*- name:\s*\S", ln)] + for k, i in enumerate(heads): + # A name line may carry an inline comment (the catalog's first entry does; H62 amendment V3). + if re.match(r"\s*- name:\s*(\S+)\s*(#.*)?$", lines[i]).group(1) != name: + continue + end = heads[k + 1] if k + 1 < len(heads) else len(lines) + ids = [j for j in range(i, end) if re.match(r"\s*model_id:", lines[j])] + if len(ids) != 1: + raise ValueError(f"{name}: {len(ids)} model_id lines") + j = ids[0] + indent, value = re.match(r"(\s*)model_id:\s*(\S+)", lines[j]).groups() + if value == new_id: + return text + if value != old_id: + raise ValueError(f"{name}: model_id is {value}, not the stock {old_id}") + lines[j] = f"{indent}model_id: {new_id}\n" + return "".join(lines) + raise ValueError(f"no catalog entry {name}") + + +def eas_edit(env_dir): + """Both edits on an env directory; returns their diffs ("" when nothing changed).""" + out = [] + for name, fn in (("config.agent-toolkit.yaml", lambda t: set_autodeploy(t, PINS["catalog_name"])), + ("models.yaml", lambda t: set_model_id(t, PINS["catalog_name"], PINS["stock_model_id"], + PINS["qat_repo"]))): + p = os.path.join(env_dir, name) + with open(p) as f: + old = f.read() + new = fn(old) + if new != old: + with open(p, "w") as f: + f.write(new) + out.append(_diff(old, new, name)) + return "".join(out) + + +def _cpulist(s): + """The CPU ids of a Linux cpulist ("0-1", "0,48", "1,3,5-7").""" + out = set() + for part in s.strip().split(","): + m = re.fullmatch(r"(\d+)(?:-(\d+))?", part.strip()) + if not m: + raise ValueError(f"not a cpulist: {s!r}") + lo, hi = int(m.group(1)), int(m.group(2) or m.group(1)) + out.update(range(lo, hi + 1)) + return out + + +def sibling_read(lists, probe_text): + """D3's sibling read. `lists` maps each CPU id to its thread_siblings_list as the guest kernel + writes it; `probe_text` is the output of EAS's own topology probe + (roles/nri_cpu_balloons/files/probe-node-topology.sh) on the same machine. + + The probe reads threads per core as the comma fields of cpu0's list. On crescent's host the + list is "0,48" (2). In the guest QEMU numbers siblings next to each other and Linux writes the + pair as a range, "0-1", which the probe reads as 1, so EAS's default 'siblings' profile falls + back to 'standard' (12 reserved CPUs) and gemma-4-26b's `cpu: auto` took 76 of 96 (H62). + When the probe reads 1 and the kernel lists 2 threads on every core, the result is the cpuset + EAS's siblings profile reserves when its probe reads SMT: the higher thread of each core. + When the probe agrees with the kernel, nothing ("") is reserved by the runner. Anything else + stops the setup (ValueError).""" + if not lists: + raise ValueError("no CPU sibling list read") + sets = {cpu: _cpulist(s) for cpu, s in lists.items()} + sizes = {len(s) for s in sets.values()} + if len(sizes) != 1: + raise ValueError(f"threads per core differ across CPUs: {sorted(sizes)}") + kernel = sizes.pop() + m = re.search(r"(?m)^NRI_TPC=(\d+)\s*$", probe_text) + if not m: + raise ValueError("no NRI_TPC line in the probe's output") + probed = int(m.group(1)) + if probed == kernel: + return "" + if (probed, kernel) != (1, 2): + raise ValueError(f"EAS's probe reads {probed} thread(s) per core, the kernel lists {kernel}") + sib = sorted(cpu for cpu, s in sets.items() if cpu != min(s)) + if len(sib) * 2 != len(sets): + raise ValueError(f"{len(sib)} sibling CPUs of {len(sets)}, want half") + return ",".join(str(c) for c in sib) + + +def read_sibling_lists(cpu_dir): + """{cpu id: thread_siblings_list} under a sysfs cpu directory (/sys/devices/system/cpu).""" + out = {} + for d in glob.glob(os.path.join(cpu_dir, "cpu[0-9]*")): + m = re.fullmatch(r"cpu(\d+)", os.path.basename(d)) + p = os.path.join(d, "topology", "thread_siblings_list") + if m and os.path.isfile(p): + with open(p) as f: + out[int(m.group(1))] = f.read().strip() + return out + + +def reserve(text, cpus): + """nri_reserved_cpu_list in EAS's global_config.yaml: the key EAS's + docs/customize/nri_cpu_balloons.md documents there ("Optional: pin reserved CPUs to a fixed + cpuset"), read by the nri_cpu_balloons role (the installer passes global_config.yaml to every + playbook as extra vars). Filled when absent or at its documented empty default; a line with + another value is refused.""" + if not re.fullmatch(r"[0-9]+([,-][0-9]+)*", cpus): # EAS's own cpuset check + raise ValueError(f"not a cpuset: {cpus!r}") + line = f'nri_reserved_cpu_list: "{cpus}"' + pat = re.compile(r'^nri_reserved_cpu_list:[ \t]*"?([^"\n]*)"?[ \t]*$', re.M) + found = pat.findall(text) + if len(found) > 1: + raise ValueError("more than one live nri_reserved_cpu_list line") + if found: + if found[0] == cpus: + return text + if found[0]: + raise ValueError(f"nri_reserved_cpu_list is already {found[0]!r}") + return pat.sub(line, text) + sep = "" if text.endswith("\n") or not text else "\n" + return (text + sep + "\n# H62r D3's sibling read: the cpuset EAS's 'siblings' profile reserves when its probe\n" + "# reads this guest's SMT (EXPERIMENTS.md \"H62r pre-registration\")\n" + line + "\n") + + +def _ready(item): + st = item.get("status") or {} + return (st.get("phase") == "Running" and not (item.get("metadata") or {}).get("deletionTimestamp") + and any(c.get("type") == "Ready" and c.get("status") == "True" for c in st.get("conditions") or [])) + + +def pick_pod(doc, old): + """(name, uid) of the newest Running, Ready, not-deleting pod whose uid is not in old.""" + new = [i for i in doc.get("items", []) if i["metadata"]["uid"] not in old and _ready(i)] + if not new: + return None + m = max(new, key=lambda i: i["metadata"].get("creationTimestamp", ""))["metadata"] + return m["name"], m["uid"] + + +def model_mount(pod, container, claim="model-store"): + """Where `container` mounts the PVC `claim` (EAS's model store).""" + vols = {v["name"] for v in pod["spec"].get("volumes", []) + if (v.get("persistentVolumeClaim") or {}).get("claimName") == claim} + for c in pod["spec"].get("containers", []): + if c["name"] == container: + for m in c.get("volumeMounts", []): + if m["name"] in vols: + return m["mountPath"] + raise ValueError(f"container {container} mounts no volume of PVC {claim}") + + +VERIFY = r'''#!/usr/bin/env python3 +# Generated by bench/h62r/h62r.py verify-script: DIR against the Hub's file list of +# @REPO@@@REV@ (H62r, B5's weights). Every LFS file's sha256, every other listed file's size, and +# the commit `hf download --local-dir` recorded per file. Last line: a JSON summary. +import glob, hashlib, json, os, sys +SIB = json.loads(@SIB@) +d = sys.argv[1] +bad = missing = checked = 0 +for s in SIB: + p = os.path.join(d, s["rfilename"]) + lfs = s.get("lfs") + if not os.path.exists(p): + print(("MISSING " if lfs else "absent ") + s["rfilename"]) + missing += bool(lfs) + continue + if lfs: + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + h.update(chunk) + ok = h.hexdigest() == lfs["sha256"] + checked += 1 + print(("ok " if ok else "BAD ") + h.hexdigest() + " " + s["rfilename"]) + else: + ok = os.path.getsize(p) == s.get("size", os.path.getsize(p)) + print(("ok " if ok else "BAD ") + "size " + str(os.path.getsize(p)) + " " + s["rfilename"]) + bad += not ok +revs = set() +for m in glob.glob(os.path.join(d, ".cache", "huggingface", "download", "**", "*.metadata"), recursive=True): + with open(m) as f: + revs.add(f.readline().strip()) +print(json.dumps({"want": "@REV@", "revisions": sorted(revs), "checked": checked, "bad": bad, "missing": missing})) +sys.exit(1 if bad or missing else 0) +''' + + +def verify_script(siblings): + """A stdlib script that checks a weights directory against `siblings` (the Hub API's + /api/models//revision/?blobs=true answer); it needs no network.""" + sib = [{k: s[k] for k in ("rfilename", "size", "lfs") if k in s} for s in siblings["siblings"]] + return (VERIFY.replace("@SIB@", repr(json.dumps(sib))).replace("@REPO@", PINS["qat_repo"]) + .replace("@REV@", PINS["qat_rev"])) + + +def _tags(ls_remote): + return {m.group(1) for m in re.finditer(r"refs/tags/([^\s^]+)(?:\^\{\})?\s*$", ls_remote, re.M)} + + +def trigger(eas_ls_remote, eat_ls_remote): + """H62r's re-run trigger: an EAS release tag after v1.2.0 (vX.Y.Z, not a release candidate), + or an EAT release tag after release-2.0.0 (release-X.Y.Z or vX.Y.Z, and EAT's hotfix form + release-X.Y.Z-hotfix, which follows the release of the same number).""" + def eas(t): + m = re.fullmatch(r"v(\d+)\.(\d+)\.(\d+)", t) + return bool(m) and tuple(int(x) for x in m.groups()) > (1, 2, 0) + + def eat(t): + m = re.fullmatch(r"(?:release-|v)(\d+)\.(\d+)\.(\d+)(-hotfix)?", t) + return bool(m) and (*(int(x) for x in m.groups()[:3]), bool(m.group(4))) > (2, 0, 0, False) + return {"eas": sorted(t for t in _tags(eas_ls_remote) if eas(t)), + "eat": sorted(t for t in _tags(eat_ls_remote) if eat(t))} + + +def pairs_left(pairs_dir, total=TASKS): + """B24 pairs not yet recorded (complete or void): pairs resume across holds (pairs.next_pair).""" + sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "eatflow")) + import pairs # noqa: E402 (bench/eatflow/pairs.py) + return max(0, total - pairs.next_pair(pairs_dir)) + + +def vmtax(h50, mine): + """Described, never gated. A1 on the host (H50's analysis) against A1 in its guest (H62r's): + the same stack, so the guest alone. And H50's B1 on the host against B5 in its guest: the same + vLLM CPU v0.28.0 image and QAT weights under another deployment, so the guest and the + deployment together.""" + def arm(a, name): + s = (a.get("within_budget_success") or {}).get(name) + if not s: + return None + return {"success": f"{s['k']}/{s['n']}", "attempt_median_s": (a.get("attempt_median_s") or {}).get(name)} + + def ratio(host, guest): + if host and guest and host["attempt_median_s"] and guest["attempt_median_s"]: + return round(guest["attempt_median_s"] / host["attempt_median_s"], 3) + return None + a_host, a_guest, b_host, b_guest = arm(h50, "A1"), arm(mine, "A1"), arm(h50, "B1"), arm(mine, "B5") + return {"a1_host": a_host, "a1_guest": a_guest, "attempt_median_guest_over_host": ratio(a_host, a_guest), + "b1_host": b_host, "b5_guest": b_guest, "b_attempt_median_guest_over_host": ratio(b_host, b_guest), + "b_note": "B1 (EAT release-1.1.0's chart) and B5 (EAS's KServe deployment) share the vLLM image and " + "weights, not the deployment: the guest and the deployment are read together", + "gated": False} + + +def on_tty(log, reply, cmd): + """Run cmd with stdin on a fresh pseudo-terminal, type reply and Enter into it, then end of + input (^D on a new line), the way a person answers a y/N prompt (H62 amendment V2: EAS's + `configure` takes no option and refuses a stdin that is not a terminal). Output and errors + are appended to log. Returns cmd's exit code.""" + import pty + import subprocess + master, slave = pty.openpty() + try: + with open(log, "ab") as out: + p = subprocess.Popen(cmd, stdin=slave, stdout=out, stderr=subprocess.STDOUT) + os.close(slave) + slave = None + os.write(master, reply.encode() + b"\n\x04") + return p.wait() + finally: + if slave is not None: + os.close(slave) + os.close(master) + + +def _load(path): + with open(path) as f: + return json.load(f) + + +def _edit(path, fn): + """Apply fn to a file's text in place; returns the diff ("" when nothing changed).""" + with open(path) as f: + old = f.read() + new = fn(old) + if new == old: + return "" + with open(path, "w") as f: + f.write(new) + return _diff(old, new, os.path.basename(path)) + + +def main(argv): + if not argv: + print(__doc__) + return 2 + cmd, args = argv[0], argv[1:] + if cmd == "pin": + if args[0] not in PINS: + print(f"h62r.py: no pin {args[0]}", file=sys.stderr) + return 2 + print(PINS[args[0]]) + elif cmd == "qemu-image": + print(QEMU_IMAGE) + elif cmd == "vm": + if args[0] not in VMS or args[1] not in VMS[args[0]]: + print(f"h62r.py: no VM field {args}", file=sys.stderr) + return 2 + print(VMS[args[0]][args[1]]) + elif cmd == "arm-fields": + print(*arm_fields(args[0])) + elif cmd == "arm": + if args[0] not in ARMS or args[1] not in ARMS[args[0]]: + print(f"h62r.py: no arm field {args}", file=sys.stderr) + return 2 + print(ARMS[args[0]][args[1]]) + elif cmd == "ssh-config": + sys.stdout.write(ssh_config(args[0])) + elif cmd == "user-data": + with open(args[1]) as f: + sys.stdout.write(user_data(args[0], f.read())) + elif cmd == "meta-data": + sys.stdout.write(meta_data(args[0])) + elif cmd == "qemu-argv": + vm, state, image, uid, gid, kvm = args[:6] + print("\n".join(qemu_argv(vm, state, image, share=args[6] if len(args) > 6 else None, + uid=int(uid), gid=int(gid), kvm_gid=int(kvm)))) + elif cmd == "host-facts": + print(json.dumps(host_facts())) + elif cmd == "ir-verify": + lines, bad = ir_verify(args[0]) + print("\n".join(lines)) + return 1 if bad else 0 + elif cmd == "gates": + res = gates(args[0], _load(args[1])) + for ok, msg in res: + print(("PASS " if ok else "FAIL ") + msg) + return 0 if all(ok for ok, _ in res) else 1 + elif cmd == "eas-check": + with open(args[0]) as f: + for line in eas_check(f.read()): + print("ok " + line) + elif cmd == "eas-edit": + sys.stdout.write(eas_edit(args[0])) + elif cmd == "sibling-read": + with open(args[1]) as f: + cpus = sibling_read(read_sibling_lists(args[0]), f.read()) + if cpus: + print(cpus) + elif cmd == "reserve": + sys.stdout.write(_edit(args[1], lambda t: reserve(t, args[0]))) + elif cmd == "uids": + print(",".join(i["metadata"]["uid"] for i in json.load(sys.stdin).get("items", []))) + elif cmd == "pick-pod": + p = pick_pod(json.load(sys.stdin), {u for u in (args[0] if args else "").split(",") if u}) + if not p: + return 1 + print(*p) + elif cmd == "model-mount": + print(model_mount(json.load(sys.stdin), args[0])) + elif cmd == "verify-script": + sys.stdout.write(verify_script(_load(args[0]))) + elif cmd == "trigger": + with open(args[0]) as f, open(args[1]) as g: + t = trigger(f.read(), g.read()) + fired = [f"{k} {v}" for k in ("eas", "eat") for v in t[k]] + print("fired: " + (", ".join(fired) if fired else "none")) + elif cmd == "on-tty": + return on_tty(args[0], args[1], args[2:]) + elif cmd == "pairs-left": + print(pairs_left(args[0])) + elif cmd == "vmtax": + print(json.dumps(vmtax(_load(args[0]), _load(args[1])), indent=1)) + else: + print(__doc__) + return 2 + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except (ValueError, KeyError, IndexError, OSError) as e: + print(f"h62r.py: {e}", file=sys.stderr) + sys.exit(2) diff --git a/bench/h62r/pairs-hold.sh b/bench/h62r/pairs-hold.sh new file mode 100755 index 00000000..5da3faff --- /dev/null +++ b/bench/h62r/pairs-hold.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# pairs-hold.sh N [smoke]: one H62r hold on crescent (EXPERIMENTS.md "H62r pre-registration") under +# bench/eatflow/crescent/hold.sh: the lock, the hold-start rule, the 97% disk guard, the serving +# sampler and the cleanup, LIVE_CAP 2370 inside a 2400 s timeout. live.sh runs H50's hold with both +# servers in their KVM guests (arm-A1, arm-B5): each hold restarts both serving pods first. +# Records: ~/k3dgpu-bench/results/H62r/hold, pairs in ~/k3dgpu-bench/results/H62r/pairs, the +# hold's own records in ~/k3dgpu-bench/runs/H62r-hold. +# smoke: the wiring smoke instead (H62r-smoke, the pair-start guard past the hold so no pair starts, +# records under ~/k3dgpu-bench/smoke/H62r); it is not one of the 8 holds. +set -uo pipefail +N=${1:?usage: pairs-hold.sh N [smoke]} +B=${HOME}/k3dgpu-bench +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +case ${2:-hold} in + hold) name=H62r-hold${N} R=${B}/results/H62r guard=600 ;; + smoke) name=H62r-smoke R=${B}/smoke/H62r guard=99999 ;; + *) echo "usage: pairs-hold.sh N [smoke]"; exit 2 ;; +esac +mkdir -p "${R}/pairs" +# The guests are this campaign's arms: the serving covariate records them as vm_cpu (serving.py). +export SERVING_VMS=h62r-vm-a,h62r-vm-b +exec "${HERE}/../eatflow/crescent/hold.sh" "${name}" 2400 env \ + LIVE_MODE=pairs LIVE_CAP=2370 LIVE_HOLD="${N}" LIVE_PAIRS_DIR="${R}/pairs" LIVE_PAIRS_GUARD="${guard}" \ + LIVE_MODEL_SET=gemma4 LIVE_MODELS="${B}/models" LIVE_BLUEPRINT="${B}/eab" LIVE_PY="${B}/mafvenv/bin/python" \ + LIVE_REMOTE_A="${HERE}/arm-A1" LIVE_REMOTE_B="${HERE}/arm-B5" H62R_STATE="${H62R_STATE:-${B}/h62r}" \ + PATH="${B}/bin:${PATH}" \ + bash "${HERE}/../eatflow/live.sh" H62r "${R}/hold${N}" diff --git a/bench/h62r/qemu.Dockerfile b/bench/h62r/qemu.Dockerfile new file mode 100644 index 00000000..70b083d0 --- /dev/null +++ b/bench/h62r/qemu.Dockerfile @@ -0,0 +1,12 @@ +# The QEMU that runs H62r's two KVM guests on crescent (EXPERIMENTS.md "H62r pre-registration", +# D3), in a container so the host gets no package: Ubuntu 26.04's qemu-system-x86 (QEMU 10.2, +# which passes AMX through to a guest from QEMU 7.0 on), qemu-img and genisoimage for the +# cloud-init seed. Every version is a pin of bench/h62r/h62r.py (PINS); bench/h62r/vm.sh builds it. +FROM ubuntu:26.04@sha256:6a06111c3b0a8600feadb270aec771dc239bf407b25625f93f44ab00e11f4d3c +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + qemu-system-x86=1:10.2.1+ds-1ubuntu3.3 \ + qemu-utils=1:10.2.1+ds-1ubuntu3.3 \ + genisoimage=9:1.1.11-5 \ + && dpkg-query -W > /h62r-packages.txt \ + && rm -rf /var/lib/apt/lists/* diff --git a/bench/h62r/setup.sh b/bench/h62r/setup.sh new file mode 100755 index 00000000..bc8e33a6 --- /dev/null +++ b/bench/h62r/setup.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# setup.sh: H62r's setup phase on crescent (EXPERIMENTS.md "H62r pre-registration", "Setup"), not a +# hold, inside the bench lock. Run from the H62r checkout at the deploy commit. +# 1. host gate: MemAvailable for both guests plus 48 GiB, / at most 90% used, Hugging Face and the +# registries reachable; re-read every 10 min for up to 60 min +# 2. A1's IR against its manifest.json; the Hub's file list of B5's weights (the QAT checkpoint +# at f1e06dc5) and the script that checks them +# 3. both guests (vm.sh build, fetch, up, wait), then the guest gate: amx_tile on all 96 vCPUs +# 4. VM a's A1 (guest-a.sh, 45 min ceiling) and VM b's EAS install (guest-b.sh, 120 min ceiling), +# side by side; their records copied back +# 5. the NFD gate (VM b's node carries cpu-cpuid.AMXBF16=true) and the weight gate (B5's files +# equal the QAT checkpoint's) +# Exit 0: both arms serve. 4: lock not taken. 6: the host gate still failed after 60 min. 7: not +# deliverable (a ceiling passed, or no AMX in a guest). 8: the NFD or weight gate failed (no hold +# until a dated amendment). 1: anything else. Records: ~/k3dgpu-bench/results/H62r/setup. +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(cd "${HERE}/../.." && pwd) +P() { python3 "${HERE}/h62r.py" "$@"; } +B=${HOME}/k3dgpu-bench +export H62R_STATE=${H62R_STATE:-${B}/h62r} +R=${H62R_RESULTS:-${B}/results/H62r}/setup +A_CAP=2700 B_CAP=7200 +mkdir -p "${R}" +log() { echo "== setup $* $(date -u +%FT%TZ)" | tee -a "${R}/setup.log"; } +# Every ssh, scp and vm.sh below runs with fd 9 closed: an ssh ControlMaster it starts outlives +# this script, and must not keep the bench lock. +G() { ssh -F "${H62R_STATE}/ssh_config" "$(P vm "$1" host)" "${@:2}" 9>&-; } +put() { scp -q -F "${H62R_STATE}/ssh_config" "${@:2}" "$(P vm "$1" host):h62r/" 9>&-; } +vm() { bash "${HERE}/vm.sh" "$@" 9>&-; } +exec 9> "${HOME}/k3dgpu-bench.lock" +flock -w 7200 9 || { log "lock not taken in 7200 s"; exit 4; } +log "lock taken" +git -C "${REPO}" rev-parse HEAD > "${R}/commit.txt" +git -C "${REPO}" status --porcelain > "${R}/commit-status.txt" + +# 1. Host gate. +for i in 0 1 2 3 4 5 6; do + P host-facts > "${R}/host-facts.json" + P gates host "${R}/host-facts.json" > "${R}/gate-host.txt" && break + [ "${i}" -lt 6 ] || { log "host gate failed after 60 min"; cat "${R}/gate-host.txt"; exit 6; } + log "host gate not passed: $(grep -c FAIL "${R}/gate-host.txt") item(s); re-read in 10 min" + sleep 600 +done +log "host gate passed" + +# 2. A1's IR and B5's reference list. +IR=$(readlink -f "${H62R_IR:-${B}/models/gemma-4-26B-A4B-it-qat-int4-ov}") +P ir-verify "${IR}" > "${R}/ir-verify.txt" || { log "A1's IR does not match its manifest"; exit 1; } +curl -fsSL "https://huggingface.co/api/models/$(P pin qat_repo)/revision/$(P pin qat_rev)?blobs=true" \ + > "${R}/qat-siblings.json" || { log "no Hub file list"; exit 1; } +P verify-script "${R}/qat-siblings.json" > "${R}/verify-weights.py" + +# 3. The guests. +log "guests" +vm build > "${R}/vm-build.log" 2>&1 || { log "QEMU image build failed"; exit 1; } +vm fetch > "${R}/vm-fetch.log" 2>&1 || { log "cloud image failed its check"; exit 1; } +for vm in a b; do + H62R_IR=${IR} vm up "${vm}" > "${R}/vm-up-${vm}.log" 2>&1 || { log "VM ${vm} did not start"; exit 1; } +done +for vm in a b; do + vm wait "${vm}" 900 >> "${R}/vm-up-${vm}.log" 2>&1 || { log "VM ${vm} did not boot"; exit 1; } +done +fa=$(vm facts a) && fb=$(vm facts b) || { log "no guest facts"; exit 1; } +python3 -c 'import json, sys; a, b = (json.loads(x) for x in sys.argv[1:3]) +print(json.dumps({"amx_tile": {"a": a["amx_tile"], "b": b["amx_tile"]}, "nproc": {"a": a["nproc"], "b": b["nproc"]}, + "a": a, "b": b}))' "${fa}" "${fb}" > "${R}/guest-facts.json" +P gates guest "${R}/guest-facts.json" > "${R}/gate-guest.txt" || { log "guest gate failed: not deliverable"; exit 7; } + +# 4. Each guest's setup, side by side under its ceiling. +for vm in a b; do + G "${vm}" mkdir -p h62r/bin + put "${vm}" "${HERE}/h62r.py" "${HERE}/guest-${vm}.sh" +done +scp -q -F "${H62R_STATE}/ssh_config" "${B}/bin/k3d" "${B}/bin/kubectl" h62r-a:h62r/bin/ 9>&- +git -C "${REPO}" archive --format=tar HEAD | G a "mkdir -p h62r/k3d-gpu && tar -x -C h62r/k3d-gpu" +log "guest-a (cap ${A_CAP} s) and guest-b (cap ${B_CAP} s)" +timeout -k 60 "${A_CAP}" ssh -F "${H62R_STATE}/ssh_config" h62r-a 'bash h62r/guest-a.sh' > "${R}/guest-a.log" 2>&1 9>&- & +pa=$! +timeout -k 60 "${B_CAP}" ssh -F "${H62R_STATE}/ssh_config" h62r-b 'bash h62r/guest-b.sh' > "${R}/guest-b.log" 2>&1 9>&- & +pb=$! +wait "${pa}"; xa=$? +wait "${pb}"; xb=$? +for vm in a b; do + scp -q -r -F "${H62R_STATE}/ssh_config" "$(P vm "${vm}" host):h62r/records" "${R}/guest-${vm}" 2>/dev/null 9>&- || true +done +log "guest-a exit ${xa}, guest-b exit ${xb}" +if [ "${xa}" = 124 ] || [ "${xb}" = 124 ]; then log "a ceiling passed: not deliverable"; exit 7; fi +[ "${xa}" = 0 ] && [ "${xb}" = 0 ] || exit 1 + +# 5. NFD and weight gates on VM b. +amx=$(G b ". ~/h62r/env && kubectl get nodes -o jsonpath='{.items[0].metadata.labels.feature\\.node\\.kubernetes\\.io/cpu-cpuid\\.AMXBF16}'") +python3 -c 'import json, sys; print(json.dumps({"amxbf16": sys.argv[1] or None}))' "${amx}" > "${R}/nfd-facts.json" +P gates nfd "${R}/nfd-facts.json" > "${R}/gate-nfd.txt" || { log "NFD gate failed"; exit 8; } +ns=$(P arm B5 ns) sel=$(P arm B5 selector) +read -r pod _ <<< "$(G b ". ~/h62r/env && kubectl -n ${ns} get pods -l ${sel} -o json" | P pick-pod "")" +[ -n "${pod:-}" ] || { log "no Ready B5 pod"; exit 1; } +mnt=$(G b ". ~/h62r/env && kubectl -n ${ns} get pod ${pod} -o json" | P model-mount main) || { log "no model mount"; exit 1; } +G b ". ~/h62r/env && kubectl -n ${ns} exec -i ${pod} -c main -- python3 - ${mnt}" \ + < "${R}/verify-weights.py" > "${R}/weights-verify.txt" 2>&1 +tail -1 "${R}/weights-verify.txt" > "${R}/weights-facts.json" +P gates weights "${R}/weights-facts.json" > "${R}/gate-weights.txt" || { log "weight gate failed"; exit 8; } +log "setup done: both arms serve" diff --git a/bench/h62r/smoke.sh b/bench/h62r/smoke.sh new file mode 100755 index 00000000..8b34ec75 --- /dev/null +++ b/bench/h62r/smoke.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# smoke.sh: H62r's wiring smoke (EXPERIMENTS.md "H62r pre-registration", "Wiring smoke"), after +# setup.sh and before hold 1. It scores nothing, is not one of the 8 holds, and is disclosed in +# the run section. +# 1. B5 through EAS's own stack, in VM b, inside the bench lock: `es_auto_installer.sh validate +# agent-toolkit`, then integrations/eat/eat-check.sh in EAS mode (PR #260, merged) through EAS's +# gateway on gemma-4-26b: the stack's readiness (integrations/eas/eas-ready.py), a chat, a +# get_weather tool call, the guardrail's block, and a sandbox run that prints 385. Both files +# come from this checkout (the deploy commit) and are recorded by sha256 +# 2. one H62r hold with the pair-start guard past its end (pairs-hold.sh 0 smoke): both serving +# pods restart and answer through the harness's LiteLLM, both warm-ups answer, no pair starts +# Exit 0 when both pass; 1 otherwise; 4 when the lock is not taken. Records: ~/k3dgpu-bench/smoke/H62r. +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(cd "${HERE}/../.." && pwd) +P() { python3 "${HERE}/h62r.py" "$@"; } +B=${HOME}/k3dgpu-bench +export H62R_STATE=${H62R_STATE:-${B}/h62r} +S=${B}/smoke/H62r +MODEL=$(P pin catalog_name) +CHECK=${REPO}/integrations/eat/eat-check.sh READY=${REPO}/integrations/eas/eas-ready.py +mkdir -p "${S}" +log() { echo "== smoke $* $(date -u +%FT%TZ)" | tee -a "${S}/smoke.log"; } +G() { ssh -F "${H62R_STATE}/ssh_config" h62r-b "$@" 9>&-; } +grep -qE '^[[:space:]]+eas\)' "${CHECK}" || { log "eat-check.sh has no EAS mode: no smoke"; exit 1; } +sha256sum "${CHECK}" "${READY}" > "${S}/check-sha256.txt" +exec 9> "${HOME}/k3dgpu-bench.lock" +flock -w 7200 9 || { log "lock not taken in 7200 s"; exit 4; } +log "lock taken; step 1 in VM b" +G "sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -q jq=$(P pin jq)" > "${S}/jq.log" 2>&1 \ + || { log "jq not installed in VM b"; exit 1; } +# eat-check.sh reads eas-ready.py at ../eas/ beside itself, as in the repository. +G "mkdir -p h62r/integrations/eat h62r/integrations/eas" +scp -q -F "${H62R_STATE}/ssh_config" "${CHECK}" h62r-b:h62r/integrations/eat/ 9>&- +scp -q -F "${H62R_STATE}/ssh_config" "${READY}" h62r-b:h62r/integrations/eas/ 9>&- +G "cd ~/enterprise-ai-solutions && ./es_auto_installer.sh validate agent-toolkit" > "${S}/validate.log" 2>&1 +xv=$? +G ". ~/h62r/env && EAT_MODE=eas EAT_MODEL=${MODEL} bash ~/h62r/integrations/eat/eat-check.sh" > "${S}/eat-check.log" 2>&1 +xc=$? +log "validate exit ${xv}, eat-check exit ${xc}" +exec 9>&- +log "step 2: H62r-smoke" +bash "${HERE}/pairs-hold.sh" 0 smoke > "${S}/hold.log" 2>&1 +xh=$? +warm=$(python3 -c 'import json, sys +rows = [json.loads(x) for x in open(sys.argv[1]) if x.strip()] +print(sum(1 for r in rows if not r.get("error")))' "${S}/pairs/warmups.jsonl" 2>/dev/null || echo 0) +pairs=$(grep -c . "${S}/pairs/pairs.jsonl" 2>/dev/null || true) +log "hold exit ${xh}, warm-ups answered ${warm} of 2, pairs ${pairs:-0}" +[ "${xv}" = 0 ] && [ "${xc}" = 0 ] && [ "${xh}" = 0 ] && [ "${warm}" = 2 ] && [ "${pairs:-0}" = 0 ] || exit 1 +log "wiring smoke passed" diff --git a/bench/h62r/teardown.sh b/bench/h62r/teardown.sh new file mode 100755 index 00000000..2e8389e3 --- /dev/null +++ b/bench/h62r/teardown.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# teardown.sh: H62r's teardown (EXPERIMENTS.md "H62r pre-registration"), after the last hold or any +# abort. Both guests are powered off and removed (vm.sh down), and their disks, the cloud image and +# the QEMU image are deleted (vm.sh purge); ~/k3dgpu-bench/results/H62r stays. Serving pairs that +# campaign.sh stopped are started again (gpu-serving.sh start) and then checked through Dynamo +# :8100 as crescent amendment E2 item 7 does: a 16-token chat answers with content and that pair's +# SGLang num_requests_total rises, polled for up to 180 s. The result is recorded; it gates nothing. +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(cd "${HERE}/../.." && pwd) +B=${HOME}/k3dgpu-bench +R=${B}/results/H62r +export H62R_STATE=${H62R_STATE:-${B}/h62r} +mkdir -p "${R}" +for vm in a b; do bash "${HERE}/vm.sh" down "${vm}"; done +bash "${HERE}/vm.sh" purge +if [ -s "${R}/serving-stopped.txt" ]; then + while read -r c; do + bash "${REPO}/bench/eatflow/crescent/gpu-serving.sh" start "${c}" >> "${R}/serving.log" 2>&1 + done < "${R}/serving-stopped.txt" + mv "${R}/serving-stopped.txt" "${R}/serving-restored.txt" + n() { curl -s -m 8 "http://127.0.0.1:$1/metrics" | awk '/^sglang:num_requests_total/{s+=$2} END{print s+0}'; } + for port in 30000 30001; do + before=$(n "${port}") end=$(( SECONDS + 180 )) ok=0 c="" + until [ "${SECONDS}" -ge "${end}" ]; do + c=$(curl -s -m 60 http://127.0.0.1:8100/v1/chat/completions -H 'Content-Type: application/json' \ + -d '{"model":"Qwen3.8-27B","messages":[{"role":"user","content":"Say OK"}],"max_tokens":16,"temperature":0,"chat_template_kwargs":{"enable_thinking":false}}' \ + | python3 -c 'import json,sys;print(json.load(sys.stdin)["choices"][0]["message"]["content"].strip())' 2>/dev/null) + if [ -n "${c}" ] && [ "$(n "${port}")" -gt "${before}" ]; then ok=1; break; fi + sleep 5 + done + echo "$(date -u +%FT%TZ) :${port} pass=${ok} before=${before} after=$(n "${port}") content=${c:-none}" \ + | tee -a "${R}/frontdoor-post.txt" + done +fi diff --git a/bench/h62r/test_h62r.py b/bench/h62r/test_h62r.py new file mode 100644 index 00000000..f3822dd9 --- /dev/null +++ b/bench/h62r/test_h62r.py @@ -0,0 +1,907 @@ +#!/usr/bin/env python3 +"""Tests for bench/h62r (EXPERIMENTS.md "H62r pre-registration"): H62's runner re-pinned to the +GA tags (EAS v1.2.0, EAT release-2.0.0, enterprise-inference v1.1.0). The pins, the guests' QEMU +command lines and cloud-init, the setup gates, D1 as a check (the GA layer manifest names the +commits itself), the documented edit and D2, D3's sibling read (EAS's probe against the guest +kernel's sibling lists, and the reserved cpuset it writes), the pod and cgroup reads behind the +idle-arm check, B5's weight check, the re-run trigger, and the shell scripts against stubs. No VM, +no cluster, no network.""" +import hashlib +import json +import os +import shutil +import stat +import subprocess +import sys +import tempfile +import textwrap +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import h62r # noqa: E402 + +REPO = os.path.normpath(os.path.join(HERE, "..", "..")) +GIB_KIB = 1024 * 1024 + + +def read(*path): + with open(os.path.join(*path)) as f: + return f.read() + + +def run(*argv, stdin=None, env=None): + return subprocess.run([sys.executable, os.path.join(HERE, "h62r.py"), *argv], input=stdin, + capture_output=True, text=True, env=env) + + +# EAS v1.2.0 (a051f00) configs/repos/repos.agent-toolkit.yaml, lines 12-27, verbatim (the +# agent-toolkit comment carries a trailing space). +MANIFEST = "".join(ln + "\n" for ln in ( + 'repos:', + ' - layer: "inference"', + ' url: "https://github.com/intel/enterprise-inference"', + ' dest: "enterprise.ai-inference"', + ' rev: "d035b1ef1981fbf5084ff05c9a337e620478bb87" # v1.1.0', + ' deployment_subdir: ""', + ' config_dir: ""', + ' default_flavour: ""', + ' model_catalog: "model_manager/models.yaml"', + '', + ' - layer: "agent-toolkit"', + ' url: "https://github.com/intel/enterprise-agent-toolkit"', + ' dest: "enterprise.agent-toolkit"', + ' rev: "7b827e2d95cec826a20c7e3f5a30eb4818ed2992" # release-2.0.0 ', + ' deployment_subdir: ""', + ' config_dir: ""', + ' default_flavour: ""')) + +# EAT release-2.0.0 (7b827e2) config.yaml, lines 33-36 (unchanged from 036b662; the file init +# seeds as env/local/config.agent-toolkit.yaml), and the same key commented in a doc block. +CONFIG = textwrap.dedent('''\ + # Model(s) to deploy, from the catalog (env/local/models.yaml) + # llm_services_autodeploy_models: ["qwen3-coder-30b", "bge-base-en"] + llm_services_autodeploy_models: ["qwen2-5-coder-14b"] + redis_stack_enabled: true + ''') + +# enterprise-inference v1.1.0 (d035b1e) model_manager/models.yaml (unchanged from 170eb3a), its +# first entry's commented name line (H62 amendment V3) and the gemma-4-26b entry between two +# others, at the file's indentation. +MODELS = ('models:\n - name: qwen3-0-6b # deploy name + the "model" id used in API requests\n' + ' model_id: Qwen/Qwen3-0.6B\n\n' + textwrap.indent(textwrap.dedent('''\ + - name: gemma-4-26b-tp2 + model_id: google/gemma-4-26B-A4B-it + category: llm + + - name: gemma-4-26b + model_id: google/gemma-4-26B-A4B-it + category: llm + runtime: vllm + server_version: "0.28.0" + cpu: auto # one NUMA node per TP rank, per node + memory: 128Gi + + - name: qwen3guard + model_id: Qwen/Qwen3Guard-Gen-0.6B + category: llm + '''), " ")) + +# The global_config.yaml init seeds (EAS v1.2.0 configs/defaults/global_config.yaml, trimmed): +# no nri_reserved_cpu_list line, so EAS's role default ("") applies. +GLOBAL = textwrap.dedent('''\ + # --- CPU Policy --- + kubernetes_cpu_policy: "nri-balloons" + + kubernetes_accelerator: "cpu" + ''') + +# thread_siblings_list of each CPU. GUEST: H62's QEMU guest (-smp 96,sockets=1,cores=48,threads=2), +# siblings adjacent, so Linux writes each pair as a range ("0-1"; read in a spike guest of that +# shape, 2026-11-06). HOST: crescent itself (read 2026-11-07T03:36Z: cpu0 "0,48", cpu1 "1,49"). +GUEST = {i: f"{i - i % 2}-{i - i % 2 + 1}" for i in range(96)} +HOST = {i: (f"{i},{i + 48}" if i < 48 else f"{i - 48},{i}") for i in range(96)} +NOSMT = {i: str(i) for i in range(96)} +ODD = ",".join(str(i) for i in range(1, 96, 2)) + + +def probe(tpc, logical=96): + """EAS v1.2.0 roles/nri_cpu_balloons/files/probe-node-topology.sh's first four lines.""" + return f"NRI_NUMA=1\nNRI_LOGICAL={logical}\nNRI_TPC={tpc}\nNRI_PHYSICAL={logical // tpc}\nNRI_PKGNODE={{0:[0]}}\n" + + +class PinsTest(unittest.TestCase): + """Every pin the pre-registration names, in one table the scripts read: B5 at the GA tags, + A1 and the guests as H62 had them (Docker CE at its current patch release).""" + + def test_b5_is_at_the_ga_tags(self): + p = h62r.PINS + self.assertEqual((p["eas_tag"], p["eas_sha"]), ("v1.2.0", "a051f00fcb2dd473fe763610471cd509db82c433")) + self.assertEqual((p["ei_tag"], p["ei_sha"]), ("v1.1.0", "d035b1ef1981fbf5084ff05c9a337e620478bb87")) + self.assertEqual((p["eat_tag"], p["eat_sha"]), ("release-2.0.0", "7b827e2d95cec826a20c7e3f5a30eb4818ed2992")) + self.assertEqual(p["eas_repo"], "https://github.com/intel/enterprise-ai-solutions.git") + self.assertEqual((p["catalog_name"], p["stock_model_id"]), ("gemma-4-26b", "google/gemma-4-26B-A4B-it")) + self.assertEqual(p["vllm_cpu_image"], "vllm/vllm-openai-cpu:v0.28.0@sha256:" + "197354e6475e638eb5ccca09beb078488046a82de0e204e7429bf1a804df9647") + + def test_a1_weights_and_guests(self): + p = h62r.PINS + self.assertEqual(p["qat_repo"], "google/gemma-4-26B-A4B-it-qat-q4_0-unquantized") + self.assertEqual(p["qat_rev"], "f1e06dc520982d9b9edd76859fdb7ab209449949") + self.assertEqual(p["node_image"], "cryptoandcoffee/k3d-gpu:0.1.61@sha256:" + "8cd264aa68c77681d7fa474693944de0db8d55de5881253e4029bf72a67e175c") + self.assertEqual(p["ovms_image"], "openvino/model_server:2026.4.1-gpu@sha256:" + "a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58") + self.assertEqual(p["cloud_image_sha256"], "04e8f2461a16a44e78a370284471f29616cdd9baf1893df25807b71f985b6c94") + self.assertTrue(p["cloud_image_url"].endswith("/release-20261017/ubuntu-24.04-server-cloudimg-amd64.img")) + self.assertEqual(p["cloud_image_bytes"], 621284352) + self.assertEqual(p["qemu_base"], "ubuntu:26.04@sha256:6a06111c3b0a8600feadb270aec771dc239bf407b25625f93f44ab00e11f4d3c") + self.assertEqual((p["qemu_pkg"], p["genisoimage_pkg"]), ("1:10.2.1+ds-1ubuntu3.3", "9:1.1.11-5")) + self.assertEqual(p["docker_ce"], "5:29.9.3-1~ubuntu.24.04~noble") + self.assertEqual(p["containerd"], "2.4.2-3~ubuntu.24.04~noble") + self.assertEqual(p["docker_gpg_fpr"], "9DC858229FC7DD38854AE2D88D81803C0EBFCD88") + self.assertEqual((p["jq"], p["python3_yaml"]), ("1.7.1-3ubuntu0.24.04.2", "6.0.1-2build2")) + self.assertEqual(h62r.QEMU_IMAGE, "h62r-qemu:1-10.2.1-ds-1ubuntu3.3") + + def test_no_branch_pins_left(self): + # D1 at rc.1 rewrote two branch revs; the GA manifest names commits, so no branch is pinned. + self.assertFalse({"ei_branch", "eat_branch"} & set(h62r.PINS)) + + def test_cli_prints_one_pin(self): + r = run("pin", "eas_sha") + self.assertEqual((r.returncode, r.stdout), (0, "a051f00fcb2dd473fe763610471cd509db82c433\n")) + self.assertEqual(run("pin", "nope").returncode, 2) + + def test_the_qemu_image_is_built_from_the_pins(self): + df = read(HERE, "qemu.Dockerfile") + self.assertIn(f"FROM {h62r.PINS['qemu_base']}\n", df) + for pkg in ("qemu-system-x86", "qemu-utils"): + self.assertIn(f"{pkg}={h62r.PINS['qemu_pkg']}", df) + self.assertIn(f"genisoimage={h62r.PINS['genisoimage_pkg']}", df) + + def test_the_cloud_image_is_checked_before_use(self): + vm = read(HERE, "vm.sh") + self.assertIn('P pin cloud_image_sha256', vm) + self.assertIn('sha256sum -c', vm) + + +class VmSpecTest(unittest.TestCase): + def test_two_guests_of_h62s_shape_under_their_own_names(self): + a, b = h62r.VMS["a"], h62r.VMS["b"] + self.assertEqual((a["arm"], a["mem_gib"], a["disk_gib"], b["arm"], b["mem_gib"], b["disk_gib"]), + ("A1", 96, 200, "B5", 192, 400)) + self.assertEqual(h62r.SMP, "96,sockets=1,cores=48,threads=2") + self.assertEqual(h62r.VCPUS, 96) + self.assertEqual((a["name"], a["host"], b["name"], b["host"]), ("h62r-vm-a", "h62r-a", "h62r-vm-b", "h62r-b")) + # No k3dgpu prefix: hold.sh's cleanup never removes them; another row's serving covariate + # counts them as foreign load. + for v in (a, b): + self.assertFalse(v["name"].startswith("k3dgpu"), v["name"]) + ports = {a["ssh_port"], a["serve_port"], b["ssh_port"], b["serve_port"]} + self.assertEqual(ports, {18842, 18843, 18851, 18852}) + self.assertFalse(ports & {18822, 18823, 18831, 18832}) # H62's ports + + def test_memory_gate_is_both_guests_plus_48_gib(self): + self.assertEqual(h62r.mem_needed_kib(), (96 + 192 + 48) * GIB_KIB) + + +class QemuArgvTest(unittest.TestCase): + def test_vm_b(self): + argv = h62r.qemu_argv("b", "/st", "h62r-qemu:x", uid=1000, gid=1000, kvm_gid=992) + self.assertEqual(argv[:5], ["docker", "run", "-d", "--name", "h62r-vm-b"]) + for pair in (["--network", "host"], ["--device", "/dev/kvm"], ["--user", "1000:1000"], + ["--group-add", "992"], ["-v", "/st:/h62r"], ["-cpu", "host"], ["-m", "192G"], + ["-smp", "96,sockets=1,cores=48,threads=2"], ["-machine", "q35,accel=kvm"], + ["--label", "io.k3d-gpu.h62r=vm"]): + self.assertTrue(any(argv[i:i + 2] == pair for i in range(len(argv))), pair) + net = argv[argv.index("-netdev") + 1] + self.assertEqual(net, "user,id=n0,hostfwd=tcp:127.0.0.1:18843-:22,hostfwd=tcp:127.0.0.1:18852-:18852") + self.assertNotIn("0.0.0.0", " ".join(argv)) + self.assertNotIn("-virtfs", argv) + self.assertIn("file=/h62r/b/disk.qcow2,if=virtio,format=qcow2", argv) + self.assertIn("file=/h62r/b/seed.iso,if=virtio,format=raw,readonly=on", argv) + self.assertEqual(argv[argv.index("h62r-qemu:x") + 1], "qemu-system-x86_64") + + def test_vm_a_shares_the_ir_read_only(self): + argv = h62r.qemu_argv("a", "/st", "img", share="/home/u/ir", uid=1, gid=1, kvm_gid=2) + self.assertIn("/home/u/ir:/share:ro", argv) + self.assertEqual(argv[argv.index("-virtfs") + 1], + "local,path=/share,mount_tag=h62rshare,security_model=none,readonly=on") + self.assertEqual(argv[argv.index("-m") + 1], "96G") + with self.assertRaises(ValueError): + h62r.qemu_argv("b", "/st", "img", share="/x", uid=1, gid=1, kvm_gid=2) + with self.assertRaises(ValueError): + h62r.qemu_argv("a", "/st", "img", uid=1, gid=1, kvm_gid=2) # A1's IR is required + + def test_cli_one_argument_per_line(self): + r = run("qemu-argv", "b", "/st", "img", "1000", "1000", "992") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout.splitlines(), h62r.qemu_argv("b", "/st", "img", uid=1000, gid=1000, kvm_gid=992)) + + +class CloudInitTest(unittest.TestCase): + KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExample h62r" + + def test_user_data(self): + u = h62r.user_data("b", self.KEY) + self.assertTrue(u.startswith("#cloud-config\n")) + self.assertIn(f" - {self.KEY}\n", u) + self.assertIn("ssh_pwauth: false\n", u) + self.assertIn("package_upgrade: false\n", u) + self.assertIn("hostname: h62r-b\n", u) + for unit in ("unattended-upgrades.service", "apt-daily.timer", "apt-daily-upgrade.timer"): + self.assertIn(unit, u) + + def test_meta_data_and_key_checks(self): + self.assertEqual(h62r.meta_data("a"), "instance-id: h62r-a\nlocal-hostname: h62r-a\n") + for bad in ("", "ssh-ed25519 AAAA\nrm -rf /", "not-a-key"): + with self.assertRaises(ValueError): + h62r.user_data("a", bad) + + +class HostFactsTest(unittest.TestCase): + def test_reads_meminfo_df_and_probes(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + mi = os.path.join(tmp, "meminfo") + with open(mi, "w") as f: + f.write("MemTotal: 527700000 kB\nMemAvailable: 374341632 kB\n") + df = "Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/x 3600000000 1100000000 2400000000 31% /\n" + seen = [] + f = h62r.host_facts(mi, df, lambda url: seen.append(url) or "registry.k8s.io" not in url) + self.assertEqual((f["mem_available_kib"], f["root_used_pct"]), (374341632, 31)) + self.assertEqual(f["reach"], {"huggingface.co": True, "registry-1.docker.io": True, "registry.k8s.io": False}) + self.assertEqual(len(seen), 3) + + +class IrVerifyTest(unittest.TestCase): + def test_manifest_check(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + files = {"openvino_model.bin": b"bin" * 10, "openvino_model.xml": b""} + man = {"files": {}} + for n, data in files.items(): + with open(os.path.join(tmp, n), "wb") as f: + f.write(data) + man["files"][n] = {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} + with open(os.path.join(tmp, "manifest.json"), "w") as f: + json.dump(man, f) + r = run("ir-verify", tmp) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("2 files, 0 bad", r.stdout) + with open(os.path.join(tmp, "openvino_model.xml"), "ab") as f: + f.write(b" ") + r = run("ir-verify", tmp) + self.assertEqual(r.returncode, 1) + self.assertIn("1 bad", r.stdout) + + +class SshConfigTest(unittest.TestCase): + def test_both_guests(self): + c = h62r.ssh_config("/home/u/k3dgpu-bench/h62r") + for host, port in (("h62r-a", 18842), ("h62r-b", 18843)): + blk = c[c.index(f"Host {host}\n"):] + self.assertIn(f" Port {port}\n", blk) + for line in (" HostName 127.0.0.1\n", " User ubuntu\n", " IdentityFile /home/u/k3dgpu-bench/h62r/id_ed25519\n", + " UserKnownHostsFile /home/u/k3dgpu-bench/h62r/known_hosts\n", " StrictHostKeyChecking accept-new\n", + " ControlMaster auto\n", " ControlPath /home/u/k3dgpu-bench/h62r/cm-%C\n", " BatchMode yes\n"): + self.assertEqual(c.count(line), 2, line) + + +class GatesTest(unittest.TestCase): + HOST = {"mem_available_kib": 357 * GIB_KIB, "root_used_pct": 31, + "reach": {"huggingface.co": True, "registry-1.docker.io": True, "registry.k8s.io": True}} + + def failed(self, phase, facts): + return [m for ok, m in h62r.gates(phase, facts) if not ok] + + def test_host(self): + self.assertEqual(self.failed("host", self.HOST), []) + self.assertEqual(self.failed("host", dict(self.HOST, mem_available_kib=h62r.mem_needed_kib())), []) + self.assertEqual(len(self.failed("host", dict(self.HOST, mem_available_kib=h62r.mem_needed_kib() - 1))), 1) + self.assertEqual(self.failed("host", dict(self.HOST, root_used_pct=90)), []) + self.assertEqual(len(self.failed("host", dict(self.HOST, root_used_pct=91))), 1) + unreach = dict(self.HOST, reach={"huggingface.co": False, "registry-1.docker.io": True, "registry.k8s.io": True}) + self.assertIn("huggingface.co", self.failed("host", unreach)[0]) + + def test_guest_amx_on_every_vcpu(self): + self.assertEqual(self.failed("guest", {"amx_tile": {"a": 96, "b": 96}, "nproc": {"a": 96, "b": 96}}), []) + self.assertEqual(len(self.failed("guest", {"amx_tile": {"a": 96, "b": 0}, "nproc": {"a": 96, "b": 96}})), 1) + self.assertEqual(len(self.failed("guest", {"amx_tile": {"a": 96}, "nproc": {"a": 96}})), 2) # b unread + + def test_nfd_and_weights(self): + self.assertEqual(self.failed("nfd", {"amxbf16": "true"}), []) + self.assertEqual(len(self.failed("nfd", {"amxbf16": None})), 1) + self.assertEqual(self.failed("weights", {"bad": 0, "missing": 0, "checked": 3}), []) + self.assertEqual(len(self.failed("weights", {"bad": 1, "missing": 0, "checked": 3})), 1) + self.assertEqual(len(self.failed("weights", {"bad": 0, "missing": 0, "checked": 0})), 1) + with self.assertRaises(ValueError): + h62r.gates("other", {}) + + def test_cli(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + p = os.path.join(tmp, "f.json") + with open(p, "w") as f: + json.dump(self.HOST, f) + r = run("gates", "host", p) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("PASS", r.stdout) + with open(p, "w") as f: + json.dump(dict(self.HOST, root_used_pct=95), f) + r = run("gates", "host", p) + self.assertEqual(r.returncode, 1) + self.assertIn("FAIL", r.stdout) + + +class EasCheckTest(unittest.TestCase): + """D1 at the GA tags: the layer manifest already names the release notes' commits, with the + release on each line's comment, so it is checked, not rewritten.""" + + def test_the_ga_manifest_passes_unchanged(self): + self.assertEqual(h62r.eas_check(MANIFEST), + ["inference d035b1ef1981fbf5084ff05c9a337e620478bb87 (v1.1.0)", + "agent-toolkit 7b827e2d95cec826a20c7e3f5a30eb4818ed2992 (release-2.0.0)"]) + + def test_refuses_anything_else(self): + for bad in (MANIFEST.replace('rev: "d035b1ef', 'rev: "0000000f'), # another commit + MANIFEST.replace("# v1.1.0", "# v1.0.1"), # another release named + MANIFEST.replace('"7b827e2d95cec826a20c7e3f5a30eb4818ed2992" # release-2.0.0 ', + '"release-2.0.0"'), # a branch, as at rc.1 + MANIFEST.replace('- layer: "inference"', '- layer: "other"'), # a layer missing + MANIFEST + ' rev: "x"\n'): # a second rev in a block + with self.assertRaises(ValueError): + h62r.eas_check(bad) + + def test_cli_reads_without_writing(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + p = os.path.join(tmp, "repos.agent-toolkit.yaml") + with open(p, "w") as f: + f.write(MANIFEST) + r = run("eas-check", p) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("ok inference d035b1ef1981fbf5084ff05c9a337e620478bb87 (v1.1.0)", r.stdout) + self.assertEqual(read(p), MANIFEST) + with open(p, "w") as f: + f.write(MANIFEST.replace("# v1.1.0", "# main")) + self.assertEqual(run("eas-check", p).returncode, 2) + self.assertEqual(run("eas-pin", p).returncode, 2) # no rewrite command + + +class EasEditTest(unittest.TestCase): + """The one documented edit (the catalog value) and D2 (gemma-4-26b's weights), as in H62.""" + + def test_autodeploy_gemma_only_on_the_live_line(self): + out = h62r.set_autodeploy(CONFIG, "gemma-4-26b") + self.assertIn('\nllm_services_autodeploy_models: ["gemma-4-26b"]\n', out) + self.assertIn('# llm_services_autodeploy_models: ["qwen3-coder-30b", "bge-base-en"]\n', out) + self.assertEqual(h62r.set_autodeploy(out, "gemma-4-26b"), out) + with self.assertRaises(ValueError): + h62r.set_autodeploy(CONFIG.replace("llm_services_autodeploy_models: [\"qwen", "#x: [\"qwen"), "gemma-4-26b") + + def test_d2_moves_only_gemma_4_26b_to_the_qat_weights(self): + out = h62r.set_model_id(MODELS, "gemma-4-26b", "google/gemma-4-26B-A4B-it", + "google/gemma-4-26B-A4B-it-qat-q4_0-unquantized") + self.assertEqual(sum(1 for a, b in zip(MODELS.splitlines(), out.splitlines()) if a != b), 1) + self.assertIn("model_id: Qwen/Qwen3-0.6B\n", out) # commented name line parsed + blocks = out.split("\n\n") + self.assertIn("model_id: google/gemma-4-26B-A4B-it\n", blocks[1]) # the tp2 entry keeps its id + self.assertIn("model_id: google/gemma-4-26B-A4B-it-qat-q4_0-unquantized\n", blocks[2]) + with self.assertRaises(ValueError): + h62r.set_model_id(MODELS, "nope", "a", "b") + with self.assertRaises(ValueError): + h62r.set_model_id(MODELS, "gemma-4-26b", "google/other", "b") + + def test_cli_edits_an_env_and_prints_the_diffs(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + for name, text in (("config.agent-toolkit.yaml", CONFIG), ("models.yaml", MODELS), ("global_config.yaml", GLOBAL)): + with open(os.path.join(tmp, name), "w") as f: + f.write(text) + r = run("eas-edit", tmp) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('+llm_services_autodeploy_models: ["gemma-4-26b"]', r.stdout) + self.assertIn("+ model_id: google/gemma-4-26B-A4B-it-qat-q4_0-unquantized", r.stdout) + self.assertEqual(read(tmp, "global_config.yaml"), GLOBAL) # not this step's file + self.assertEqual(run("eas-edit", tmp).stdout, "") + + +class SiblingReadTest(unittest.TestCase): + """D3's sibling read. EAS v1.2.0's probe counts the comma fields of cpu0's + thread_siblings_list. In the guest Linux writes each adjacent sibling pair as a range ("0-1"), + so the probe reads 1 thread per core and EAS's default 'siblings' profile falls back to + 'standard' (12 reserved CPUs): H62's 76-CPU gemma pod. On crescent's host ("0,48") it reads 2 + and reserves the 48 siblings. The sibling read supplies that reservation through EAS's own + nri_reserved_cpu_list, and only when the probe and the kernel disagree in that way.""" + + def test_guest_where_the_probe_misreads_gets_the_odd_vcpus(self): + self.assertEqual(h62r.sibling_read(GUEST, probe(1)), ODD) + self.assertEqual(len(ODD.split(",")), 48) + + def test_host_numbering_needs_nothing(self): + self.assertEqual(h62r.sibling_read(HOST, probe(2)), "") + + def test_no_smt_needs_nothing(self): + self.assertEqual(h62r.sibling_read(NOSMT, probe(1)), "") + + def test_any_other_disagreement_stops(self): + for lists, text in ((NOSMT, probe(2)), # the probe sees SMT the kernel does not + ({**GUEST, 95: "95"}, probe(1)), # uneven threads per core + ({i: f"{i - i % 4}-{i - i % 4 + 3}" for i in range(96)}, probe(1)), # 4 threads per core + (GUEST, "NRI_NUMA=1\n"), # no NRI_TPC line + ({}, probe(1))): # no CPU read + with self.assertRaises(ValueError): + h62r.sibling_read(lists, text) + + def test_cli_reads_a_sysfs_cpu_tree(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + for i, s in GUEST.items(): + d = os.path.join(tmp, "cpu", f"cpu{i}", "topology") + os.makedirs(d) + with open(os.path.join(d, "thread_siblings_list"), "w") as f: + f.write(s + "\n") + for extra in ("cpufreq", "cpuidle", "cpu.txt"): # siblings of cpuN in sysfs + os.makedirs(os.path.join(tmp, "cpu", extra), exist_ok=True) + p = os.path.join(tmp, "probe.txt") + with open(p, "w") as f: + f.write(probe(1)) + r = run("sibling-read", os.path.join(tmp, "cpu"), p) + self.assertEqual((r.returncode, r.stdout), (0, ODD + "\n"), r.stderr) + with open(p, "w") as f: + f.write(probe(2)) # the probe agrees: nothing + self.assertEqual(run("sibling-read", os.path.join(tmp, "cpu"), p).stdout, "") + with open(p, "w") as f: + f.write(probe(4)) # it disagrees otherwise: stop + self.assertEqual(run("sibling-read", os.path.join(tmp, "cpu"), p).returncode, 2) + + +class ReserveTest(unittest.TestCase): + """The reserved cpuset goes into env/local/global_config.yaml as nri_reserved_cpu_list, the + key EAS's docs/customize/nri_cpu_balloons.md documents there (the installer passes + global_config.yaml to every playbook as extra vars).""" + + def test_appends_the_documented_key(self): + out = h62r.reserve(GLOBAL, ODD) + self.assertTrue(out.startswith(GLOBAL)) + self.assertIn(f'\nnri_reserved_cpu_list: "{ODD}"\n', out) + self.assertEqual(out.count("nri_reserved_cpu_list:"), 1) + self.assertEqual(h62r.reserve(out, ODD), out) # a second run changes nothing + + def test_fills_the_documented_empty_default(self): + out = h62r.reserve(GLOBAL + 'nri_reserved_cpu_list: ""\n', ODD) + self.assertEqual(out, GLOBAL + f'nri_reserved_cpu_list: "{ODD}"\n') + + def test_refuses_another_value_or_a_bad_cpuset(self): + with self.assertRaises(ValueError): + h62r.reserve(GLOBAL + 'nri_reserved_cpu_list: "0-3"\n', ODD) + for bad in ("", "1,3,", "a-b", '1"\nx: y'): + with self.assertRaises(ValueError): + h62r.reserve(GLOBAL, bad) + + def test_cli_edits_in_place_and_prints_the_diff(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + p = os.path.join(tmp, "global_config.yaml") + with open(p, "w") as f: + f.write(GLOBAL) + r = run("reserve", ODD, p) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn(f'+nri_reserved_cpu_list: "{ODD}"', r.stdout) + self.assertEqual(read(p), h62r.reserve(GLOBAL, ODD)) + self.assertEqual(run("reserve", ODD, p).stdout, "") + + +def pod(name, uid, ready=True, deleting=False, created="2026-10-30T00:00:00Z", phase="Running"): + meta = {"name": name, "uid": uid, "creationTimestamp": created} + if deleting: + meta["deletionTimestamp"] = "2026-10-30T00:01:00Z" + return {"metadata": meta, "status": {"phase": phase, + "conditions": [{"type": "Ready", "status": "True" if ready else "False"}]}} + + +class PodTest(unittest.TestCase): + def test_pick_a_new_ready_pod(self): + old = pod("p0", "u0", deleting=True) + self.assertIsNone(h62r.pick_pod({"items": [old, pod("p1", "u1", ready=False)]}, {"u0"})) + self.assertEqual(h62r.pick_pod({"items": [old, pod("p1", "u1")]}, {"u0"}), ("p1", "u1")) + self.assertIsNone(h62r.pick_pod({"items": [pod("p0", "u0")]}, {"u0"})) + self.assertEqual(h62r.pick_pod({"items": [pod("p0", "u0")]}, set()), ("p0", "u0")) + two = {"items": [pod("p1", "u1", created="2026-10-30T00:00:01Z"), pod("p2", "u2", created="2026-10-30T00:00:09Z")]} + self.assertEqual(h62r.pick_pod(two, set()), ("p2", "u2")) + self.assertIsNone(h62r.pick_pod({"items": [pod("p1", "u1", phase="Pending")]}, set())) + + def test_cli(self): + doc = json.dumps({"items": [pod("p0", "u0", deleting=True), pod("p1", "u1")]}) + self.assertEqual(run("uids", stdin=doc).stdout, "u0,u1\n") + r = run("pick-pod", "u0", stdin=doc) + self.assertEqual((r.returncode, r.stdout), (0, "p1 u1\n")) + self.assertEqual(run("pick-pod", "u0,u1", stdin=doc).returncode, 1) + + def test_model_mount(self): + p = {"spec": {"volumes": [{"name": "home", "emptyDir": {}}, + {"name": "model-cache", "persistentVolumeClaim": {"claimName": "model-store"}}], + "containers": [{"name": "main", "volumeMounts": [ + {"name": "home", "mountPath": "/home"}, + {"name": "model-cache", "mountPath": "/mnt/models", "subPath": "gemma-4-26b"}]}]}} + self.assertEqual(h62r.model_mount(p, "main"), "/mnt/models") + with self.assertRaises(ValueError): + h62r.model_mount(p, "other") + + +class VerifyScriptTest(unittest.TestCase): + """B5's weights, read in its pod: every LFS file of the QAT checkpoint at f1e06dc5.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.d = os.path.join(self.tmp, "models") + os.makedirs(os.path.join(self.d, ".cache", "huggingface", "download")) + files = {"model-00001.safetensors": b"w1" * 1000, "config.json": b"{}"} + sib = [] + for n, data in files.items(): + with open(os.path.join(self.d, n), "wb") as f: + f.write(data) + with open(os.path.join(self.d, ".cache", "huggingface", "download", n + ".metadata"), "w") as f: + f.write(h62r.PINS["qat_rev"] + "\netag\n1730000000.0\n") + s = {"rfilename": n, "size": len(data)} + if n.endswith(".safetensors"): + s["lfs"] = {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)} + sib.append(s) + self.sib = {"sha": h62r.PINS["qat_rev"], "siblings": sib} + self.script = os.path.join(self.tmp, "verify.py") + with open(self.script, "w") as f: + f.write(h62r.verify_script(self.sib)) + + def check(self): + r = subprocess.run([sys.executable, self.script, self.d], capture_output=True, text=True) + return r.returncode, json.loads(r.stdout.splitlines()[-1]) + + def test_matching_weights_pass_and_name_their_commit(self): + rc, s = self.check() + self.assertEqual((rc, s["bad"], s["missing"], s["checked"], s["revisions"]), (0, 0, 0, 1, [h62r.PINS["qat_rev"]])) + self.assertIn("bench/h62r/h62r.py", read(self.script)) + + def test_a_changed_weight_fails(self): + with open(os.path.join(self.d, "model-00001.safetensors"), "ab") as f: + f.write(b"x") + self.assertEqual(self.check()[0], 1) + + +class TriggerTest(unittest.TestCase): + """H62r's re-run trigger: an EAS release after v1.2.0 (no release candidate), or an EAT + release after release-2.0.0 (its hotfix of 2.0.0 too).""" + + EAS = ("986847891a835136591b48e08e784250cc275a15\trefs/tags/v1.0.0\n" + "1d5686e75f8ec0b1edac206e9ca4db7ddde32250\trefs/tags/v1.2.0-rc.1\n" + "a051f00fcb2dd473fe763610471cd509db82c433\trefs/tags/v1.2.0\n") + EAT = ("0c726f576595971c53cc45b94e08ba969beb4a4a\trefs/tags/release-1.1.0\n" + "0c7504eec7224556d87e173aa530c0999dc68925\trefs/tags/release-2.0.0\n" + "7b827e2d95cec826a20c7e3f5a30eb4818ed2992\trefs/tags/release-2.0.0^{}\n") + + def test_nothing_fired_at_the_pinned_releases(self): + self.assertEqual(h62r.trigger(self.EAS, self.EAT), {"eas": [], "eat": []}) + + def test_a_later_release_fires(self): + for tag in ("v1.2.1", "v1.3.0", "v2.0.0", "v1.10.0"): + self.assertEqual(h62r.trigger(self.EAS + f"e\trefs/tags/{tag}\n", self.EAT)["eas"], [tag], tag) + for tag in ("v1.3.0-rc.1", "v1.2.0", "v1.1.1"): + self.assertEqual(h62r.trigger(self.EAS + f"e\trefs/tags/{tag}\n", self.EAT)["eas"], [], tag) + for tag in ("release-2.0.0-hotfix", "release-2.0.1", "release-2.1.0", "v3.0.0"): + self.assertEqual(h62r.trigger(self.EAS, self.EAT + f"f\trefs/tags/{tag}\n")["eat"], [tag], tag) + for tag in ("release-2.0.0", "release-1.2.0-hotfix", "release-2.1.0-rc1"): + self.assertEqual(h62r.trigger(self.EAS, self.EAT + f"f\trefs/tags/{tag}\n")["eat"], [], tag) + + def test_cli(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + for n, t in (("eas", self.EAS), ("eat", self.EAT + "f\trefs/tags/release-2.0.1\n")): + with open(os.path.join(tmp, n), "w") as f: + f.write(t) + r = run("trigger", os.path.join(tmp, "eas"), os.path.join(tmp, "eat")) + self.assertEqual((r.returncode, r.stdout), (0, "fired: eat release-2.0.1\n")) + + +class CampaignMathTest(unittest.TestCase): + def test_pairs_left(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + self.assertEqual(h62r.pairs_left(tmp), 24) + with open(os.path.join(tmp, "pairs.jsonl"), "w") as f: + f.writelines(json.dumps({"j": j, "status": "complete"}) + "\n" for j in range(10)) + self.assertEqual(run("pairs-left", tmp).stdout, "14\n") + + def test_vmtax_is_a_description(self): + h50 = {"within_budget_success": {"A1": {"k": 19, "n": 24}, "B1": {"k": 21, "n": 24}}, + "attempt_median_s": {"A1": 8.0, "B1": 12.0}} + mine = {"within_budget_success": {"A1": {"k": 20, "n": 24}, "B5": {"k": 18, "n": 24}}, + "attempt_median_s": {"A1": 9.2, "B5": 15.0}} + t = h62r.vmtax(h50, mine) + self.assertEqual(t["a1_guest"], {"success": "20/24", "attempt_median_s": 9.2}) + self.assertAlmostEqual(t["attempt_median_guest_over_host"], 1.15) + self.assertAlmostEqual(t["b_attempt_median_guest_over_host"], 1.25) + self.assertEqual(t["gated"], False) + + +def stub(path, body): + with open(path, "w") as f: + f.write("#!/usr/bin/env bash\n" + body) + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + + +class ArmScriptTest(unittest.TestCase): + """bench/h62r/arm.sh against a stub ssh: the restart wait, the cgroup spec the idle-arm check + reads, cpustat and the ports.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.log = os.path.join(self.tmp, "ssh.log") + stub(os.path.join(self.tmp, "ssh"), textwrap.dedent(f'''\ + cmd="${{@: -1}}" + echo "$*" >> "{self.log}" + case "${{cmd}}" in + *"get pods -l"*) + n=$(cat "{self.tmp}/n" 2>/dev/null || echo 0) + f="{self.tmp}/pods-${{n}}.json"; [ -f "$f" ] || f=$(ls "{self.tmp}"/pods-*.json | sort -V | tail -1) + cat "$f"; echo $(( n + 1 )) > "{self.tmp}/n" ;; + *"find /sys/fs/cgroup"*) echo /sys/fs/cgroup/kubepods.slice/kubepods-podu1.slice ;; + *"cat /x/cpu.stat"*) printf 'usage_usec 77\\n' ;; + *) : ;; + esac + ''')) + self.env = dict(os.environ, H62R_SSH=os.path.join(self.tmp, "ssh"), H62R_STATE=self.tmp, H62R_POLL_S="0.1") + + def arm(self, *args): + return subprocess.run(["bash", os.path.join(HERE, "arm.sh"), *args], capture_output=True, text=True, env=self.env) + + def pods(self, *docs): + for i, d in enumerate(docs): + with open(os.path.join(self.tmp, f"pods-{i}.json"), "w") as f: + json.dump({"items": d}, f) + + def test_one_lookup_per_call(self): + self.assertEqual(run("arm-fields", "A1").stdout, + "a h62r-a 18851 k3d-gpu-agent app=ovms svc/ovms 8000 ovms \n") + self.assertEqual(h62r.arm_fields("B5")[-1], "llminferenceservice/gemma-4-26b") + self.assertEqual(read(HERE, "arm.sh").count("P arm"), 1) + + def test_ports_and_unknown_arm(self): + self.assertEqual(self.arm("A1", "port").stdout, "18851\n") + self.assertEqual(self.arm("B5", "port").stdout, "18852\n") + self.assertEqual(self.arm("C9", "port").returncode, 2) + + def test_restart_waits_for_a_new_ready_pod(self): + self.pods([pod("p0", "u0")], [pod("p0", "u0", deleting=True)], [pod("p1", "u1", ready=False)], [pod("p1", "u1")]) + r = self.arm("B5", "restart", "30") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("ready: p1 u1", r.stdout) + log = read(self.log) + self.assertIn("h62r-b", log) + self.assertIn("kubectl -n llm-inference delete pod -l app.kubernetes.io/part-of=llminferenceservice," + "app.kubernetes.io/name=gemma-4-26b,kserve.io/component=workload --wait=false", log) + + def test_restart_not_ready_in_time_exits_3(self): + self.pods([pod("p0", "u0")], [pod("p1", "u1", ready=False)]) + self.assertEqual(self.arm("A1", "restart", "1").returncode, 3) + self.assertEqual(self.arm("A1", "restart", "0").returncode, 3) + + def test_cgroup_spec_and_cpustat(self): + self.pods([pod("p1", "u1")]) + r = self.arm("B5", "cgroup") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout, f"exec:{os.path.join(HERE, 'arm.sh')} B5 cpustat " + "/sys/fs/cgroup/kubepods.slice/kubepods-podu1.slice\n") + self.assertEqual(self.arm("B5", "cpustat", "/x").stdout, "usage_usec 77\n") + + def test_forward_restarts_the_guest_port_forward(self): + r = self.arm("A1", "forward") + self.assertEqual(r.returncode, 0, r.stderr) + log = read(self.log) + self.assertIn("port-forward --address 0.0.0.0 svc/ovms 18851:8000", log) + self.assertIn("~/h62r/pf-A1.pid", log) + self.arm("A1", "unforward") + self.assertIn("kill", read(self.log).splitlines()[-1]) + + +class OnTtyTest(unittest.TestCase): + """H62 amendment V2, kept: EAS v1.2.0's es_auto_installer.sh is byte-identical to rc.1's, so + configure still takes no option and its y/N prompt still refuses a stdin that is not a terminal.""" + + def test_stdin_is_a_terminal_and_the_reply_answers_the_prompt(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + log = os.path.join(tmp, "out.log") + r = subprocess.run([sys.executable, os.path.join(HERE, "h62r.py"), "on-tty", log, "y", "bash", "-c", + '[ -t 0 ] || { echo notty; exit 3; }; read -rp "Proceed? [y/N] " r; echo "reply=$r"; exit 5'], + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 5) + self.assertIn("reply=y", read(log)) + + +class ScriptsTest(unittest.TestCase): + """The shell scripts: syntax, and what each one is pinned to run.""" + + SCRIPTS = ("arm.sh", "vm.sh", "guest-a.sh", "guest-b.sh", "setup.sh", "smoke.sh", "pairs-hold.sh", + "campaign.sh", "teardown.sh", "upstream.sh", "topology-spike.sh") + + def test_syntax(self): + for s in self.SCRIPTS: + r = subprocess.run(["bash", "-n", os.path.join(HERE, s)], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, f"{s}: {r.stderr}") + + def test_shellcheck_when_present(self): + if not shutil.which("shellcheck"): + self.skipTest("no shellcheck") + r = subprocess.run(["shellcheck", "-S", "warning", *(os.path.join(HERE, s) for s in self.SCRIPTS)], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout) + + def test_every_script_is_h62rs_own(self): + # H62's runner is historical: nothing here reads its scripts, its state or its results. + for s in self.SCRIPTS + ("arm-A1", "arm-B5", "qemu.Dockerfile", "h62r.py"): + text = read(HERE, s) + for old in ("bench/h62/", "/h62/", "results/H62/", "results/H62\n", "smoke/H62\n", "H62_STATE", + "h62.py", "k3dgpu-e-h62\n", "h62-vm-", "H62-hold", "H62-smoke"): + self.assertNotIn(old, text, f"{s}: {old!r}") + + def test_vm_up_runs_the_argv_h62r_py_builds(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + with open(os.path.join(tmp, "id_ed25519"), "w") as f: + f.write("private\n") + with open(os.path.join(tmp, "id_ed25519.pub"), "w") as f: + f.write(CloudInitTest.KEY + "\n") + env = dict(os.environ, H62R_DRY="1", H62R_STATE=tmp, H62R_KVM_GID="992") + r = subprocess.run(["bash", os.path.join(HERE, "vm.sh"), "up", "b"], capture_output=True, text=True, env=env) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + want = h62r.qemu_argv("b", tmp, h62r.QEMU_IMAGE, uid=os.getuid(), gid=os.getgid(), kvm_gid=992) + self.assertIn(" ".join(want) + "\n", r.stdout) + self.assertIn("qemu-img create -f qcow2 -F qcow2 -b /h62r/base.img /h62r/b/disk.qcow2 400G", r.stdout) + self.assertIn("genisoimage -output /h62r/b/seed.iso -volid cidata -joliet -rock", r.stdout) + self.assertEqual(read(tmp, "b", "user-data"), h62r.user_data("b", CloudInitTest.KEY)) + self.assertEqual(read(tmp, "ssh_config"), h62r.ssh_config(tmp)) + + def test_pairs_hold_is_h50s_hold_with_remote_arms(self): + sh = read(HERE, "pairs-hold.sh") + self.assertIn('crescent/hold.sh" "${name}" 2400 env', sh) + self.assertIn("hold) name=H62r-hold${N} R=${B}/results/H62r guard=600 ;;", sh) + self.assertIn("smoke) name=H62r-smoke R=${B}/smoke/H62r guard=99999 ;;", sh) + for item in ("LIVE_MODE=pairs", "LIVE_CAP=2370", 'LIVE_HOLD="${N}"', "LIVE_MODEL_SET=gemma4", + 'bash "${HERE}/../eatflow/live.sh" H62r', "SERVING_VMS=h62r-vm-a,h62r-vm-b", + 'LIVE_REMOTE_A="${HERE}/arm-A1"', 'LIVE_REMOTE_B="${HERE}/arm-B5"'): + self.assertIn(item, sh, item) + for absent in ("LIVE_MOUNT_B", "LIVE_EAT_CHART", "K3D_GPU_IMAGE"): + self.assertNotIn(absent, sh, absent) + for arm in ("A1", "B5"): + self.assertIn(f'exec "$(dirname "$0")/arm.sh" {arm} "$@"', read(HERE, f"arm-{arm}")) + self.assertTrue(os.access(os.path.join(HERE, f"arm-{arm}"), os.X_OK)) + + def test_guest_b_is_intels_documented_path_at_the_ga_tags(self): + sh = read(HERE, "guest-b.sh") + steps = ['git clone -q -b "${TAG}" https://github.com/intel/enterprise-ai-solutions.git', + "P eas-check configs/repos/repos.agent-toolkit.yaml", + 'P on-tty "${R}/configure.log" y ./es_auto_installer.sh configure', + "./es_auto_installer.sh init agent-toolkit", + "P eas-edit env/local", + "sh roles/nri_cpu_balloons/files/probe-node-topology.sh", + 'P sibling-read /sys/devices/system/cpu "${R}/probe.txt"', + 'P reserve "${cpus}" env/local/global_config.yaml', + "./es_auto_installer.sh install agent-toolkit"] + pos = [sh.index(s) for s in steps] + self.assertEqual(pos, sorted(pos), "steps in the registered order") + self.assertIn('[ "$(git rev-parse HEAD)" = "${SHA}" ]', sh) + self.assertNotIn("eas-pin", sh) # D1 rewrites nothing now + self.assertIn("fs.inotify.max_user_instances = 8192", sh) + self.assertIn("fs.inotify.max_user_watches = 1048576", sh) + # global_config.yaml: written only by D3's sibling read, then copied into the records. + uses = [ln.strip() for ln in sh.splitlines() if "global_config.yaml" in ln] + self.assertEqual(uses, ['[ -z "${cpus}" ] || P reserve "${cpus}" env/local/global_config.yaml > "${R}/d3-siblings.diff"', + "for f in global_config.yaml config.agent-toolkit.yaml config.inference.yaml " + "models.yaml inventory/hosts.yaml; do"]) + self.assertIn("HF_TOKEN", sh) + self.assertNotIn("--only", sh) + self.assertNotIn("existing_kubernetes", sh) + + def test_guest_b_records_the_sizing_whether_or_not_the_install_passes(self): + sh = read(HERE, "guest-b.sh") + i = sh.index('if ! ./es_auto_installer.sh install agent-toolkit > "${R}/install.log" 2>&1; then') + block = sh[i:sh.index("\nfi\n", i)] + for f in ("fail-nodes.txt", "fail-pods.txt", "fail-events.txt", "fail-describe.txt"): # H62 amendment V4 + self.assertIn(f, block) + self.assertIn("sizing fail", block) + self.assertTrue(block.rstrip().endswith("exit 1")) + self.assertIn("sizing ok", sh[i + len(block):]) + sizing = sh[sh.index("sizing() {"):sh.index("\n}\n", sh.index("sizing() {"))] + for item in ("balloonspolicies.config.nri", "resources.requests.cpu", "sandbox-router-deployment", + "describe nodes"): + self.assertIn(item, sizing, item) + self.assertNotIn("KUBECONFIG", sh[:i]) # set only after the install + + def test_guest_a_deploys_h50s_a1(self): + sh = read(HERE, "guest-a.sh") + live = read(REPO, "bench", "eatflow", "live.sh") + for item in ("K3D_GPU_AGENT_DEVICE=CPU", "K3D_GPU_AGENT_MODEL=k3dgpu/gemma-4-26B-A4B-it-qat-int4-ov", + "K3D_GPU_AGENT_MODEL_NAME=gemma-4-26b-a4b-it", "K3D_GPU_AGENT_DRAFT=off", + "K3D_GPU_AGENT_TOOL_PARSER=gemma4", "K3D_GPU_AGENT_REASONING_PARSER=''", "K3D_GPU_AUTOQUANT=off"): + self.assertIn(item, sh, item) + self.assertNotIn("K3D_GPU_AGENT_BACKEND", sh) # H50's and H62's command + self.assertIn("TGT=k3dgpu/gemma-4-26B-A4B-it-qat-int4-ov TGT_DIR=gemma-4-26B-A4B-it-qat-int4-ov A_TOOL=gemma4 A_DRAFT=off", live) + # A1 stays H50's OVMS 2026.4.1: a deploy commit that renders another image stops here. + self.assertIn('grep -qF "image: $(P pin ovms_image)" "${R}/agent.yaml"', sh) + self.assertIn("K3D_GPU_CLUSTER=k3dgpu-e-h62r", sh) + self.assertIn("h62rshare /mnt/h62r-ir", sh) + self.assertIn('K3D_GPU_IMAGE="$(P pin node_image)"', sh) + self.assertIn("eviction-hard=memory.available<100Mi,nodefs.available<1%,imagefs.available<1%,nodefs.inodesFree<1%", sh) + for pkg in ("docker-ce", "docker-ce-cli"): + self.assertIn(f'{pkg}="$(P pin docker_ce)"', sh) + self.assertIn('containerd.io="$(P pin containerd)"', sh) + + def test_smoke_runs_mains_eas_check_in_vm_b(self): + sh = read(HERE, "smoke.sh") + self.assertIn("./es_auto_installer.sh validate agent-toolkit", sh) + self.assertIn("EAT_MODE=eas EAT_MODEL=${MODEL}", sh) + self.assertIn("MODEL=$(P pin catalog_name)", sh) + # PR #260 merged: the check is integrations/eat/eat-check.sh at the deploy commit, beside + # the integrations/eas/eas-ready.py it reads, both recorded by sha256. + self.assertIn('CHECK=${REPO}/integrations/eat/eat-check.sh READY=${REPO}/integrations/eas/eas-ready.py', sh) + self.assertIn("h62r-b:h62r/integrations/eat/", sh) + self.assertIn("h62r-b:h62r/integrations/eas/", sh) + self.assertIn('sha256sum "${CHECK}" "${READY}"', sh) + self.assertRegex(read(REPO, "integrations", "eat", "eat-check.sh"), r"(?m)^\s+eas\)") + self.assertIn('ready="${HERE}/../eas/eas-ready.py"', read(REPO, "integrations", "eat", "eat-check.sh")) + self.assertIn('"${HERE}/pairs-hold.sh" 0 smoke', sh) + + def test_campaign_order_caps_and_attempt_bound(self): + sh = read(HERE, "campaign.sh") + pos = [sh.index(s) for s in ("ATTEMPTS_MAX=3", "trap teardown EXIT", '"${HERE}/setup.sh"', '"${HERE}/smoke.sh"', + '"${HERE}/pairs-hold.sh" "${n}"', "pairs-report")] + self.assertEqual(pos, sorted(pos)) + self.assertIn("MAX_HOLDS=8", sh) + self.assertIn("NOT_DELIVERABLE_MAX=2", sh) + self.assertIn('h62r.py" pairs-left', sh) + self.assertIn("--hyp H62r --arm-a A1 --arm-b B5", sh) + self.assertIn("R=${B}/results/H62r", sh) + + def test_campaign_refuses_a_fourth_setup_attempt(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + res = os.path.join(tmp, "k3dgpu-bench", "results", "H62r") + os.makedirs(res) + with open(os.path.join(res, "attempts.txt"), "w") as f: + f.write("a\nb\nc\n") + r = subprocess.run(["bash", os.path.join(HERE, "campaign.sh")], capture_output=True, text=True, + env=dict(os.environ, HOME=tmp), timeout=60) + self.assertEqual(r.returncode, 10, r.stdout + r.stderr) + self.assertIn("setup attempt bound", read(res, "campaign.log")) + self.assertEqual(read(res, "attempts.txt"), "a\nb\nc\n") + self.assertFalse(os.path.exists(os.path.join(res, "teardown.log"))) # nothing was started + + def test_setup_caps_and_gates(self): + sh = read(HERE, "setup.sh") + self.assertIn('flock -w 7200 9', sh) + self.assertIn("A_CAP=2700", sh) + self.assertIn("B_CAP=7200", sh) + for phase in ("host", "guest", "nfd", "weights"): + self.assertIn(f"P gates {phase}", sh) + self.assertIn('put "${vm}" "${HERE}/h62r.py" "${HERE}/guest-${vm}.sh"', sh) + + def test_topology_spike_runs_eas_own_probe_in_h62rs_guest_shape(self): + sh = read(HERE, "topology-spike.sh") + self.assertIn("https://raw.githubusercontent.com/intel/enterprise-ai-solutions/${SHA}/roles/nri_cpu_balloons/" + "files/probe-node-topology.sh", sh) + self.assertIn('SHA=$(python3 "${HERE}/h62r.py" pin eas_sha)', sh) + self.assertIn("print(h62r.SMP)", sh) + self.assertIn("-machine q35,accel=kvm -cpu host", sh) + self.assertRegex(sh, r"ALPINE=alpine:[0-9.]+@sha256:[0-9a-f]{64}\n") + + def test_upstream_reads_both_repositories(self): + sh = read(HERE, "upstream.sh") + self.assertIn("git ls-remote --tags https://github.com/intel/enterprise-ai-solutions.git", sh) + self.assertIn("git ls-remote --tags https://github.com/intel/enterprise-agent-toolkit.git", sh) + self.assertIn('h62r.py" trigger', sh) + + +if __name__ == "__main__": + unittest.main(verbosity=1) diff --git a/bench/h62r/topology-spike.sh b/bench/h62r/topology-spike.sh new file mode 100755 index 00000000..58a1c828 --- /dev/null +++ b/bench/h62r/topology-spike.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# topology-spike.sh: what EAS v1.2.0's node topology probe reads in a KVM guest of H62r's shape +# (EXPERIMENTS.md "H62r pre-registration", D3's sibling read). It boots a guest with +# -smp 96,sockets=1,cores=48,threads=2 from an initramfs (alpine's busybox userland and EAS's +# roles/nri_cpu_balloons/files/probe-node-topology.sh at the pinned EAS commit), prints cpu1's +# thread_siblings_list and the probe's NRI_NUMA, NRI_LOGICAL, NRI_TPC and NRI_PHYSICAL lines, and +# powers off. Not a hold and not a measurement of either arm. +# Needs docker, qemu-system-x86_64 and /dev/kvm; TOPO_KERNEL is the guest kernel (default: the +# first /boot/vmlinuz*); TOPO_SMP overrides the shape. The guest has no network and no disk. +set -euo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ALPINE=alpine:3.24.4@sha256:93b41ef40b1677fce8fcc64ff4af09daddfacf5040afde618bc46c1e2d1cc7d4 +SHA=$(python3 "${HERE}/h62r.py" pin eas_sha) +KERNEL=${TOPO_KERNEL:-$(find /boot -maxdepth 1 -name 'vmlinuz*' | sort | head -1)} +SMP=${TOPO_SMP:-$(python3 -c 'import sys; sys.path.insert(0, sys.argv[1]); import h62r; print(h62r.SMP)' "${HERE}")} +[ -r "${KERNEL}" ] || { echo "no readable guest kernel (set TOPO_KERNEL)"; exit 2; } +d=$(mktemp -d) +cid="" +cleanup() { [ -z "${cid}" ] || docker rm "${cid}" > /dev/null 2>&1 || true; rm -rf "${d}"; } +trap cleanup EXIT +mkdir -p "${d}/root" +cid=$(docker create "${ALPINE}") +docker export "${cid}" | tar -x -C "${d}/root" +curl -fsSLo "${d}/root/probe.sh" \ + "https://raw.githubusercontent.com/intel/enterprise-ai-solutions/${SHA}/roles/nri_cpu_balloons/files/probe-node-topology.sh" +cat > "${d}/root/init" <<'EOF' +#!/bin/sh +mount -t devtmpfs devtmpfs /dev +exec > /dev/ttyS0 2>&1 +mount -t proc proc /proc +mount -t sysfs sysfs /sys +echo "SPIKE cpu1 thread_siblings_list=$(cat /sys/devices/system/cpu/cpu1/topology/thread_siblings_list)" +sh /probe.sh | grep -E '^NRI_(NUMA|LOGICAL|TPC|PHYSICAL)=' | sed 's/^/SPIKE /' +poweroff -f +EOF +chmod +x "${d}/root/init" +(cd "${d}/root" && find . | cpio -o -H newc 2> /dev/null | gzip -1) > "${d}/initrd.gz" +echo "qemu: $(qemu-system-x86_64 --version | head -1); kernel ${KERNEL}; -smp ${SMP}; probe at EAS ${SHA}" +timeout 180 qemu-system-x86_64 -machine q35,accel=kvm -cpu host -smp "${SMP}" -m 1G -kernel "${KERNEL}" \ + -initrd "${d}/initrd.gz" -append "console=ttyS0 quiet rdinit=/init" -nographic -no-reboot 2>&1 \ + | tr -d '\r' | sed -n 's/.*SPIKE //p' diff --git a/bench/h62r/upstream.sh b/bench/h62r/upstream.sh new file mode 100755 index 00000000..c96400ed --- /dev/null +++ b/bench/h62r/upstream.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# upstream.sh: H62r's re-run trigger (EXPERIMENTS.md "H62r pre-registration", "Re-run trigger"). +# Reads the tags of EAS and EAT with `git ls-remote` and prints "fired: none", or the release tags +# that fired it: an EAS release after v1.2.0 (not a release candidate) or an EAT release after +# release-2.0.0. +set -euo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +d=$(mktemp -d) +trap 'rm -rf "${d}"' EXIT +git ls-remote --tags https://github.com/intel/enterprise-ai-solutions.git > "${d}/eas" +git ls-remote --tags https://github.com/intel/enterprise-agent-toolkit.git > "${d}/eat" +python3 "${HERE}/h62r.py" trigger "${d}/eas" "${d}/eat" diff --git a/bench/h62r/vm.sh b/bench/h62r/vm.sh new file mode 100755 index 00000000..40e064e5 --- /dev/null +++ b/bench/h62r/vm.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# vm.sh build | fetch | up VM | wait VM SECONDS | facts VM | ssh VM CMD... | down VM | purge +# H62r's two KVM guests on crescent (EXPERIMENTS.md "H62r pre-registration"): VM a serves A1, VM b +# serves B5. QEMU runs in the pinned container of qemu.Dockerfile, so the host gets no package. +# Disks, seed images, the ssh key and ssh_config live under H62R_STATE (default +# ~/k3dgpu-bench/h62r); results do not. Shapes, ports and pins come from h62r.py. +# build the QEMU image (h62r.py qemu-image); its package list into H62R_STATE +# fetch the Ubuntu 24.04 cloud image into H62R_STATE/base.img, checked against its sha256 +# up VM the guest's qcow2 overlay, its cloud-init seed and its QEMU container; VM a also +# gets A1's IR read-only (H62R_IR, default the IR live.sh's gemma4 set names) +# wait VM SECONDS until the guest answers ssh and cloud-init has finished +# facts VM one JSON line: nproc, amx_tile count, MemTotal and kernel (the guest gate's facts) +# ssh VM CMD... a command in the guest +# down VM power the guest off (at most 120 s), then remove its container +# purge the guests' disks, the cloud image and the QEMU image (refused while a guest runs) +# H62R_DRY=1 prints each docker command instead of running it. +set -euo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +P() { python3 "${HERE}/h62r.py" "$@"; } +STATE=${H62R_STATE:-${HOME}/k3dgpu-bench/h62r} +IMAGE=$(P qemu-image) +DRY=${H62R_DRY:-0} +run() { if [ "${DRY}" = 1 ]; then echo "$*"; else "$@"; fi; } +in_qemu() { run docker run --rm --user "$(id -u):$(id -g)" -v "${STATE}:/h62r" "${IMAGE}" "$@"; } +gssh() { ssh -F "${STATE}/ssh_config" "$(P vm "$1" host)" "${@:2}"; } +exists() { [ "${DRY}" != 1 ] && docker ps -a --format '{{.Names}}' | grep -x "$1" > /dev/null; } +mkdir -p "${STATE}" +chmod 700 "${STATE}" +case ${1:?usage: vm.sh build|fetch|up VM|wait VM SECONDS|facts VM|ssh VM CMD...|down VM|purge} in +build) + run docker build -t "${IMAGE}" -f "${HERE}/qemu.Dockerfile" "${HERE}" + if [ "${DRY}" != 1 ]; then + docker image inspect --format '{{.Id}}' "${IMAGE}" > "${STATE}/qemu-image.txt" + docker run --rm "${IMAGE}" cat /h62r-packages.txt > "${STATE}/qemu-packages.txt" + fi ;; +fetch) + if [ ! -f "${STATE}/base.img" ]; then + curl -fsSLo "${STATE}/base.img.part" "$(P pin cloud_image_url)" + mv "${STATE}/base.img.part" "${STATE}/base.img" + fi + echo "$(P pin cloud_image_sha256) ${STATE}/base.img" | sha256sum -c + [ "$(stat -c %s "${STATE}/base.img")" = "$(P pin cloud_image_bytes)" ] || { echo "base.img: wrong size"; exit 1; } + chmod 444 "${STATE}/base.img" ;; +up) + vm=${2:?usage: vm.sh up a|b} + name=$(P vm "${vm}" name) + if exists "${name}"; then echo "${name} exists: vm.sh down ${vm} first"; exit 1; fi + d=${STATE}/${vm} + mkdir -p "${d}" + [ -f "${STATE}/id_ed25519" ] || ssh-keygen -q -t ed25519 -N '' -C h62r -f "${STATE}/id_ed25519" + P ssh-config "${STATE}" > "${STATE}/ssh_config" + # A new guest has a new host key on the same forwarded port. + [ ! -f "${STATE}/known_hosts" ] || ssh-keygen -q -R "[127.0.0.1]:$(P vm "${vm}" ssh_port)" \ + -f "${STATE}/known_hosts" > /dev/null 2>&1 || true + P user-data "${vm}" "${STATE}/id_ed25519.pub" > "${d}/user-data" + P meta-data "${vm}" > "${d}/meta-data" + rm -f "${d}/seed.iso" "${d}/disk.qcow2" + in_qemu genisoimage -output "/h62r/${vm}/seed.iso" -volid cidata -joliet -rock "/h62r/${vm}/user-data" "/h62r/${vm}/meta-data" + in_qemu qemu-img create -f qcow2 -F qcow2 -b /h62r/base.img "/h62r/${vm}/disk.qcow2" "$(P vm "${vm}" disk_gib)G" + share=() + if [ "$(P vm "${vm}" share)" = True ]; then + share=("$(readlink -f "${H62R_IR:-${HOME}/k3dgpu-bench/models/gemma-4-26B-A4B-it-qat-int4-ov}")") + fi + kvm=${H62R_KVM_GID:-$(stat -c %g /dev/kvm)} + mapfile -t argv < <(P qemu-argv "${vm}" "${STATE}" "${IMAGE}" "$(id -u)" "$(id -g)" "${kvm}" "${share[@]}") + [ "${#argv[@]}" -gt 10 ] || { echo "no QEMU command line for VM ${vm}"; exit 1; } + run "${argv[@]}" ;; +wait) + vm=${2:?usage: vm.sh wait a|b SECONDS} + end=$(( SECONDS + ${3:?usage: vm.sh wait a|b SECONDS} )) + until gssh "${vm}" true > /dev/null 2>&1; do + [ "${SECONDS}" -lt "${end}" ] || { echo "VM ${vm}: no ssh within ${3} s"; exit 1; } + sleep 5 + done + gssh "${vm}" cloud-init status --wait > /dev/null ;; +facts) + gssh "${2:?usage: vm.sh facts a|b}" python3 - <<'PY' +import json, os, platform +flags = [ln for ln in open("/proc/cpuinfo") if ln.startswith("flags")] +mem = {k: v for k, _, v in (ln.partition(":") for ln in open("/proc/meminfo"))} +print(json.dumps({"nproc": os.cpu_count(), "amx_tile": sum(" amx_tile" in ln for ln in flags), + "mem_total_kib": int(mem["MemTotal"].split()[0]), "kernel": platform.release()})) +PY + ;; +ssh) + gssh "${2:?usage: vm.sh ssh a|b CMD...}" "${@:3}" ;; +down) + vm=${2:?usage: vm.sh down a|b} + name=$(P vm "${vm}" name) + gssh "${vm}" sudo poweroff > /dev/null 2>&1 || true + for _ in $(seq 60); do + [ "$(docker inspect -f '{{.State.Running}}' "${name}" 2>/dev/null)" = true ] || break + sleep 2 + done + run docker rm -f "${name}" > /dev/null 2>&1 || true ;; +purge) + for vm in a b; do + if exists "$(P vm "${vm}" name)"; then echo "VM ${vm} still exists: vm.sh down ${vm} first"; exit 1; fi + done + rm -rf "${STATE:?}/a" "${STATE:?}/b" "${STATE}/base.img" "${STATE}/base.img.part" "${STATE}/known_hosts" + run docker image rm "${IMAGE}" > /dev/null 2>&1 || true ;; +*) + echo "usage: vm.sh build|fetch|up VM|wait VM SECONDS|facts VM|ssh VM CMD...|down VM|purge"; exit 2 ;; +esac diff --git a/docs/intel/technical-report.md b/docs/intel/technical-report.md index d587d431..9a2cab20 100644 --- a/docs/intel/technical-report.md +++ b/docs/intel/technical-report.md @@ -1,6 +1,6 @@ # k3d-gpu on Intel hardware: technical report -Status: working report, 2026-11-01. Ledger: [EXPERIMENTS.md](../../EXPERIMENTS.md), verdicts as of the H57dp, H54q, H53, H62, H58, H59, H60, H55 and H59d runs (82 rows). +Status: working report, 2026-11-01. Ledger: [EXPERIMENTS.md](../../EXPERIMENTS.md), verdicts as of the H57dp, H54q, H53, H62, H58, H59, H60, H55 and H59d runs (83 rows). ## Abstract @@ -11,7 +11,7 @@ This report covers what was measured, and how. Every result but H52, H52b, H7x, - Nineteen hypotheses are SUPPORTED. Sixteen are scoped to that host; H25x (H25 replicated), H59 and H60 are scoped to the Xeon 6741P. H60 is the newest: Qwen3.8-Flash-Next, served from the Xeon's DRAM by llama.cpp v0.8.0, decoded 16.0 tok/s at one stream and read 0.82 of a same-hold Gemma 4 QAT control's share of the DRAM bandwidth ceiling (CI 0.819-0.831, band 0.7-1.3), with GSM8K 50/50 [ledger:H60]. Before it, H59: on the Xeon 6741P at four concurrent requests, llama.cpp with Google's QAT Q4_0 GGUF and the QAT MTP drafter decoded Gemma 4 26B-A4B 3.50x faster per request than OVMS with OpenVINO's base-model int4 IR (95% CI 3.42-3.61), faster in each of 3 holds, at GSM8K 49 against 50 of 50. So with no backend set, `agent up` now serves llama.cpp on the CPU of that class [ledger:H59]. - Twenty are REFUTED. Six of them changed shipped defaults. The newest, H28x, is H28 replicated on the Xeon 6741P: eight pods that mmap one 1 GiB model through an image volume again shared one page-cache copy (PSS 132.6 MiB per pod, 1.00x the size on the node against 9.00x for per-pod copies), but they were ready in 0.714 of the copy arm's time, not the 0.5 or less the row needed, so only the memory part replicated [ledger:H28x]. Before it, H7x is H7 run to its full protocol on the Xeon 6741P: at matched 16-bit weights (Qwen3-4B, 16 CPUs), OVMS 2026.4.1 decoded at 1.22x the rate of EAT 1.1.0's stock vLLM CPU engine (v0.28.0) in all five block pairs, so OVMS's decode lead is not only weight precision; vLLM prefilled the long prompt faster [ledger:H7x]. Before it, H58 is the zero-touch thesis on the Xeon 6741P, with the two serving pairs holding about half of every B70. With no flag, `k3d-gpu up` advertised three of the four B70s and no DSA work queue. `k3d-gpu agent up --model` then refused all three models. Gemma 4 26B-A4B and Qwen3.8-27B went to OVMS on the GPU, where their int4 IRs missed the best card's free-memory budget by 0.1 and 0.5 GiB. Qwen3.8-Flash-Next was refused because no DRAM bandwidth was cached. So EAT's banking flow never came up [ledger:H58]. Before it, H52b is H52 after `k3d-gpu host-setup --apply` on the Xeon 6741P: DSA was advertised and reached a pod, but IAA got no work queue, the QAT plugin advertised none of the VFs, and `up` advertised two of the four B70s: the serving pairs hold about half the VRAM of every card, and its busy-card filter (half or more) left out the two just over half and kept the two just under it [ledger:H52b]. Before it, H51 ran the graded agents-per-box ramp on the Xeon 6741P: the k3d-gpu CPU stack failed at one agent (turn p95 10.41 s against the 10 s SLO), and so did EAT's stock vLLM CPU arm [ledger:H51]. H50 ran inside EAT's banking flow on the same host: EAT's current stock vLLM CPU arm (release-1.1.0) delivered at least as many tasks within the turn budget as the k3d-gpu CPU stack: 21 of 24 against 19 of 24 [ledger:H50]. - 34 are INCONCLUSIVE, mostly because the shared host throttles or its load was unmatched. H62 (the k3d-gpu CPU stack against Intel's new default, EAS v1.2.0-rc.1 installing EAT 2.0, each in a KVM guest on the Xeon 6741P) is INCONCLUSIVE because Intel's own install did not finish: enterprise-inference's `cpu: auto` sized the Gemma 4 pod to 76 of the guest's 95.4 allocatable CPUs, so EAT's sandbox router could not be scheduled, twice, and no hold ran. H54q (the QAT-matched int4 IR of Gemma 4 26B-A4B against the bf16 QAT weights and OpenVINO's base-model IR, on GSM8K items 0-199 on the Xeon 6741P) is INCONCLUSIVE: the QAT-matched IR stayed inside the pre-registered margin of the bf16 reference, but the base-model IR scored no lower than it, with every build near the item set's ceiling. H50o (the k3d-gpu CPU stack against EAT's own OVMS chart on the Xeon 6741P) is INCONCLUSIVE because that chart's OVMS 2025.4 crashed on loading Gemma 4 26B-A4B, so no pair ran. H54a and H54d (Gemma 4 26B-A4B QAT on the Xeon 6741P) are INCONCLUSIVE because the stock arm, EAT 1.0.3's vLLM CPU v0.24.0, as this harness rendered EAT's entry (no CPU balloon, requests or limits), stalled in both of H54a's holds, so no arm could be scored against it. EAT 1.1.0's vLLM CPU v0.28.0 was not measured in H54a; H50 ran it in EAT's banking flow. H54b and H54c (one Arc Pro B70 against two, and experts in DRAM with the rest on one B70) are INCONCLUSIVE because their "one card" pods saw all four cards, so no arm was delivered as pre-registered. H53 (EAT's banking flow on the B70s: k3d-gpu's one-card OVMS pick against EAT 1.1.0's vLLM XPU path across all four cards) is INCONCLUSIVE because the EAT arm did not start in a k3d-gpu cluster: oneCCL needs `/dev/dri/by-path` for its cross-card collective, and the k3d node has none, so no hold ran. H57da (k3d-gpu's OVMS pick vs llama.cpp for Gemma 4 QAT on the i9-13900T) did not resolve: the strictly pinned llama.cpp arms tracked host load; H57db did not run (memory). H57dp and H57de are INCONCLUSIVE: they re-ran that comparison with an unpinned llama.cpp arm, and the load gate failed on OVMS's low foreign P-core load (0.45 CPU-s/s against the llama.cpp arms' 3.1-3.4); the unpinned arm was 1.108x the pinned one and 1.369x OVMS, not gated, and OVMS ran without its P-core balloon. H55 (SLO-gated admission of an E-core neighbour on the i9-13900T) is INCONCLUSIVE: all 25 runs completed on a busy host, but in one round the controller's run closed three request windows, one short of its gate; not gated, the controller held the neighbour at its lowest quota in four of five rounds, and no arm kept prefill within the row's bar of quiet. H59d (the engine choice on the i9-13900T: llama.cpp with Google's QAT Q4_0 GGUF and the QAT MTP drafter against OVMS with OpenVINO's base-model int4 IR) is INCONCLUSIVE because the llama.cpp arm was not deliverable in two of its four holds: in both holds where it followed OVMS, its server was OOMKilled at the fit's 45Gi pod limit (4 x 262,144 tokens), while the same render served in both holds where it ran first. Not gated, in the one performance hold it served it decoded 1.05x OVMS per request at four concurrent requests, and GSM8K read 50 against 49 of 50. So with no backend set, the i9-13900T keeps OVMS. H56 (two holds at once in cpuset slices of the Xeon 6741P) is INCONCLUSIVE: in every hold with a sibling, the sibling's banking block ended shortly after the measured block began, so all four were VOID and no interference statistic was scored; crescent rows stay at one hold at a time. -- 9 are PENDING: the engine-choice row H59w (i7-13700), and the second client host's (wildct, i7-13700) rows H52w, H18w, H25w, H28w, H34w, H57wa, H57wm and H57wb, pre-registered and not yet run. H52, the first row run on that host (a Xeon 6741P with four Arc Pro B70s), is REFUTED. H27x, H27 re-run on the Xeon (AMX), is REFUTED as on the i9. H34x, H34 re-run there, is REFUTED: the CPU Fast Draft that sped up the i9 slowed decode on the Xeon. +- 10 are PENDING: H62r (H62 again at the GA tags, EAS v1.2.0 installing EAT 2.0, each in a KVM guest on the Xeon 6741P), the engine-choice row H59w (i7-13700), and the second client host's (wildct, i7-13700) rows H52w, H18w, H25w, H28w, H34w, H57wa, H57wm and H57wb, pre-registered and not yet run. H52, the first row run on that host (a Xeon 6741P with four Arc Pro B70s), is REFUTED. H27x, H27 re-run on the Xeon (AMX), is REFUTED as on the i9. H34x, H34 re-run there, is REFUTED: the CPU Fast Draft that sped up the i9 slowed decode on the Xeon. - Every other Intel family is detected and configured in fixture tests only. None has a measurement. No world-first, superlative or priority claim is made; [docs/provenance.md](../provenance.md) supports none. diff --git a/pins/classes b/pins/classes index 95c50d46..76db8f2f 100644 --- a/pins/classes +++ b/pins/classes @@ -75,10 +75,10 @@ historical-ref bench/h54/h54.py "ovms_eat": "2025.4@sha256:c370720007564bbdcbca1 # --- Ledger rows: runners of PENDING rows and their finished siblings ---------------- bench/eatflow/agent.lock ledger H40,H41,H42,H43,H44,H45,H46,H47,H48,H50,H50o,H53,H56 amend=Crescent the agent lock: H40-H48, then crescent M1's Gemma 4 and H53's Qwen3.8 sampling -bench/eatflow/live.sh ledger H40,H41,H42,H43,H44,H45,H46,H47,H48,H47b,H49,H49b,H50,H50o,H51,H52,H52b,H53,H56,H58,H62 amend=Crescent,H52,H62 the agent-flow runner (desktop rows, then crescent) +bench/eatflow/live.sh ledger H40,H41,H42,H43,H44,H45,H46,H47,H48,H47b,H49,H49b,H50,H50o,H51,H52,H52b,H53,H56,H58,H62,H62r amend=Crescent,H52,H62,H62r the agent-flow runner (desktop rows, then crescent) bench/eatflow/crescent/serving.py ledger H50,H50o,H51,H52,H52b,H53,H56,H58,H59,H7x,H25x,H27x,H28x,H34x,H60 amend=Crescent,H52,H59,Cross-host crescent's serving sampler; the cross-host holds and H59's crescent holds run it -bench/eatflow/crescent/** ledger H50,H50o,H51,H52,H52b,H53,H56,H58,H62 amend=Crescent,H52,H62 the crescent wrappers, setup and holds -bench/eatflow/** ledger H40,H41,H42,H43,H44,H45,H46,H47,H48,H47b,H49,H49b,H50,H50o,H51,H52,H52b,H53,H56,H58,H62 amend=Crescent,H52,H62 the agent-flow harness (pairs, solo, BFCL, tasks) +bench/eatflow/crescent/** ledger H50,H50o,H51,H52,H52b,H53,H56,H58,H62,H62r amend=Crescent,H52,H62,H62r the crescent wrappers, setup and holds +bench/eatflow/** ledger H40,H41,H42,H43,H44,H45,H46,H47,H48,H47b,H49,H49b,H50,H50o,H51,H52,H52b,H53,H56,H58,H62,H62r amend=Crescent,H52,H62,H62r the agent-flow harness (pairs, solo, BFCL, tasks) bench/apb/h7x* ledger H7x amend=Cross-host H7x's runner bench/apb/test_h7x.py ledger H7x amend=Cross-host H7x's runner tests bench/xhost/fetch_models.py ledger H7x,H25x,H27x,H28x,H34x,H60,H18w,H25w,H28w,H34w,H52w,H57wa,H57wm,H57wb,H59w amend=Cross-host,Wildct the cross-host model fetch; wildct's setup-models.sh runs it @@ -91,6 +91,7 @@ bench/h57d/h57d.py ledger H57da,H57dm,H57db,H57dp,H57de,H57 bench/h57d/** ledger H57da,H57dm,H57db,H57dp,H57de,H57wa,H57wm,H57wb amend=H57d,Wildct the H57d runner; wildct's H57w rows run it (bench/wildct/row.sh h57w) bench/h59/** ledger H59,H59d,H59w amend=H59 H59's scorer bench/h62/** ledger H62 amend=H62 H62's runner (the EAS and EAT 2.0 VM pairs on crescent) +bench/h62r/** ledger H62r amend=H62r H62r's runner (H62 at the GA tags: EAS v1.2.0 with EAT release-2.0.0) bench/flashnext/** ledger H60 amend=H60 H60's runner bench/h58/** ledger H58 amend=H58 H58's runner (the zero-touch thesis on crescent, amendment R1) bench/admission/** ledger H55 amend=H55 H55's SLO-gated admission harness From af083753b6a2dc5caa5e2b9d3917421f28e112bd Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:15:32 -0800 Subject: [PATCH 2/8] ledger(H62r): quote H62's row-naming sentence exactly --- EXPERIMENTS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index 855d4a82..8658fd0f 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -5157,10 +5157,10 @@ merged before the data it governs. **Why.** -- H62's pre-registration names this row: when an EAS release at v1.2.0 or later or an EAT - release at 2.0.0 or later is tagged, "a new row, H62r, is pre-registered: the same design with - B5 at the tagged release (its release notes' component table)". H62 ended at setup with no hold - (INCONCLUSIVE, not deliverable), so it keeps its verdict and its records stay as they are. +- H62's pre-registration names this row for an EAS release at v1.2.0 or later or an EAT release + at 2.0.0 or later: "A new row, H62r, is pre-registered: the same design with B5 at the tagged + release (its release notes' component table)." H62 ended at setup with no hold (INCONCLUSIVE, + not deliverable), so it keeps its verdict and its records stay as they are. - The trigger has fired. `bench/h62/upstream.sh`, run 2026-11-07T04:06Z: `fired: eas v1.2.0, eat release-2.0.0`. - H62's question stands at the release Intel now ships: does k3d-gpu's CPU stack deliver far more From 668c210dd38b5957269c1f9d51cdb08298b6d545 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:48:19 -0800 Subject: [PATCH 3/8] bench(h62r): campaign.sh reads the re-run trigger before setup and before each hold The pre-registration says the trigger runs before setup and before each hold, with its output in the records, and that a trigger fired before hold 1 needs a dated amendment re-pinning B5 before that hold. campaign.sh never ran upstream.sh, so nothing enforced either. It now reads the trigger (up to 3 reads, 60 s apart) into upstream.txt before setup and before each hold. Fired or unreadable before setup: exit 11 before anything starts, with no setup attempt spent. Fired before hold 1: exit 11 with no hold. Unreadable before a hold, or fired after hold 1: recorded, and the holds go on at these pins. The ledger's trigger and reproduce text say the same. --- EXPERIMENTS.md | 18 ++++++---- bench/h62r/campaign.sh | 37 ++++++++++++++++++-- bench/h62r/test_h62r.py | 77 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index 8658fd0f..4c88624b 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -5365,10 +5365,14 @@ amendment V4's records. It fires on an EAS release tag after v1.2.0 (`vX.Y.Z`, not a release candidate) or an EAT release tag after `release-2.0.0` (`release-X.Y.Z`, `vX.Y.Z`, or the `-hotfix` form, including `release-2.0.0-hotfix`). On 2026-11-07T04:06Z it read `fired: none`. It runs before setup and -before each hold, and its output goes into the records. +before each hold, and its output goes into the records: `campaign.sh` runs it there (up to 3 +reads, 60 s apart) and writes each read to `upstream.txt`. +- Fired or unreadable before setup: nothing starts, and no setup attempt is spent. - Fired before hold 1: a dated amendment re-pins B5 to the tagged release, by amendment E1's logic. If the tag's components equal the pinned commits, the amendment only records that. + `campaign.sh` stops before hold 1 for it. +- Unreadable before a hold: recorded as unreadable, and the hold runs. - Fired after hold 1: H62r's holds finish at these pins, and H62r keeps its verdict. A new row is pre-registered for the tagged release. @@ -5378,8 +5382,8 @@ before each hold, and its output goes into the records. differences: the pins (`h62r.py PINS`), the guests' names, ports, label, share tag and state directory, D1 as a check (`eas-check`), D3's sibling read (`sibling-read`, `reserve`) and the sizing records in `guest-b.sh`, A1's image check in `guest-a.sh`, main's EAS check in - `smoke.sh`, the setup bound in `campaign.sh`, and the trigger's thresholds. `topology-spike.sh` - is the spike of item 3 above. + `smoke.sh`, the setup bound and the trigger's reads in `campaign.sh`, and the trigger's + thresholds. `topology-spike.sh` is the spike of item 3 above. - `bench/eatflow/pairs.py`: `DESIGNS["H62r"] = ("H49", "L3")` and its order seed. `bench/eatflow/live.sh`: `ARM=H62r` runs H62's remote-arm branch. - `pins/classes`: `bench/h62r/**` is H62r's ledger line, and H62r joins the `bench/eatflow` lines @@ -5387,10 +5391,10 @@ before each hold, and its output goes into the records. - Tests: `bench/h62r/test_h62r.py` (in CI) and `H62rTest` in `bench/eatflow/test_pairs.py`. **Reproduce (planned).** On crescent, from `~/k3dgpu-bench/k3d-gpu-h62r` at the deploy commit: -`bench/h62r/upstream.sh`, then `bench/h62r/campaign.sh` (setup, smoke, holds, score, teardown). -Records go to `~/k3dgpu-bench/results/H62r/`, with the hold records in -`~/k3dgpu-bench/runs/H62r-hold/` and the smoke in `~/k3dgpu-bench/smoke/H62r/`. They come back -under `bench/eatflow/results/2026-11-06/H62r/`. +`bench/h62r/campaign.sh` (the trigger, setup, smoke, holds, score, teardown). Records go to +`~/k3dgpu-bench/results/H62r/`, with the hold records in `~/k3dgpu-bench/runs/H62r-hold/` and +the smoke in `~/k3dgpu-bench/smoke/H62r/`. They come back under +`bench/eatflow/results/2026-11-06/H62r/`. **Also found, offered upstream.** EAS v1.2.0's probe reads one thread per core on any Linux node that lists a core's threads as a range, as this KVM guest does. Counting the CPUs of the cpulist, diff --git a/bench/h62r/campaign.sh b/bench/h62r/campaign.sh index 8d3d659c..1125ef40 100755 --- a/bench/h62r/campaign.sh +++ b/bench/h62r/campaign.sh @@ -5,13 +5,18 @@ # also after an abort. # - at most 3 setup attempts in all (attempts.txt, one line per start); a fourth start exits 10 # before anything is started +# - the re-run trigger (upstream.sh, up to 3 reads 60 s apart) before setup and before each +# hold, into upstream.txt. Fired or unreadable before setup: exit 11 before anything starts, +# and no attempt is spent. Fired before hold 1: exit 11 with no hold, for the dated amendment +# that re-pins B5. Unreadable before a hold, or fired after hold 1: recorded, and the holds go +# on at these pins # - setup exit 6 (the host gate still failed after 60 min): the two qwen38 serving pairs are # stopped under the box owner's grant (gpu-serving.sh stop, never rm) and setup runs once more; # teardown.sh starts them again # - a hold that exits 3 (a server not ready with 900 s left) counts as one of the 8 and runs no # pair; after 2 such holds in a row no further hold runs (INCONCLUSIVE, not deliverable) -# Records: ~/k3dgpu-bench/results/H62r (campaign.log, attempts.txt, setup/, pairs/, hold/, -# report.txt, vmtax.json). +# Records: ~/k3dgpu-bench/results/H62r (campaign.log, attempts.txt, upstream.txt, setup/, pairs/, +# hold/, report.txt, vmtax.json). set -uo pipefail HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO=$(cd "${HERE}/../.." && pwd) @@ -20,11 +25,30 @@ R=${B}/results/H62r MAX_HOLDS=8 NOT_DELIVERABLE_MAX=2 ATTEMPTS_MAX=3 mkdir -p "${R}" log() { echo "== campaign $* $(date -u +%FT%TZ)" | tee -a "${R}/campaign.log"; } +# trigger PHASE: the re-run trigger's line into upstream.txt. Status 0 fired, 1 not, 2 unreadable. +trigger() { + local out="" i + for i in 1 2 3; do + if out=$(bash "${HERE}/upstream.sh" 2>&1); then + echo "$(date -u +%FT%TZ) $1 ${out}" >> "${R}/upstream.txt" + [ "${out}" != "fired: none" ] + return + fi + [ "${i}" = 3 ] || sleep "${H62R_TRIGGER_RETRY_S:-60}" + done + echo "$(date -u +%FT%TZ) $1 unreadable: ${out//$'\n'/ }" >> "${R}/upstream.txt" + return 2 +} tries=$(grep -c . "${R}/attempts.txt" 2>/dev/null) if [ "${tries:-0}" -ge "${ATTEMPTS_MAX}" ]; then log "setup attempt bound reached (${tries:-0} of ${ATTEMPTS_MAX}): no further attempt" exit 10 fi +trigger setup +case $? in + 0) log "re-run trigger fired before setup ($(tail -1 "${R}/upstream.txt")): no setup until a dated amendment re-pins B5"; exit 11 ;; + 2) log "re-run trigger unreadable before setup: no setup"; exit 11 ;; +esac echo "$(date -u +%FT%TZ) $(git -C "${REPO}" rev-parse HEAD)" >> "${R}/attempts.txt" teardown() { bash "${HERE}/teardown.sh" >> "${R}/teardown.log" 2>&1; log "teardown exit $?"; } trap teardown EXIT @@ -49,6 +73,15 @@ nd=0 for n in $(seq 1 "${MAX_HOLDS}"); do left=$(python3 "${HERE}/h62r.py" pairs-left "${R}/pairs") [ "${left}" -gt 0 ] || break + trigger "hold${n}" + case $? in + 0) if [ "${n}" = 1 ]; then + log "re-run trigger fired before hold 1 ($(tail -1 "${R}/upstream.txt")): no hold until a dated amendment re-pins B5" + exit 11 + fi + log "re-run trigger fired after hold 1: H62r's holds finish at these pins" ;; + 2) log "re-run trigger unreadable before hold ${n}: recorded" ;; + esac log "hold ${n}: ${left} pair(s) left" bash "${HERE}/pairs-hold.sh" "${n}" rc=$? diff --git a/bench/h62r/test_h62r.py b/bench/h62r/test_h62r.py index f3822dd9..6fc80d99 100644 --- a/bench/h62r/test_h62r.py +++ b/bench/h62r/test_h62r.py @@ -855,8 +855,9 @@ def test_smoke_runs_mains_eas_check_in_vm_b(self): def test_campaign_order_caps_and_attempt_bound(self): sh = read(HERE, "campaign.sh") - pos = [sh.index(s) for s in ("ATTEMPTS_MAX=3", "trap teardown EXIT", '"${HERE}/setup.sh"', '"${HERE}/smoke.sh"', - '"${HERE}/pairs-hold.sh" "${n}"', "pairs-report")] + pos = [sh.index(s) for s in ("ATTEMPTS_MAX=3", "trigger setup", "trap teardown EXIT", '"${HERE}/setup.sh"', + '"${HERE}/smoke.sh"', 'trigger "hold${n}"', '"${HERE}/pairs-hold.sh" "${n}"', + "pairs-report")] self.assertEqual(pos, sorted(pos)) self.assertIn("MAX_HOLDS=8", sh) self.assertIn("NOT_DELIVERABLE_MAX=2", sh) @@ -878,6 +879,78 @@ def test_campaign_refuses_a_fourth_setup_attempt(self): self.assertEqual(read(res, "attempts.txt"), "a\nb\nc\n") self.assertFalse(os.path.exists(os.path.join(res, "teardown.log"))) # nothing was started + def campaign_with_stubs(self, reads, holds_left=3): + """campaign.sh copied beside stub setup, smoke, holds, teardown, h62r.py and upstream.sh. + `reads` is upstream.sh's answer per call, in order ("ERR" exits 1). Returns the run, the + results directory and the order of the calls.""" + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + here = os.path.join(tmp, "repo", "bench", "h62r") + os.makedirs(here) + shutil.copy(os.path.join(HERE, "campaign.sh"), here) + calls = os.path.join(tmp, "calls") + with open(os.path.join(tmp, "reads"), "w") as f: + f.write("".join(r + "\n" for r in reads)) + stubs = { + "setup.sh": f'echo setup >> "{calls}"', + "smoke.sh": f'echo smoke >> "{calls}"', + "teardown.sh": f'echo teardown >> "{calls}"', + "pairs-hold.sh": f'echo "hold$1" >> "{calls}"; echo $(( $(cat "{tmp}/left") - 1 )) > "{tmp}/left"', + "upstream.sh": (f'echo upstream >> "{calls}"; r=$(head -1 "{tmp}/reads"); sed -i 1d "{tmp}/reads"; ' + '[ "${r}" != ERR ] || { echo "ls-remote failed" >&2; exit 1; }; echo "${r}"'), + } + for name, body in stubs.items(): + with open(os.path.join(here, name), "w") as f: + f.write("#!/usr/bin/env bash\n" + body + "\n") + with open(os.path.join(here, "h62r.py"), "w") as f: + f.write(f'import sys\nprint(open("{tmp}/left").read().strip()) if sys.argv[1] == "pairs-left" else None\n') + with open(os.path.join(tmp, "left"), "w") as f: + f.write(f"{holds_left}\n") + r = subprocess.run(["bash", os.path.join(here, "campaign.sh")], capture_output=True, text=True, timeout=60, + env=dict(os.environ, HOME=tmp, H62R_TRIGGER_RETRY_S="0")) + res = os.path.join(tmp, "k3dgpu-bench", "results", "H62r") + order = read(calls).split() if os.path.exists(calls) else [] + return r, res, order + + def test_campaign_reads_the_trigger_before_setup_and_before_each_hold(self): + # The pre-registration's re-run trigger: it runs before setup and before each hold, and its + # output goes into the records. + r, res, order = self.campaign_with_stubs(["fired: none"] * 4) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual(order, ["upstream", "setup", "smoke", "upstream", "hold1", "upstream", "hold2", + "upstream", "hold3", "teardown"]) + rec = read(res, "upstream.txt").splitlines() + self.assertEqual([ln.split(" ", 1)[1] for ln in rec], + ["setup fired: none", "hold1 fired: none", "hold2 fired: none", "hold3 fired: none"]) + + def test_campaign_stops_when_the_trigger_fires_before_hold_1(self): + # Fired before setup: nothing starts and no setup attempt is spent. + r, res, order = self.campaign_with_stubs(["fired: eas v1.2.1"]) + self.assertEqual(r.returncode, 11, r.stdout + r.stderr) + self.assertEqual(order, ["upstream"]) + self.assertFalse(os.path.exists(os.path.join(res, "attempts.txt"))) + self.assertIn("setup fired: eas v1.2.1", read(res, "upstream.txt")) + # Fired after setup, before hold 1: no hold runs; the guests are torn down. + r, res, order = self.campaign_with_stubs(["fired: none", "fired: eat release-2.0.1"]) + self.assertEqual(r.returncode, 11, r.stdout + r.stderr) + self.assertEqual(order, ["upstream", "setup", "smoke", "upstream", "teardown"]) + self.assertIn("amendment", read(res, "campaign.log")) + + def test_campaign_finishes_at_its_pins_when_the_trigger_fires_after_hold_1(self): + r, res, order = self.campaign_with_stubs(["fired: none", "fired: none", "fired: eas v1.2.1", "fired: eas v1.2.1"]) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual([c for c in order if c.startswith("hold")], ["hold1", "hold2", "hold3"]) + self.assertIn("hold2 fired: eas v1.2.1", read(res, "upstream.txt")) + + def test_campaign_records_an_unreadable_trigger_after_three_reads(self): + r, res, order = self.campaign_with_stubs(["fired: none"] + ["ERR"] * 3 + ["fired: none"] * 2) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual(order[:7], ["upstream", "setup", "smoke", "upstream", "upstream", "upstream", "hold1"]) + self.assertRegex(read(res, "upstream.txt"), r"(?m)^\S+ hold1 unreadable: ls-remote failed$") + # Unreadable before setup: nothing starts. + r, res, order = self.campaign_with_stubs(["ERR"] * 3) + self.assertEqual((r.returncode, order), (11, ["upstream"] * 3), r.stdout + r.stderr) + def test_setup_caps_and_gates(self): sh = read(HERE, "setup.sh") self.assertIn('flock -w 7200 9', sh) From 84d995882f875554766df3f52b9652eb3c032851 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:51:59 -0800 Subject: [PATCH 4/8] ledger(H62r): EAS's probe lists every guest vCPU as a sibling; the upstream note names both reads EAS v1.2.0's probe reads thread_siblings_list twice. The pre-registration covered its threads per core (line 31) but not its sibling list (line 74), which takes a list's first comma field as the core's lowest thread. In the guest that field is the range "0-1", so NRI_SIBLINGS names all 96 vCPUs. topology-spike.sh now prints that line too; run again on the desktop (QEMU 10.2.1, -smp 96,sockets=1,cores=48,threads=2, the probe at a051f00) it printed NRI_SIBLINGS=0,1,...,95. So the upstream note's fix (count the cpulist with ranges expanded) was incomplete: with only the count changed, the siblings profile would reserve every vCPU, because it reserves NRI_SIBLINGS as read. The note now names both reads. D3's cpuset is described as the set the probe defines as siblings, read with ranges expanded (what EAS reserves on crescent's host), not as what EAS would reserve in this guest; with nri_reserved_cpu_list set the role does not read NRI_SIBLINGS. The risk list says the probe's records will show all 96. No prediction, gate or cap changes. --- EXPERIMENTS.md | 38 ++++++++++++++++++++++++------------ bench/h62r/guest-b.sh | 5 +++-- bench/h62r/h62r.py | 6 ++++-- bench/h62r/test_h62r.py | 4 ++++ bench/h62r/topology-spike.sh | 7 ++++--- 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index 4c88624b..c071759d 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -69,7 +69,7 @@ setup too weak to decide). Only `SUPPORTED` backs a claim. | H50o | (Model, arms and sampling superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20): Gemma 4 26B-A4B QAT.) H50's comparison with arm B3, EAT's own OVMS chart (`core/helm-charts/ovms` at release-1.0.3-hotfix, stock values, `openvino/model_server` 2025.4, the closest OpenVINO/ model it pulls, `OpenVINO/Qwen3-4B-int4-ov`) instead of B1: A1 delivers far more B24 tasks within budget than B3. With H50 it splits B1 -> A1 into EAT switching its own engine (B1 -> B3) and what k3d-gpu adds on top (B3 -> A1). | Superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20), which restates the predictions; as first written: Written 2026-10-20, before any measurement. B3's stock graph has no tool parser (the chart's `ovms --pull` writes none) and its model reasons by default, so its tool calls arrive as text: B3 within-budget success <= 0.10 (0.7), paired A1 - B3 >= +60 points (0.65). Pair yield: a B3 attempt should end within 30-90 s (a failed tool call or a reasoned text answer), so a 40-min hold should run 15-25 pairs and 24 pairs fit 2-3 holds; L2 passes (0.6), so SUPPORTED (0.5), INCONCLUSIVE (0.45). | As H50 with arm B = B3 (same pairs, gates, thresholds, statistic, cluster per hold `k3dgpu-e-h50o`); holds of at most 40 min until 24 pairs or 6 holds. The B1 -> B3 and B3 -> A1 splits are described from the two rows' matched pairs, never gated. `bench/eatflow/crescent/pairs-hold.sh H50o `, `agentbench pairs-report --hyp H50o --arm-a A1 --arm-b B3`. Model and predictions from [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20). Analysis rule L3 from [amendment A3](#crescent-amendment-a3-analysis-rule-l3-for-the-crescent-pairs-rows-2026-10-20). B3 is unchanged under [amendment E1](#crescent-amendment-e1-eat-stock-is-release-110-for-h50-h50o-and-h51-2026-10-29) (EAT release-1.1.0's OVMS chart renders the same objects); node image, deploy commit and order per [amendment E2](#crescent-amendment-e2-pins-deploy-commits-order-and-the-restore-check-for-the-pending-crescent-rows-2026-10-29); H50o's deploy commit, B3's records and its wiring smoke per [H50o amendment O1](#h50o-amendment-o1-deploy-commit-b3s-records-and-a-b3-wiring-smoke-2026-10-29). Result ([H50o run](#h50o-run-xeon-6741p-2026-10-29)): B3 was not ready in hold 1, so under M1's rule H50o runs no further hold and no pair ran (1 of the 6 permitted holds). On k3d, B3's PVC bound, its pod was scheduled and its init container pulled `OpenVINO/gemma-4-26b-a4b-it-int4-ov` at `4e5885c3` (every file equal to the Hub's), but OVMS 2025.4's server failed on each of its 7 starts, going into CrashLoopBackOff; the last start's recorded exit is 139 (SIGSEGV), a second into loading the model. The same server in a plain container, outside k3d, on the same revision, exited 139 the same way with and without the chart's limits. Not deliverable: EAT's OVMS chart as shipped does not serve this model. | INCONCLUSIVE | | H51 | (Model, arms and sampling superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20): Gemma 4 26B-A4B QAT.) Graded agents per box on the Xeon 6741P: H45's ramp, gates and scoring (B15-RO, typical load, 300 s steps, pooled p95 turn <= 10 s, success >= the arm's N=1 success - 10 points, errors <= 1%, >= 60 window requests), extended to N=16, for A1 and B1. | Superseded by [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20), which restates the predictions; as first written: Written 2026-10-20, before any measurement. A1 in [1, 8] (0.5). B1 in [0, 4] (0.5). B1 <= A1 (0.55). | H45's `apb` runner and scorer (`apb-score --hyp H51`, ramp 1, 2, 4, 8, 16), one fresh cluster per hold, at most 4 holds of at most 40 min. A prediction holds when the arm's bracket lies inside its range and is refuted when it lies outside; B1 <= A1 is decided only on matched load. REFUTED when any prediction is refuted, SUPPORTED when all three hold, else INCONCLUSIVE. Runs only if H50's holds end within their cap. Model and predictions from [amendment M1](#crescent-amendment-m1-gemma-4-26b-a4b-qat-on-the-cpu-rows-2026-10-20). B1 as H50, from [amendment E1](#crescent-amendment-e1-eat-stock-is-release-110-for-h50-h50o-and-h51-2026-10-29); node image, deploy commit and order per [amendment E2](#crescent-amendment-e2-pins-deploy-commits-order-and-the-restore-check-for-the-pending-crescent-rows-2026-10-29); deploy commit, M1's ranges in the scorer, one hold per arm and the wiring smokes per [H51 amendment S1](#h51-amendment-s1-deploy-commit-scorer-ranges-ramp-per-hold-and-wiring-smokes-2026-10-29). Result ([H51 run](#h51-run-xeon-6741p-2026-10-29)): one hold per arm, 2 of the 4 permitted, none VOID. Both arms failed N=1. A1: 65 window requests, turn p95 10.41 s against the 10 s SLO, success 30/32. B1: 58 window requests (under 60), p95 11.52 s, success 27/29. Agents per box A1 0, outside its predicted [1, 4], so REFUTED. B1's 0 lies inside [0, 4], and B1 <= A1 holds on matched load (0 <= 0). | REFUTED | | H62 | Against Intel's new default. On the Xeon 6741P, inside EAT's banking flow (B24) with Gemma 4 26B-A4B on its QAT weights, the k3d-gpu CPU stack A1 (H50's A1 unchanged: `agent manifest`, OVMS 2026.4.1 int4 on the snapped QAT IR, CPU, no draft) delivers far more tasks within the 120 s turn budget than B5, Intel's documented single-node default for its agent toolkit: EAS v1.2.0-rc.1 (`5c27fb3`) installing EAT 2.0 (`release-2.0.0` `036b662`) and enterprise-inference `release-1.1` (`170eb3a`) with Kubespray onto one node, serving its catalog entry `gemma-4-26b` (vLLM CPU v0.28.0 on KServe, NRI balloons) on the same QAT weights. Each stack runs in its own KVM guest of one shape on the same host; both are live in one hold, interleaved. Deviations from stock are named (D1-D4; D4: the harness reaches B5's workload Service, not EAS's LiteLLM gateway and guardrail). A stack-vs-stack claim (amendment 16). | Written 2026-10-29, before any H62 step and after H50 (A1 19/24, B1 21/24) and H51 were known. A1 within-budget success >= 0.70 (0.65). B5 >= 0.70 (0.6). Paired A1 - B5 in [-0.20, +0.15] (0.6). Balance gate passes (0.9). REFUTED (0.6), INCONCLUSIVE (0.35: mostly B5 not deliverable in its guest within 120 min, a failed wiring smoke or a gate), SUPPORTED (0.05). Not gated: median attempt A1/B5 <= 0.85 (0.5); A1's median attempt in its guest over H50's 8.0 s on the host in [1.0, 1.3] (0.6). | [H62 pre-registration](#h62-pre-registration-k3d-gpu-against-intels-new-default-eas-with-eat-20-on-the-xeon-6741p-2026-10-29): H50's design, statistic, gates and verdict rule (rule L3, amendment A3, seed `H62-crescent-2026-10-29`), two KVM guests on crescent (VM a 96 vCPU and 96 GiB for A1, VM b 96 vCPU and 192 GiB for B5), the harness's LiteLLM per arm on the host, holds of at most 40 min until 24 pairs or 8 holds, each hold restarting both serving pods. Setup and wiring-smoke gates (memory, disk, AMX in each guest, NFD label, B5's weights equal to the QAT checkpoint, EAS's own `validate` and gateway check) come before hold 1. SUPPORTED: paired difference CI lower bound >= +30 points and A1 >= 0.6, with every gate passed. REFUTED: CI upper bound < +30 points. Otherwise INCONCLUSIVE. `bench/h62/campaign.sh`; `agentbench pairs-report --hyp H62 --arm-a A1 --arm-b B5`. Re-run as a new row when an EAS release at v1.2.0 or later or an EAT release at 2.0.0 or later is tagged (`bench/h62/upstream.sh`). Mechanics per [H62 amendment V1](#h62-amendment-v1-the-wiring-smokes-eas-check-without-the-rest-of-260-2026-11-01) (the smoke's EAS check), [V2](#h62-amendment-v2-eass-configure-answered-on-a-terminal-2026-11-01) (configure on a terminal), [V3](#h62-amendment-v3-d2-reads-the-catalogs-commented-name-lines-2026-11-01) (D2's catalog parse) and [V4](#h62-amendment-v4-the-clusters-state-is-recorded-when-b5s-install-fails-2026-11-01) (records on an install failure; the last setup attempt). Result ([H62 run](#h62-run-xeon-6741p-2026-11-01)): no hold ran, 0 of the 8 permitted, and no smoke ran. A1 served in its guest within its 45 min ceiling (`guest-a.sh` done at 35.7 and 34.7 min). Intel's install of B5 failed twice at the same step, inside its 120 min ceiling: EAT's sandbox-router could not be scheduled ("Insufficient cpu") because enterprise-inference's `cpu: auto` had sized gemma-4-26b to 76 of the node's 95.4 allocatable CPUs, beside Qwen3Guard's 8 and the platform's requests, which left 0.25 CPU for the router's 0.5. B5 is not deliverable in its guest, so H62 is INCONCLUSIVE by amendment V4's rule. | INCONCLUSIVE | -| H62r | H62's hypothesis, verbatim but for B5's release identifiers (the GA tags): Against Intel's new default. On the Xeon 6741P, inside EAT's banking flow (B24) with Gemma 4 26B-A4B on its QAT weights, the k3d-gpu CPU stack A1 (H50's A1 unchanged: `agent manifest`, OVMS 2026.4.1 int4 on the snapped QAT IR, CPU, no draft) delivers far more tasks within the 120 s turn budget than B5, Intel's documented single-node default for its agent toolkit: EAS v1.2.0 (`a051f00`) installing EAT 2.0 (`release-2.0.0` `7b827e2`) and enterprise-inference `v1.1.0` (`d035b1e`) with Kubespray onto one node, serving its catalog entry `gemma-4-26b` (vLLM CPU v0.28.0 on KServe, NRI balloons) on the same QAT weights. Each stack runs in its own KVM guest of one shape on the same host; both are live in one hold, interleaved. Deviations from stock are named (D1-D4; D4: the harness reaches B5's workload Service, not EAS's LiteLLM gateway and guardrail). A stack-vs-stack claim (amendment 16). | Written 2026-11-06, before any H62r step, after H62 (INCONCLUSIVE: no hold ran, B5's install not deliverable in its guest) was known. H62's predictions, copied verbatim and not tuned: A1 within-budget success >= 0.70 (0.65). B5 >= 0.70 (0.6). Paired A1 - B5 in [-0.20, +0.15] (0.6). Balance gate passes (0.9). REFUTED (0.6), INCONCLUSIVE (0.35: mostly B5 not deliverable in its guest within 120 min, a failed wiring smoke or a gate), SUPPORTED (0.05). Not gated: median attempt A1/B5 <= 0.85 (0.5); A1's median attempt in its guest over H50's 8.0 s on the host in [1.0, 1.3] (0.6). | [H62r pre-registration](#h62r-pre-registration-h62-at-the-ga-tags-eas-v120-with-eat-20-on-the-xeon-6741p-2026-11-06): H62's design, statistic, gates, caps, budget and verdict rule, unchanged (rule L3, amendment A3, its own seed `H62r-crescent-2026-11-06`; VM a 96 vCPU and 96 GiB for A1, VM b 96 vCPU and 192 GiB for B5; the harness's LiteLLM per arm on the host; holds of at most 40 min until 24 pairs or 8 holds, each restarting both serving pods; VM a's setup 45 min and VM b's install 120 min). B5 at the GA tags: EAS `v1.2.0`, whose layer manifest names enterprise-inference `v1.1.0` and EAT `release-2.0.0` by commit itself, so D1 is a check. From H62's commits to these, only docs, `VERSION`, the layer manifests and enterprise-inference's LiteLLM (v1.84.0) changed. Mechanics for H62's failure, registered before any measurement: EAS's topology probe counts the comma fields of cpu0's `thread_siblings_list`, reads the guest's adjacent pair (`0-1`) as one thread per core and falls back to its `standard` reservation (12 CPUs), so `cpu: auto` gave gemma-4-26b 96 - 12 - 8 = 76 CPUs and starved EAT's sandbox router; on crescent's host (`0,48`) the probe reads two and reserves the 48 siblings. D3's sibling read: when EAS's probe reads one thread per core where the guest kernel lists two, `nri_reserved_cpu_list` (EAS's documented setting in `global_config.yaml`) names the guest's 48 sibling vCPUs, the reservation EAS's default `siblings` profile makes when it reads SMT; expected, not gated, gemma-4-26b at 40 CPUs. A1's render must carry H50's OVMS image. At most 3 setup attempts. SUPPORTED: paired difference CI lower bound >= +30 points and A1 >= 0.6, with every gate passed. REFUTED: CI upper bound < +30 points. Otherwise INCONCLUSIVE. `bench/h62r/campaign.sh`; `agentbench pairs-report --hyp H62r --arm-a A1 --arm-b B5`. Re-run as a new row when an EAS release after v1.2.0 or an EAT release after `release-2.0.0` is tagged (`bench/h62r/upstream.sh`). | PENDING | +| H62r | H62's hypothesis, verbatim but for B5's release identifiers (the GA tags): Against Intel's new default. On the Xeon 6741P, inside EAT's banking flow (B24) with Gemma 4 26B-A4B on its QAT weights, the k3d-gpu CPU stack A1 (H50's A1 unchanged: `agent manifest`, OVMS 2026.4.1 int4 on the snapped QAT IR, CPU, no draft) delivers far more tasks within the 120 s turn budget than B5, Intel's documented single-node default for its agent toolkit: EAS v1.2.0 (`a051f00`) installing EAT 2.0 (`release-2.0.0` `7b827e2`) and enterprise-inference `v1.1.0` (`d035b1e`) with Kubespray onto one node, serving its catalog entry `gemma-4-26b` (vLLM CPU v0.28.0 on KServe, NRI balloons) on the same QAT weights. Each stack runs in its own KVM guest of one shape on the same host; both are live in one hold, interleaved. Deviations from stock are named (D1-D4; D4: the harness reaches B5's workload Service, not EAS's LiteLLM gateway and guardrail). A stack-vs-stack claim (amendment 16). | Written 2026-11-06, before any H62r step, after H62 (INCONCLUSIVE: no hold ran, B5's install not deliverable in its guest) was known. H62's predictions, copied verbatim and not tuned: A1 within-budget success >= 0.70 (0.65). B5 >= 0.70 (0.6). Paired A1 - B5 in [-0.20, +0.15] (0.6). Balance gate passes (0.9). REFUTED (0.6), INCONCLUSIVE (0.35: mostly B5 not deliverable in its guest within 120 min, a failed wiring smoke or a gate), SUPPORTED (0.05). Not gated: median attempt A1/B5 <= 0.85 (0.5); A1's median attempt in its guest over H50's 8.0 s on the host in [1.0, 1.3] (0.6). | [H62r pre-registration](#h62r-pre-registration-h62-at-the-ga-tags-eas-v120-with-eat-20-on-the-xeon-6741p-2026-11-06): H62's design, statistic, gates, caps, budget and verdict rule, unchanged (rule L3, amendment A3, its own seed `H62r-crescent-2026-11-06`; VM a 96 vCPU and 96 GiB for A1, VM b 96 vCPU and 192 GiB for B5; the harness's LiteLLM per arm on the host; holds of at most 40 min until 24 pairs or 8 holds, each restarting both serving pods; VM a's setup 45 min and VM b's install 120 min). B5 at the GA tags: EAS `v1.2.0`, whose layer manifest names enterprise-inference `v1.1.0` and EAT `release-2.0.0` by commit itself, so D1 is a check. From H62's commits to these, only docs, `VERSION`, the layer manifests and enterprise-inference's LiteLLM (v1.84.0) changed. Mechanics for H62's failure, registered before any measurement: EAS's topology probe counts the comma fields of cpu0's `thread_siblings_list`, reads the guest's adjacent pair (`0-1`) as one thread per core and falls back to its `standard` reservation (12 CPUs), so `cpu: auto` gave gemma-4-26b 96 - 12 - 8 = 76 CPUs and starved EAT's sandbox router; on crescent's host (`0,48`) the probe reads two and reserves the 48 siblings. D3's sibling read: when EAS's probe reads one thread per core where the guest kernel lists two, `nri_reserved_cpu_list` (EAS's documented setting in `global_config.yaml`) names the guest's 48 sibling vCPUs, the reservation EAS's default `siblings` profile makes on crescent's host; expected, not gated, gemma-4-26b at 40 CPUs. A1's render must carry H50's OVMS image. At most 3 setup attempts. SUPPORTED: paired difference CI lower bound >= +30 points and A1 >= 0.6, with every gate passed. REFUTED: CI upper bound < +30 points. Otherwise INCONCLUSIVE. `bench/h62r/campaign.sh`; `agentbench pairs-report --hyp H62r --arm-a A1 --arm-b B5`. Re-run as a new row when an EAS release after v1.2.0 or an EAT release after `release-2.0.0` is tagged (`bench/h62r/upstream.sh`). | PENDING | | H52 | A plain `k3d-gpu up` (no flags, the auto-adapt default) on the Xeon 6741P host as found detects and enables every engine actually present and advertises each usable one to the scheduler, with nothing missing and nothing phantom; engines present but not configured on the host are named, not advertised. | Written 2026-10-20 from an inventory read from sysfs, lspci and CPUID, not from k3d-gpu. 4 x Arc Pro B70 (xe) as `gpu.intel.com/xe: 4`; AMX TILE, BF16, INT8, FP16 and 14 AVX-512/VNNI labels; no hybrid, NUMA, NVIDIA, Gaudi or NPU objects; DSA, IAA and QAT named as present but unconfigured (0.4: `up` may not name them). | The table in [H50-H52 pre-registration](#h50-h52-pre-registration-the-xeon-6741p-2026-10-20): `k3d-gpu doctor`, `K3D_GPU_CLUSTER=k3dgpu-h52 k3d-gpu up`, `kubectl get node -o json`, `k3d-gpu test`, and one enumeration pod per B70. SUPPORTED: every row matches. REFUTED: any row does not. Run 2026-10-20 ([H52 run](#h52-run-xeon-6741p-2026-10-20)): the four B70s (`gpu.intel.com/xe: 4`, `xmx=true`), AMX (TILE, BF16, INT8, FP16) and all 14 AVX-512/VNNI labels detected and reached pods, nothing phantom; but no media engine label on `xe`, and `up` does not name the unconfigured DSA, IAA and QAT (`doctor` does). | REFUTED | | H52b | After `k3d-gpu host-setup --apply` configures the host's DSA, IAA and QAT (the fix that adds that path, merged first), a plain `k3d-gpu up` also advertises them, and each one reaches a pod; the host is then restored exactly. | Written 2026-10-20, before the fix exists. DSA, IAA and QAT allocatable > 0, each at the count host-setup reports configuring; their test pods see `/dev/dsa`, `/dev/iax`, `/dev/vfio` (0.6). The other rows read as H52's expected table, media labels included ([amendment D1](#h52-amendment-d1-the-fixes-and-h52b-2026-10-20)). | Runs after H52, once the host-setup fix is merged; same procedure plus `host-setup --apply` before `up` and its printed revert after. SUPPORTED: the three engines advertised and reached, the H52 rows unchanged, and the host inventory read back equal to the one taken before. REFUTED: any of these fails. The host-setup command (`--apply --with-kernel-wqs`), the plan read first, the counts, the plain `up`'s node image and the revert per [amendment D2](#h52-amendment-d2-h52bs-host-setup-step-on-crescent-2026-10-29); the runner and deploy commit per [amendment D3](#h52-amendment-d3-h52bs-runner-and-deploy-commit-2026-10-29). Result ([H52b run](#h52b-run-xeon-6741p-2026-10-29)): one hold at k3d-gpu 0.1.79; the plan passed D2's gate (8 commands) and applied. DSA advertised 3 of the 3 configured work queues and its pod saw `/dev/dsa`; IAA got no work queue and no resource (as D2 item 4 disclosed); QAT got 48 VFs, but its plugin crash-looped and advertised none; `gpu.intel.com/xe` read 2, not 4: the serving pairs hold about half the VRAM of all four cards, and `up`'s busy-card filter (half the VRAM) left out the two just over half (16.1, 16.2 GiB) and kept the two just under it (15.4, 15.5 GiB), which the pairs also hold, so two enumeration pods ran, not four. Media labels (`vcs` 2, `vecs` 2), AMX, the 14 AVX-512/VNNI labels, CPU 96 and no other vendor read as expected. The third inventory equals the first. Three clauses fail, so REFUTED. | REFUTED | | H53 | Inside EAT's banking flow on the Arc Pro B70s with Qwen3.8-27B (thinking off), k3d-gpu's own GPU choice AG (OVMS int4 on one card, `agent manifest`, auto device) delivers far more B24 tasks within the 120 s turn budget than B2, "EAT PR #21 nearest profile (Qwen3.6-27B) on Qwen3.8 weights, fp16" (vLLM XPU v0.28.0, TP 4). A stack-vs-stack claim. | Written 2026-10-20, before any H53 measurement. AG >= 0.70 (0.6); B2 boots by ladder rung (b) (0.6); B2 >= 0.50 (0.5); block mean AG - B2 in [-0.15, +0.30] (0.5). INCONCLUSIVE (0.6), REFUTED (0.3), SUPPORTED (0.1). | Solo holds in the order AG, B2, B2, AG, AG, B2 (up to 8), blocks = hold pairs (2k-1, 2k), block-level inference: mean of per-block d over >= 3 counted blocks, t-interval. SUPPORTED: lower bound >= +30 points and AG >= 0.6, L3 gates passed. REFUTED: upper bound < +30 points. Else INCONCLUSIVE. `bench/eatflow/crescent/solo-hold.sh N ARM`, `agentbench solo-report --hyp H53 --arm-a AG --arm-b B2`. See [H53 pre-registration](#h53-pre-registration-eats-banking-flow-on-arc-pro-b70-2026-10-20). Node image, deploy commit, order and the restore check per [amendment E2](#crescent-amendment-e2-pins-deploy-commits-order-and-the-restore-check-for-the-pending-crescent-rows-2026-10-29); the solo runner fix, setup, the wiring smokes, the window and its futility stop per [H53 amendment R1](#h53-amendment-r1-the-solo-runner-fix-setup-wiring-smokes-and-the-gpu-window-2026-10-30); tools per [amendment E4](#crescent-amendment-e4-kubectl-v1372-helm-v431-and-uv-0136-in-the-crescent-workspace-2026-10-29). Result ([H53 run](#h53-run-arc-pro-b70-2026-10-30)): no hold ran, 0 of the 8 permitted. AG's wiring smoke passed (OVMS on one card, warm-up answered). B2 needed rung (b) (`setvars.sh` is not in the v0.28.0 image), then failed in its first cross-card collective: oneCCL opens `/dev/dri/by-path`, which the k3d node and pod lack. No further rung applies, so B2 is not deliverable and H53 is INCONCLUSIVE by its pre-registered rule. | INCONCLUSIVE | @@ -5215,6 +5215,10 @@ measurement. network) and runs the probe fetched from EAS at `a051f00`. It printed `cpu1 thread_siblings_list=0-1`, `NRI_NUMA=1`, `NRI_LOGICAL=96`, `NRI_TPC=1` and `NRI_PHYSICAL=96`. That matches H62's read of VM b (H62 run: `numa=1 physical_cpus=96 threads_per_core=1`). + Run again on 2026-11-07 with the probe's sibling line, it also printed `NRI_SIBLINGS` with all + 96 vCPUs, `0,1,...,95`: the probe's sibling read (line 74) takes the first comma field of each + CPU's list as that core's lowest thread, and in the guest that field is the whole range `0-1`, + which matches no CPU id. On crescent's host the same read gives `48,...,95`. 4. With one thread per core, the default `siblings` profile of `roles/nri_cpu_balloons/files/compute_reserved_cpus_advanced.sh` logs "SMT/Hyperthreading disabled — no siblings to reserve" and falls back to `standard` (line 264): 12 reserved CPUs. @@ -5275,11 +5279,15 @@ and agent-sandbox v0.5.0. `base_domain_name` stays `solutions.ai`. BalloonsPolicy (`roles/nri_cpu_balloons/tasks/install_advanced.yaml` line 50, "Use operator-supplied reserved CPU list"). `nri_deployment_profile` stays `siblings`. - The lean. The setting restores the reservation EAS makes on crescent's own numbering, so the - guest stops changing Intel's sizing. It does not tune B5 to a number. Expected, not gated: - `cpu: auto` renders gemma-4-26b at 40 CPUs, where H62's stock read rendered 76 and left the - stack undeliverable. A1 serves on 48 CPUs in its guest (`agent manifest`'s count in H62's - setup). The guests' vCPUs float over the host's CPUs, so the guest's sibling split is not the - host's; that is part of the VM tax, described for both arms. + guest stops changing Intel's sizing. It does not tune B5 to a number. The cpuset is the set + EAS's probe defines as siblings ("the non-lowest thread of each physical core", its lines + 16-17), read with the ranges expanded. It is not the probe's own `NRI_SIBLINGS`, which in this + guest names all 96 vCPUs (item 3 above). With the cpuset set, the role does not read that line: + only `compute_advanced_reserved_cpus.yaml` does, and `install_advanced.yaml` skips it. + Expected, not gated: `cpu: auto` renders gemma-4-26b at 40 CPUs, where H62's stock read + rendered 76 and left the stack undeliverable. A1 serves on 48 CPUs in its guest (`agent + manifest`'s count in H62's setup). The guests' vCPUs float over the host's CPUs, so the + guest's sibling split is not the host's; that is part of the VM tax, described for both arms. - **D4, the request path:** unchanged. **A1: H50's A1 unchanged.** The same guest steps and the same `agent manifest` command as H62 (no @@ -5351,8 +5359,9 @@ amendment V4's records. **Risks, named before measuring.** H62's list, plus these. - The operator cpuset path has not run here. The NRI role still generates its balloon types from - the probe's read (96 CPUs, one thread per core), now beside a reserved cpuset of 48. A failure - there goes to the setup bound above. + the probe's read (96 CPUs, one thread per core), now beside a reserved cpuset of 48, and the + probe's records will list all 96 vCPUs as siblings (item 3 above). A failure there goes to the + setup bound above. - B5 runs vLLM on about 40 vCPUs, not 76. If B5 then misses the 120 s turn budget more often, that is the result of Intel's sizing on this host's numbering, and it is reported with the rendered request. @@ -5396,10 +5405,15 @@ reads, 60 s apart) and writes each read to `upstream.txt`. the smoke in `~/k3dgpu-bench/smoke/H62r/`. They come back under `bench/eatflow/results/2026-11-06/H62r/`. -**Also found, offered upstream.** EAS v1.2.0's probe reads one thread per core on any Linux node -that lists a core's threads as a range, as this KVM guest does. Counting the CPUs of the cpulist, -with ranges expanded, instead of its comma fields would read two. It is an upstreamable finding -for Intel's EAS team; nothing is filed from this row. +**Also found, offered upstream.** EAS v1.2.0's probe does not expand the range form of +`thread_siblings_list`, which Linux writes when a core's threads are numbered next to each other, +as in this KVM guest. Two of its reads use that list, and both need the change. Its threads per +core (line 31) counts the list's comma fields and reads one. Its sibling list (line 74) takes the +first comma field as the core's lowest thread and lists every CPU. With only the count changed, +the `siblings` profile would reserve all 96 vCPUs here, because it reserves `NRI_SIBLINGS` as +read. With the ranges expanded in both reads, the probe gives two threads per core and the 48 +higher threads. It is an upstreamable finding for Intel's EAS team; nothing is filed +from this row. ### H53 pre-registration: EAT's banking flow on Arc Pro B70 (2026-10-20) diff --git a/bench/h62r/guest-b.sh b/bench/h62r/guest-b.sh index 924a5cfc..86389ef4 100755 --- a/bench/h62r/guest-b.sh +++ b/bench/h62r/guest-b.sh @@ -12,8 +12,9 @@ # model_id is the QAT checkpoint) # - D3's sibling read: EAS's own topology probe (from this clone) and the kernel's # thread_siblings_list; when the probe reads one thread per core where the kernel lists two, -# nri_reserved_cpu_list (EAS's global config, env/local) names the cpuset EAS's default -# 'siblings' profile reserves when it reads SMT (the higher thread of each core) +# nri_reserved_cpu_list (EAS's global config, env/local) names the siblings as the probe +# defines them (the higher thread of each core), what EAS's default 'siblings' profile +# reserves on crescent's host # - install agent-toolkit, wait for the gemma-4-26b workload pod, then the records; the sizing # records (balloon policies, every pod's CPU request, the nodes, the sandbox router) pass or # fail; on a failure also H62 amendment V4's records diff --git a/bench/h62r/h62r.py b/bench/h62r/h62r.py index c16ff559..6e707f6e 100644 --- a/bench/h62r/h62r.py +++ b/bench/h62r/h62r.py @@ -363,8 +363,10 @@ def sibling_read(lists, probe_text): list is "0,48" (2). In the guest QEMU numbers siblings next to each other and Linux writes the pair as a range, "0-1", which the probe reads as 1, so EAS's default 'siblings' profile falls back to 'standard' (12 reserved CPUs) and gemma-4-26b's `cpu: auto` took 76 of 96 (H62). - When the probe reads 1 and the kernel lists 2 threads on every core, the result is the cpuset - EAS's siblings profile reserves when its probe reads SMT: the higher thread of each core. + When the probe reads 1 and the kernel lists 2 threads on every core, the result is the set the + probe defines as siblings, the non-lowest thread of each core, read with the ranges expanded: + what EAS's siblings profile reserves on crescent's host. (The probe's own NRI_SIBLINGS line + names every CPU in this guest, because it takes "0-1" as the lowest thread.) When the probe agrees with the kernel, nothing ("") is reserved by the runner. Anything else stops the setup (ValueError).""" if not lists: diff --git a/bench/h62r/test_h62r.py b/bench/h62r/test_h62r.py index 6fc80d99..65509893 100644 --- a/bench/h62r/test_h62r.py +++ b/bench/h62r/test_h62r.py @@ -968,6 +968,10 @@ def test_topology_spike_runs_eas_own_probe_in_h62rs_guest_shape(self): self.assertIn("print(h62r.SMP)", sh) self.assertIn("-machine q35,accel=kvm -cpu host", sh) self.assertRegex(sh, r"ALPINE=alpine:[0-9.]+@sha256:[0-9a-f]{64}\n") + # Both of the probe's reads of thread_siblings_list: its threads per core (line 31) and its + # sibling list (line 74, which the 'siblings' profile reserves when it reads SMT). + self.assertIn("NRI_(NUMA|LOGICAL|TPC|PHYSICAL)=", sh) + self.assertIn("NRI_SIBLINGS=", sh) def test_upstream_reads_both_repositories(self): sh = read(HERE, "upstream.sh") diff --git a/bench/h62r/topology-spike.sh b/bench/h62r/topology-spike.sh index 58a1c828..c0872263 100755 --- a/bench/h62r/topology-spike.sh +++ b/bench/h62r/topology-spike.sh @@ -3,8 +3,9 @@ # (EXPERIMENTS.md "H62r pre-registration", D3's sibling read). It boots a guest with # -smp 96,sockets=1,cores=48,threads=2 from an initramfs (alpine's busybox userland and EAS's # roles/nri_cpu_balloons/files/probe-node-topology.sh at the pinned EAS commit), prints cpu1's -# thread_siblings_list and the probe's NRI_NUMA, NRI_LOGICAL, NRI_TPC and NRI_PHYSICAL lines, and -# powers off. Not a hold and not a measurement of either arm. +# thread_siblings_list, the probe's NRI_NUMA, NRI_LOGICAL, NRI_TPC and NRI_PHYSICAL lines and its +# NRI_SIBLINGS line (the cpuset the 'siblings' profile reserves when NRI_TPC reads 2), and powers +# off. Not a hold and not a measurement of either arm. # Needs docker, qemu-system-x86_64 and /dev/kvm; TOPO_KERNEL is the guest kernel (default: the # first /boot/vmlinuz*); TOPO_SMP overrides the shape. The guest has no network and no disk. set -euo pipefail @@ -30,7 +31,7 @@ exec > /dev/ttyS0 2>&1 mount -t proc proc /proc mount -t sysfs sysfs /sys echo "SPIKE cpu1 thread_siblings_list=$(cat /sys/devices/system/cpu/cpu1/topology/thread_siblings_list)" -sh /probe.sh | grep -E '^NRI_(NUMA|LOGICAL|TPC|PHYSICAL)=' | sed 's/^/SPIKE /' +sh /probe.sh | grep -E '^NRI_(NUMA|LOGICAL|TPC|PHYSICAL)=|^NRI_SIBLINGS=' | sed 's/^/SPIKE /' poweroff -f EOF chmod +x "${d}/root/init" From 9736c695cb0e5193f9205d4321efa52d838b5a24 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:53:41 -0800 Subject: [PATCH 5/8] bench(h62r): record every pod's image ID per hold; keep holds and earlier smokes intact across attempts Three runner gaps found in review, each with a test that failed first. - arm.sh records now writes image-ids.txt, the image ID of every container in the guest. The pre-registration's risks say each hold records the sandbox router's image ID (EAT's floating staging tag), and no record carried one: pods.txt is `-o wide` and pod.json is the serving pod's. - campaign.sh refuses to start (exit 11, not counted as an attempt) once a hold directory exists. It numbers holds from 1, so a later attempt after a hold would have written over hold1/ and runs/H62r-hold1/ and restarted the count of 8 holds. - smoke.sh moves an earlier attempt's smoke/H62r and runs/H62r-smoke aside before it starts. warmups.jsonl is appended, so after a first smoke whose hold answered both warm-ups, every later smoke read 4 of 2 and failed, and validate.log and eat-check.log of the earlier smoke were overwritten. The H62r section says the same for the setup bound and the smoke. --- EXPERIMENTS.md | 14 ++++--- bench/h62r/arm.sh | 5 ++- bench/h62r/campaign.sh | 10 +++++ bench/h62r/smoke.sh | 9 ++++- bench/h62r/test_h62r.py | 83 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 8 deletions(-) diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index 8658fd0f..4434be59 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -5331,16 +5331,18 @@ amendment V4's records. `origin/main` commit at or after this PR's merge. PR #260 merged on 2026-11-01, so there is no run-branch condition. VM a gets the same commit (`git archive`), and every record names it. - Setup: at most 3 attempts in all. `campaign.sh` counts its starts in `attempts.txt` and - refuses a fourth before anything starts. Between attempts, a dated amendment names the cause and - the change. If the records trace a failed install to stock B5 itself (EAS's or EAT's own sizing, - images or defaults in this guest with D3's sibling read), H62r is INCONCLUSIVE (not - deliverable) with no further attempt, as H62 amendment V4 ruled. A router that is still not - placed with the sibling read is that case. + refuses a fourth before anything starts. It also refuses any start once a hold has run (its + holds number from 1, and the 8 holds are counted in all), and that refusal is not an attempt. + Between attempts, a dated amendment names the cause and the change. If the records trace a + failed install to stock B5 itself (EAS's or EAT's own sizing, images or defaults in this guest + with D3's sibling read), H62r is INCONCLUSIVE (not deliverable) with no further attempt, as H62 + amendment V4 ruled. A router that is still not placed with the sibling read is that case. - Wiring smoke (`bench/h62r/smoke.sh`): `./es_auto_installer.sh validate agent-toolkit`, then `integrations/eat/eat-check.sh` with `EAT_MODE=eas` through EAS's own gateway on `gemma-4-26b` (#260's check, now on main, with `integrations/eas/eas-ready.py` beside it), both from the deploy commit and recorded by sha256. Then one hold, `H62r-smoke`, with the pair-start guard past its - end. All of it must pass before hold 1. + end. All of it must pass before hold 1. An earlier attempt's smoke records are moved aside first, + so none is overwritten and the warm-up count is this smoke's own. - Holds (`bench/h62r/pairs-hold.sh N` under `bench/eatflow/crescent/hold.sh`): at most 40 min each, until 24 pairs or 8 holds, each restarting both serving pods with H62's 900 s readiness rule, two not-deliverable holds in a row ending the row, the 600 s pair-start guard and the hold-start diff --git a/bench/h62r/arm.sh b/bench/h62r/arm.sh index 6fe2d43c..19d9a929 100755 --- a/bench/h62r/arm.sh +++ b/bench/h62r/arm.sh @@ -5,7 +5,8 @@ # restart SECONDS delete its serving pod, then wait until a new pod is Ready; exit 3 when none is # Ready within SECONDS (the hold's 900 s readiness rule passes the time left) # records DIR the pod, the CPU ISA it sees (amx_tile lines), its cpuset and nproc, the nodes, -# the guest's load and memory, and B5's LLMInferenceService +# the guest's load and memory, B5's LLMInferenceService, and the image ID of every +# container in the guest (B5's sandbox router runs a floating tag) # logs DIR every pod, events, the pod description and the server log (a hold that fails) # forward (re)start `kubectl port-forward` of its Service in the guest, on the serve port # unforward stop that port-forward @@ -58,6 +59,8 @@ records) g "uptime; free -g; grep -c amx_tile /proc/cpuinfo; nproc" > "${d}/guest.txt" 2>&1 [ -z "${EXTRA}" ] || g "kubectl -n ${NS} get ${EXTRA} -o yaml" > "${d}/extra.yaml" 2>&1 g "kubectl get pods -A -o wide" > "${d}/pods.txt" 2>&1 + g "kubectl get pods -A -o 'custom-columns=NS:.metadata.namespace,POD:.metadata.name,IMAGE_ID:.status.containerStatuses[*].imageID'" \ + > "${d}/image-ids.txt" 2>&1 echo "records: ${d}" ;; logs) d=${3:?usage: arm.sh ARM logs DIR} diff --git a/bench/h62r/campaign.sh b/bench/h62r/campaign.sh index 8d3d659c..a0800a50 100755 --- a/bench/h62r/campaign.sh +++ b/bench/h62r/campaign.sh @@ -5,6 +5,8 @@ # also after an abort. # - at most 3 setup attempts in all (attempts.txt, one line per start); a fourth start exits 10 # before anything is started +# - a start after any hold has run (a hold directory in the records) exits 11 before anything +# is started, and is not an attempt # - setup exit 6 (the host gate still failed after 60 min): the two qwen38 serving pairs are # stopped under the box owner's grant (gpu-serving.sh stop, never rm) and setup runs once more; # teardown.sh starts them again @@ -20,6 +22,14 @@ R=${B}/results/H62r MAX_HOLDS=8 NOT_DELIVERABLE_MAX=2 ATTEMPTS_MAX=3 mkdir -p "${R}" log() { echo "== campaign $* $(date -u +%FT%TZ)" | tee -a "${R}/campaign.log"; } +# Holds count toward the 8 in all and their records are measured data. This script numbers its +# holds from 1, so once any hold has run it refuses to start: a dated amendment says how the row +# goes on. +held=$(find "${R}" -mindepth 1 -maxdepth 1 -type d -name 'hold[0-9]*' -printf '%f\n' | sort -V | paste -sd' ') +if [ -n "${held}" ]; then + log "holds already recorded (${held}): no start" + exit 11 +fi tries=$(grep -c . "${R}/attempts.txt" 2>/dev/null) if [ "${tries:-0}" -ge "${ATTEMPTS_MAX}" ]; then log "setup attempt bound reached (${tries:-0} of ${ATTEMPTS_MAX}): no further attempt" diff --git a/bench/h62r/smoke.sh b/bench/h62r/smoke.sh index 8b34ec75..f6ad2c16 100755 --- a/bench/h62r/smoke.sh +++ b/bench/h62r/smoke.sh @@ -9,7 +9,8 @@ # come from this checkout (the deploy commit) and are recorded by sha256 # 2. one H62r hold with the pair-start guard past its end (pairs-hold.sh 0 smoke): both serving # pods restart and answer through the harness's LiteLLM, both warm-ups answer, no pair starts -# Exit 0 when both pass; 1 otherwise; 4 when the lock is not taken. Records: ~/k3dgpu-bench/smoke/H62r. +# Exit 0 when both pass; 1 otherwise; 4 when the lock is not taken. Records: ~/k3dgpu-bench/smoke/H62r; +# an earlier attempt's smoke records (and runs/H62r-smoke) are first moved to .. set -uo pipefail HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO=$(cd "${HERE}/../.." && pwd) @@ -19,6 +20,12 @@ export H62R_STATE=${H62R_STATE:-${B}/h62r} S=${B}/smoke/H62r MODEL=$(P pin catalog_name) CHECK=${REPO}/integrations/eat/eat-check.sh READY=${REPO}/integrations/eas/eas-ready.py +# An earlier setup attempt's smoke keeps its records as they were: they move aside, so this smoke +# overwrites none of them and its gate reads only the warm-ups and pairs its own hold writes. +stamp=$(date -u +%Y%m%dT%H%M%SZ) +for d in "${S}" "${B}/runs/H62r-smoke"; do + [ ! -e "${d}" ] || mv "${d}" "${d}.${stamp}" +done mkdir -p "${S}" log() { echo "== smoke $* $(date -u +%FT%TZ)" | tee -a "${S}/smoke.log"; } G() { ssh -F "${H62R_STATE}/ssh_config" h62r-b "$@" 9>&-; } diff --git a/bench/h62r/test_h62r.py b/bench/h62r/test_h62r.py index f3822dd9..a18d0f7d 100644 --- a/bench/h62r/test_h62r.py +++ b/bench/h62r/test_h62r.py @@ -704,6 +704,20 @@ def test_forward_restarts_the_guest_port_forward(self): self.arm("A1", "unforward") self.assertIn("kill", read(self.log).splitlines()[-1]) + def test_records_name_every_pods_image_id(self): + # Pre-registration, risks: the sandbox router runs EAT's floating staging tag, so each + # hold records the image ID of every container in the guest, the router's included. + self.pods([pod("p1", "u1")]) + d = os.path.join(self.tmp, "rec") + r = self.arm("B5", "records", d) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + cmds = [ln for ln in read(self.log).splitlines() if "imageID" in ln] + self.assertEqual(len(cmds), 1, read(self.log)) + self.assertIn("kubectl get pods -A", cmds[0]) + self.assertIn(".status.containerStatuses[*].imageID", cmds[0]) + self.assertIn("'", cmds[0]) # the [*] reaches kubectl quoted + self.assertTrue(os.path.isfile(os.path.join(d, "image-ids.txt"))) + class OnTtyTest(unittest.TestCase): """H62 amendment V2, kept: EAS v1.2.0's es_auto_installer.sh is byte-identical to rc.1's, so @@ -878,6 +892,75 @@ def test_campaign_refuses_a_fourth_setup_attempt(self): self.assertEqual(read(res, "attempts.txt"), "a\nb\nc\n") self.assertFalse(os.path.exists(os.path.join(res, "teardown.log"))) # nothing was started + def tree(self, scripts): + """A copy of bench/h62r (scripts from this checkout) under a temp HOME, with every other + script it calls replaced by a stub that logs its call and exits 1.""" + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + d = os.path.join(tmp, "repo", "bench", "h62r") + os.makedirs(d) + for s in scripts + ("h62r.py",): + shutil.copy2(os.path.join(HERE, s), d) + for s in ("setup.sh", "smoke.sh", "pairs-hold.sh", "teardown.sh"): + if s not in scripts: + stub(os.path.join(d, s), f'echo "{s} $*" >> "{tmp}/calls"; exit 1\n') + return tmp, d + + def test_campaign_never_numbers_holds_over_recorded_ones(self): + # Holds are measured data and count toward the 8 in all: a start after any hold has run + # would number its holds from 1 again, over hold/ and runs/H62r-hold/, so it is + # refused before anything starts (a dated amendment says how the row goes on). + tmp, d = self.tree(("campaign.sh",)) + res = os.path.join(tmp, "k3dgpu-bench", "results", "H62r") + os.makedirs(os.path.join(res, "hold1")) + with open(os.path.join(res, "attempts.txt"), "w") as f: + f.write("a\n") + r = subprocess.run(["bash", os.path.join(d, "campaign.sh")], capture_output=True, text=True, + env=dict(os.environ, HOME=tmp), timeout=60) + self.assertEqual(r.returncode, 11, r.stdout + r.stderr) + self.assertIn("hold1", read(res, "campaign.log")) + self.assertEqual(read(res, "attempts.txt"), "a\n") # not an attempt + self.assertFalse(os.path.exists(os.path.join(tmp, "calls"))) # nothing was started + + def test_smoke_counts_only_its_own_warm_ups(self): + # An earlier attempt's smoke (its hold appended 2 answered warm-ups) is kept as it was and + # moved aside; this smoke's gate reads only the warm-ups its own hold wrote. + tmp, d = self.tree(("smoke.sh",)) + repo = os.path.join(tmp, "repo") + for sub, name, text in (("eat", "eat-check.sh", "case x in\n eas) ;;\nesac\n"), ("eas", "eas-ready.py", "")): + os.makedirs(os.path.join(repo, "integrations", sub)) + with open(os.path.join(repo, "integrations", sub, name), "w") as f: + f.write(text) + smoke = os.path.join(tmp, "k3dgpu-bench", "smoke", "H62r") + ok = '{"hold": 0, "arm": "%s", "seconds": 1.0, "error": null, "answer": "ok"}\n' + os.makedirs(os.path.join(smoke, "pairs")) + with open(os.path.join(smoke, "pairs", "warmups.jsonl"), "w") as f: + f.write(ok % "A1" + ok % "B5") + with open(os.path.join(smoke, "eat-check.log"), "w") as f: + f.write("attempt 1\n") + runs = os.path.join(tmp, "k3dgpu-bench", "runs") + os.makedirs(os.path.join(runs, "H62r-smoke")) + with open(os.path.join(runs, "H62r-smoke", "hold.log"), "w") as f: + f.write("attempt 1\n") + stub(os.path.join(d, "pairs-hold.sh"), textwrap.dedent(f'''\ + p="$HOME/k3dgpu-bench/smoke/H62r/pairs"; mkdir -p "$p" + printf '{ok % "A1"}{ok % "B5"}' >> "$p/warmups.jsonl" + ''')) + bin_ = os.path.join(tmp, "bin") + os.makedirs(bin_) + for tool in ("ssh", "scp"): + stub(os.path.join(bin_, tool), "exit 0\n") + r = subprocess.run(["bash", os.path.join(d, "smoke.sh")], capture_output=True, text=True, timeout=60, + env=dict(os.environ, HOME=tmp, PATH=bin_ + os.pathsep + os.environ["PATH"])) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr + read(smoke, "smoke.log")) + self.assertIn("warm-ups answered 2 of 2", read(smoke, "smoke.log")) + kept = [p for p in os.listdir(os.path.dirname(smoke)) if p != "H62r"] + self.assertEqual(len(kept), 1, kept) + self.assertEqual(read(os.path.dirname(smoke), kept[0], "eat-check.log"), "attempt 1\n") + self.assertEqual(read(os.path.dirname(smoke), kept[0], "pairs", "warmups.jsonl"), ok % "A1" + ok % "B5") + self.assertEqual(os.listdir(runs), [kept[0].replace("H62r", "H62r-smoke", 1)]) + self.assertEqual(read(runs, os.listdir(runs)[0], "hold.log"), "attempt 1\n") + def test_setup_caps_and_gates(self): sh = read(HERE, "setup.sh") self.assertIn('flock -w 7200 9', sh) From ea50fbec2336cbeb8987951d95c853c885922299 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:54:17 -0800 Subject: [PATCH 6/8] bench(h62r): the trigger read decides on upstream.sh's own line, not its stderr campaign.sh captured upstream.sh's stderr with its output, so a warning from git ls-remote next to "fired: none" would have read as a fired trigger and stopped the campaign. It now reads only the "fired: ..." line on stdout; a read without that line counts as unreadable, and the errors go into upstream.txt. --- bench/h62r/campaign.sh | 9 +++++---- bench/h62r/test_h62r.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/bench/h62r/campaign.sh b/bench/h62r/campaign.sh index 1125ef40..bbce8cb9 100755 --- a/bench/h62r/campaign.sh +++ b/bench/h62r/campaign.sh @@ -25,18 +25,19 @@ R=${B}/results/H62r MAX_HOLDS=8 NOT_DELIVERABLE_MAX=2 ATTEMPTS_MAX=3 mkdir -p "${R}" log() { echo "== campaign $* $(date -u +%FT%TZ)" | tee -a "${R}/campaign.log"; } -# trigger PHASE: the re-run trigger's line into upstream.txt. Status 0 fired, 1 not, 2 unreadable. +# trigger PHASE: the re-run trigger's line ("fired: ...", upstream.sh's stdout) into upstream.txt. +# Status 0 fired, 1 not, 2 unreadable (no such line in 3 reads; its errors are recorded). trigger() { - local out="" i + local out="" err="${R}/upstream.err" i for i in 1 2 3; do - if out=$(bash "${HERE}/upstream.sh" 2>&1); then + if out=$(bash "${HERE}/upstream.sh" 2> "${err}") && [[ "${out}" == "fired: "* ]]; then echo "$(date -u +%FT%TZ) $1 ${out}" >> "${R}/upstream.txt" [ "${out}" != "fired: none" ] return fi [ "${i}" = 3 ] || sleep "${H62R_TRIGGER_RETRY_S:-60}" done - echo "$(date -u +%FT%TZ) $1 unreadable: ${out//$'\n'/ }" >> "${R}/upstream.txt" + echo "$(date -u +%FT%TZ) $1 unreadable: $(paste -sd' ' "${err}")${out:+ (output: ${out//$'\n'/ })}" >> "${R}/upstream.txt" return 2 } tries=$(grep -c . "${R}/attempts.txt" 2>/dev/null) diff --git a/bench/h62r/test_h62r.py b/bench/h62r/test_h62r.py index 65509893..fb3efdab 100644 --- a/bench/h62r/test_h62r.py +++ b/bench/h62r/test_h62r.py @@ -897,7 +897,9 @@ def campaign_with_stubs(self, reads, holds_left=3): "teardown.sh": f'echo teardown >> "{calls}"', "pairs-hold.sh": f'echo "hold$1" >> "{calls}"; echo $(( $(cat "{tmp}/left") - 1 )) > "{tmp}/left"', "upstream.sh": (f'echo upstream >> "{calls}"; r=$(head -1 "{tmp}/reads"); sed -i 1d "{tmp}/reads"; ' - '[ "${r}" != ERR ] || { echo "ls-remote failed" >&2; exit 1; }; echo "${r}"'), + '[ "${r}" != ERR ] || { echo "ls-remote failed" >&2; exit 1; }; ' + '[ "${r}" != WARN ] || { echo "warning: redirecting to https://example/" >&2; r="fired: none"; }; ' + 'echo "${r}"'), } for name, body in stubs.items(): with open(os.path.join(here, name), "w") as f: @@ -951,6 +953,18 @@ def test_campaign_records_an_unreadable_trigger_after_three_reads(self): r, res, order = self.campaign_with_stubs(["ERR"] * 3) self.assertEqual((r.returncode, order), (11, ["upstream"] * 3), r.stdout + r.stderr) + def test_campaign_reads_the_triggers_line_not_its_warnings(self): + # git ls-remote may warn on stderr; only upstream.sh's own line decides whether it fired. + r, res, order = self.campaign_with_stubs(["WARN", "WARN", "fired: none", "fired: none"]) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertEqual([c for c in order if c.startswith("hold")], ["hold1", "hold2", "hold3"]) + self.assertEqual([ln.split(" ", 1)[1] for ln in read(res, "upstream.txt").splitlines()][:2], + ["setup fired: none", "hold1 fired: none"]) + # A read that prints no trigger line is unreadable, not fired. + r, res, order = self.campaign_with_stubs(["", "", ""]) + self.assertEqual((r.returncode, order), (11, ["upstream"] * 3), r.stdout + r.stderr) + self.assertIn("setup unreadable", read(res, "upstream.txt")) + def test_setup_caps_and_gates(self): sh = read(HERE, "setup.sh") self.assertIn('flock -w 7200 9', sh) From c94129f24add41670a6fc3693e0784976c20d067 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:56:32 -0800 Subject: [PATCH 7/8] ledger(H62r): the host-numbering sibling read is from a sysfs tree, not from crescent Item 3 said the probe's sibling read gives 48,...,95 on crescent's host. That value comes from running the probe on a sysfs tree with the host's numbering; crescent was read only for cpu0 and cpu1. The sentence now says so. --- EXPERIMENTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index a1e84c07..4d50103c 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -5218,7 +5218,8 @@ measurement. Run again on 2026-11-07 with the probe's sibling line, it also printed `NRI_SIBLINGS` with all 96 vCPUs, `0,1,...,95`: the probe's sibling read (line 74) takes the first comma field of each CPU's list as that core's lowest thread, and in the guest that field is the whole range `0-1`, - which matches no CPU id. On crescent's host the same read gives `48,...,95`. + which matches no CPU id. On lists in the host's numbering (`0,48`, `1,49`, ...) the same read + gives `48,...,95` (the probe run on such a sysfs tree, not on crescent). 4. With one thread per core, the default `siblings` profile of `roles/nri_cpu_balloons/files/compute_reserved_cpus_advanced.sh` logs "SMT/Hyperthreading disabled — no siblings to reserve" and falls back to `standard` (line 264): 12 reserved CPUs. @@ -5414,8 +5415,8 @@ core (line 31) counts the list's comma fields and reads one. Its sibling list (l first comma field as the core's lowest thread and lists every CPU. With only the count changed, the `siblings` profile would reserve all 96 vCPUs here, because it reserves `NRI_SIBLINGS` as read. With the ranges expanded in both reads, the probe gives two threads per core and the 48 -higher threads. It is an upstreamable finding for Intel's EAS team; nothing is filed -from this row. +higher threads. It is an upstreamable finding for Intel's EAS team; nothing is filed from this +row. ### H53 pre-registration: EAT's banking flow on Arc Pro B70 (2026-10-20) From 19bd518184e8abb31760ff7d0b018b267ca835fa Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Fri, 6 Nov 2026 20:57:48 -0800 Subject: [PATCH 8/8] bench(h62r): reserve reads a key line with a trailing comment as the key reserve() matched the nri_reserved_cpu_list line only without a trailing comment, so a live line like `nri_reserved_cpu_list: "" # ...` was missed and a second key was appended after it, and a commented line holding another cpuset was not refused. It now reads any live key line (double- quoted, single-quoted or bare value, optional comment): an empty value is filled, the same cpuset is left alone, and any other value or an unreadable line stops the setup. EAS v1.2.0's seeded global_config.yaml carries no such line, so this run was not exposed. The comment it appends now describes the cpuset as the probe's siblings, as the ledger does. --- bench/h62r/h62r.py | 22 ++++++++++++++-------- bench/h62r/test_h62r.py | 9 +++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/bench/h62r/h62r.py b/bench/h62r/h62r.py index 6e707f6e..858aa2be 100644 --- a/bench/h62r/h62r.py +++ b/bench/h62r/h62r.py @@ -411,19 +411,25 @@ def reserve(text, cpus): if not re.fullmatch(r"[0-9]+([,-][0-9]+)*", cpus): # EAS's own cpuset check raise ValueError(f"not a cpuset: {cpus!r}") line = f'nri_reserved_cpu_list: "{cpus}"' - pat = re.compile(r'^nri_reserved_cpu_list:[ \t]*"?([^"\n]*)"?[ \t]*$', re.M) - found = pat.findall(text) + key = re.compile(r"^nri_reserved_cpu_list:(.*)$", re.M) + found = key.findall(text) if len(found) > 1: raise ValueError("more than one live nri_reserved_cpu_list line") if found: - if found[0] == cpus: + # The value: double-quoted, single-quoted or bare, then an optional comment. + m = re.fullmatch(r"""\s*(?:"([^"\n]*)"|'([^'\n]*)'|([^\s#"']*))\s*(?:#.*)?""", found[0]) + if not m: + raise ValueError(f"nri_reserved_cpu_list line not read: {found[0].strip()!r}") + have = next((g for g in m.groups() if g is not None), "") + if have == cpus: return text - if found[0]: - raise ValueError(f"nri_reserved_cpu_list is already {found[0]!r}") - return pat.sub(line, text) + if have: + raise ValueError(f"nri_reserved_cpu_list is already {have!r}") + return key.sub(lambda _: line, text, count=1) sep = "" if text.endswith("\n") or not text else "\n" - return (text + sep + "\n# H62r D3's sibling read: the cpuset EAS's 'siblings' profile reserves when its probe\n" - "# reads this guest's SMT (EXPERIMENTS.md \"H62r pre-registration\")\n" + line + "\n") + return (text + sep + "\n# H62r D3's sibling read: the siblings EAS's probe defines (the higher thread of\n" + "# each core), as its 'siblings' profile reserves them on crescent's host\n" + "# (EXPERIMENTS.md \"H62r pre-registration\")\n" + line + "\n") def _ready(item): diff --git a/bench/h62r/test_h62r.py b/bench/h62r/test_h62r.py index 58843403..9e2cd34e 100644 --- a/bench/h62r/test_h62r.py +++ b/bench/h62r/test_h62r.py @@ -473,6 +473,15 @@ def test_fills_the_documented_empty_default(self): out = h62r.reserve(GLOBAL + 'nri_reserved_cpu_list: ""\n', ODD) self.assertEqual(out, GLOBAL + f'nri_reserved_cpu_list: "{ODD}"\n') + def test_a_key_line_with_a_comment_is_the_same_key(self): + # A live key line may carry a trailing comment (as the role's defaults write it); it is + # filled or refused like any other, never followed by a second key of the same name. + out = h62r.reserve(GLOBAL + 'nri_reserved_cpu_list: "" # empty: sized by the profile\n', ODD) + self.assertEqual(out.count("nri_reserved_cpu_list:"), 1, out) + self.assertIn(f'nri_reserved_cpu_list: "{ODD}"', out) + with self.assertRaises(ValueError): + h62r.reserve(GLOBAL + 'nri_reserved_cpu_list: "0-3" # pinned\n', ODD) + def test_refuses_another_value_or_a_bad_cpuset(self): with self.assertRaises(ValueError): h62r.reserve(GLOBAL + 'nri_reserved_cpu_list: "0-3"\n', ODD)