From 031593fd3b3a3abb37abbc7636fa1e4094212448 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sat, 7 Nov 2026 21:13:01 -0800 Subject: [PATCH 1/5] tests: the ledger drift guard fails a doc that calls a decided row pre-registered or not yet run A row the ledger has decided can still be described in a doc as pre-registered or not yet run, and the guard read only verdict words, so the stale line passed. H62r is REFUTED on main and three docs still say it is pre-registered. The new form reads every id before "not yet run", "not run yet" or "is/are pre-registered" in the same clause, across a paragraph's line breaks, and the one id of a table row. Each such row must be PENDING. With this change the guard fails on main at docs/platforms.md:252, docs/intel/partner-update-2026-11.md:50 and docs/intel/technical-report.md:152; the next commit fixes those docs. --- tests/ledger_drift_check.py | 54 ++++++++++++++++++++++++++++++++ tests/ledger_drift_check_test.py | 46 +++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/tests/ledger_drift_check.py b/tests/ledger_drift_check.py index 237034ba..23500f4e 100644 --- a/tests/ledger_drift_check.py +++ b/tests/ledger_drift_check.py @@ -21,6 +21,11 @@ that opens with a verdict word and a colon; every id on it. count "Six hypotheses are SUPPORTED", "Eight are PENDING": the number must equal the ledger's count for that verdict. + not run "H62r is pre-registered and not yet run", "rows H28w and H57wa, + pre-registered and not yet run": every id before "not yet run", "not + run yet" or "is/are pre-registered" in the same clause (no '.', ';' or + '|' between; a paragraph's line breaks allowed), or the one id of a + table row, must be PENDING in the ledger. Docs only, not scripts. A row id may carry one lowercase letter (H1b, a re-run row of H1). @@ -45,6 +50,9 @@ LIST_LINE = re.compile(r"^\s*(?:[-*]\s+)?\**(" + V + r")\b[^:]{0,40}:") COUNT = re.compile(r"\b(\d+|" + "|".join(NUMBERS) + r")\s+(?:hypotheses\s+|rows\s+)?(?:are|is)\s+(" + V + r")\b", re.IGNORECASE) +NOT_RUN = re.compile(r"\b(?:not yet run|not run yet|(?:is|are) pre-registered)\b") +BULLET = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s") +SENTENCE_END = re.compile(r"\.(?=\s|$)") def ledger(root): @@ -158,6 +166,52 @@ def expect(n, hid, said, how): have = sum(1 for v in rows.values() if v == verdict) if said != have: errors.append(f"{rel}:{n}: says {said} {verdict} but EXPERIMENTS.md has {have}") + return errors + check_not_run(path.read_text(), rel, rows) + + +def units(text): + """(start line, [(line number, text), ...]) for each table row, paragraph and list + item: blank lines, headings, skipped lines and a new list item end a unit.""" + unit, last = [], 0 + for n, line in visible_lines(text): + table, new = line.lstrip().startswith("|"), BULLET.match(line) + if unit and (n != last + 1 or not line.strip() or line.startswith("#") or table or new): + yield unit + unit = [] + last = n + if table: + yield [(n, line)] + elif line.strip() and not line.startswith("#"): + unit.append((n, line)) + if unit: + yield unit + + +def check_not_run(text, rel, rows): + """A row that a doc calls pre-registered or not yet run must be PENDING. The row + is every id before the words in their clause (no '.', ';' or '|' between, line + breaks inside a paragraph or list item allowed), or the one id of a table row.""" + errors = [] + for unit in units(text): + joined, starts = "", [] + for n, line in unit: + starts.append((len(joined), n)) + joined += line + " " + cuts = [m.end() for m in SENTENCE_END.finditer(joined)] + cuts += [i + 1 for i, c in enumerate(joined) if c in ";|"] + table = joined.lstrip().startswith("|") + for said in NOT_RUN.finditer(joined): + at = said.start() + n = [ln for off, ln in starts if off <= at][-1] + clause = joined[max([c for c in cuts if c <= at], default=0):at] + named = ids_in(clause) + if table and len(ids_in(joined)) == 1: + named |= ids_in(joined) + for hid in sorted(named): + have = rows.get(hid) + error = f"{rel}:{n}: {hid} reads not yet run (not run) but EXPERIMENTS.md says {have}" + if have not in (None, "PENDING") and error not in errors: + errors.append(error) return errors diff --git a/tests/ledger_drift_check_test.py b/tests/ledger_drift_check_test.py index e4cd856b..684aab37 100644 --- a/tests/ledger_drift_check_test.py +++ b/tests/ledger_drift_check_test.py @@ -124,6 +124,31 @@ def test_a_sentence_end_breaks_the_clause(self): def test_another_id_between_breaks_the_clause(self): self.assertClean("H3 and H2 is REFUTED.\n") + def test_a_pending_row_not_yet_run(self): + self.assertClean("H4 is pre-registered and not yet run.\n") + + def test_not_yet_run_in_another_sentence(self): + self.assertClean("H3 ran on the desktop. F4 is pre-registered and not yet run.\n") + + def test_did_not_run_is_not_pending(self): + self.assertClean("H1 did not run: its arm was not deliverable.\n") + + def test_not_yet_run_after_a_paragraph_break(self): + self.assertClean("about H2\n\nnot yet run elsewhere\n") + + def test_not_yet_run_names_the_rows_before_it_in_its_clause(self): + self.assertClean("That path is pre-registered as F4 and has not run yet; the old pin passed [ledger:H3].\n") + self.assertClean("The old pin passed [ledger:H3]; the new one is F4 and not run yet.\n") + + def test_a_table_row_naming_two_rows_reads_each_cell(self): + self.assertClean("| H2, H4 | H4 is not yet run |\n") + + def test_a_bullet_ends_a_not_yet_run_sentence(self): + self.assertClean("- H2 ran on the Xeon\n- H4 is pre-registered and not yet run.\n") + + def test_not_yet_run_in_a_comment(self): + self.assertClean("\n") + class Drift(unittest.TestCase): def assertDrift(self, doc, pattern, path="docs/guide.md"): @@ -160,6 +185,27 @@ def test_verdict_list_line(self): def test_count(self): self.assertDrift("- Six are PENDING.\n", r"says 6 PENDING but EXPERIMENTS.md has 1") + def test_decided_row_not_yet_run(self): + self.assertDrift("H2 is pre-registered and not yet run.\n", + r"guide.md:1: H2 reads not yet run \(not run\).*REFUTED") + + def test_decided_row_pre_registered(self): + self.assertDrift("H3 is pre-registered on the new tags.\n", r"H3 reads not yet run \(not run\).*SUPPORTED") + + def test_not_run_yet(self): + self.assertDrift("H5 has not run yet.\n", r"H5 reads not yet run \(not run\).*INCONCLUSIVE") + + def test_not_yet_run_across_lines(self): + self.assertDrift("After the campaign, H3 repeats H4 at the\nnew tags, and it is pre-registered and not yet run.\n", + r"guide.md:2: H3 reads not yet run \(not run\).*SUPPORTED") + + def test_not_yet_run_in_a_bullet(self): + self.assertDrift("- H4 ran.\n- the rows H2 and H4, pre-registered and not yet run.\n", + r"guide.md:2: H2 reads not yet run") + + def test_not_yet_run_in_a_table_row(self): + self.assertDrift("| Row | State |\n|---|---|\n| H2 | not yet run |\n", r"guide.md:3: H2 reads not yet run") + def test_lettered_id_reference(self): rc, out = run("Status: PENDING [ledger:H1b].\n", LEDGER.replace( "| H2 | two", "| H1b | one, re-run | p | probe | SUPPORTED |\n| H2 | two")) From f4f4a4774089c52e4cc040049bd44c5594b45d77 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sat, 7 Nov 2026 21:19:52 -0800 Subject: [PATCH 2/5] docs: H62r's result, the wildct rows and the vLLM agent status, up to date with main (2026-11-07) H62r is REFUTED on main (#343), but the partner update, docs/intel.md and docs/platforms.md still called it pre-registered, and the technical report's limitations still said it had not run. Those lines now give the result: at the GA tags, with EAS's documented nri_reserved_cpu_list naming the guest's 48 sibling vCPUs, Intel's install finished in its KVM guest and, inside EAT's banking flow, delivered as many tasks within the turn budget as k3d-gpu's CPU stack, 21 of 24 each. The partner update states it as a result that helps EAT, names the EAS topology probe reading behind H62 (probe-node-topology.sh at v1.2.0, lines 30-31), and the EI cpu: auto draft in upstream-eat.md says where that gap shows. Other stale statements: - wildct: H28w has run and its result is in review (#344); H57wa, H57wm, H57wb and H59w are still pre-registered and not yet run. All five stay PENDING. - docs/intel.md: the strict hybrid test first ran on hardware in H52w, and H18w backs sandboxes on E-cores alongside H18. F5 is SUPPORTED in the EAT 2.0 install row. - Partner update: the canary serves engines in a plain container with the launcher's arguments, not in a pod; a pin move also moves its test goldens (#328); EAS #50 merged into EAS main on 2026-11-02 (not in v1.2.0), and the same is noted in docs/intel/eas.md. - Partner update, open items: in release 0.1.93 agent up does not serve through vLLM on Arc (the service-link VLLM_PORT and the warm-up content type, fixed in #339, in review), and the Gaudi path shares that manifest. - README: H25w in the Intel summary, H62r beside F5 in the EAT section, H18w beside H18 in the sandbox table. tests/ledger_drift_check.py, tests/ledger_drift_check_test.py, tests/claims_check.sh, tests/claims_check_test.sh, tests/docs_test.sh (mkdocs from docs-build/requirements.txt), tests/platforms_doc_test.sh, tests/pe_premise_check.sh, tests/image_pins_test.sh and tests/ledger_freshness_test.py pass. --- README.md | 11 +++-- docs/intel.md | 22 +++++----- docs/intel/eas.md | 2 +- docs/intel/partner-update-2026-11.md | 60 +++++++++++++++++++--------- docs/intel/technical-report.md | 8 ++-- docs/intel/upstream-eat.md | 8 +++- docs/platforms.md | 8 +++- 7 files changed, 80 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 2b8622a1..97236f2c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ A Docker-based [rancher/k3s](https://hub.docker.com/r/rancher/k3s) node image on > i9-13900T host. On a second client host, an i7-13700 whose owner opts its GPUs > out, a plain `up` found the hybrid P/E cores and the ISA extensions the launcher > labels and nothing the host lacks [ledger:H52w], and `k3d-gpu sandbox up` kept -> agent sandboxes on its E-cores behind default-deny egress [ledger:H18w]. +> agent sandboxes on its E-cores behind default-deny egress [ledger:H18w]. There, +> OVMS's shipped prefix caching gave an added agent its first chunk 14.3x sooner [ledger:H25w]. > On a Xeon 6741P with 4x Arc Pro B70, a plain `up` found the > B70s (`xe`), AMX and AVX-512 [ledger:H52]. In a second run there, `xe` advertised > 2 of the 4 B70s: a serving workload held about half the VRAM of each card, and @@ -1449,7 +1450,11 @@ check` and the teardown passed, but enterprise-inference's Keycloak step put the admin password on a process command line, where the run's secret scan found it. F5 ran it at the GA tags on the Xeon 6741P and is SUPPORTED ([EXPERIMENTS.md](EXPERIMENTS.md#f5-run-xeon-6741p-2026-11-07)): `eat up` with `failed=0` and -no hand edit, `eat check`, a secret scan of 0 and `eat down` all passed. The commands are `k3d-gpu eat up` (`--dry-run`, +no hand edit, `eat check`, a secret scan of 0 and `eat down` all passed. H62r ran Intel's own +documented install of the same tags in a KVM guest on that Xeon: with EAS's documented +`nri_reserved_cpu_list` it finished, and inside EAT's banking flow it delivered as many tasks +within the turn budget as k3d-gpu's CPU stack, 21 of 24 each +([EXPERIMENTS.md](EXPERIMENTS.md#h62r-run-xeon-6741p-2026-11-07)). The commands are `k3d-gpu eat up` (`--dry-run`, `--explain`; `--legacy` runs EAT 1.x), `eat status`, `eat check` and `eat down`, or `k3d-gpu eas` with the layer named. See [docs/intel/eas.md](docs/intel/eas.md), and [docs/intel.md](docs/intel.md#upstream-findings) for the upstream findings and issue drafts. @@ -1581,7 +1586,7 @@ What works on k3d, and what does not: | Piece | On k3d-gpu | |-------|------------| | Controller, Sandbox, SandboxTemplate, SandboxClaim, SandboxWarmPool | Yes. Plain manifests, no webhook or cert-manager. | -| NetworkPolicy | Enforced by k3s's embedded kube-router controller. The template allows ingress on 8888 only from pods labelled `k3d-gpu.io/sandbox-client=true` (and upstream's `sandbox-router`). Egress is limited to cluster DNS and public addresses. RFC1918, CGNAT `100.64.0.0/10` (tailscale) and link-local are blocked, so a sandbox cannot reach pods, services, the k3d docker network, tailnet peers or the host by those addresses. A blocked connect fails at once with `Connection refused` (kube-router rejects it), not a timeout [ledger:H18]. The host's public IPs stay reachable, so bind inference servers to a private address or firewall them. Ingress is gated on a pod label that any namespace can set: fine for a single-user dev cluster, not for a shared one. | +| NetworkPolicy | Enforced by k3s's embedded kube-router controller. The template allows ingress on 8888 only from pods labelled `k3d-gpu.io/sandbox-client=true` (and upstream's `sandbox-router`). Egress is limited to cluster DNS and public addresses. RFC1918, CGNAT `100.64.0.0/10` (tailscale) and link-local are blocked, so a sandbox cannot reach pods, services, the k3d docker network, tailnet peers or the host by those addresses. A blocked connect fails at once with `Connection refused` (kube-router rejects it), not a timeout [ledger:H18], also on a second hybrid host [ledger:H18w]. The host's public IPs stay reachable, so bind inference servers to a private address or firewall them. Ingress is gated on a pod label that any namespace can set: fine for a single-user dev cluster, not for a shared one. | | gVisor / Kata (`runtimeClassName`) | Not available. The node image ships neither, and Kata needs `/dev/kvm` in the node container. Sandboxes run under runc, hardened instead: non-root, all capabilities dropped, no privilege escalation, `RuntimeDefault` seccomp, no service-account token. That is container isolation, not a VM or user-space kernel boundary. | | sandbox-router, sandboxd | Not installed. Upstream publishes them only as staging `latest-main` images, which cannot be pinned to a release. Clients reach a sandbox by pod IP (from a `k3d-gpu.io/sandbox-client=true` pod) or `kubectl port-forward`, which containerd opens inside the pod's network namespace, so the policy does not block it. | diff --git a/docs/intel.md b/docs/intel.md index 33f68d29..c7e2d041 100644 --- a/docs/intel.md +++ b/docs/intel.md @@ -108,13 +108,14 @@ the plugins, and asserts the result, with no manual step. `nvidia=2, intel=1, hybrid=1`. That run used commit `109154d`. The hybrid `test` there only checked that each pod printed its `Cpus_allowed_list`; the stricter check that the list sits inside the P-core or E-core list landed - after it (`fb5c17d`) and has not yet run on hardware. The second is a Xeon 6741P + after it (`fb5c17d`); its first hardware run was on the third host, below. The second is a Xeon 6741P (Granite Rapids, 48 cores, AMX, AVX-512, 3 DSA, 3 IAA, 3 QAT 4xxx) with 4x Arc Pro B70 on `xe`. A plain `up` ran there on 2026-10-20 [ledger:H52], and `up` after `k3d-gpu host-setup --apply` on 2026-10-29 [ledger:H52b]. The third is wildct, an i7-13700 (P-cores 0-15, E-cores 16-23) with a UHD 770 iGPU and 2 NVIDIA GPUs, all three opted out by the box owner (`K3D_GPU_VENDORS=auto,-nvidia,-intel`), so only its CPU rows - have run. A plain `up` of release 0.1.89 ran there on 2026-11-07 [ledger:H52w]. + have run. A plain `up` of release 0.1.89 ran there on 2026-11-07, and the strict `test` + passed: its P-core pod ran on CPU 0 and its E-core pod on CPU 17 [ledger:H52w]. - **Verified by sim**: `tests/detect_test.sh` builds a fake sysfs/devfs tree, sets `K3D_GPU_SYSROOT`, and runs `detect`, `doctor` or `test` against it, with `docker`, `k3d` and `kubectl` stubbed. 1028 cases pass (2026-10-20). Sim covers detection, @@ -215,7 +216,7 @@ That split, orchestration and serving on fast cores with agent tools on many sma is the placement k3d-gpu sets with one pod label, not isolation: package power, LLC and DRAM bandwidth stay shared, so E-core load still slowed CPU-served prefill on the P-cores [ledger:H3]. Sandboxes on E-cores behind default-deny egress is SUPPORTED on the -i9-13900T [ledger:H18]. No Xeon run backs it yet. +i9-13900T [ledger:H18] and on wildct's i7-13700 [ledger:H18w]. No Xeon run backs it yet. ### EAT by default vs with k3d-gpu @@ -234,7 +235,7 @@ and installs through EAS. | Hosts | Xeon. Minimum 48 cores and 32 GB RAM, recommended 96+ cores and 64 GB; Docker mode asks for 96+ cores and 128 GB | Any Intel host it detects: client Core and Core Ultra, Xeon, Arc, Gaudi (detection and config fixture-tested per family, [platforms.md](platforms.md)). EAT ran end to end on one i9-13900T with no AMX and no AVX-512, n=1 [ledger:H1b] | | OS | Ubuntu 22.04 / 24.04, x86_64 | Any Linux with Docker that runs k3d; `host-setup` prints pinned driver steps for Ubuntu 24.04/26.04, Fedora, Arch and Manjaro | | Cluster setup | Kubespray onto the host, or brownfield on an existing cluster; the `ei-*-eligible` labels come only from the Kubespray path | One `k3d-gpu up`; `K3D_GPU_NODE_LABELS` sets EAT's labels, `K3D_GPU_BALLOONS=external` hands the NRI policy to EAT [ledger:H14] | -| EAT 2.0 install | EAS `es_auto_installer.sh install agent-toolkit`, onto Kubespray or an existing cluster (`existing_kubernetes`) | `k3d-gpu eat up` (`eas up --agent-toolkit`): the existing-cluster mode at the GA tags (EAS `v1.2.0`, EAT `release-2.0.0`), with the k3d settings and patches chosen from the cluster ([intel/eas.md](intel/eas.md)). F5 checks it on the Xeon 6741P. F3 (EAS `main`) is REFUTED ([run](../EXPERIMENTS.md#f3-run-i9-13900t-2026-11-01)): it installed, checked and tore down, and the secret scan found the Keycloak admin password on a command line | +| EAT 2.0 install | EAS `es_auto_installer.sh install agent-toolkit`, onto Kubespray or an existing cluster (`existing_kubernetes`) | `k3d-gpu eat up` (`eas up --agent-toolkit`): the existing-cluster mode at the GA tags (EAS `v1.2.0`, EAT `release-2.0.0`), with the k3d settings and patches chosen from the cluster ([intel/eas.md](intel/eas.md)). F5 ran it once on the Xeon 6741P and is SUPPORTED ([run](../EXPERIMENTS.md#f5-run-xeon-6741p-2026-11-07)). F3 (EAS `main`) is REFUTED ([run](../EXPERIMENTS.md#f3-run-i9-13900t-2026-11-01)): it installed, checked and tore down, and the secret scan found the Keycloak admin password on a command line | | Engines | vLLM CPU (`vllm/vllm-openai-cpu:v0.24.0`, bf16), 9 model profiles in the single-node table (7 chat, 1 embedding, 1 reranker). An OVMS Helm chart ships but no default path deploys it: `openvino/model_server:2025.4`, plus an unpinned `latest-py` tag for its model-export containers | `agent up` picks per host: vLLM on a Gaudi, OVMS 2026.4.1 on an XMX GPU, else OVMS on the P-cores, or, on a CPU class where an engine-choice row is SUPPORTED (the Xeon 6741P today), llama.cpp with the model's int4 GGUF (for Gemma 4 26B-A4B, the one model measured, Google's QAT Q4_0 GGUF and QAT MTP drafter) [ledger:H59]; vLLM XPU (`v0.32.0`) or llm-scaler on Arc by flag. Its images are pinned by digest in `images.lock` | | Accelerators | None requested: no `gpu.intel.com`, `habana.ai` or `/dev/dri` on the default path | Intel GPU, NPU, Gaudi, DSA, IAA and QAT plugins started per detected device, and `up` checks that each is schedulable (on the Xeon 6741P it warned when QAT advertised none [ledger:H52b]) | | CPU placement | NRI balloons sized for Xeon NUMA; on a hybrid CPU EAT's reservation needs the patch in [`contrib/eat/`](../contrib/eat) | Balloons baked for hybrid P/E and multi-NUMA hosts; vLLM on P-cores and agent services on E-cores through `contrib/eat/` values | @@ -268,9 +269,11 @@ stated as the ledger graded it; only the SUPPORTED rows back a claim. | H51 | Graded agents per box: neither CPU stack served one graded agent within the 10 s SLO ([run](../EXPERIMENTS.md#h51-run-xeon-6741p-2026-10-29)) | REFUTED | Detection (H52 REFUTED, H52b REFUTED) is in the [capability matrix](#capability-matrix). -H54d and H54q are in the table below. After the campaign, H62r repeats H62 at Intel's GA -tags of 2026-11-06 (EAS v1.2.0 installing EAT `release-2.0.0` with enterprise-inference -v1.1.0); it is pre-registered and not yet run +H54d and H54q are in the table below. After the campaign, H62r, in the table above, repeated +H62 at Intel's GA tags of 2026-11-06 (EAS v1.2.0 installing EAT `release-2.0.0` with +enterprise-inference v1.1.0). It ran on 2026-11-07 with EAS's documented +`nri_reserved_cpu_list` set to the guest's sibling vCPUs, because in a KVM guest of that shape +EAS's topology probe reads one thread per core ([pre-registration](../EXPERIMENTS.md#h62r-pre-registration-h62-at-the-ga-tags-eas-v120-with-eat-20-on-the-xeon-6741p-2026-11-06)). H54 rows on that host: Gemma 4 26B-A4B (QAT weights) on the Xeon CPU, and split with a B70 @@ -299,8 +302,9 @@ in its registered order. Each row is stated as the ledger graded it. | [ledger:H25w] | H25 replicated: with OVMS's shipped prefix caching on, an added agent's first chunk arrived 14.3x sooner than with it off (CI 13.38-14.89), under matched `typical` load ([run](../EXPERIMENTS.md#h25w-run-wildct-i7-13700-2026-11-07)) | SUPPORTED | | [ledger:H34w] | H34 replicated: OVMS's CPU Fast Draft against the 4B target alone under matched `typical` load. Neither the pre-registered gain nor its exclusion was resolved, so the i7-13700 class gets no CPU draft by default ([run](../EXPERIMENTS.md#h34w-run-wildct-i7-13700-2026-11-07)) | INCONCLUSIVE | -H28w, H57wa, H57wm, H57wb and the engine-choice row H59w are pre-registered there and not -yet run. +H28w has run there; its result is in review +([#344](https://github.com/88plug/k3d-gpu/pull/344)) and not yet in the ledger. H57wa, H57wm, +H57wb and the engine-choice row H59w are pre-registered there and not yet run. Facts from `0ecf67a` that shape a k3d run: diff --git a/docs/intel/eas.md b/docs/intel/eas.md index ebcdced8..2236d86b 100644 --- a/docs/intel/eas.md +++ b/docs/intel/eas.md @@ -220,7 +220,7 @@ Open EAS PRs that change a decision when they merge (`overlay.py --watch`, check | EAS PR | Effect on the overlay | |---|---| -| [#50](https://github.com/intel/enterprise-ai-solutions/pull/50) and [#52](https://github.com/intel/enterprise-ai-solutions/pull/52) (OpenShift) | 0006 no longer applies: the selector moves into `_eg_k8s_provider`, still `control-plane: ""`. That is a STOP naming #50. Once a k3s vars file lands, 0006 becomes drop-fixed | +| [#50](https://github.com/intel/enterprise-ai-solutions/pull/50) and [#52](https://github.com/intel/enterprise-ai-solutions/pull/52) (OpenShift) | 0006 no longer applies: the selector moves into `_eg_k8s_provider`, still `control-plane: ""`. That is a STOP naming #50. Once a k3s vars file lands, 0006 becomes drop-fixed. #50 merged into EAS `main` on 2026-11-02 and #52 is open (read 2026-11-07); neither is in `v1.2.0`, so only a move of the `main` profile past #50 meets the STOP | | [#61](https://github.com/intel/enterprise-ai-solutions/pull/61) (Headlamp SSO), merged 2026-10-30 | 0002 fails on its `global_config.yaml` hunk context: a STOP naming #61, so 0002 needs regenerating. 0004 (skipped on one node) and 0005 (docs, never applied) fail there too. 0006 still applies. EAS `main` is `0f7bc2b` since (read 2026-11-01); the pin stays `97f6157`, where the decisions above hold, and moving it past #61 needs 0002 regenerated first (read from #61's head, not re-checked on the merge) | ## Pins diff --git a/docs/intel/partner-update-2026-11.md b/docs/intel/partner-update-2026-11.md index 48e44f19..d9ab2ebe 100644 --- a/docs/intel/partner-update-2026-11.md +++ b/docs/intel/partner-update-2026-11.md @@ -35,7 +35,7 @@ F3's Keycloak finding does not arise on this path; the inference layer on its ow 8 E-cores, AVX-VNNI, no AVX-512 or AMX) with a UHD 770 iGPU and two NVIDIA cards. The box owner opts all three GPUs out, so its rows are CPU rows. Each ran on release 0.1.89 under the [wildct pre-registration](../../EXPERIMENTS.md#wildct-pre-registration-client-raptor-lake-i7-13700-2026-10-20). -Four rows have run there so far, and three of them are SUPPORTED: +Four rows there have a verdict, and three of them are SUPPORTED: - Detection: a plain `k3d-gpu up` found the hybrid P/E cores and the six ISA extensions the launcher labels, and nothing the host lacks. `k3d-gpu test` put its P-core pod on CPU 0 and its E-core pod on CPU 17, and `up` named both GPU vendors as detected and opted out [ledger:H52w]. - Agent sandboxes, H18 replicated: in 3 of 3 runs `k3d-gpu sandbox up` kept the sandbox on an E-core behind default-deny egress. The host's tailscale and LAN addresses were refused from the sandbox, and PyPI was reachable [ledger:H18w]. @@ -43,15 +43,34 @@ Four rows have run there so far, and three of them are SUPPORTED: The fourth, H34w, is INCONCLUSIVE: with H34 replicated there under matched `typical` load, OVMS's CPU Fast Draft against the target alone resolved neither the pre-registered gain nor its -exclusion, so the i7-13700 class gets no CPU draft by default. The rest of the host's rows -(H28w, H57wa, H57wm, H57wb and the engine-choice row H59w) are pre-registered and not yet run. -Until a row for the i7-13700 class says otherwise, that class keeps OVMS as its CPU engine. - -**H62 at Intel's GA tags is pre-registered.** H62r repeats H62 on the GA tags: k3d-gpu's CPU stack against Intel's -documented single-node default (EAS v1.2.0 installing EAT 2.0 with Kubespray), each in its own -KVM guest of one shape on the Xeon 6741P, inside EAT's banking flow on Gemma 4 26B-A4B's QAT -weights. Its predictions are H62's, copied verbatim +exclusion, so the i7-13700 class gets no CPU draft by default. A fifth, H28w (H28's shared +page cache for models mounted as image volumes), has run; its result is in review +([#344](https://github.com/88plug/k3d-gpu/pull/344)) and not yet in the ledger. H57wa, H57wm, +H57wb and the engine-choice row H59w are pre-registered and not yet run. Until a row for the +i7-13700 class says otherwise, that class keeps OVMS as its CPU engine. + +**Intel's documented EAT 2.0 install, at the GA tags, finished in a KVM guest and served EAT's +banking flow as well as k3d-gpu's CPU stack.** H62r repeated H62 at the GA tags: Intel's +documented single-node default (EAS v1.2.0 installing EAT 2.0 and enterprise-inference v1.1.0 +with Kubespray, gemma-4-26b on vLLM CPU v0.28.0) against k3d-gpu's CPU stack (OVMS with an int4 IR +of the same Gemma 4 26B-A4B QAT weights), each in its own KVM guest of one shape on the Xeon 6741P +([H62r run](../../EXPERIMENTS.md#h62r-run-xeon-6741p-2026-11-07)): + +- Intel's install finished in its guest in one attempt, all three PLAY RECAPs at `failed=0`. EAS's documented operator setting `nri_reserved_cpu_list` ([nri_cpu_balloons.md#L71-L74](https://github.com/intel/enterprise-ai-solutions/blob/a051f00fcb2dd473fe763610471cd509db82c433/docs/customize/nri_cpu_balloons.md#L71-L74)) named the guest's 48 sibling vCPUs, so `cpu: auto` sized gemma-4-26b to 40 CPUs and EAT's sandbox router was placed. In H62 that router could not be scheduled. +- `validate agent-toolkit` passed, and a chat, a tool call, the guardrail's block and a sandbox run went through EAS's gateway. +- In 24 interleaved pairs of EAT's banking tasks, each stack finished 21 of 24 within the turn budget. The paired difference was 0.000 (CI -0.125 to +0.125), and every gate passed. + +The row predicted k3d-gpu's stack far ahead, so the ledger grades it REFUTED. For EAT users the +reading is simple: on this Xeon, Intel's GA install, run as Intel documents it, does as well in +EAT's own banking flow as k3d-gpu's own CPU stack. + +One finding from the run is for EAS: in a KVM guest whose kernel numbers a core's two threads +next to each other, EAS's topology probe counts the comma fields of `thread_siblings_list` +([probe-node-topology.sh#L30-L31](https://github.com/intel/enterprise-ai-solutions/blob/a051f00fcb2dd473fe763610471cd509db82c433/roles/nri_cpu_balloons/files/probe-node-topology.sh#L30-L31)) +and reads one thread per core. EAS then reserves 12 CPUs instead of the 48 siblings, which is how +H62's install came to size the model to 76 CPUs ([H62r pre-registration](../../EXPERIMENTS.md#h62r-pre-registration-h62-at-the-ga-tags-eas-v120-with-eat-20-on-the-xeon-6741p-2026-11-06)). +The reserve list above is the documented way around it. **Pins move on their own, and each move is checked before it merges.** One registry, `pins/registry.toml`, now owns every version, digest and model revision that k3d-gpu's own @@ -59,7 +78,8 @@ files and its EAT stock path pin. The Freshness workflow proposes each move to u newest stable release, every 6 hours, as one pull request per group ([pins/README.md](../../pins/README.md)). - The merge policy is computed from the diff. A patch or minor move of a plugin or a CPU engine merges on its own once CI and the canary pass. A minor move of a component that a SUPPORTED ledger row measured (OVMS and llama.cpp today) waits for a person, as do a major, a host driver and an EAT stock pin. -- The Canary workflow runs nightly and on every pin change ([canary/README.md](../../canary/README.md)). It serves a pinned Qwen3-0.6B on vLLM CPU, OVMS and llama.cpp the way the launcher runs them. It checks every flag and environment variable the launcher emits against the vLLM XPU, llm-scaler, Gaudi vLLM, llama.cpp SYCL and Vulkan, DLStreamer, sglang Xeon, NPU UMD and OVMS images, and it starts the node image the way `up` does. It runs on GitHub-hosted runners, so it needs no device. +- A pin move also moves the test goldens and fixtures that record the old value, so a bump's pull request does not fail on a stale fixture ([#328](https://github.com/88plug/k3d-gpu/pull/328)). +- The Canary workflow runs nightly and on every pin change ([canary/README.md](../../canary/README.md)). It serves a pinned Qwen3-0.6B on vLLM CPU, OVMS and llama.cpp in a plain container with the launcher's arguments, not in a pod. It checks every flag and environment variable the launcher emits against the vLLM XPU, llm-scaler, Gaudi vLLM, llama.cpp SYCL and Vulkan, DLStreamer, sglang Xeon, NPU UMD and OVMS images, and it starts the node image the way `up` does. It runs on GitHub-hosted runners, so it needs no device. - The hourly bot sweep merges one such pull request at a time, only while main's CI is green, and then dispatches one batched release. - A pre-registered row's arms stay put. A PENDING row's runner pins move only by a dated ledger amendment, so H59w's registered llama.cpp build and drafter-conversion packages are held. EAT 1.x's pin holds below `release-2.0.0`, which removed the installer `eat-up.sh` drives; EAT 2.0 goes through `k3d-gpu eas`. @@ -132,22 +152,24 @@ SUPPORTED rows, each scoped to its host: - On the i9-13900T: sixteen rows, among them EAT `release-1.0.3-hotfix` deployed and served on a default k3d-gpu cluster, n=1 [ledger:H1b], EAT's NRI balloons chart installing over a k3d-gpu cluster [ledger:H14], and agent sandboxes behind default-deny egress [ledger:H18]. The full list is in the [technical report](technical-report.md#supported-quotable-scoped-to-the-i9-13900t-h25x-h59-and-h60-to-the-xeon-6741p-h52w-h18w-and-h25w-to-wildcts-i7-13700). - On wildct, an i7-13700 client host: detection of its hybrid cores and ISA extensions with its GPUs opted out [ledger:H52w], H18 replicated, agent sandboxes on its E-cores behind default-deny egress [ledger:H18w], and H25 replicated, an added agent's first chunk 14.3x sooner with OVMS's shipped prefix caching on [ledger:H25w]. -## Two results that help EAT +## Three results that help EAT -Both are stated as the ledger graded them. Inside EAT's banking flow -on the Xeon 6741P, EAT 1.1.0's stock vLLM CPU arm delivered at least as many tasks within -the turn budget as k3d-gpu's CPU stack (H50 REFUTED), and EAT 1.1.0's move to vLLM CPU -v0.28.0 is past the v0.24.0 CPU attention hang that stalled EAT 1.0.3's stock arm in H54a. +Each is stated as the ledger graded it. Inside EAT's banking flow on the Xeon 6741P: + +- At the GA tags, Intel's documented install of EAS v1.2.0 with EAT 2.0 finished in a KVM guest and delivered as many tasks within the turn budget as k3d-gpu's CPU stack, 21 of 24 each (H62r REFUTED, above). +- EAT 1.1.0's stock vLLM CPU arm delivered at least as many tasks within the turn budget as k3d-gpu's CPU stack (H50 REFUTED). +- EAT 1.1.0's move to vLLM CPU v0.28.0 is past the v0.24.0 CPU attention hang that stalled EAT 1.0.3's stock arm in H54a. ## What is still open - **The Keycloak argv secret.** F3 stays REFUTED. With the agent toolkit at the GA tags (`auth_provider: litellm`), EAS installs no Keycloak, and F5 found no Keycloak secret; the inference layer on its own keeps Keycloak. `k3d-gpu eas up` now patches enterprise-inference's `keycloak_config` role so that neither the admin password nor the client secret is a `kcadm.sh` argument ([State and secrets](eas.md#state-and-secrets)). The patch is checked with a `ps` sampler against a stand-in for kcadm; no live run has checked it yet. A fresh `eas up --agent-toolkit` with F3's secret scan would. -- **Intel's default install in a guest (H62, H62r).** H62's install stopped where `cpu: auto` left too little CPU for EAT's sandbox router. H62r's pre-registration traces that, from source at the GA tags, to EAS's topology probe reading a KVM guest's threads as one per core, and re-runs the row at those tags. +- **EAS's topology probe in a KVM guest.** H62r answered H62's question at the GA tags, with EAS's documented reserve list set. The probe's reading of a guest's threads as one per core is unchanged at `v1.2.0`, and we have not drafted an EAS issue for it yet. The `cpu: auto` issue draft stands for an install whose reserve leaves the pods outside the balloons too little CPU. +- **The vLLM agent path on Arc and Gaudi.** In release 0.1.93, `k3d-gpu agent up` does not serve through vLLM. On an Arc Pro B70 the engine exits at start because the pod's Kubernetes service link sets `VLLM_PORT` to a URI, and the warm-up request carries no JSON content type. [#339](https://github.com/88plug/k3d-gpu/pull/339) fixes both, and with its pod spec vLLM v0.33.0 served Qwen3-4B on one B70; it is in review and in no release yet. The Gaudi path shares the manifest and has not run on hardware. On Arc, `agent up` with no backend set serves OVMS, which this does not affect; with `--model`, the fit can pick vLLM. - **k3d-gpu gaps found on the Xeon.** `/dev/dri/by-path` for oneCCL across B70s (H53); the fit refusing rather than falling back to the CPU, and no DRAM bandwidth cached on a new host (H58); placing one-card arms on one card (H54b, H54c). -- **Replications.** H59 on the i9-13900T (H59d) is INCONCLUSIVE: llama.cpp's arm was OOMKilled at the fit's pod limit and not deliverable in two holds, so no statistic was scored. H59 on the i7-13700 (H59w) is PENDING, as are the wildct client-host rows after H34w. H34 on the i7-13700 (H34w) is INCONCLUSIVE: the CPU Fast Draft's gain over the target alone was neither the pre-registered win nor ruled out, so that class keeps no default CPU draft. Until a row is SUPPORTED for its class, those CPU classes keep OVMS. +- **Replications.** H59 on the i9-13900T (H59d) is INCONCLUSIVE: llama.cpp's arm was OOMKilled at the fit's pod limit and not deliverable in two holds, so no statistic was scored. H59 on the i7-13700 (H59w) is PENDING, as are the wildct rows H57wa, H57wm and H57wb; H28w's result is in review. H34 on the i7-13700 (H34w) is INCONCLUSIVE: the CPU Fast Draft's gain over the target alone was neither the pre-registered win nor ruled out, so that class keeps no default CPU draft. Until a row is SUPPORTED for its class, those CPU classes keep OVMS. - **EAT 1.1.0 through `eat-up.sh`.** Pinned since 2026-10-29 and pre-registered as F4; not run yet. -- **EAS upstream moves.** `k3d-gpu eas` defaults to the GA tags; the `main` profile (F3) and `v1.2.0-rc.1` (H62) stay as recorded. When Intel tags the next EAS release, the `eas-release` pin moves the default profile, and the overlay's verified cases and config goldens for the new tree still need a person to add them. EAS #61 merged on 2026-10-30 (EAS `main` `0f7bc2b`). The `main` profile stays at `97f6157`; moving past #61 needs patch 0002 regenerated. EAS #50 and #52 (OpenShift) are open and would move 0006. EAS #71 (teardown hang) is open; `eas down` uses its workaround. -- **ERAG and Arc under EAS.** EAS's Enterprise RAG layer is not wired into `k3d-gpu eas` yet, and EAS has no Intel GPU path on this branch; Arc and Gaudi stay with `k3d-gpu agent up`. +- **EAS upstream moves.** `k3d-gpu eas` defaults to the GA tags; the `main` profile (F3) and `v1.2.0-rc.1` (H62) stay as recorded. When Intel tags the next EAS release, the `eas-release` pin moves the default profile, and the overlay's verified cases and config goldens for the new tree still need a person to add them. EAS #61 merged on 2026-10-30 (EAS `main` `0f7bc2b`). The `main` profile stays at `97f6157`; moving past #61 needs patch 0002 regenerated. EAS #50 (OpenShift) merged into EAS `main` on 2026-11-02 and #52 is open (read 2026-11-07); neither is in `v1.2.0`, and moving the `main` profile past #50 needs 0006 reworked. EAS #71 (teardown hang) is open; `eas down` uses its workaround. +- **ERAG and Arc under EAS.** EAS's Enterprise RAG layer is not wired into `k3d-gpu eas` yet, and EAS has no Intel GPU path on this branch; Arc and Gaudi stay with `k3d-gpu agent up` (see the vLLM item above). ## Related diff --git a/docs/intel/technical-report.md b/docs/intel/technical-report.md index 1d1eb63e..24601e1d 100644 --- a/docs/intel/technical-report.md +++ b/docs/intel/technical-report.md @@ -11,7 +11,7 @@ This report covers what was measured, and how. Every result but H52, H52b, H7x, - 22 hypotheses are SUPPORTED. Sixteen are scoped to that host; H25x (H25 replicated), H59 and H60 are scoped to the Xeon 6741P, and H52w, H18w and H25w to wildct, an i7-13700. H25w is the newest: H25 replicated on that host. With OVMS's shipped prefix caching on, an added agent's first chunk arrived 14.3x sooner than with it off (2.20 s vs 31.44 s, CI 13.38-14.89) under matched `typical` load; one of the six rounds was cut by the level's time cap [ledger:H25w]. Before it, H18w: H18 replicated on that host. In 3 of 3 runs `k3d-gpu sandbox up` from the release kept the sandbox on an E-core (CPU 17) behind default-deny egress: the host's tailscale and LAN addresses were refused from the sandbox and reached from a control pod, and `pip install --user` from PyPI worked [ledger:H18w]. Before it, H52w: on that client Raptor Lake host, with the box owner's opt-out of its two NVIDIA cards and its iGPU, a plain `k3d-gpu up` found the hybrid P/E cores and the six ISA extensions the launcher labels, and nothing the host lacks. It named both GPU vendors as detected and opted out, and no pod mounted or asked for a GPU [ledger:H52w]. Before it, H60: Qwen3.8-Flash-Next, served from the Xeon's DRAM by llama.cpp v0.8.0, decoded 16.0 tok/s at one stream and read 0.82 of a same-hold Gemma 4 QAT control's share of the DRAM bandwidth ceiling (CI 0.819-0.831, band 0.7-1.3), with GSM8K 50/50 [ledger:H60]. Before it, H59: on the Xeon 6741P at four concurrent requests, llama.cpp with Google's QAT Q4_0 GGUF and the QAT MTP drafter decoded Gemma 4 26B-A4B 3.50x faster per request than OVMS with OpenVINO's base-model int4 IR (95% CI 3.42-3.61), faster in each of 3 holds, at GSM8K 49 against 50 of 50. So with no backend set, `agent up` now serves llama.cpp on the CPU of that class [ledger:H59]. - 21 are REFUTED. Six of them changed shipped defaults. The newest, H62r, is H62 at the GA tags, each stack in a KVM guest of one shape on the Xeon 6741P. With EAS's documented `nri_reserved_cpu_list` naming the guest's sibling vCPUs, Intel's documented install of EAS v1.2.0 with EAT 2.0 finished, and inside EAT's banking flow it delivered as many tasks within the turn budget as the k3d-gpu CPU stack: 21 of 24 each, a paired difference of 0.00 [ledger:H62r]. Before it, H28x is H28 replicated on the Xeon 6741P: eight pods that mmap one 1 GiB model through an image volume again shared one page-cache copy (PSS 132.6 MiB per pod, 1.00x the size on the node against 9.00x for per-pod copies), but they were ready in 0.714 of the copy arm's time, not the 0.5 or less the row needed, so only the memory part replicated [ledger:H28x]. Before it, H7x is H7 run to its full protocol on the Xeon 6741P: at matched 16-bit weights (Qwen3-4B, 16 CPUs), OVMS 2026.4.1 decoded at 1.22x the rate of EAT 1.1.0's stock vLLM CPU engine (v0.28.0) in all five block pairs, so OVMS's decode lead is not only weight precision; vLLM prefilled the long prompt faster [ledger:H7x]. Before it, H58 is the zero-touch thesis on the Xeon 6741P, with the two serving pairs holding about half of every B70. With no flag, `k3d-gpu up` advertised three of the four B70s and no DSA work queue. `k3d-gpu agent up --model` then refused all three models. Gemma 4 26B-A4B and Qwen3.8-27B went to OVMS on the GPU, where their int4 IRs missed the best card's free-memory budget by 0.1 and 0.5 GiB. Qwen3.8-Flash-Next was refused because no DRAM bandwidth was cached. So EAT's banking flow never came up [ledger:H58]. Before it, H52b is H52 after `k3d-gpu host-setup --apply` on the Xeon 6741P: DSA was advertised and reached a pod, but IAA got no work queue, the QAT plugin advertised none of the VFs, and `up` advertised two of the four B70s: the serving pairs hold about half the VRAM of every card, and its busy-card filter (half or more) left out the two just over half and kept the two just under it [ledger:H52b]. Before it, H51 ran the graded agents-per-box ramp on the Xeon 6741P: the k3d-gpu CPU stack failed at one agent (turn p95 10.41 s against the 10 s SLO), and so did EAT's stock vLLM CPU arm [ledger:H51]. H50 ran inside EAT's banking flow on the same host: EAT's current stock vLLM CPU arm (release-1.1.0) delivered at least as many tasks within the turn budget as the k3d-gpu CPU stack: 21 of 24 against 19 of 24 [ledger:H50]. - 35 are INCONCLUSIVE, mostly because the shared host throttles or its load was unmatched. The newest, H34w, is INCONCLUSIVE: with H34 replicated on wildct's i7-13700 under matched `typical` load, the CPU Fast Draft decoded 1.064x the target alone (CI 0.984-1.208) at 0.951x its package J/token (CI 0.848-1.025), so neither the pre-registered win nor its exclusion was resolved, and that CPU class keeps no default draft [ledger:H34w]. H62 (the k3d-gpu CPU stack against Intel's new default, EAS v1.2.0-rc.1 installing EAT 2.0, each in a KVM guest on the Xeon 6741P) is INCONCLUSIVE because Intel's own install did not finish: enterprise-inference's `cpu: auto` sized the Gemma 4 pod to 76 of the guest's 95.4 allocatable CPUs, so EAT's sandbox router could not be scheduled, twice, and no hold ran. H62r ran the same design at the GA tags and is REFUTED (above). H54q (the QAT-matched int4 IR of Gemma 4 26B-A4B against the bf16 QAT weights and OpenVINO's base-model IR, on GSM8K items 0-199 on the Xeon 6741P) is INCONCLUSIVE: the QAT-matched IR stayed inside the pre-registered margin of the bf16 reference, but the base-model IR scored no lower than it, with every build near the item set's ceiling. H50o (the k3d-gpu CPU stack against EAT's own OVMS chart on the Xeon 6741P) is INCONCLUSIVE because that chart's OVMS 2025.4 crashed on loading Gemma 4 26B-A4B, so no pair ran. H54a and H54d (Gemma 4 26B-A4B QAT on the Xeon 6741P) are INCONCLUSIVE because the stock arm, EAT 1.0.3's vLLM CPU v0.24.0, as this harness rendered EAT's entry (no CPU balloon, requests or limits), stalled in both of H54a's holds, so no arm could be scored against it. EAT 1.1.0's vLLM CPU v0.28.0 was not measured in H54a; H50 ran it in EAT's banking flow. H54b and H54c (one Arc Pro B70 against two, and experts in DRAM with the rest on one B70) are INCONCLUSIVE because their "one card" pods saw all four cards, so no arm was delivered as pre-registered. H53 (EAT's banking flow on the B70s: k3d-gpu's one-card OVMS pick against EAT 1.1.0's vLLM XPU path across all four cards) is INCONCLUSIVE because the EAT arm did not start in a k3d-gpu cluster: oneCCL needs `/dev/dri/by-path` for its cross-card collective, and the k3d node has none, so no hold ran. H57da (k3d-gpu's OVMS pick vs llama.cpp for Gemma 4 QAT on the i9-13900T) did not resolve: the strictly pinned llama.cpp arms tracked host load; H57db did not run (memory). H57dp and H57de are INCONCLUSIVE: they re-ran that comparison with an unpinned llama.cpp arm, and the load gate failed on OVMS's low foreign P-core load (0.45 CPU-s/s against the llama.cpp arms' 3.1-3.4); the unpinned arm was 1.108x the pinned one and 1.369x OVMS, not gated, and OVMS ran without its P-core balloon. H55 (SLO-gated admission of an E-core neighbour on the i9-13900T) is INCONCLUSIVE: all 25 runs completed on a busy host, but in one round the controller's run closed three request windows, one short of its gate; not gated, the controller held the neighbour at its lowest quota in four of five rounds, and no arm kept prefill within the row's bar of quiet. H59d (the engine choice on the i9-13900T: llama.cpp with Google's QAT Q4_0 GGUF and the QAT MTP drafter against OVMS with OpenVINO's base-model int4 IR) is INCONCLUSIVE because the llama.cpp arm was not deliverable in two of its four holds: in both holds where it followed OVMS, its server was OOMKilled at the fit's 45Gi pod limit (4 x 262,144 tokens), while the same render served in both holds where it ran first. Not gated, in the one performance hold it served it decoded 1.05x OVMS per request at four concurrent requests, and GSM8K read 50 against 49 of 50. So with no backend set, the i9-13900T keeps OVMS. H56 (two holds at once in cpuset slices of the Xeon 6741P) is INCONCLUSIVE: in every hold with a sibling, the sibling's banking block ended shortly after the measured block began, so all four were VOID and no interference statistic was scored; crescent rows stay at one hold at a time. -- 5 are PENDING: the engine-choice row H59w (i7-13700), and the second client host's (wildct, i7-13700) rows H28w, H57wa, H57wm and H57wb, pre-registered and not yet run. H52, the first row run on that host (a Xeon 6741P with four Arc Pro B70s), is REFUTED. H27x, H27 re-run on the Xeon (AMX), is REFUTED as on the i9. H34x, H34 re-run there, is REFUTED: the CPU Fast Draft that sped up the i9 slowed decode on the Xeon. +- 5 are PENDING, all on wildct's i7-13700: the engine-choice row H59w and the rows H28w, H57wa, H57wm and H57wb. H28w has run there, and its result is in review ([#344](https://github.com/88plug/k3d-gpu/pull/344)), not yet in the ledger; the other four are pre-registered and not yet run. H52, the first row run on the Xeon 6741P with four Arc Pro B70s, is REFUTED. H27x, H27 re-run on the Xeon (AMX), is REFUTED as on the i9. H34x, H34 re-run there, is REFUTED: the CPU Fast Draft that sped up the i9 slowed decode on the Xeon. - Every other Intel family is detected and configured in fixture tests only. None has a measurement. No world-first, superlative or priority claim is made; [docs/provenance.md](../provenance.md) supports none. @@ -34,7 +34,7 @@ A verdict is one of `PENDING`, `SUPPORTED`, `REFUTED` or `INCONCLUSIVE`. Most ro - a sourced provenance row; - a primary-source URL. -`tests/ledger_drift_check.py` also runs in CI. It fails when a doc states a verdict for a row (`H2 is REFUTED`, `PENDING [ledger:H1]`, a verdict table cell or a verdict section heading) that differs from the ledger, and when a verdict count in this report's summary is off. +`tests/ledger_drift_check.py` also runs in CI. It fails when a doc states a verdict for a row (`H2 is REFUTED`, `PENDING [ledger:H1]`, a verdict table cell or a verdict section heading) that differs from the ledger, when a doc calls a row the ledger has decided pre-registered or not yet run, and when a verdict count in this report's summary is off. ### Matched-load design @@ -149,11 +149,11 @@ The external bake-offs are in the tables above: the Ollama, llama.cpp and plain ## Limitations -1. **Three hosts.** Every measurement comes from the i9-13900T, the Xeon 6741P (Granite Rapids, AMX) with 4x Arc Pro B70, or wildct's i7-13700, and the abstract names which rows ran on the Xeon and on wildct. No result transfers to another host or family without its own row. On the Xeon, the Gemma 4 microbench row H54a has no scored stock arm, the B70 rows H54b and H54c ran with a placement failure (every pod saw all four cards), and H62r is pre-registered and not yet run. On wildct, whose GPUs the box owner opted out, only CPU rows run. +1. **Three hosts.** Every measurement comes from the i9-13900T, the Xeon 6741P (Granite Rapids, AMX) with 4x Arc Pro B70, or wildct's i7-13700, and the abstract names which rows ran on the Xeon and on wildct. No result transfers to another host or family without its own row. On the Xeon, the Gemma 4 microbench row H54a has no scored stock arm, the B70 rows H54b and H54c ran with a placement failure (every pod saw all four cards), and H62 and H62r ran each stack in its own KVM guest of one shape, not on the host itself. On wildct, whose GPUs the box owner opted out, only CPU rows run. 2. **Shared, throttling host.** The host is live and shared, and the package throttles under the probes' own load. Most INCONCLUSIVE verdicts come from this. H22 throttles itself, so it is barred from this host. 3. **The XMX half of the device rule is untested.** H6 supports "stay on the P-cores when the iGPU has no XMX". Whether the iGPU wins when it has XMX is an open prediction. 4. **Small n.** Most SUPPORTED rows are n=3 per arm, so they show direction and correctness, not effect-size precision. -5. **EAT end to end is one run.** H1b deployed and served the full EAT stack once on this host. H1 stays INCONCLUSIVE: its wait could not pass on k3s v1.37. Nothing is known for other client CPUs. +5. **EAT end to end, one run per path.** H1b deployed and served EAT `release-1.0.3-hotfix` once on the i9-13900T. On the Xeon 6741P, F5 installed and checked EAT 2.0 at Intel's GA tags once through `k3d-gpu eat` ([F5 run](../../EXPERIMENTS.md#f5-run-xeon-6741p-2026-11-07)), and H62r's Intel arm ran Intel's documented install of the same tags once, in a KVM guest [ledger:H62r]. H1 stays INCONCLUSIVE: its wait could not pass on k3s v1.37. Nothing is known for other client CPUs. 6. **Power envelope.** The i9-13900T is a 35 W T-SKU by rating only. Firmware sets PL1 = PL2 = 117 W, so every result from that host, H22's throttle included, is at that envelope. No i9-13900T result transfers to a T-SKU held at Intel's 35 W / 106 W limits. wildct's i7-13700 ran H25w with its package held at a 35 W PL1 [ledger:H25w]. ## Reproduce diff --git a/docs/intel/upstream-eat.md b/docs/intel/upstream-eat.md index 3eb5a985..1bbad419 100644 --- a/docs/intel/upstream-eat.md +++ b/docs/intel/upstream-eat.md @@ -434,7 +434,13 @@ install (EAS `v1.2.0-rc.1` with Kubespray in a 96-vCPU, one-NUMA KVM guest) gave `gemma-4-26b` 76 of 95.4 allocatable CPUs and Qwen3Guard 8; the rest of the stack requested about 11.2. EAT 2.0's sandbox router, applied after the models, then stayed Pending with `Insufficient cpu`, in two of two attempts -([H62 run](../../EXPERIMENTS.md#h62-run-xeon-6741p-2026-11-01)). +([H62 run](../../EXPERIMENTS.md#h62-run-xeon-6741p-2026-11-01)). In H62r, at the GA tags in +the same guest shape, EAS's documented `nri_reserved_cpu_list` named the guest's 48 sibling +vCPUs; `gemma-4-26b` was sized to 40 CPUs and the router was placed +([H62r run](../../EXPERIMENTS.md#h62r-run-xeon-6741p-2026-11-07)). So the gap shows where the +reserve is small: one thread per core, or a KVM guest whose adjacent sibling numbering EAS's +topology probe reads as one thread per core +([probe-node-topology.sh#L30-L31](https://github.com/intel/enterprise-ai-solutions/blob/a051f00fcb2dd473fe763610471cd509db82c433/roles/nri_cpu_balloons/files/probe-node-topology.sh#L30-L31)). **Suggested fix:** subtract the CPU already requested on the node by non-balloon pods, and keep back the requests of the layers the installer has still to apply (the agent toolkit's diff --git a/docs/platforms.md b/docs/platforms.md index 9d94c161..eaf9a469 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -249,8 +249,12 @@ two ways, one per EAT generation. Its pins: F3 ran the `main` profile once end to end and is REFUTED ([EXPERIMENTS.md](../EXPERIMENTS.md#f3-run-i9-13900t-2026-11-01)): the install, `eas check` and the teardown passed, and the secret scan found the Keycloak admin password on a process - command line (enterprise-inference's `keycloak_config` role). H62r is pre-registered on the - GA tags. + command line (enterprise-inference's `keycloak_config` role). H62r ran Intel's own + documented install of the GA tags in a KVM guest on the Xeon 6741P: with EAS's documented + `nri_reserved_cpu_list` it finished, and in EAT's banking flow it delivered as many tasks + within the turn budget as k3d-gpu's CPU stack, 21 of 24 each. So H62r, which predicted + k3d-gpu's stack far ahead, is REFUTED + ([EXPERIMENTS.md](../EXPERIMENTS.md#h62r-run-xeon-6741p-2026-11-07)). - `docs/intel.md`, `contrib/eat/` and `integrations/eat/upstream/` read `main` at [`0ecf67a`](https://github.com/intel/enterprise-agent-toolkit/tree/0ecf67a8331e8b502ef41e08bf6d2881416178c5). From f6ad64cdbd626274c424b96ccecd18ca7de756f7 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sat, 7 Nov 2026 21:51:06 -0800 Subject: [PATCH 3/5] docs(intel): the partner update names release 0.1.94, which ships #339's vLLM fix #339 merged while this branch's CI ran, and release 0.1.94 carries it: the 0.1.94 launcher renders enableServiceLinks: false for the vLLM pod. The partner update's open item now says the vLLM agent path did not serve up to 0.1.93, that 0.1.94 ships the fix, and that a full agent up run of a release on a B70 is not recorded yet. The status line moves to release 0.1.94. --- docs/intel/partner-update-2026-11.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intel/partner-update-2026-11.md b/docs/intel/partner-update-2026-11.md index d9ab2ebe..15fa6345 100644 --- a/docs/intel/partner-update-2026-11.md +++ b/docs/intel/partner-update-2026-11.md @@ -1,6 +1,6 @@ # k3d-gpu partner update, November 2026 -Status: 2026-11-07. k3d-gpu release 0.1.93. Ledger: [EXPERIMENTS.md](../../EXPERIMENTS.md), +Status: 2026-11-07. k3d-gpu release 0.1.94. Ledger: [EXPERIMENTS.md](../../EXPERIMENTS.md), 83 rows. Only SUPPORTED rows are quoted as results; REFUTED and INCONCLUSIVE rows are stated as the ledger graded them. Every number here is in the ledger with its protocol and reproduce block, unless its line cites a merged pull request as the source. Nothing in this page has been sent to an `intel/*` repository. @@ -164,7 +164,7 @@ Each is stated as the ledger graded it. Inside EAT's banking flow on the Xeon 67 - **The Keycloak argv secret.** F3 stays REFUTED. With the agent toolkit at the GA tags (`auth_provider: litellm`), EAS installs no Keycloak, and F5 found no Keycloak secret; the inference layer on its own keeps Keycloak. `k3d-gpu eas up` now patches enterprise-inference's `keycloak_config` role so that neither the admin password nor the client secret is a `kcadm.sh` argument ([State and secrets](eas.md#state-and-secrets)). The patch is checked with a `ps` sampler against a stand-in for kcadm; no live run has checked it yet. A fresh `eas up --agent-toolkit` with F3's secret scan would. - **EAS's topology probe in a KVM guest.** H62r answered H62's question at the GA tags, with EAS's documented reserve list set. The probe's reading of a guest's threads as one per core is unchanged at `v1.2.0`, and we have not drafted an EAS issue for it yet. The `cpu: auto` issue draft stands for an install whose reserve leaves the pods outside the balloons too little CPU. -- **The vLLM agent path on Arc and Gaudi.** In release 0.1.93, `k3d-gpu agent up` does not serve through vLLM. On an Arc Pro B70 the engine exits at start because the pod's Kubernetes service link sets `VLLM_PORT` to a URI, and the warm-up request carries no JSON content type. [#339](https://github.com/88plug/k3d-gpu/pull/339) fixes both, and with its pod spec vLLM v0.33.0 served Qwen3-4B on one B70; it is in review and in no release yet. The Gaudi path shares the manifest and has not run on hardware. On Arc, `agent up` with no backend set serves OVMS, which this does not affect; with `--model`, the fit can pick vLLM. +- **The vLLM agent path on Arc and Gaudi.** Up to release 0.1.93, `k3d-gpu agent up` did not serve through vLLM. On an Arc Pro B70 the engine exited at start because the pod's Kubernetes service link set `VLLM_PORT` to a URI, and the warm-up request carried no JSON content type. [#339](https://github.com/88plug/k3d-gpu/pull/339) fixes both and ships in release 0.1.94; with its pod spec and warm-up request, vLLM v0.33.0 served Qwen3-4B on one B70. A full `agent up` run of a release on a B70 is not recorded yet. The Gaudi path shares the manifest and has not run on hardware. On Arc, `agent up` with no backend set serves OVMS, which this did not affect; with `--model`, the fit can pick vLLM. - **k3d-gpu gaps found on the Xeon.** `/dev/dri/by-path` for oneCCL across B70s (H53); the fit refusing rather than falling back to the CPU, and no DRAM bandwidth cached on a new host (H58); placing one-card arms on one card (H54b, H54c). - **Replications.** H59 on the i9-13900T (H59d) is INCONCLUSIVE: llama.cpp's arm was OOMKilled at the fit's pod limit and not deliverable in two holds, so no statistic was scored. H59 on the i7-13700 (H59w) is PENDING, as are the wildct rows H57wa, H57wm and H57wb; H28w's result is in review. H34 on the i7-13700 (H34w) is INCONCLUSIVE: the CPU Fast Draft's gain over the target alone was neither the pre-registered win nor ruled out, so that class keeps no default CPU draft. Until a row is SUPPORTED for its class, those CPU classes keep OVMS. - **EAT 1.1.0 through `eat-up.sh`.** Pinned since 2026-10-29 and pre-registered as F4; not run yet. From 4be955583829e5b6feee0a534ce374fb9adebd93 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sat, 7 Nov 2026 22:16:50 -0800 Subject: [PATCH 4/5] tests: the not-run form reads its phrases in any case, so a table cell "Not yet run" is checked A status cell such as "| H2 | Not yet run |" starts with a capital, and the guard matched the phrases in lower case only, so that cell passed for a decided row. The phrases now match in any case. The new test failed before the change. No doc in the repo changes its result: 43 files agree with 83 ledger rows. --- tests/ledger_drift_check.py | 9 +++++---- tests/ledger_drift_check_test.py | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/ledger_drift_check.py b/tests/ledger_drift_check.py index 23500f4e..0632b235 100644 --- a/tests/ledger_drift_check.py +++ b/tests/ledger_drift_check.py @@ -23,9 +23,10 @@ must equal the ledger's count for that verdict. not run "H62r is pre-registered and not yet run", "rows H28w and H57wa, pre-registered and not yet run": every id before "not yet run", "not - run yet" or "is/are pre-registered" in the same clause (no '.', ';' or - '|' between; a paragraph's line breaks allowed), or the one id of a - table row, must be PENDING in the ledger. Docs only, not scripts. + run yet" or "is/are pre-registered", in any case, in the same clause (no + '.', ';' or '|' between; a paragraph's line breaks allowed), or the one + id of a table row ("| H28w | Not yet run |"), must be PENDING in the + ledger. Docs only, not scripts. A row id may carry one lowercase letter (H1b, a re-run row of H1). @@ -50,7 +51,7 @@ LIST_LINE = re.compile(r"^\s*(?:[-*]\s+)?\**(" + V + r")\b[^:]{0,40}:") COUNT = re.compile(r"\b(\d+|" + "|".join(NUMBERS) + r")\s+(?:hypotheses\s+|rows\s+)?(?:are|is)\s+(" + V + r")\b", re.IGNORECASE) -NOT_RUN = re.compile(r"\b(?:not yet run|not run yet|(?:is|are) pre-registered)\b") +NOT_RUN = re.compile(r"\b(?:not yet run|not run yet|(?:is|are) pre-registered)\b", re.IGNORECASE) BULLET = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s") SENTENCE_END = re.compile(r"\.(?=\s|$)") diff --git a/tests/ledger_drift_check_test.py b/tests/ledger_drift_check_test.py index 684aab37..e15645e4 100644 --- a/tests/ledger_drift_check_test.py +++ b/tests/ledger_drift_check_test.py @@ -206,6 +206,10 @@ def test_not_yet_run_in_a_bullet(self): def test_not_yet_run_in_a_table_row(self): self.assertDrift("| Row | State |\n|---|---|\n| H2 | not yet run |\n", r"guide.md:3: H2 reads not yet run") + def test_not_yet_run_capitalized(self): + self.assertDrift("| Row | State |\n|---|---|\n| H2 | Not yet run |\n", r"guide.md:3: H2 reads not yet run") + self.assertDrift("| H3 | Pre-registered. Not run yet |\n", r"guide.md:1: H3 reads not yet run.*SUPPORTED") + def test_lettered_id_reference(self): rc, out = run("Status: PENDING [ledger:H1b].\n", LEDGER.replace( "| H2 | two", "| H1b | one, re-run | p | probe | SUPPORTED |\n| H2 | two")) From 982e1e172ac25e32005b691a74b1330a99652d5b Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sat, 7 Nov 2026 22:17:56 -0800 Subject: [PATCH 5/5] docs(intel): the partner update scopes H62r as the ledger does and names the vLLM image 0.1.94 ships The H62r paragraph said Intel's GA install ran "as Intel documents it". The ledger records that the install ran with the reserve list set (D3) and that the harness sent Intel's arm its turns at the model's Service, past EAS's LiteLLM gateway and guardrail (D4), in one hold. The paragraph now says so. The reserve list is called EAS's documented setting that works around the probe, not a documented workaround. The vLLM agent item said that with #339's pod spec and warm-up, vLLM v0.33.0 served on a B70, next to "ships in release 0.1.94". That run applied the fix by hand on the v0.33.0 bump (#326, closed unmerged); 0.1.94 ships v0.32.0. The item now says both. --- docs/intel/partner-update-2026-11.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/intel/partner-update-2026-11.md b/docs/intel/partner-update-2026-11.md index 15fa6345..54b2a6aa 100644 --- a/docs/intel/partner-update-2026-11.md +++ b/docs/intel/partner-update-2026-11.md @@ -60,9 +60,11 @@ of the same Gemma 4 26B-A4B QAT weights), each in its own KVM guest of one shape - `validate agent-toolkit` passed, and a chat, a tool call, the guardrail's block and a sandbox run went through EAS's gateway. - In 24 interleaved pairs of EAT's banking tasks, each stack finished 21 of 24 within the turn budget. The paired difference was 0.000 (CI -0.125 to +0.125), and every gate passed. -The row predicted k3d-gpu's stack far ahead, so the ledger grades it REFUTED. For EAT users the -reading is simple: on this Xeon, Intel's GA install, run as Intel documents it, does as well in -EAT's own banking flow as k3d-gpu's own CPU stack. +The row predicted k3d-gpu's stack far ahead, so the ledger grades it REFUTED. For EAT users: on +this Xeon, Intel's GA install, with the documented reserve list set, did as well in EAT's own +banking flow as k3d-gpu's own CPU stack. The scope is the ledger's: one hold, both stacks in KVM +guests, and the harness sent Intel's arm its turns at the model's Service, past EAS's LiteLLM +gateway and guardrail, a deviation H62 named before it ran (D4). One finding from the run is for EAS: in a KVM guest whose kernel numbers a core's two threads next to each other, EAS's topology probe counts the comma fields of `thread_siblings_list` @@ -70,7 +72,7 @@ next to each other, EAS's topology probe counts the comma fields of `thread_sibl and reads one thread per core. EAS then reserves 12 CPUs instead of the 48 siblings, which is how H62's install came to size the model to 76 CPUs ([H62r pre-registration](../../EXPERIMENTS.md#h62r-pre-registration-h62-at-the-ga-tags-eas-v120-with-eat-20-on-the-xeon-6741p-2026-11-06)). -The reserve list above is the documented way around it. +Setting EAS's documented reserve list, as above, works around it. **Pins move on their own, and each move is checked before it merges.** One registry, `pins/registry.toml`, now owns every version, digest and model revision that k3d-gpu's own @@ -164,7 +166,7 @@ Each is stated as the ledger graded it. Inside EAT's banking flow on the Xeon 67 - **The Keycloak argv secret.** F3 stays REFUTED. With the agent toolkit at the GA tags (`auth_provider: litellm`), EAS installs no Keycloak, and F5 found no Keycloak secret; the inference layer on its own keeps Keycloak. `k3d-gpu eas up` now patches enterprise-inference's `keycloak_config` role so that neither the admin password nor the client secret is a `kcadm.sh` argument ([State and secrets](eas.md#state-and-secrets)). The patch is checked with a `ps` sampler against a stand-in for kcadm; no live run has checked it yet. A fresh `eas up --agent-toolkit` with F3's secret scan would. - **EAS's topology probe in a KVM guest.** H62r answered H62's question at the GA tags, with EAS's documented reserve list set. The probe's reading of a guest's threads as one per core is unchanged at `v1.2.0`, and we have not drafted an EAS issue for it yet. The `cpu: auto` issue draft stands for an install whose reserve leaves the pods outside the balloons too little CPU. -- **The vLLM agent path on Arc and Gaudi.** Up to release 0.1.93, `k3d-gpu agent up` did not serve through vLLM. On an Arc Pro B70 the engine exited at start because the pod's Kubernetes service link set `VLLM_PORT` to a URI, and the warm-up request carried no JSON content type. [#339](https://github.com/88plug/k3d-gpu/pull/339) fixes both and ships in release 0.1.94; with its pod spec and warm-up request, vLLM v0.33.0 served Qwen3-4B on one B70. A full `agent up` run of a release on a B70 is not recorded yet. The Gaudi path shares the manifest and has not run on hardware. On Arc, `agent up` with no backend set serves OVMS, which this did not affect; with `--model`, the fit can pick vLLM. +- **The vLLM agent path on Arc and Gaudi.** Up to release 0.1.93, `k3d-gpu agent up` did not serve through vLLM. On an Arc Pro B70 the engine exited at start because the pod's Kubernetes service link set `VLLM_PORT` to a URI, and the warm-up request carried no JSON content type. [#339](https://github.com/88plug/k3d-gpu/pull/339) fixes both and ships in release 0.1.94. With its pod spec and warm-up request applied by hand, vLLM v0.33.0 served Qwen3-4B on one B70; 0.1.94 ships v0.32.0, and a full `agent up` run of a release on a B70 is not recorded yet. The Gaudi path shares the manifest and has not run on hardware. On Arc, `agent up` with no backend set serves OVMS, which this did not affect; with `--model`, the fit can pick vLLM. - **k3d-gpu gaps found on the Xeon.** `/dev/dri/by-path` for oneCCL across B70s (H53); the fit refusing rather than falling back to the CPU, and no DRAM bandwidth cached on a new host (H58); placing one-card arms on one card (H54b, H54c). - **Replications.** H59 on the i9-13900T (H59d) is INCONCLUSIVE: llama.cpp's arm was OOMKilled at the fit's pod limit and not deliverable in two holds, so no statistic was scored. H59 on the i7-13700 (H59w) is PENDING, as are the wildct rows H57wa, H57wm and H57wb; H28w's result is in review. H34 on the i7-13700 (H34w) is INCONCLUSIVE: the CPU Fast Draft's gain over the target alone was neither the pre-registered win nor ruled out, so that class keeps no default CPU draft. Until a row is SUPPORTED for its class, those CPU classes keep OVMS. - **EAT 1.1.0 through `eat-up.sh`.** Pinned since 2026-10-29 and pre-registered as F4; not run yet.