ledger(H62r): pre-register H62 at the GA tags (EAS v1.2.0, EAT release-2.0.0), with its runner

.github/workflows/ci.yml

@@ -301,6 +301,8 @@ jobs:
       # fixtures, arm.sh against a local stub server, its records scored by h59.py.
       - run: python3 bench/h59/test_run.py
       - run: python3 bench/h62/test_h62.py
+      # H62r's runner: H62's at the GA tags, with D1 as a check and D3's sibling read.
+      - run: python3 bench/h62r/test_h62r.py
       # H58's runner (amendment R1): h58.py's interlock, placement and clause scorers, and run.sh
       # end to end against stand-ins for k3d-gpu, kubectl, docker and the Hub API.
       - run: python3 bench/h58/test_h58.py

bench/eatflow/live.sh

@@ -47,6 +47,7 @@
 #                   pairs: ARM = H47, H49, H47b, H49b, H50 or H50o, both arms live in cluster k3dgpu-e-<arm>, one hold
 #                   of interleaved pairs into LIVE_PAIRS_DIR (LIVE_HOLD = this hold's number);
 #                   ARM = H62: both servers in KVM guests, driven by LIVE_REMOTE_A and LIVE_REMOTE_B (bench/h62);
+#                   ARM = H62r: the same, with H62r's guests (bench/h62r);
 #                   code: H43 (amendment W1), the coding agent on C12 with `k3d-gpu sandbox up`, one
 #                   block per level in LIVE_CODE_LEVELS (default "typical heavy heavy typical") into
 #                   LIVE_CODE_DIR/h43-<arm>-<level> (shared by an arm's holds, so a level's halves resume);
@@ -498,6 +499,7 @@ if [ "${MODE}" = pairs ]; then
     DESIGN=${FAMILY}
     [ "${ARM}" = H50o ] && DESIGN=H50o
     [ "${ARM}" = H62 ] && DESIGN=H62
+    [ "${ARM}" = H62r ] && DESIGN=H62   # H62 re-run at the GA tags: the same remote arms (bench/h62r)
     [ "${ARM}" = H50 ] && KV_GIB=${LIVE_EAT_KV_GIB:-40}   # EAT's shipped KV cache (40 GiB)
     mkdir -p "${PD}"
     PD=$(cd "${PD}" && pwd)

bench/eatflow/pairs.py

@@ -17,7 +17,7 @@
 pairs' pre-windows. bg_cpu, package W and throttle during the attempt are reported, never
 gated.
 
-Load rule L3 (H50, H50o, H53, H62; EXPERIMENTS.md "Crescent amendment A3"): the same
+Load rule L3 (H50, H50o, H53, H62, H62r; EXPERIMENTS.md "Crescent amendment A3"): the same
 pre-windows, no match filter. Every complete pair enters the statistic; a balance gate (a
 sign-flip test of arm on each pre covariate, p < 0.01 gates) replaces it; a two-regressor
 adjustment and L2's matched subset are reported, never gated. The arm order of pair j is drawn
@@ -538,16 +538,17 @@ def boot_share(d, pred, boot=stats.BOOT, seed=stats.SEED, stat=None):
 # Rule L3 rows: the seed of their arm order, fixed in the pre-registration (amendment A3).
 ORDER_SEEDS = {"H50": "H50-crescent-2026-10-20", "H50o": "H50o-crescent-2026-10-20",
                "H53": "H53-crescent-2026-10-20", "H62": "H62-crescent-2026-10-29",
-               "H58": "H58-crescent-2026-11-01"}
+               "H58": "H58-crescent-2026-11-01", "H62r": "H62r-crescent-2026-11-06"}
 
 # Hypothesis id -> (design, load rule). H47b and H49b are H47 and H49 under rule L2; H50 is
 # H49b's design on the Xeon 6741P, H50o the same with arm B3 (EXPERIMENTS.md "H50-H52 pre-registration");
 # H53 the H49 statistic over solo-hold block pairs on the Arc Pro B70s (solo.py; block-level verdict, "H53 pre-registration");
 # H62 H50's design with arm B5, Intel's EAS default, each server in a KVM guest ("H62 pre-registration");
-# H58 one solo hold of arm Z, the zero-touch agent, for its clause (c) ("H58 amendment R1").
+# H58 one solo hold of arm Z, the zero-touch agent, for its clause (c) ("H58 amendment R1");
+# H62r H62's design with B5 at the GA tags ("H62r pre-registration").
 DESIGNS = {"H47": ("H47", "L1"), "H49": ("H49", "L1"), "H47b": ("H47", "L2"), "H49b": ("H49", "L2"),
            "H50": ("H49", "L3"), "H50o": ("H49", "L3"), "H53": ("H49", "L3"), "H62": ("H49", "L3"),
-           "H58": ("H49", "L3")}
+           "H58": ("H49", "L3"), "H62r": ("H49", "L3")}
 
 
 # The ledger a rule-L3 row must be pre-registered in before any of its holds runs.

bench/eatflow/test_pairs.py

@@ -1092,6 +1092,45 @@ def test_live_sh_runs_h62_against_remote_arms(self):
         self.assertIn("REMOTE=()", live[:live.index("cleanup() {")])
 
 
+class H62rTest(unittest.TestCase):
+    """H62r (EXPERIMENTS.md "H62r pre-registration"): H62 re-run with B5 at the GA tags. Its own
+    row, seed and pairs registration; live.sh runs it through H62's remote-arm branch."""
+
+    REG = {"hyp": "H62r", "rule": "L3", "seed": "H62r-crescent-2026-11-06"}
+
+    def test_h62r_is_h62s_design_and_rule_with_its_own_seed(self):
+        self.assertEqual(PR.design("H62r"), ("H49", "L3"))
+        self.assertEqual(PR.ORDER_SEEDS["H62r"], self.REG["seed"])
+        self.assertIn("H62r", AB.PAIRS_HYPS)
+        at = [dict(r, arm="B5") if r["arm"] == "B1" else r for r in L2Test.h49_like(L2Test())]
+        res = PR.analyze(at, pairs_of(24), "H62r", "A1", "B5", registration=self.REG)
+        ref = PR.analyze(at, pairs_of(24), "H62", "A1", "B5", registration=H62Test.REG)
+        self.assertEqual((res["rule"], res["arms"]), ("L3", ["A1", "B5"]))
+        self.assertEqual((res["verdict"], res["paired_diff"]), (ref["verdict"], ref["paired_diff"]))   # H62's statistic
+        with self.assertRaises(ValueError):                           # H62's registration is not H62r's
+            PR.analyze(at, pairs_of(24), "H62r", "A1", "B5", registration=H62Test.REG)
+
+    def test_h62r_draws_its_own_arm_order(self):
+        h62 = [PR.order(j, "A1", "B5", PR.ORDER_SEEDS["H62"]) for j in range(24)]
+        h62r = [PR.order(j, "A1", "B5", PR.ORDER_SEEDS["H62r"]) for j in range(24)]
+        self.assertNotEqual(h62, h62r)
+        self.assertEqual(h62r, [PR.order(j, "A1", "B5", "H62r-crescent-2026-11-06") for j in range(24)])
+
+    def test_registration_names_h62r(self):
+        tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp)
+        self.assertEqual(PR.register(tmp, "H62r"), self.REG)
+        with self.assertRaises(PR.HoldAborted):                       # an H62 hold cannot reuse the directory
+            PR.register(tmp, "H62")
+
+    def test_live_sh_runs_h62r_through_the_remote_arm_branch(self):
+        with open(os.path.join(HERE, "live.sh")) as f:
+            live = f.read()
+        self.assertIn('[ "${ARM}" = H62r ] && DESIGN=H62', live)
+        pairs = live[live.index('if [ "${MODE}" = pairs ]; then'):]
+        self.assertLess(pairs.index('[ "${ARM}" = H62r ] && DESIGN=H62'), pairs.index("case ${DESIGN} in"))
+
+
 class B3RecordsTest(unittest.TestCase):
     """H50o amendment O1 item 2: the records of B3's pulled model that the pre-registration names
     (the init container's log, every file's sha256, the graph each hold serves), read from the

bench/h62r/arm-A1

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# arm-A1: bench/h62r/arm.sh for arm A1, as one word (live.sh's LIVE_REMOTE_A).
+exec "$(dirname "$0")/arm.sh" A1 "$@"

bench/h62r/arm-B5

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# arm-B5: bench/h62r/arm.sh for arm B5, as one word (live.sh's LIVE_REMOTE_B).
+exec "$(dirname "$0")/arm.sh" B5 "$@"

bench/h62r/arm.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# arm.sh ARM CMD [ARG]: one H62r arm's server in its KVM guest (EXPERIMENTS.md "H62r
+# pre-registration"). bench/eatflow/live.sh's remote-arm mode calls it through arm-A1 and arm-B5.
+#   port               the host port its Service is forwarded to (the guest's hostfwd)
+#   restart SECONDS    delete its serving pod, then wait until a new pod is Ready; exit 3 when none is
+#                      Ready within SECONDS (the hold's 900 s readiness rule passes the time left)
+#   records DIR        the pod, the CPU ISA it sees (amx_tile lines), its cpuset and nproc, the nodes,
+#                      the guest's load and memory, B5's LLMInferenceService, and the image ID of every
+#                      container in the guest (B5's sandbox router runs a floating tag)
+#   logs DIR           every pod, events, the pod description and the server log (a hold that fails)
+#   forward            (re)start `kubectl port-forward` of its Service in the guest, on the serve port
+#   unforward          stop that port-forward
+#   cgroup             "exec:<this script> ARM cpustat <dir>": its pod's cgroup in the guest, as the
+#                      idle-arm check reads it (bench/eatflow/pairs.py)
+#   cpustat DIR        that cgroup's cpu.stat
+# ARM: A1 (VM a: k3d-gpu's agent OVMS) or B5 (VM b: EAS's gemma-4-26b LLMInferenceService).
+# H62R_STATE (default ~/k3dgpu-bench/h62r) holds ssh_config; H62R_SSH is the ssh client (tests stub
+# it); H62R_POLL_S is the restart poll period (default 5 s). Each guest's ~/h62r/env sets PATH and
+# KUBECONFIG for kubectl there.
+set -uo pipefail
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+SELF=${HERE}/arm.sh
+P() { python3 "${HERE}/h62r.py" "$@"; }
+ARM=${1:?usage: arm.sh A1|B5 CMD [ARG]} CMD=${2:?usage: arm.sh A1|B5 CMD [ARG]}
+fields=$(P arm-fields "${ARM}" 2>/dev/null) || { echo "arm.sh: no arm ${ARM} (A1 or B5)" >&2; exit 2; }
+read -r _ HOST PORT NS SEL SVC SPORT CTR EXTRA <<< "${fields}"
+STATE=${H62R_STATE:-${HOME}/k3dgpu-bench/h62r}
+# g CMD: one command in the guest, after its ~/h62r/env.
+g() { ${H62R_SSH:-ssh} -F "${STATE}/ssh_config" "${HOST}" ". ~/h62r/env && $*"; }
+pods() { g "kubectl -n ${NS} get pods -l ${SEL} -o json"; }
+ready_pod() { pods | P pick-pod "${1:-}"; }    # "<name> <uid>" of a Ready pod whose uid is not in $1
+case ${CMD} in
+port)
+    echo "${PORT}" ;;
+restart)
+    left=${3:-}
+    [ -n "${left}" ] || { echo "usage: arm.sh ${ARM} restart SECONDS" >&2; exit 2; }
+    [ "${left}" -gt 0 ] 2>/dev/null || { echo "no time left to restart"; exit 3; }
+    end=$(( SECONDS + left ))
+    old=$(pods | P uids) || { echo "cannot list the pods"; exit 1; }
+    g "kubectl -n ${NS} delete pod -l ${SEL} --wait=false" || { echo "cannot delete the pods"; exit 1; }
+    while [ "${SECONDS}" -lt "${end}" ]; do
+        if p=$(ready_pod "${old}"); then echo "ready: ${p}"; exit 0; fi
+        sleep "${H62R_POLL_S:-5}"
+    done
+    echo "not ready within ${left} s"
+    exit 3 ;;
+records)
+    d=${3:?usage: arm.sh ARM records DIR}
+    mkdir -p "${d}"
+    read -r name uid <<< "$(ready_pod "")"
+    [ -n "${name:-}" ] || { echo "no Ready pod"; exit 1; }
+    echo "${name} ${uid}" > "${d}/pod.txt"
+    g "kubectl -n ${NS} get pod ${name} -o json" > "${d}/pod.json"
+    g "kubectl -n ${NS} exec ${name} -c ${CTR} -- grep -c amx_tile /proc/cpuinfo" > "${d}/isa.txt" 2>&1
+    g "kubectl -n ${NS} exec ${name} -c ${CTR} -- sh -c 'cat /sys/fs/cgroup/cpuset.cpus.effective; nproc'" \
+        > "${d}/cpuset.txt" 2>&1
+    g "kubectl get nodes -o json" > "${d}/nodes.json"
+    g "uptime; free -g; grep -c amx_tile /proc/cpuinfo; nproc" > "${d}/guest.txt" 2>&1
+    [ -z "${EXTRA}" ] || g "kubectl -n ${NS} get ${EXTRA} -o yaml" > "${d}/extra.yaml" 2>&1
+    g "kubectl get pods -A -o wide" > "${d}/pods.txt" 2>&1
+    g "kubectl get pods -A -o 'custom-columns=NS:.metadata.namespace,POD:.metadata.name,IMAGE_ID:.status.containerStatuses[*].imageID'" \
+        > "${d}/image-ids.txt" 2>&1
+    echo "records: ${d}" ;;
+logs)
+    d=${3:?usage: arm.sh ARM logs DIR}
+    mkdir -p "${d}"
+    g "kubectl get pods -A -o wide" > "${d}/pods.txt" 2>&1
+    g "kubectl get events -A --sort-by=.lastTimestamp" > "${d}/events.txt" 2>&1
+    g "kubectl -n ${NS} describe pods -l ${SEL}" > "${d}/describe.txt" 2>&1
+    g "kubectl -n ${NS} logs -l ${SEL} -c ${CTR} --tail=1000" > "${d}/server.log" 2>&1
+    echo "logs: ${d}" ;;
+forward)
+    g "pid=\$(cat ~/h62r/pf-${ARM}.pid 2>/dev/null) && kill \${pid} 2>/dev/null;" \
+      "setsid nohup kubectl -n ${NS} port-forward --address 0.0.0.0 ${SVC} ${PORT}:${SPORT}" \
+      "> ~/h62r/pf-${ARM}.log 2>&1 < /dev/null & echo \$! > ~/h62r/pf-${ARM}.pid" ;;
+unforward)
+    g "pid=\$(cat ~/h62r/pf-${ARM}.pid 2>/dev/null) && kill \${pid} 2>/dev/null; rm -f ~/h62r/pf-${ARM}.pid" ;;
+cgroup)
+    read -r _ uid <<< "$(ready_pod "")"
+    [ -n "${uid:-}" ] || { echo "no Ready pod" >&2; exit 1; }
+    # cgroupfs driver (k3s in VM a's k3d node): pod<uid>; systemd driver (Kubespray's containerd
+    # in VM b): ...-pod<uid with underscores>.slice.
+    dir=$(g "find /sys/fs/cgroup -maxdepth 9 -type d \\( -name pod${uid} -o -name '*pod${uid//-/_}.slice' \\) -print -quit")
+    [ -n "${dir}" ] || { echo "no cgroup for pod ${uid}" >&2; exit 1; }
+    echo "exec:${SELF} ${ARM} cpustat ${dir}" ;;
+cpustat)
+    g "cat ${3:?usage: arm.sh ARM cpustat DIR}/cpu.stat" ;;
+*)
+    echo "usage: arm.sh A1|B5 port|restart SECONDS|records DIR|logs DIR|forward|unforward|cgroup|cpustat DIR" >&2
+    exit 2 ;;
+esac

bench/h62r/campaign.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# campaign.sh: H62r end to end on crescent (EXPERIMENTS.md "H62r pre-registration"), from the H62r
+# checkout at the deploy commit: setup.sh, smoke.sh, then holds (pairs-hold.sh 1, 2, ...) until
+# B24's 24 pairs are recorded or 8 holds have run, then the score. teardown.sh runs at the end,
+# also after an abort.
+#   - at most 3 setup attempts in all (attempts.txt, one line per start); a fourth start exits 10
+#     before anything is started
+#   - a start after any hold has run (a hold<N> directory in the records) exits 11 before anything
+#     is started, and is not an attempt
+#   - the re-run trigger (upstream.sh, up to 3 reads 60 s apart) before setup and before each
+#     hold, into upstream.txt. Fired or unreadable before setup: exit 12 before anything starts,
+#     and no attempt is spent. Fired before hold 1: exit 12 with no hold, for the dated amendment
+#     that re-pins B5. Unreadable before a hold, or fired after hold 1: recorded, and the holds go
+#     on at these pins
+#   - setup exit 6 (the host gate still failed after 60 min): the two qwen38 serving pairs are
+#     stopped under the box owner's grant (gpu-serving.sh stop, never rm) and setup runs once more;
+#     teardown.sh starts them again
+#   - a hold that exits 3 (a server not ready with 900 s left) counts as one of the 8 and runs no
+#     pair; after 2 such holds in a row no further hold runs (INCONCLUSIVE, not deliverable)
+# Records: ~/k3dgpu-bench/results/H62r (campaign.log, attempts.txt, upstream.txt, setup/, pairs/,
+# hold<N>/, report.txt, vmtax.json).
+set -uo pipefail
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+REPO=$(cd "${HERE}/../.." && pwd)
+B=${HOME}/k3dgpu-bench
+R=${B}/results/H62r
+MAX_HOLDS=8 NOT_DELIVERABLE_MAX=2 ATTEMPTS_MAX=3
+mkdir -p "${R}"
+log() { echo "== campaign $* $(date -u +%FT%TZ)" | tee -a "${R}/campaign.log"; }
+# trigger PHASE: the re-run trigger's line ("fired: ...", upstream.sh's stdout) into upstream.txt.
+# Status 0 fired, 1 not, 2 unreadable (no such line in 3 reads; its errors are recorded).
+trigger() {
+    local out="" err="${R}/upstream.err" i
+    for i in 1 2 3; do
+        if out=$(bash "${HERE}/upstream.sh" 2> "${err}") && [[ "${out}" == "fired: "* ]]; then
+            echo "$(date -u +%FT%TZ) $1 ${out}" >> "${R}/upstream.txt"
+            [ "${out}" != "fired: none" ]
+            return
+        fi
+        [ "${i}" = 3 ] || sleep "${H62R_TRIGGER_RETRY_S:-60}"
+    done
+    echo "$(date -u +%FT%TZ) $1 unreadable: $(paste -sd' ' "${err}")${out:+ (output: ${out//$'\n'/ })}" >> "${R}/upstream.txt"
+    return 2
+}
+# Holds count toward the 8 in all and their records are measured data. This script numbers its
+# holds from 1, so once any hold has run it refuses to start: a dated amendment says how the row
+# goes on.
+held=$(find "${R}" -mindepth 1 -maxdepth 1 -type d -name 'hold[0-9]*' -printf '%f\n' | sort -V | paste -sd' ')
+if [ -n "${held}" ]; then
+    log "holds already recorded (${held}): no start"
+    exit 11
+fi
+tries=$(grep -c . "${R}/attempts.txt" 2>/dev/null)
+if [ "${tries:-0}" -ge "${ATTEMPTS_MAX}" ]; then
+    log "setup attempt bound reached (${tries:-0} of ${ATTEMPTS_MAX}): no further attempt"
+    exit 10
+fi
+trigger setup
+case $? in
+    0) log "re-run trigger fired before setup ($(tail -1 "${R}/upstream.txt")): no setup until a dated amendment re-pins B5"; exit 12 ;;
+    2) log "re-run trigger unreadable before setup: no setup"; exit 12 ;;
+esac
+echo "$(date -u +%FT%TZ) $(git -C "${REPO}" rev-parse HEAD)" >> "${R}/attempts.txt"
+teardown() { bash "${HERE}/teardown.sh" >> "${R}/teardown.log" 2>&1; log "teardown exit $?"; }
+trap teardown EXIT
+log "start (attempt $(( ${tries:-0} + 1 )) of ${ATTEMPTS_MAX}) at $(git -C "${REPO}" rev-parse HEAD)"
+bash "${HERE}/setup.sh"
+rc=$?
+if [ "${rc}" = 6 ]; then
+    log "host gate failed: stopping the qwen38 serving pairs (owner's grant; docker stop, never rm)"
+    for c in qwen38-a-hr qwen38-b-hr; do
+        if bash "${REPO}/bench/eatflow/crescent/gpu-serving.sh" stop "${c}" >> "${R}/serving.log" 2>&1; then
+            echo "${c}" >> "${R}/serving-stopped.txt"
+        fi
+    done
+    bash "${HERE}/setup.sh"
+    rc=$?
+fi
+[ "${rc}" = 0 ] || { log "setup exit ${rc}: no hold"; exit "${rc}"; }
+bash "${HERE}/smoke.sh"
+rc=$?
+[ "${rc}" = 0 ] || { log "wiring smoke failed (exit ${rc}): no hold"; exit 9; }
+nd=0
+for n in $(seq 1 "${MAX_HOLDS}"); do
+    left=$(python3 "${HERE}/h62r.py" pairs-left "${R}/pairs")
+    [ "${left}" -gt 0 ] || break
+    trigger "hold${n}"
+    case $? in
+        0) if [ "${n}" = 1 ]; then
+               log "re-run trigger fired before hold 1 ($(tail -1 "${R}/upstream.txt")): no hold until a dated amendment re-pins B5"
+               exit 12
+           fi
+           log "re-run trigger fired after hold 1: H62r's holds finish at these pins" ;;
+        2) log "re-run trigger unreadable before hold ${n}: recorded" ;;
+    esac
+    log "hold ${n}: ${left} pair(s) left"
+    bash "${HERE}/pairs-hold.sh" "${n}"
+    rc=$?
+    log "hold ${n} exit ${rc}"
+    if [ "${rc}" = 3 ]; then
+        nd=$(( nd + 1 ))
+        [ "${nd}" -lt "${NOT_DELIVERABLE_MAX}" ] || { log "${nd} holds in a row not deliverable: no further hold"; break; }
+    else
+        nd=0
+    fi
+done
+python3 "${REPO}/bench/eatflow/agentbench.py" pairs-report "${R}/pairs" --hyp H62r --arm-a A1 --arm-b B5 \
+    > "${R}/report.txt" 2>&1
+log "pairs-report exit $?"
+h50=${REPO}/bench/eatflow/results/2026-10-20/H50/pairs/analysis.json
+if [ -f "${R}/pairs/analysis.json" ]; then
+    python3 "${HERE}/h62r.py" vmtax "${h50}" "${R}/pairs/analysis.json" > "${R}/vmtax.json"
+fi