From 258ce7b6c278ba844f2ee27e631ecf47e3ac946f Mon Sep 17 00:00:00 2001 From: "Arda M. Ekiz" <72472003+ArdaEkiz0@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:40:05 +0300 Subject: [PATCH 1/2] windows: one-command benchmark submission block (fixes #139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn r3_bench.py into the community-ready Windows benchmark tracker by the Hardware Benchmarks campaign (discussion #142): - `--submission`: after the usual discarded warmup, run N timed reps (default 2) on the ~3k-token long-context prompt — identical construction and text to the Python bench's BENCH_LONG — average the numbers, and print the ready-to-paste submission block with `runtime: windows`. - `--reporter` / `--notes` fill their fields; `--json` prints the block as one JSON line; per-rep detail lands in benchmarks/r3/bench__submission.json. - Metadata auto-filled, stdlib only (registry + kernel32/psapi): chip, ram_gb, os (build), device (computer name), engine commit (vendor.llama.pin), gpu (Vulkan device from the engine log). - peak_active_gib = engine process peak working set during the timed reps, probed via the open process handle after communicate() so the OS high-water mark survives process exit. - Default path unchanged (3 reps, median summary); rows gain additive gpu / peak fields. README §2.1 documents the flow and the memory field. --- windows/README.md | 21 ++++ windows/tools/r3_bench.py | 225 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 235 insertions(+), 11 deletions(-) diff --git a/windows/README.md b/windows/README.md index 3eda58a..e00ac25 100644 --- a/windows/README.md +++ b/windows/README.md @@ -129,6 +129,27 @@ Context to keep when reading these numbers: - Same-condition A/B against unpatched vanilla llama.cpp (vendor tree with the patch bands skipped) shows the patch set costs nothing (8B: 41.5 vs 39.5 decode; 35B: 27.2 vs 27.4 — within machine drift). - Expect day-to-day drift of a few percent on this class of machine; compare like-for-like, same day, same process if you benchmark. +### 2.1 Community benchmark submission (Hardware Benchmarks #142) + +To add a Windows row to the Hardware Benchmarks discussion (#142), build the +engine and download a tier as in §1, then run: + + python tools/r3_bench.py --tier 8b --submission --reporter @you + +The script discards the short warmup, runs 2 timed reps (`--reps N` to +override) on the ~3k-token long-context prompt (same construction as the +Python bench's BENCH_LONG), and prints a ready-to-paste submission block — +the mean of the timed reps, with `device` / `chip` / `gpu` (the Vulkan device +from the engine log) / `ram_gb` / `os` / `engine_commit` (vendor.llama.pin) +filled in automatically. `--json` additionally prints the block as a single +JSON line for aggregation, and `benchmarks/r3/bench__submission.json` +stores the per-rep detail. + +On the memory field: `peak_active_gib` on Windows is the engine process's +peak working set during the timed reps (OS-level high-water mark) — the same +intent as the Python bench's `mlx.core.get_peak_memory()`, measured at the +process level instead of the allocator. + --- ## 3. Technical Details diff --git a/windows/tools/r3_bench.py b/windows/tools/r3_bench.py index 79b4eb6..ffa1b1f 100644 --- a/windows/tools/r3_bench.py +++ b/windows/tools/r3_bench.py @@ -8,9 +8,16 @@ window when compared against the reference — cross-window comparison is always suspect (measure on the same day / same process, or treat numbers as shape-only). +Community submission mode (Hardware Benchmarks discussion #142, issue #139): +`--submission` discards the same short warmup, then runs `reps` timed reps +(default 2) on the ~3k-token long-context prompt, averages the numbers, and +prints the ready-to-paste submission block; `--json` prints the same data as +JSON for aggregation. + Usage: python tools/r3_bench.py --tier 35b [--reps 3] + python tools/r3_bench.py --tier 8b --submission --reporter @you """ -import argparse, json, os, re, subprocess, sys, time +import argparse, ctypes, json, os, re, subprocess, sys, time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -44,6 +51,21 @@ def models_root(tier): PROMPT = ("混合专家模型（MoE）通过稀疏激活把多个专家网络与门控路由组合起来：" * 12) + +# Long-context prompt for the submission block (~3k tokens after tokenization): +# identical construction and text to the Python bench's BENCH_LONG (see +# python/examples/bench.py), so prefill numbers on both runtimes refer to the +# same measured work. Short prompts only measure fixed overhead. +_PARA_ZH = '\u6d41\u5f0f\u63a8\u7406\u6846\u67b6\u7684\u8bbe\u8ba1\u8981\u70b9\u5728\u4e8e\u628a\u78c1\u76d8\u5e26\u5bbd\u3001\u7f13\u5b58\u5c42\u7ea7\u4e0e\u8ba1\u7b97\u6838\u5fc3\u4e09\u8005\u91cd\u53e0\u8d77\u6765\u3002\u6df7\u5408\u4e13\u5bb6\u6a21\u578b\u6bcf\u4e00\u5c42\u53ea\u5728\u5c11\u6570\u4e13\u5bb6\u4e0a\u6fc0\u6d3b\uff0c\u8def\u7531\u5668\u7ed9\u51fa\u7684\u7a00\u758f\u9009\u62e9\u6070\u597d\u4e3a\u6309\u9700\u52a0\u8f7d\u63d0\u4f9b\u4e86\u5929\u7136\u7684\u6279\u7c92\u5ea6\u3002\u82e5\u80fd\u5728\u524d\u4e00\u6b65\u7684\u9690\u72b6\u6001\u57fa\u7840\u4e0a\u9884\u5224\u4e0b\u4e00\u6b65\u7684\u4e13\u5bb6\u96c6\u5408\uff0c\u56fa\u6001\u786c\u76d8\u7684\u8bfb\u53d6\u5ef6\u8fdf\u5c31\u80fd\u88ab\u8ba1\u7b97\u5b8c\u5168\u63a9\u76d6\uff0c\u8fd9\u5bf9\u8fb9\u7f18\u8bbe\u5907\u4e0a\u7684\u5927\u6a21\u578b\u90e8\u7f72\u5177\u6709\u76f4\u63a5\u610f\u4e49\u3002' +_PARA_EN = 'The design of a streaming inference framework hinges on overlapping disk bandwidth, cache hierarchy, and compute. A mixture-of-experts layer activates only a handful of experts per token, and the router sparsity provides a natural granularity for on-demand loading. If the expert set of the next step can be predicted from the previous hidden state, SSD read latency hides behind compute entirely, which matters for large-model deployment on edge hardware.' +_LONG_HEADER = '\u8bf7\u4ed4\u7ec6\u9605\u8bfb\u4ee5\u4e0b\u6750\u6599\u5e76\u7528\u4e00\u53e5\u8bdd\u603b\u7ed3\u5176\u6838\u5fc3\u601d\u60f3\uff1a' + + +def long_prompt(): + paras = [_PARA_ZH if i % 2 == 0 else _PARA_EN for i in range(40)] + return _LONG_HEADER + " ".join(paras) + + PAT = { "prompt": re.compile(r"prompt eval time =.*?,\s*([\d.]+) tokens per second"), "eval": re.compile(r"(?= 22000 else "10" + return f"Windows {label} (build {v.build})" + + +def machine_name(): + """Computer name (device) from the registry; a human brand is not exposed by winreg.""" + try: + import winreg + k = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, + r"SYSTEM\CurrentControlSet\Control\ComputerName\ComputerName") + name, _ = winreg.QueryValueEx(k, "ComputerName") + winreg.CloseKey(k) + return str(name) + except OSError: + return None + + +def engine_commit(): + """Head of vendor.llama.pin at the repo root (the pinned llama.cpp fork).""" + pin = os.path.join(ROOT, "..", "vendor.llama.pin") + try: + with open(pin, encoding="utf-8") as f: + return f.read().splitlines()[0].strip() + except OSError: + return None + + +def vulkan_device(err): + """Vulkan device name reported by the engine's own startup log.""" + m = re.search(r"Device\s*\d*\s*[:=]\s*(.+?)\s*(?:\(|\n|$)", err, re.I) + if m and re.search(r"vulkan", err[:m.start()][-400:], re.I): + return m.group(1).strip() + for line in err.splitlines(): + if re.search(r"vulkan", line, re.I): + m2 = re.search(r"(?:device|adapter|gpu)[^\n:]*[:=]\s*(.+)", line, re.I) + if m2: + return m2.group(1).strip() + return None + + +def open_handle(pid): + """Engine-process handle for the memory probe (Windows only; None otherwise).""" + if sys.platform != "win32": + return None + for _ in range(10): # the child may still be spawning + h = ctypes.windll.kernel32.OpenProcess(0x0410, False, pid) # QUERY_INFORMATION|VM_READ + if h: + return h + time.sleep(0.01) + return None + + +def peak_working_set_gib(handle): + """Peak process working set (GiB) — the OS keeps the high-water mark while + any handle to the process is open, so probe after communicate(), then close.""" + if not handle: + return None + try: + class PMC(ctypes.Structure): + _fields_ = [("cb", ctypes.c_uint), + ("PageFaultCount", ctypes.c_uint), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t)] + pmc = PMC() + pmc.cb = ctypes.sizeof(pmc) + if not ctypes.WinDLL("psapi").GetProcessMemoryInfo(handle, ctypes.byref(pmc), pmc.cb): + return None + return pmc.PeakWorkingSetSize / (1024 ** 3) + except OSError: + return None + + +def one(gguf, lora, n, tier, prompt=None): cmd = [os.path.join(SERVER_BIN, "llama-completion.exe"), "-m", gguf, "-ngl", "99", "-cmoe", "--temp", "0", "-n", str(n), "--seed", "7", - "-no-cnv", "--no-display-prompt", "-p", PROMPT] + "-no-cnv", "--no-display-prompt", "-p", prompt or PROMPT] if lora: cmd[3:3] = ["--lora", lora] t0 = time.time() - r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf8", errors="replace") - err = r.stderr or "" - out = {"wall_s": round(time.time() - t0, 1)} + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf8", errors="replace") + h = open_handle(p.pid) + _, err = p.communicate() + peak = peak_working_set_gib(h) # probed after wait while the handle is live + if h: + ctypes.windll.kernel32.CloseHandle(h) + out = {"wall_s": round(time.time() - t0, 1), + "peak_working_set_gib": round(peak, 2) if peak is not None else None, + "gpu": vulkan_device(err or "")} for k, pat in PAT.items(): - m = pat.search(err) + m = pat.search(err or "") if m: out[k] = float(m.group(1)) - ev = re.search(r"n_ctx = (\d+)", err) + ev = re.search(r"n_ctx = (\d+)", err or "") out["n_ctx"] = int(ev.group(1)) if ev else None - if "error" in err.lower() and "eval time" not in err: + if "error" in (err or "").lower() and "eval time" not in err: out["error"] = [l for l in err.splitlines() if "error" in l.lower()][:2] if "total_ms" in out and "prompt_ms" in out: dec_ms = out["total_ms"] - out["prompt_ms"] # total = generation end-to-end, excludes load (verified) @@ -77,11 +221,50 @@ def one(gguf, lora, n, tier): return out +def submission_block(tier, reporter, notes, rows): + """Ready-to-paste Hardware Benchmarks block; numbers are the mean of the timed reps.""" + evs = [r["eval"] for r in rows if "eval" in r] + prs = [r["prompt"] for r in rows if "prompt" in r] + peaks = [r["peak_working_set_gib"] for r in rows if r.get("peak_working_set_gib") is not None] + gpus = sorted({r["gpu"] for r in rows if r.get("gpu")}) + dev, chip = machine_name(), cpu_name() + ram = total_ram_gb() + data = { + "tier": f"edge0-{tier}", + "runtime": "windows", + "device": dev or "(unknown)", + "chip": chip or "(unknown)", + "gpu": gpus[0] if gpus else "(not detected)", + "ram_gb": round(ram) if ram else None, + "os": windows_build(), + "decode_tok_s": round(sum(evs) / len(evs), 2) if evs else None, + "prefill_tok_s": round(sum(prs) / len(prs), 2) if prs else None, + "peak_active_gib": round(max(peaks), 2) if peaks else None, + "engine_commit": engine_commit(), + "reporter": reporter or "", + "notes": notes or "", + } + return data + + +def print_block(d): + for k, v in d.items(): + print(f"{k:<16} {'' if v is None else v}".rstrip()) + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--tier", choices=["35b", "8b"], required=True) - ap.add_argument("--reps", type=int, default=3) + ap.add_argument("--reps", type=int, default=None, + help="timed reps (default: 2 with --submission, 3 otherwise)") + ap.add_argument("--submission", action="store_true", + help="print the Hardware Benchmarks submission block (long-context prefill)") + ap.add_argument("--json", action="store_true", + help="with --submission, also print the block as JSON") + ap.add_argument("--reporter", default="", help="@handle for the submission block") + ap.add_argument("--notes", default="", help="free-form notes field") a = ap.parse_args() + reps = a.reps or (2 if a.submission else 3) exe = os.path.join(SERVER_BIN, "llama-completion.exe") if not os.path.isfile(exe): raise SystemExit(f"llama-completion.exe not found in {SERVER_BIN} " @@ -90,10 +273,30 @@ def main(): gguf = os.path.join(md, f"edge0-{a.tier}-gguf", f"edge0-{a.tier}.gguf") lora = os.path.join(md, f"edge0-{a.tier}", f"lora_edge0_{a.tier}-gguf.gguf") + if a.submission: + warm = one(gguf, lora, 64, a.tier) # warmup (page cache + mmap fault-in), discarded + print(f"warm(discard): {warm}", flush=True) + longp = long_prompt() + rows = [] + for i in range(reps): + time.sleep(5) + m = one(gguf, lora, 256, a.tier, prompt=longp) + m["rep"] = i + rows.append(m) + print(f"rep{i}: {m}", flush=True) + data = submission_block(a.tier, a.reporter, a.notes, rows) + outp = os.path.join(ROOT, "benchmarks", "r3", f"bench_{a.tier}_submission.json") + os.makedirs(os.path.dirname(outp), exist_ok=True) + with open(outp, "w", encoding="utf8") as f: + json.dump({**data, "timed_reps": rows}, f, ensure_ascii=False, indent=1) + print_block(data) + if a.json: + print("SUBMIT_JSON " + json.dumps(data, ensure_ascii=False)) + return rows = [] warm = one(gguf, lora, 64, a.tier) # warmup (page cache + mmap fault-in), discarded print(f"warm(discard): {warm}", flush=True) - for i in range(a.reps): + for i in range(reps): time.sleep(5) m = one(gguf, lora, 256, a.tier) m["rep"] = i From ed627fcf080eb906c90bcc22d1c4c2c8915fd649 Mon Sep 17 00:00:00 2001 From: "Arda M. Ekiz" <72472003+ArdaEkiz0@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:07:59 +0300 Subject: [PATCH 2/2] windows: parse the ggml_vulkan device line; clarify peak_working_set semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine prints 'ggml_vulkan: 0 = (vendor) | ...'; the old pattern expected a 'Device 0:' form and printed '(not detected)'. Both forms now parse. peak_active_gib stays the process peak working set, but the README drop claims it is the same measurement as the MLX allocator peak — Windows has no allocator counter, so it is not directly comparable with the python-mlx column. --- windows/README.md | 6 +++--- windows/tools/r3_bench.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/windows/README.md b/windows/README.md index e00ac25..29161a8 100644 --- a/windows/README.md +++ b/windows/README.md @@ -146,9 +146,9 @@ JSON line for aggregation, and `benchmarks/r3/bench__submission.json` stores the per-rep detail. On the memory field: `peak_active_gib` on Windows is the engine process's -peak working set during the timed reps (OS-level high-water mark) — the same -intent as the Python bench's `mlx.core.get_peak_memory()`, measured at the -process level instead of the allocator. +peak working set during the timed reps (OS-level high-water mark). Windows +has no allocator-level counter, so this is not directly comparable with the +python-mlx column's `mlx.core.get_peak_memory()`. --- diff --git a/windows/tools/r3_bench.py b/windows/tools/r3_bench.py index ffa1b1f..f5a5b23 100644 --- a/windows/tools/r3_bench.py +++ b/windows/tools/r3_bench.py @@ -140,15 +140,17 @@ def engine_commit(): def vulkan_device(err): - """Vulkan device name reported by the engine's own startup log.""" - m = re.search(r"Device\s*\d*\s*[:=]\s*(.+?)\s*(?:\(|\n|$)", err, re.I) - if m and re.search(r"vulkan", err[:m.start()][-400:], re.I): - return m.group(1).strip() + """Vulkan device name from the engine's own startup log, e.g: + 'ggml_vulkan: 0 = AMD Radeon RX 9070 GRE (vendor) | uma: ...'.""" for line in err.splitlines(): - if re.search(r"vulkan", line, re.I): - m2 = re.search(r"(?:device|adapter|gpu)[^\n:]*[:=]\s*(.+)", line, re.I) - if m2: - return m2.group(1).strip() + if not re.search(r"vulkan", line, re.I): + continue + m = re.search(r"\d+\s*=\s*([^\n|()]+)", line) + if m: + return m.group(1).strip() + m = re.search(r"Device\s*\d*\s*[:=]\s*([^\n|(),]+)", line, re.I) + if m: + return m.group(1).strip() return None