From 62d921d2aff239d7c2461d27b6ff7db5fb807632 Mon Sep 17 00:00:00 2001 From: zilch Date: Sat, 12 Sep 2026 02:16:30 +0800 Subject: [PATCH] bench: render one console results table after benchmark completes --- bench/bench_env.py | 55 +++++++++++++++-------- bench/bench_physics.py | 100 ++++++++++++++++++++++++++--------------- bench/report.py | 53 ++++++++++++++++++++++ 3 files changed, 152 insertions(+), 56 deletions(-) create mode 100644 bench/report.py diff --git a/bench/bench_env.py b/bench/bench_env.py index 8ca7a37..561301f 100644 --- a/bench/bench_env.py +++ b/bench/bench_env.py @@ -21,6 +21,7 @@ os.environ.setdefault("GOMP_SPINCOUNT", "0") import numpy as np +from report import render_table import motrix_envs # noqa: F401 registers built-in environments from motrix_env_core import registry @@ -167,15 +168,30 @@ def _measure_env( return result, actions, action_spec, profile -def _print_result(result: dict[str, object], speedup: float | None) -> None: +_RESULT_HEADERS = [ + "environment", + "num_envs", + "p10 ms", + "median ms", + "p90 ms", + "env steps/s", + "CPU %", + "speedup", +] + + +def _result_row(result: dict[str, object], speedup: float | None) -> list[str]: speedup_cell = "—" if speedup is None else f"{speedup:.2f}x" - print( - f"{str(result['env']):<28} | {int(result['num_envs']):>9} | " - f"{float(result['p10_ms']):>10.3f} | {float(result['median_ms']):>10.3f} | " - f"{float(result['p90_ms']):>10.3f} | {float(result['env_steps_per_s']):>14,.0f} | " - f"{float(result['cpu_util_percent']):>9.1f}% | " - f"{speedup_cell:>9}" - ) + return [ + str(result["env"]), + str(int(result["num_envs"])), + f"{float(result['p10_ms']):.3f}", + f"{float(result['median_ms']):.3f}", + f"{float(result['p90_ms']):.3f}", + f"{float(result['env_steps_per_s']):,.0f}", + f"{float(result['cpu_util_percent']):.1f}%", + speedup_cell, + ] def main() -> None: @@ -183,7 +199,7 @@ def main() -> None: parser.add_argument("--env", default="cartpole", help="Reference registered environment name.") parser.add_argument("--compare-env", help="Optional registered environment to compare with --env.") parser.add_argument("--mode", default="train", help="Environment mode (train/play).") - parser.add_argument("--num-envs", type=int, nargs="+", default=[1], help="Batch sizes to benchmark.") + parser.add_argument("--num-envs", type=int, nargs="+", default=[2048], help="Batch sizes to benchmark.") parser.add_argument("--steps", type=int, default=1000, help="Timed env.step() calls per measurement.") parser.add_argument("--warmup", type=int, default=10, help="Untimed env.step() calls per measurement.") parser.add_argument("--seed", type=int, default=20260808) @@ -203,12 +219,8 @@ def main() -> None: if args.json and args.breakdown: parser.error("--json and --breakdown cannot be used together") - if not args.json: - print( - f"{'environment':<28} | {'num_envs':>9} | {'p10 ms':>10} | {'median ms':>10} | " - f"{'p90 ms':>10} | {'env steps/s':>14} | {'CPU %':>10} | {'speedup':>9}" - ) - + rows: list[list[str]] = [] + perf_sections: list[tuple[str, int, _ProfileResult]] = [] for num_envs in args.num_envs: reference, actions, action_spec, reference_profile = _measure_env( args.env, @@ -238,12 +250,17 @@ def main() -> None: if args.json: print(json.dumps(result, sort_keys=True)) else: - _print_result(result, speedup) + rows.append(_result_row(result, speedup)) for name, profile in profiles: - if profile is None: - continue - print(f"\n{name}, num_envs={num_envs} step breakdown ({profile.elapsed_seconds:.3f}s wall time)") + if profile is not None: + perf_sections.append((name, num_envs, profile)) + + if not args.json: + print(render_table(_RESULT_HEADERS, rows)) + for name, num_envs, profile in perf_sections: + print() + print(f"{name}, num_envs={num_envs} step breakdown ({profile.elapsed_seconds:.3f}s wall time)") _print_perf_tree(profile.root) diff --git a/bench/bench_physics.py b/bench/bench_physics.py index 08cc6e6..d6d01f3 100644 --- a/bench/bench_physics.py +++ b/bench/bench_physics.py @@ -38,6 +38,7 @@ import motrixsim as mtx import numpy as np +from report import render_table import motrix_envs # noqa: F401 registers robots and scene assets from motrix_env_core import registry @@ -158,17 +159,32 @@ def reset_state() -> None: } -def _print_result(label: str, r: dict, context: dict) -> None: - print(f"--- {label} ---") - print(f"robot: {r['robot']} num_envs: {r['num_envs']} actuators: {r['actuators']} links: {r['links']}") - print(f"sim: dt={r['dt']}s solver_iterations={r['solver_iterations']} nstep={r['nstep']} iters={r['iters']}") - print(f"step_n time (ms): median={r['median_ms']:.3f} p10={r['p10_ms']:.3f} p90={r['p90_ms']:.3f}") - print(f"batch steps/s: {r['batch_steps_per_s']:.1f} | total env steps/s: {r['total_env_steps_per_s']:.0f}") - print(f"context: cpus={context['num_cpus']} numa_nodes={context['num_numa_nodes']} binding=[{context['binding']}]") +_RESULT_HEADERS = [ + "mode", + "robot", + "num_envs", + "median ms", + "p10 ms", + "p90 ms", + "batch steps/s", + "total env steps/s", +] + + +def _result_row(label: str, r: dict) -> list[str]: + return [ + label, + str(r["robot"]), + str(r["num_envs"]), + f"{r['median_ms']:.3f}", + f"{r['p10_ms']:.3f}", + f"{r['p90_ms']:.3f}", + f"{r['batch_steps_per_s']:.1f}", + f"{r['total_env_steps_per_s']:,.0f}", + ] -def run_single(args) -> None: - context = _device_context() +def run_single(args) -> tuple[list[list[str]], list[str]]: r = measure( robot=args.robot, num_envs=args.num_envs, @@ -178,7 +194,7 @@ def run_single(args) -> None: warmup=args.warmup, iters=args.iters, ) - _print_result(f"single-process ({args.numa})", r, context) + return ([_result_row(f"single-process ({args.numa})", r)], []) def _worker_entry(args) -> None: @@ -236,22 +252,22 @@ def _measure_via_subprocess(args, node: int, num_envs: int, out_json: str) -> di return json.loads(Path(out_json).read_text()) -def run_node(args) -> None: +def run_node(args) -> tuple[list[list[str]], list[str]]: """Re-exec the measurement pinned to NUMA node 0.""" + notes: list[str] = [] num_nodes = _detect_num_nodes() if num_nodes < 2: - print(f"only {num_nodes} NUMA node(s) detected; --numa node is a no-op here") + notes.append(f"only {num_nodes} NUMA node(s) detected; --numa node is a no-op here") with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: out_json = f.name try: r = _measure_via_subprocess(args, node=0, num_envs=args.num_envs, out_json=out_json) - context = _device_context() - _print_result("node-pinned (numactl --cpunodebind=0 --membind=0)", r, context) + return ([_result_row("node-pinned (cpunodebind=0, membind=0)", r)], notes) finally: Path(out_json).unlink(missing_ok=True) -def run_shard(args) -> None: +def run_shard(args) -> tuple[list[list[str]], list[str]]: """Spawn one pinned worker per NUMA node, each running num_envs/num_nodes envs.""" num_nodes = _detect_num_nodes() if num_nodes < 2: @@ -259,9 +275,6 @@ def run_shard(args) -> None: if args.num_envs % num_nodes != 0: raise ValueError(f"--num-envs ({args.num_envs}) must be divisible by num_nodes ({num_nodes})") per_node = args.num_envs // num_nodes - print(f"sharding {args.num_envs} envs across {num_nodes} NUMA nodes -> {per_node} envs/worker") - print(f"context: cpus={os.cpu_count()} numa_nodes={num_nodes} (driver binding=[{_current_binding()}])") - tmp_paths = [] procs = [] for node in range(num_nodes): @@ -289,7 +302,7 @@ def run_shard(args) -> None: "--warmup", str(args.warmup), "--iters", - str(args.iters), + args.iters if isinstance(args.iters, str) else str(args.iters), "--out-json", tmp_path, ] @@ -306,17 +319,16 @@ def run_shard(args) -> None: aggregate_env_steps = sum(r["total_env_steps_per_s"] for r in results) slowest_median = max(r["median_ms"] for r in results) - print() - for r in results: - _print_result(f"shard node {r['node']}", r, _device_context()) - print() - print(f"=== aggregate ({len(results)} workers x {per_node} envs = {args.num_envs} total) ===") - print(f"sum total env steps/s: {aggregate_env_steps:.0f}") - print( - f"(slowest worker median {slowest_median:.3f} ms -> " - f"{args.num_envs / slowest_median * 1000:.0f} env steps/s at wall-clock)" - ) - print("compare: single-process total env steps/s is the number to beat; see wiki/design/numba-server-perf.md 6.3") + notes = [ + f"sharded {args.num_envs} envs across {num_nodes} NUMA nodes -> {per_node} envs/worker", + f"aggregate ({len(results)} workers x {per_node} envs = {args.num_envs} total): " + f"sum total env steps/s {aggregate_env_steps:,.0f}", + f"slowest worker median {slowest_median:.3f} ms -> " + f"{args.num_envs / slowest_median * 1000:,.0f} env steps/s at wall-clock", + "compare: single-process total env steps/s is the number to beat; see wiki/design/numba-server-perf.md 6.3", + ] + rows = [_result_row(f"shard node {r['node']}", r) for r in results] + return rows, notes def main() -> None: @@ -356,16 +368,30 @@ def main() -> None: return num_envs_values = args.num_envs - for index, num_envs in enumerate(num_envs_values): - if index: - print() + rows: list[list[str]] = [] + notes: list[str] = [] + for num_envs in num_envs_values: args.num_envs = num_envs if args.numa == "single": - run_single(args) + batch_rows, batch_notes = run_single(args) elif args.numa == "node": - run_node(args) - elif args.numa == "shard": - run_shard(args) + batch_rows, batch_notes = run_node(args) + else: + batch_rows, batch_notes = run_shard(args) + rows.extend(batch_rows) + for note in batch_notes: + if note not in notes: + notes.append(note) + + context = _device_context() + print(render_table(_RESULT_HEADERS, rows)) + print( + f"sim: robot={args.robot} dt={args.dt}s solver_iterations={args.solver_iterations} " + f"nstep={args.nstep} warmup={args.warmup} iters={args.iters}" + ) + print(f"context: cpus={context['num_cpus']} numa_nodes={context['num_numa_nodes']} binding=[{context['binding']}]") + for note in notes: + print(note) if __name__ == "__main__": diff --git a/bench/report.py b/bench/report.py new file mode 100644 index 0000000..22753d9 --- /dev/null +++ b/bench/report.py @@ -0,0 +1,53 @@ +# Copyright Motphys Technology Co., Ltd. 2025, 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Console table rendering for bench scripts. + +Bench scripts collect result rows while measurements run and render a single +aligned table after the run finishes, so engine log output emitted during the +benchmark cannot interleave with the report. +""" + +from __future__ import annotations + +import re + +_NUMERIC_CELL = re.compile(r"-?[\d.,]+(%|x)?") + + +def _is_numeric(cell: str) -> bool: + return _NUMERIC_CELL.fullmatch(cell) is not None + + +def _is_dash(cell: str) -> bool: + return cell in {"—", "-"} + + +def render_table(headers: list[str], rows: list[list[str]]) -> str: + """Render an ASCII table; numeric-looking columns are right-aligned.""" + widths = [len(header) for header in headers] + for row in rows: + for index, cell in enumerate(row): + widths[index] = max(widths[index], len(cell)) + + right_align = [ + all(_is_numeric(row[column]) or _is_dash(row[column]) for row in rows) for column in range(len(headers)) + ] + + def border(left: str, middle: str, right: str) -> str: + return left + middle.join("-" * (width + 2) for width in widths) + right + + def format_row(cells: list[str]) -> str: + parts = [] + for cell, width, numeric in zip(cells, widths, right_align): + parts.append(cell.rjust(width) if numeric else cell.ljust(width)) + return "| " + " | ".join(parts) + " |" + + lines = [ + border("+", "+", "+"), + format_row(headers), + border("+", "+", "+"), + *(format_row(row) for row in rows), + border("+", "+", "+"), + ] + return "\n".join(lines)