From cf0cab1a8552b2187062681129b2ad29665e1359 Mon Sep 17 00:00:00 2001 From: dadhi <39516+dadhi@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:34:33 +0000 Subject: [PATCH] Fix benchmark report text encoding Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Regression/compare.py | 8 ++++---- .../Regression/tests/test_policy.py | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/compare.py b/test/FastExpressionCompiler.Benchmarks/Regression/compare.py index d258f16a9..b327ed47b 100644 --- a/test/FastExpressionCompiler.Benchmarks/Regression/compare.py +++ b/test/FastExpressionCompiler.Benchmarks/Regression/compare.py @@ -77,16 +77,16 @@ def render(report, artifact_url=None, warning=3.0, regression=5.0): confirmed = sum(classify(r, warning, regression)[2] for r in rows) title = "Inconclusive" if not complete else "Potential regressions detected" if confirmed else "Comparison complete" lines = [f"### {c['runtime']}: {title} (reporting only)", "", - f"Base `{c['base_sha'][:12]}` → PR `{c['head_sha'][:12]}`; SDK `{c['sdk']}`; harness `{c['harness_sha'][:12]}` ({c['harness_source']}).", ""] + f"Base `{c['base_sha'][:12]}` -> PR `{c['head_sha'][:12]}`; SDK `{c['sdk']}`; harness `{c['harness_sha'][:12]}` ({c['harness_source']}).", ""] if not complete: lines += ["Missing or invalid benchmark results. This is not evidence of a performance pass.", ""] if c["harness_source"] == "bootstrap": lines += ["Bootstrap: the base does not contain this harness yet. One candidate harness was used for both revisions; later PRs use the base-branch harness.", ""] - lines += ["| Backend / phase / workload | Base ns | PR ns | Time change | B/op: base → PR | Assessment |", "| --- | ---: | ---: | ---: | ---: | --- |"] + lines += ["| Backend / phase / workload | Base ns | PR ns | Time change | B/op: base -> PR | Assessment |", "| --- | ---: | ---: | ---: | ---: | --- |"] for r in rows: means = {v: statistics.median(rd[v]["mean"] for rd in r["rounds"]) for v in ("baseline", "candidate")} allocations = {v: statistics.median(rd[v]["bytes"] for rd in r["rounds"]) for v in means} delta, status, _ = classify(r, warning, regression) - lines.append(f"| {r['backend']} / {r['method']} / {r['shape']} | {means['baseline']:.2f} | {means['candidate']:.2f} | {delta:+.1f}% | {allocations['baseline']:.2f} → {allocations['candidate']:.2f} | {status} |") - lines += ["", f"Two opposite-order process rounds on one GitHub-hosted runner; BDN confidence intervals must be separated in both rounds to classify a >{regression:g}% time regression. A ≥3% change without that evidence is a warning, not proof. Five measurement iterations/round; hosted-runner noise remains.", "", + lines.append(f"| {r['backend']} / {r['method']} / {r['shape']} | {means['baseline']:.2f} | {means['candidate']:.2f} | {delta:+.1f}% | {allocations['baseline']:.2f} -> {allocations['candidate']:.2f} | {status} |") + lines += ["", f"Two opposite-order process rounds on one GitHub-hosted runner; BDN confidence intervals must be separated in both rounds to classify a >{regression:g}% time regression. A >=3% change without that evidence is a warning, not proof. Five measurement iterations/round; hosted-runner noise remains.", "", "Compile excludes delegate invocation; CompileAndFirstInvoke includes the first call/JIT of a new delegate. Invoke is warmed; closure invocation includes creating and calling the returned delegate. MemoryDiagnoser reports managed B/op. Loop/checksum overhead is shared by both revisions."] if artifact_url: lines += ["", f"[Workflow run and complete raw artifacts]({artifact_url})"] return "\n".join(lines) + "\n" diff --git a/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py b/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py index c79dce675..80a0a7034 100644 --- a/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py +++ b/test/FastExpressionCompiler.Benchmarks/Regression/tests/test_policy.py @@ -29,6 +29,12 @@ def test_baseline_control_is_complete(self): self.assertTrue(compare.validate_report(report())) self.assertIn("Comparison complete", compare.render(report())) + def test_rendered_report_is_ascii(self): + rendered = compare.render(report()) + self.assertTrue(rendered.isascii()) + self.assertIn("-> PR", rendered) + self.assertIn("B/op: base -> PR", rendered) + def test_significance_required_in_both_rounds(self): row = report()["rows"][0] row["rounds"][0]["candidate"] = stats(120,119,121)