Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions test/FastExpressionCompiler.Benchmarks/Regression/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,16 @@ def render(report, artifact_url=None, warning=3.0, regression=5.0):
confirmed = sum(classify(r, warning, regression)[2] for r in rows)
title = "Inconclusive" if not complete else "Potential regressions detected" if confirmed else "Comparison complete"
lines = [f"### {c['runtime']}: {title} (reporting only)", "",
f"Base `{c['base_sha'][:12]}` → PR `{c['head_sha'][:12]}`; SDK `{c['sdk']}`; harness `{c['harness_sha'][:12]}` ({c['harness_source']}).", ""]
f"Base `{c['base_sha'][:12]}` -> PR `{c['head_sha'][:12]}`; SDK `{c['sdk']}`; harness `{c['harness_sha'][:12]}` ({c['harness_source']}).", ""]
if not complete: lines += ["Missing or invalid benchmark results. This is not evidence of a performance pass.", ""]
if c["harness_source"] == "bootstrap": lines += ["Bootstrap: the base does not contain this harness yet. One candidate harness was used for both revisions; later PRs use the base-branch harness.", ""]
lines += ["| Backend / phase / workload | Base ns | PR ns | Time change | B/op: base → PR | Assessment |", "| --- | ---: | ---: | ---: | ---: | --- |"]
lines += ["| Backend / phase / workload | Base ns | PR ns | Time change | B/op: base -> PR | Assessment |", "| --- | ---: | ---: | ---: | ---: | --- |"]
for r in rows:
means = {v: statistics.median(rd[v]["mean"] for rd in r["rounds"]) for v in ("baseline", "candidate")}
allocations = {v: statistics.median(rd[v]["bytes"] for rd in r["rounds"]) for v in means}
delta, status, _ = classify(r, warning, regression)
lines.append(f"| {r['backend']} / {r['method']} / {r['shape']} | {means['baseline']:.2f} | {means['candidate']:.2f} | {delta:+.1f}% | {allocations['baseline']:.2f} → {allocations['candidate']:.2f} | {status} |")
lines += ["", f"Two opposite-order process rounds on one GitHub-hosted runner; BDN confidence intervals must be separated in both rounds to classify a >{regression:g}% time regression. A ≥3% change without that evidence is a warning, not proof. Five measurement iterations/round; hosted-runner noise remains.", "",
lines.append(f"| {r['backend']} / {r['method']} / {r['shape']} | {means['baseline']:.2f} | {means['candidate']:.2f} | {delta:+.1f}% | {allocations['baseline']:.2f} -> {allocations['candidate']:.2f} | {status} |")
lines += ["", f"Two opposite-order process rounds on one GitHub-hosted runner; BDN confidence intervals must be separated in both rounds to classify a >{regression:g}% time regression. A >=3% change without that evidence is a warning, not proof. Five measurement iterations/round; hosted-runner noise remains.", "",
"Compile excludes delegate invocation; CompileAndFirstInvoke includes the first call/JIT of a new delegate. Invoke is warmed; closure invocation includes creating and calling the returned delegate. MemoryDiagnoser reports managed B/op. Loop/checksum overhead is shared by both revisions."]
if artifact_url: lines += ["", f"[Workflow run and complete raw artifacts]({artifact_url})"]
return "\n".join(lines) + "\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ def test_baseline_control_is_complete(self):
self.assertTrue(compare.validate_report(report()))
self.assertIn("Comparison complete", compare.render(report()))

def test_rendered_report_is_ascii(self):
rendered = compare.render(report())
self.assertTrue(rendered.isascii())
self.assertIn("-> PR", rendered)
self.assertIn("B/op: base -> PR", rendered)

def test_significance_required_in_both_rounds(self):
row = report()["rows"][0]
row["rounds"][0]["candidate"] = stats(120,119,121)
Expand Down
Loading