diff --git a/dokimi/changelog.d/20261101_100000_napetrov_exact-stats.md b/dokimi/changelog.d/20261101_100000_napetrov_exact-stats.md new file mode 100644 index 00000000..94f88feb --- /dev/null +++ b/dokimi/changelog.d/20261101_100000_napetrov_exact-stats.md @@ -0,0 +1,8 @@ + + +### Added + +- **Unrounded statistics next to the rounded ones** — every paired significance block (`summary.per_arm[arm].significance` in `arms.v1`, and the cells that copy it) gains `exact: {delta_mean, ci_lo, ci_hi, p_adjusted}`: the values before rounding. `delta_mean` (2 decimals), `ci_lo`/`ci_hi` (4) and `p_adjusted` (4) are unchanged, so existing readers see the same numbers; a consumer that publishes the effect or re-derives a verdict reads `exact`. diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index e6b169f5..6f292fab 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -92,8 +92,14 @@ def _is_degenerate(mean: float, sd: float) -> bool: return sd <= ZERO_VARIANCE_REL_TOL * max(1.0, abs(mean)) +def _rnd(x: float, ndigits: Optional[int]) -> float: + """``round(x, ndigits)``, or ``x`` itself when ``ndigits`` is None.""" + return x if ndigits is None else round(x, ndigits) + + def bootstrap_ci( - values, confidence: float = 0.95, n_resamples: int = 10_000, seed: int = 0 + values, confidence: float = 0.95, n_resamples: int = 10_000, seed: int = 0, + ndigits: Optional[int] = 4, ) -> dict: """Percentile bootstrap confidence interval for the mean of ``values``. @@ -116,7 +122,7 @@ def bootstrap_ci( if n == 0: return {"mean": None, "lo": None, "hi": None, "n": 0} if n == 1: - return {"mean": round(vals[0], 4), "lo": round(vals[0], 4), "hi": round(vals[0], 4), "n": 1} + return {"mean": _rnd(vals[0], ndigits), "lo": _rnd(vals[0], ndigits), "hi": _rnd(vals[0], ndigits), "n": 1} rng = random.Random(seed) means = [] @@ -131,15 +137,15 @@ def bootstrap_ci( lo_idx = max(0, int(round(alpha * (n_resamples - 1)))) hi_idx = min(n_resamples - 1, int(round((1.0 - alpha) * (n_resamples - 1)))) return { - "mean": round(sum(vals) / n, 4), - "lo": round(means[lo_idx], 4), - "hi": round(means[hi_idx], 4), + "mean": _rnd(sum(vals) / n, ndigits), + "lo": _rnd(means[lo_idx], ndigits), + "hi": _rnd(means[hi_idx], ndigits), "n": n, "method": "percentile_bootstrap", } -def mean_ci(values, confidence: float = 0.95, seed: int = 0) -> dict: +def mean_ci(values, confidence: float = 0.95, seed: int = 0, ndigits: Optional[int] = 4) -> dict: """Confidence interval for the mean: Student-t when SciPy is available. The t interval is both better calibrated at small n and *coherent* with the @@ -158,20 +164,20 @@ def mean_ci(values, confidence: float = 0.95, seed: int = 0) -> dict: if n == 0: return {"mean": None, "lo": None, "hi": None, "n": 0, "method": "none"} if not HAS_SCIPY or n < 2: - return bootstrap_ci(vals, confidence=confidence, seed=seed) + return bootstrap_ci(vals, confidence=confidence, seed=seed, ndigits=ndigits) mu = _mean(vals) sd = _sample_sd(vals) if _is_degenerate(mu, sd): # Every value identical: the interval is the point itself, and a # bootstrap would report the same thing after 10k pointless resamples. - return {"mean": round(mu, 4), "lo": round(mu, 4), "hi": round(mu, 4), + return {"mean": _rnd(mu, ndigits), "lo": _rnd(mu, ndigits), "hi": _rnd(mu, ndigits), "n": n, "method": "degenerate"} half = _sp_stats.t.ppf(1.0 - (1.0 - confidence) / 2.0, n - 1) * sd / (n ** 0.5) return { - "mean": round(mu, 4), - "lo": round(mu - half, 4), - "hi": round(mu + half, 4), + "mean": _rnd(mu, ndigits), + "lo": _rnd(mu - half, ndigits), + "hi": _rnd(mu + half, ndigits), "n": n, "method": "student_t", } @@ -508,7 +514,8 @@ def _describe_deltas( # zero: reporting 0.0 would label a flawless, perfectly consistent # shift "negligible". None renders as "unknown" — what we actually know. cohens_d = None - ci = mean_ci(deltas, confidence=confidence, seed=seed) + exact_ci = mean_ci(deltas, confidence=confidence, seed=seed, ndigits=None) + ci = {k: round(v, 4) if k in ("lo", "hi") and v is not None else v for k, v in exact_ci.items()} result: Dict[str, Any] = { "n": len(deltas), @@ -520,6 +527,9 @@ def _describe_deltas( "ci_hi": ci["hi"], "ci_confidence": confidence, "ci_method": ci.get("method"), + # The fields above are rounded for display; a consumer that re-derives a verdict + # or publishes the effect reads these. p_adjusted is set with the family correction. + "exact": {"delta_mean": d_mean, "ci_lo": exact_ci["lo"], "ci_hi": exact_ci["hi"]}, } result.update(_one_sample_pvalues(deltas)) @@ -615,6 +625,14 @@ def arm_significance( return out +def set_exact_p_adjusted(results: Sequence[Dict[str, Any]], *, method: str) -> None: + """Set ``exact.p_adjusted`` to the family correction of ``p_ttest`` before rounding.""" + exact = adjust_pvalues([r["p_ttest"] for r in results], method=method, rounded=False) + for res, p in zip(results, exact, strict=True): + if "exact" in res: + res["exact"]["p_adjusted"] = p + + def _apply_correction( results: Sequence[Dict[str, Any]], *, correction: str, label: str ) -> None: @@ -626,6 +644,7 @@ def _apply_correction( the multiplier on both. """ decided = correct_family([r["p_ttest"] for r in results], method=correction) + set_exact_p_adjusted(results, method=correction) tested = sum(1 for r in results if r["p_ttest"] is not None) for res, (adj, adj_significant) in zip(results, decided, strict=True): res["p_adjusted"] = adj diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index 06dc3161..cc0f20fa 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -996,10 +996,9 @@ def _adjusted_significance( for sig, (p_adj, adj_significant) in zip(tested, decided, strict=True): sig["p_adjusted"] = p_adj # Same conjunction as the raw verdict, with the adjusted verdict in place of the raw one. - sig["significant_adjusted"] = bool( - adj_significant - and (sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA) - ) + sig["significant_adjusted"] = bool(adj_significant and ( + sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA)) + arm_stats.set_exact_p_adjusted(tested, method=correction) # Correct on the exact p-values, then round: 3 * round(0.016666, 4) is 0.0501. return {rid: sig and arm_stats._finalize(sig) for rid, sig in sigs.items()}, len(tested) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d751c464..08d667d9 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -4,7 +4,7 @@ "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, - "dokimi/report/model_compare.py": {"lines": 1200, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, + "dokimi/report/model_compare.py": {"lines": 1199, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1235, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_model_compare_correction.py b/dokimi/tests/test_model_compare_correction.py new file mode 100644 index 00000000..758ad9b5 --- /dev/null +++ b/dokimi/tests/test_model_compare_correction.py @@ -0,0 +1,31 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The run family correction of the model comparison keeps the exact p-values.""" + +from __future__ import annotations + +from dokimi.eval import stats +from dokimi.report.model_compare import _adjusted_significance + + +def test_adjusted_significance_corrects_exact_p_only_across_testable_runs(): + def run(deltas): + return [{"with_docs": 1.0 + d, "without_docs": 1.0} for d in deltas] + + scores = { + "a": run([1.0, 2.5, 3.1, 4.7, 5.2, 6.0, 7.1]), + "b": run([0.3, 0.5]), # untestable: fewer than MIN_PAIRS_FOR_TEST pairs + "c": run([2.0, 1.5, 2.2, 1.9, 2.4, 1.1, 2.8]), + } + sigs, n_tested = _adjusted_significance(["a", "b", "c"], scores, "holm") + + assert n_tested == 2 + assert sigs["b"] is None # significance_test is None below MIN_PAIRS_FOR_TEST + # sigs are rounded for display (p_ttest included), so take the raw p-values from paired_test. + raw = [stats.paired_test([1.0 + d for d in ds], [1.0] * len(ds), rounded=False)["p_ttest"] + for ds in ([1.0, 2.5, 3.1, 4.7, 5.2, 6.0, 7.1], [2.0, 1.5, 2.2, 1.9, 2.4, 1.1, 2.8])] + assert None not in raw + exact = stats.adjust_pvalues(raw, method="holm", rounded=False) + assert sigs["a"]["exact"]["p_adjusted"] == exact[0] + assert sigs["c"]["exact"]["p_adjusted"] == exact[1] diff --git a/dokimi/tests/test_stats.py b/dokimi/tests/test_stats.py index 3bab27a6..30e333af 100644 --- a/dokimi/tests/test_stats.py +++ b/dokimi/tests/test_stats.py @@ -633,7 +633,7 @@ def fake(vals, confidence=None, **kw): monkeypatch.setattr(stats, "HAS_SCIPY", False) monkeypatch.setattr(stats, "bootstrap_ci", fake) stats.mean_ci([1.0, 2.0, 3.0], confidence=0.8, seed=7) - assert seen == {"seed": 7, "confidence": 0.8} + assert seen == {"seed": 7, "confidence": 0.8, "ndigits": 4} def test_bootstrap_ci_defaults_are_pinned(): diff --git a/dokimi/tests/test_stats_exact.py b/dokimi/tests/test_stats_exact.py new file mode 100644 index 00000000..9a6ef122 --- /dev/null +++ b/dokimi/tests/test_stats_exact.py @@ -0,0 +1,41 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The unrounded statistics a paired test keeps next to its display fields.""" + +from __future__ import annotations + +import pytest + +from dokimi.eval import stats + + +def test_exact_holds_the_unrounded_delta_and_interval(): + res = stats.paired_test([1.0, 2.0, 4.0, 8.0], [0.0, 0.0, 0.0, 1.0 / 3]) + deltas = [1.0, 2.0, 4.0, 8.0 - 1.0 / 3] + + assert res["exact"]["delta_mean"] == pytest.approx(sum(deltas) / 4, abs=1e-12) + assert res["delta_mean"] == round(res["exact"]["delta_mean"], 2) + assert res["ci_lo"] == round(res["exact"]["ci_lo"], 4) + assert res["ci_hi"] == round(res["exact"]["ci_hi"], 4) + assert res["exact"]["ci_lo"] != res["ci_lo"] # the rounding is real on this input + + +def test_exact_p_adjusted_is_the_family_correction_before_rounding(): + # Seven pairs per comparison: below MIN_PAIRS_FOR_TEST (6) there is no p to correct, + # and the assertions would pass on None == None. + results = [ + stats.paired_test([1.0, 2.5, 3.1, 4.7, 5.2, 6.0, 7.1], [0.0, 0.1, 0.0, 0.2, 0.3, 0.1, 0.0], rounded=False), + stats.paired_test([0.3, -0.2, 0.9, 0.1, 0.5, 0.4, 0.8], [0.0] * 7, rounded=False), + stats.paired_test([2.0, 1.5, 2.2, 1.9, 2.4, 1.1, 2.8], [1.0, 0.9, 1.2, 1.0, 1.1, 0.8, 1.5], rounded=False), + ] + assert all(r["p_ttest"] is not None for r in results) + stats._apply_correction(results, correction="holm", label="arms") + exact = stats.adjust_pvalues([r["p_ttest"] for r in results], method="holm", rounded=False) + + for res, p in zip(results, exact, strict=True): + assert res["exact"]["p_adjusted"] is not None + assert res["exact"]["p_adjusted"] == p + assert res["p_adjusted"] == stats._round_p(p) + # The first comparison's exact corrected p is not its displayed value: the rounding is real here. + assert results[0]["exact"]["p_adjusted"] != results[0]["p_adjusted"]