feat(stats): keep the unrounded delta, CI and corrected p next to the rounded ones

dokimi/changelog.d/20261101_100000_napetrov_exact-stats.md

@@ -0,0 +1,8 @@
+<!--
+Copyright (C) 2026 Intel Corporation
+SPDX-License-Identifier: Apache-2.0
+-->
+
+### Added
+
+- **Unrounded statistics next to the rounded ones** — every paired significance block (`summary.per_arm[arm].significance` in `arms.v1`, and the cells that copy it) gains `exact: {delta_mean, ci_lo, ci_hi, p_adjusted}`: the values before rounding. `delta_mean` (2 decimals), `ci_lo`/`ci_hi` (4) and `p_adjusted` (4) are unchanged, so existing readers see the same numbers; a consumer that publishes the effect or re-derives a verdict reads `exact`.

dokimi/dokimi/eval/stats.py

@@ -92,8 +92,14 @@ def _is_degenerate(mean: float, sd: float) -> bool:
     return sd <= ZERO_VARIANCE_REL_TOL * max(1.0, abs(mean))
 
 
+def _rnd(x: float, ndigits: Optional[int]) -> float:
+    """``round(x, ndigits)``, or ``x`` itself when ``ndigits`` is None."""
+    return x if ndigits is None else round(x, ndigits)
+
+
 def bootstrap_ci(
-    values, confidence: float = 0.95, n_resamples: int = 10_000, seed: int = 0
+    values, confidence: float = 0.95, n_resamples: int = 10_000, seed: int = 0,
+    ndigits: Optional[int] = 4,
 ) -> dict:
     """Percentile bootstrap confidence interval for the mean of ``values``.
 
@@ -116,7 +122,7 @@ def bootstrap_ci(
     if n == 0:
         return {"mean": None, "lo": None, "hi": None, "n": 0}
     if n == 1:
-        return {"mean": round(vals[0], 4), "lo": round(vals[0], 4), "hi": round(vals[0], 4), "n": 1}
+        return {"mean": _rnd(vals[0], ndigits), "lo": _rnd(vals[0], ndigits), "hi": _rnd(vals[0], ndigits), "n": 1}
 
     rng = random.Random(seed)
     means = []
@@ -131,15 +137,15 @@ def bootstrap_ci(
     lo_idx = max(0, int(round(alpha * (n_resamples - 1))))
     hi_idx = min(n_resamples - 1, int(round((1.0 - alpha) * (n_resamples - 1))))
     return {
-        "mean": round(sum(vals) / n, 4),
-        "lo": round(means[lo_idx], 4),
-        "hi": round(means[hi_idx], 4),
+        "mean": _rnd(sum(vals) / n, ndigits),
+        "lo": _rnd(means[lo_idx], ndigits),
+        "hi": _rnd(means[hi_idx], ndigits),
         "n": n,
         "method": "percentile_bootstrap",
     }
 
 
-def mean_ci(values, confidence: float = 0.95, seed: int = 0) -> dict:
+def mean_ci(values, confidence: float = 0.95, seed: int = 0, ndigits: Optional[int] = 4) -> dict:
     """Confidence interval for the mean: Student-t when SciPy is available.
 
     The t interval is both better calibrated at small n and *coherent* with the
@@ -158,20 +164,20 @@ def mean_ci(values, confidence: float = 0.95, seed: int = 0) -> dict:
     if n == 0:
         return {"mean": None, "lo": None, "hi": None, "n": 0, "method": "none"}
     if not HAS_SCIPY or n < 2:
-        return bootstrap_ci(vals, confidence=confidence, seed=seed)
+        return bootstrap_ci(vals, confidence=confidence, seed=seed, ndigits=ndigits)
 
     mu = _mean(vals)
     sd = _sample_sd(vals)
     if _is_degenerate(mu, sd):
         # Every value identical: the interval is the point itself, and a
         # bootstrap would report the same thing after 10k pointless resamples.
-        return {"mean": round(mu, 4), "lo": round(mu, 4), "hi": round(mu, 4),
+        return {"mean": _rnd(mu, ndigits), "lo": _rnd(mu, ndigits), "hi": _rnd(mu, ndigits),
                 "n": n, "method": "degenerate"}
     half = _sp_stats.t.ppf(1.0 - (1.0 - confidence) / 2.0, n - 1) * sd / (n ** 0.5)
     return {
-        "mean": round(mu, 4),
-        "lo": round(mu - half, 4),
-        "hi": round(mu + half, 4),
+        "mean": _rnd(mu, ndigits),
+        "lo": _rnd(mu - half, ndigits),
+        "hi": _rnd(mu + half, ndigits),
         "n": n,
         "method": "student_t",
     }
@@ -508,7 +514,8 @@ def _describe_deltas(
         # zero: reporting 0.0 would label a flawless, perfectly consistent
         # shift "negligible". None renders as "unknown" — what we actually know.
         cohens_d = None
-    ci = mean_ci(deltas, confidence=confidence, seed=seed)
+    exact_ci = mean_ci(deltas, confidence=confidence, seed=seed, ndigits=None)
+    ci = {k: round(v, 4) if k in ("lo", "hi") and v is not None else v for k, v in exact_ci.items()}
 
     result: Dict[str, Any] = {
         "n": len(deltas),
@@ -520,6 +527,9 @@ def _describe_deltas(
         "ci_hi": ci["hi"],
         "ci_confidence": confidence,
         "ci_method": ci.get("method"),
+        # The fields above are rounded for display; a consumer that re-derives a verdict
+        # or publishes the effect reads these. p_adjusted is set with the family correction.
+        "exact": {"delta_mean": d_mean, "ci_lo": exact_ci["lo"], "ci_hi": exact_ci["hi"]},
     }
     result.update(_one_sample_pvalues(deltas))
 
@@ -615,6 +625,14 @@ def arm_significance(
     return out
 
 
+def set_exact_p_adjusted(results: Sequence[Dict[str, Any]], *, method: str) -> None:
+    """Set ``exact.p_adjusted`` to the family correction of ``p_ttest`` before rounding."""
+    exact = adjust_pvalues([r["p_ttest"] for r in results], method=method, rounded=False)
+    for res, p in zip(results, exact, strict=True):
+        if "exact" in res:
+            res["exact"]["p_adjusted"] = p
+
+
 def _apply_correction(
     results: Sequence[Dict[str, Any]], *, correction: str, label: str
 ) -> None:
@@ -626,6 +644,7 @@ def _apply_correction(
     the multiplier on both.
     """
     decided = correct_family([r["p_ttest"] for r in results], method=correction)
+    set_exact_p_adjusted(results, method=correction)
     tested = sum(1 for r in results if r["p_ttest"] is not None)
     for res, (adj, adj_significant) in zip(results, decided, strict=True):
         res["p_adjusted"] = adj

dokimi/dokimi/report/model_compare.py

@@ -996,10 +996,9 @@ def _adjusted_significance(
     for sig, (p_adj, adj_significant) in zip(tested, decided, strict=True):
         sig["p_adjusted"] = p_adj
         # Same conjunction as the raw verdict, with the adjusted verdict in place of the raw one.
-        sig["significant_adjusted"] = bool(
-            adj_significant
-            and (sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA)
-        )
+        sig["significant_adjusted"] = bool(adj_significant and (
+            sig["p_wilcoxon"] is None or sig["p_wilcoxon"] < arm_stats.ALPHA))
+    arm_stats.set_exact_p_adjusted(tested, method=correction)
     # Correct on the exact p-values, then round: 3 * round(0.016666, 4) is 0.0501.
     return {rid: sig and arm_stats._finalize(sig) for rid, sig in sigs.items()}, len(tested)
 

dokimi/scripts/growth_baseline.json

@@ -4,7 +4,7 @@
     "dokimi/harnesses/benchflow.py": {"lines": 869},
     "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"},
-    "dokimi/report/model_compare.py": {"lines": 1200, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"},
+    "dokimi/report/model_compare.py": {"lines": 1199, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"},
     "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"},
     "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"},
     "tests/test_container_agent.py": {"lines": 1235, "rationale": "pre-existing when the gate was introduced"},

dokimi/tests/test_model_compare_correction.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""The run family correction of the model comparison keeps the exact p-values."""
+
+from __future__ import annotations
+
+from dokimi.eval import stats
+from dokimi.report.model_compare import _adjusted_significance
+
+
+def test_adjusted_significance_corrects_exact_p_only_across_testable_runs():
+    def run(deltas):
+        return [{"with_docs": 1.0 + d, "without_docs": 1.0} for d in deltas]
+
+    scores = {
+        "a": run([1.0, 2.5, 3.1, 4.7, 5.2, 6.0, 7.1]),
+        "b": run([0.3, 0.5]),  # untestable: fewer than MIN_PAIRS_FOR_TEST pairs
+        "c": run([2.0, 1.5, 2.2, 1.9, 2.4, 1.1, 2.8]),
+    }
+    sigs, n_tested = _adjusted_significance(["a", "b", "c"], scores, "holm")
+
+    assert n_tested == 2
+    assert sigs["b"] is None  # significance_test is None below MIN_PAIRS_FOR_TEST
+    # sigs are rounded for display (p_ttest included), so take the raw p-values from paired_test.
+    raw = [stats.paired_test([1.0 + d for d in ds], [1.0] * len(ds), rounded=False)["p_ttest"]
+           for ds in ([1.0, 2.5, 3.1, 4.7, 5.2, 6.0, 7.1], [2.0, 1.5, 2.2, 1.9, 2.4, 1.1, 2.8])]
+    assert None not in raw
+    exact = stats.adjust_pvalues(raw, method="holm", rounded=False)
+    assert sigs["a"]["exact"]["p_adjusted"] == exact[0]
+    assert sigs["c"]["exact"]["p_adjusted"] == exact[1]

dokimi/tests/test_stats.py

@@ -633,7 +633,7 @@ def fake(vals, confidence=None, **kw):
     monkeypatch.setattr(stats, "HAS_SCIPY", False)
     monkeypatch.setattr(stats, "bootstrap_ci", fake)
     stats.mean_ci([1.0, 2.0, 3.0], confidence=0.8, seed=7)
-    assert seen == {"seed": 7, "confidence": 0.8}
+    assert seen == {"seed": 7, "confidence": 0.8, "ndigits": 4}
 
 
 def test_bootstrap_ci_defaults_are_pinned():

dokimi/tests/test_stats_exact.py

@@ -0,0 +1,41 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""The unrounded statistics a paired test keeps next to its display fields."""
+
+from __future__ import annotations
+
+import pytest
+
+from dokimi.eval import stats
+
+
+def test_exact_holds_the_unrounded_delta_and_interval():
+    res = stats.paired_test([1.0, 2.0, 4.0, 8.0], [0.0, 0.0, 0.0, 1.0 / 3])
+    deltas = [1.0, 2.0, 4.0, 8.0 - 1.0 / 3]
+
+    assert res["exact"]["delta_mean"] == pytest.approx(sum(deltas) / 4, abs=1e-12)
+    assert res["delta_mean"] == round(res["exact"]["delta_mean"], 2)
+    assert res["ci_lo"] == round(res["exact"]["ci_lo"], 4)
+    assert res["ci_hi"] == round(res["exact"]["ci_hi"], 4)
+    assert res["exact"]["ci_lo"] != res["ci_lo"]  # the rounding is real on this input
+
+
+def test_exact_p_adjusted_is_the_family_correction_before_rounding():
+    # Seven pairs per comparison: below MIN_PAIRS_FOR_TEST (6) there is no p to correct,
+    # and the assertions would pass on None == None.
+    results = [
+        stats.paired_test([1.0, 2.5, 3.1, 4.7, 5.2, 6.0, 7.1], [0.0, 0.1, 0.0, 0.2, 0.3, 0.1, 0.0], rounded=False),
+        stats.paired_test([0.3, -0.2, 0.9, 0.1, 0.5, 0.4, 0.8], [0.0] * 7, rounded=False),
+        stats.paired_test([2.0, 1.5, 2.2, 1.9, 2.4, 1.1, 2.8], [1.0, 0.9, 1.2, 1.0, 1.1, 0.8, 1.5], rounded=False),
+    ]
+    assert all(r["p_ttest"] is not None for r in results)
+    stats._apply_correction(results, correction="holm", label="arms")
+    exact = stats.adjust_pvalues([r["p_ttest"] for r in results], method="holm", rounded=False)
+
+    for res, p in zip(results, exact, strict=True):
+        assert res["exact"]["p_adjusted"] is not None
+        assert res["exact"]["p_adjusted"] == p
+        assert res["p_adjusted"] == stats._round_p(p)
+    # The first comparison's exact corrected p is not its displayed value: the rounding is real here.
+    assert results[0]["exact"]["p_adjusted"] != results[0]["p_adjusted"]