fix(librarian): count rows that never reached the C.3 fit against the ECE gate (#1166)

application/tests/librarian/evaluate_harness_test.py

@@ -279,7 +279,12 @@ def test_no_graded_rows_is_not_an_error(self) -> None:
             )
 
         self.assertEqual(status, 0)
-        self.assertIn("no rows with an expected decision", out.getvalue())
+        report = out.getvalue()
+        self.assertIn("1 of 1 rows with an expected decision had no audit", report)
+        self.assertIn(
+            "none of the 1 rows with an expected decision could be graded", report
+        )
+        self.assertNotIn("no rows with an expected decision", report)
 
 
 class BoundaryGateTest(unittest.TestCase):
@@ -340,5 +345,190 @@ def test_total_boundary_rejection_fails_the_run(self) -> None:
         self.assertIn("FAILED (gates did not run)", out.getvalue())
 
 
+class CalibrationCoverageTest(unittest.TestCase):
+    """#1166: rows that never reach the C.3 fit must count against the gate."""
+
+    def _rows(self) -> list:
+        return [
+            _golden_row("p1", "positive", "alpha", ["616-305"]),
+            _golden_row("p2", "positive", "alpha2", ["616-305"]),
+            _golden_row("p3", "positive", "empty", ["616-305"]),
+            _golden_row("n1", "hard_negative", "beta", []),
+            _golden_row("n2", "hard_negative", "beta2", []),
+            _golden_row("e1", "explicit", "delta", ["616-305"]),
+        ]
+
+    def _pipe(self) -> "CountingPipeline":
+        return CountingPipeline(
+            {
+                "alpha": [("616-305", 5.0), ("999-999", 0.1)],
+                "alpha2": [("616-305", 4.0), ("999-999", 0.2)],
+                "beta": [("111-111", 3.0), ("222-222", 2.9)],
+                "beta2": [("111-111", 2.0), ("222-222", 1.9)],
+                "delta": [("616-305", 1.0)],
+                # "empty" yields no candidates on purpose
+            }
+        )
+
+    def test_eligible_counts_only_the_two_calibration_slices(self) -> None:
+        self.assertEqual(harness.calibration_eligible(self._rows()), 5)
+
+    def test_partial_drop_fails_the_gate_and_says_how_many(self) -> None:
+        rows, pipe = self._rows(), self._pipe()
+        audits = harness.live_audits(rows, pipe, pipe)
+        with _captured_stdout() as out:
+            status, scaler = harness.report_calibration(rows, audits)
+        report = out.getvalue()
+        self.assertEqual(status, 1)
+        self.assertIsNotNone(scaler, "T is still fitted so C.4 can report")
+        self.assertIn("reached calibration: 4/5", report)
+        self.assertIn("1 of 5 eligible rows did not reach the fit", report)
+        self.assertIn("FAILED (gate denominator incomplete; ", report)
+
+    def test_full_coverage_gates_on_ece_only(self) -> None:
+        rows = [r for r in self._rows() if r.id != "p3"]
+        pipe = self._pipe()
+        audits = harness.live_audits(rows, pipe, pipe)
+        with _captured_stdout() as out:
+            status, scaler = harness.report_calibration(rows, audits)
+        report = out.getvalue()
+        self.assertIsNotNone(scaler)
+        self.assertIn("reached calibration: 4/4", report)
+        self.assertNotIn("denominator incomplete", report)
+        self.assertIn("(calibrated, in-sample)", report)
+        # Expected outcome computed independently of the report text: refit T on
+        # the same calibration set and apply the 0.10 bar ourselves.
+        from application.utils.librarian.calibration.temperature import (
+            expected_calibration_error,
+            fit_temperature,
+        )
+
+        logit_sets, labels = harness.calibration_set(rows, audits)
+        scaler_check = fit_temperature(logit_sets, labels)
+        ece_check = expected_calibration_error(
+            [scaler_check.confidence(s) for s in logit_sets], labels
+        )
+        self.assertAlmostEqual(scaler.temperature, scaler_check.temperature, places=6)
+        self.assertEqual(status, 0 if ece_check < 0.10 else 1)
+
+    def _well_calibrated_pipe(self) -> "CountingPipeline":
+        # Coin-flip shortlists on a balanced set: ECE is 0 at any T, so only
+        # coverage can decide these two tests.
+        return CountingPipeline(
+            {
+                "alpha": [("616-305", 0.0), ("999-999", 0.0)],
+                "alpha2": [("616-305", 0.0), ("999-999", 0.0)],
+                "beta": [("111-111", 0.0), ("222-222", 0.0)],
+                "beta2": [("111-111", 0.0), ("222-222", 0.0)],
+            }
+        )
+
+    def test_passing_ece_with_partial_coverage_still_fails(self) -> None:
+        rows, pipe = self._rows(), self._well_calibrated_pipe()
+        audits = harness.live_audits(rows, pipe, pipe)
+        with _captured_stdout() as out:
+            status, _ = harness.report_calibration(rows, audits)
+        report = out.getvalue()
+        self.assertEqual(status, 1, "the #1166 case: ECE passes, rows are missing")
+        self.assertIn("gate ECE<0.10: FAIL", report)
+        self.assertNotIn("PASS", report)
+        self.assertIn("ECE alone met the bar", report)
+
+    def test_passing_ece_with_full_coverage_passes(self) -> None:
+        rows = [r for r in self._rows() if r.id != "p3"]
+        pipe = self._well_calibrated_pipe()
+        audits = harness.live_audits(rows, pipe, pipe)
+        with _captured_stdout() as out:
+            status, _ = harness.report_calibration(rows, audits)
+        self.assertEqual(status, 0)
+        self.assertIn("gate ECE<0.10: PASS", out.getvalue())
+
+    def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None:
+        from application.utils.librarian.calibration.temperature import (
+            TemperatureScaler,
+        )
+
+        rows = [
+            _golden_row("d1", "positive", "alpha", ["616-305"]),
+            _golden_row(
+                "d2", "hard_negative", "beta", [], reason_code="BELOW_THRESHOLD"
+            ),
+            _golden_row("d3", "positive", "never-audited", ["616-305"]),
+        ]
+        pipe = CountingPipeline(
+            {
+                "alpha": [("616-305", 20.0), ("999-999", 0.0)],
+                "beta": [("111-111", 1.0), ("222-222", 0.99)],
+            }
+        )
+        audits = harness.live_audits(rows[:2], pipe, pipe)
+        with _captured_stdout() as out:
+            status = harness.report_decision_accuracy(
+                rows, audits, TemperatureScaler(1.0), 0.80
+            )
+        report = out.getvalue()
+        self.assertEqual(status, 0, "still informational")
+        self.assertIn("1 of 3 rows with an expected decision had no audit", report)
+        self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report)
+
+    def test_degenerate_set_still_reports_coverage(self) -> None:
+        rows = [
+            _golden_row("p1", "positive", "alpha", ["616-305"]),
+            _golden_row("p2", "positive", "empty", ["616-305"]),
+            _golden_row("n1", "hard_negative", "empty2", []),
+        ]
+        pipe = CountingPipeline(
+            {"alpha": [("616-305", 5.0)]}
+        )  # the other two are empty
+        audits = harness.live_audits(rows, pipe, pipe)
+        with _captured_stdout() as out:
+            status, scaler = harness.report_calibration(rows, audits)
+        report = out.getvalue()
+        self.assertEqual(status, 1)
+        self.assertIsNone(scaler)
+        self.assertIn("reached calibration: 1/3", report)
+        self.assertIn("2 of 3 eligible rows did not reach the fit", report)
+        self.assertIn("FAILED (gate did not run)", report)
+
+    def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None:
+        from application.utils.librarian.calibration.temperature import (
+            TemperatureScaler,
+        )
+
+        rows = [
+            _golden_row("d1", "positive", "alpha", ["616-305"]),
+            _golden_row("d2", "positive", "alpha2", ["616-305"]),
+        ]
+        with _captured_stdout() as out:
+            status = harness.report_decision_accuracy(
+                rows, {}, TemperatureScaler(1.0), 0.80
+            )
+        report = out.getvalue()
+        self.assertEqual(status, 0)
+        self.assertIn("2 of 2 rows with an expected decision had no audit", report)
+        self.assertIn(
+            "none of the 2 rows with an expected decision could be graded", report
+        )
+        self.assertNotIn("no rows with an expected decision", report)
+
+    def test_selection_without_expected_decisions_keeps_the_original_line(self) -> None:
+        from application.utils.librarian.calibration.temperature import (
+            TemperatureScaler,
+        )
+
+        rows = [_golden_row("e1", "explicit", "delta", ["616-305"])]
+        rows[0] = rows[0].model_copy(
+            update={"expected": rows[0].expected.model_copy(update={"decision": None})}
+        )
+        with _captured_stdout() as out:
+            status = harness.report_decision_accuracy(
+                rows, {}, TemperatureScaler(1.0), 0.80
+            )
+        self.assertEqual(status, 0)
+        self.assertIn(
+            "no rows with an expected decision in this selection", out.getvalue()
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

docs/gsoc_2026_module_c/runbook.md

@@ -44,8 +44,9 @@ pass and one fitted `T` — the pipeline is built once per run, not once per
 report.
 
 **C.3 is the only *additional* gate a live run adds.** A failed or skipped ECE
-gate returns non-zero, so a live run cannot pass without calibration having
-actually run, and C.4 stays informational. The hermetic gates still apply on top:
+gate returns non-zero, and so does a run where eligible rows never reached the
+fit, so a live run cannot pass without calibration having actually run over the
+whole selection. C.4 stays informational. The hermetic gates still apply on top:
 a failed C.0.5 explicit slice, or a C.0 boundary that rejects the whole dataset,
 fails the run in either mode (§5).
 
@@ -201,6 +202,12 @@ A degenerate calibration set — all top-1 labels the same class — cannot iden
 `T`. The harness fails rather than skipping, so this is reported, not hidden.
 Widen the slice selection so both the positive and hard_negative slices are in.
 
+**C.3 fails with "gate denominator incomplete"**
+Eligible rows never reached the fit, so the `reached calibration: used/eligible`
+line shows `used` below `eligible`. Widening the selection will not help. Find
+the rows with no audit or an empty reranked shortlist and fix why they came back
+empty.
+
 ---
 
 ## 6. Known limitations

scripts/evaluate_librarian.py

@@ -327,6 +327,14 @@ def calibration_set(
     return logit_sets, labels
 
 
+def calibration_eligible(rows: List[GoldenDatasetRow]) -> int:
+    """Rows the C.3 fit is *supposed* to draw on: the positive + hard_negative
+    slices of the selection. ``calibration_set`` may return fewer (no audit, or an
+    empty reranked shortlist); the gate counts against this number so a drop is
+    never silent (#1166, same class as the explicit-gate fix in #1140)."""
+    return sum(1 for r in rows if r.slice.value in ("positive", "hard_negative"))
+
+
 def report_calibration(
     rows: List[GoldenDatasetRow],
     audits: Dict[str, Any],
@@ -336,12 +344,13 @@ def report_calibration(
     Reads the shared ``audits`` from ``live_audits``, the same shortlists
     ``report_retrieval_recall`` scores, so the positive slice is not retrieved and
     reranked a second time just to calibrate on it. Confidence is the top-1 mass
-    of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on
-    ECE < 0.10.
+    of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL. PASS
+    needs ECE < 0.10 and every eligible row to have reached the fit (#1166).
 
-    Returns ``(status, scaler)``. Status is 1 on a failed gate, and also on a
-    degenerate calibration set, so a live run can never exit 0 without the gate
-    actually having run. The fitted scaler is handed back (``None`` when the set
+    Returns ``(status, scaler)``. Status is 1 on a failed gate, on eligible rows
+    missing from the fit, and on a degenerate calibration set, so a live run can
+    never exit 0 without the gate actually having run over the whole selection.
+    The fitted scaler is handed back (``None`` when the set
     was degenerate) so the C.4 report thresholds on this same ``T`` instead of
     fitting its own.
     """
@@ -352,6 +361,16 @@ def report_calibration(
     )
 
     logit_sets, labels = calibration_set(rows, audits)
+    # Coverage first, so the degenerate path below also says how many of the
+    # eligible rows reached the fit instead of only "need both outcomes".
+    eligible = calibration_eligible(rows)
+    used = len(labels)
+    print(f"calibration (C.3): reached calibration: {used}/{eligible}")
+    if used < eligible:
+        print(
+            f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not "
+            "reach the fit (no audit or empty reranked shortlist)"
+        )
 
     if len(set(labels)) < 2:
         # Degenerate calibration set: single-class labels, or nothing left after
@@ -371,12 +390,23 @@ def report_calibration(
     conf_cal = [scaler.confidence(s) for s in logit_sets]
     ece_raw = expected_calibration_error(conf_raw, labels)
     ece_cal = expected_calibration_error(conf_cal, labels)
-    gate_ok = ece_cal < 0.10
+    # The gate counts against every eligible row in the selection. Rows that
+    # never reached the fit (no audit, empty shortlist) shrink the denominator
+    # silently otherwise, and a hollowed-out set can still print PASS (#1166).
+    coverage_ok = used == eligible
+    ece_ok = ece_cal < 0.10
+    gate_ok = ece_ok and coverage_ok
     print(
-        f"calibration (C.3, {len(labels)} rows): fitted T={scaler.temperature:.3f}; "
-        f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); "
-        f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}"
+        f"calibration (C.3, {used} rows): fitted T={scaler.temperature:.3f}; "
+        f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated, in-sample); "
+        f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}; "
+        f"reached calibration: {used}/{eligible}"
     )
+    if not coverage_ok:
+        print(
+            "calibration (C.3): FAILED (gate denominator incomplete; "
+            f"ECE alone {'met' if ece_ok else 'missed'} the bar)"
+        )
     return (0 if gate_ok else 1), scaler
 
 
@@ -406,9 +436,23 @@ def report_decision_accuracy(
     from application.utils.librarian.decision_engine import decide
     from application.utils.librarian.schemas import Decision
 
-    graded = [r for r in rows if r.expected.decision is not None and r.id in audits]
+    selected = [r for r in rows if r.expected.decision is not None]
+    graded = [r for r in selected if r.id in audits]
+    if len(graded) < len(selected):
+        # Informational report, but a silent drop still hides a broken pipeline;
+        # say so even when nothing at all could be graded.
+        print(
+            f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows "
+            "with an expected decision had no audit and were not graded"
+        )
     if not graded:
-        print("decision (C.4): no rows with an expected decision in this selection")
+        if selected:
+            print(
+                f"decision (C.4): none of the {len(selected)} rows with an expected "
+                "decision could be graded (no audits)"
+            )
+        else:
+            print("decision (C.4): no rows with an expected decision in this selection")
         return 0
 
     dec_match = reason_match = 0