diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ae979a177..67c7cdd54 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -279,7 +279,12 @@ def test_no_graded_rows_is_not_an_error(self) -> None: ) self.assertEqual(status, 0) - self.assertIn("no rows with an expected decision", out.getvalue()) + report = out.getvalue() + self.assertIn("1 of 1 rows with an expected decision had no audit", report) + self.assertIn( + "none of the 1 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) class BoundaryGateTest(unittest.TestCase): @@ -340,5 +345,190 @@ def test_total_boundary_rejection_fails_the_run(self) -> None: self.assertIn("FAILED (gates did not run)", out.getvalue()) +class CalibrationCoverageTest(unittest.TestCase): + """#1166: rows that never reach the C.3 fit must count against the gate.""" + + def _rows(self) -> list: + return [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "alpha2", ["616-305"]), + _golden_row("p3", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + _golden_row("n2", "hard_negative", "beta2", []), + _golden_row("e1", "explicit", "delta", ["616-305"]), + ] + + def _pipe(self) -> "CountingPipeline": + return CountingPipeline( + { + "alpha": [("616-305", 5.0), ("999-999", 0.1)], + "alpha2": [("616-305", 4.0), ("999-999", 0.2)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + "beta2": [("111-111", 2.0), ("222-222", 1.9)], + "delta": [("616-305", 1.0)], + # "empty" yields no candidates on purpose + } + ) + + def test_eligible_counts_only_the_two_calibration_slices(self) -> None: + self.assertEqual(harness.calibration_eligible(self._rows()), 5) + + def test_partial_drop_fails_the_gate_and_says_how_many(self) -> None: + rows, pipe = self._rows(), self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNotNone(scaler, "T is still fitted so C.4 can report") + self.assertIn("reached calibration: 4/5", report) + self.assertIn("1 of 5 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate denominator incomplete; ", report) + + def test_full_coverage_gates_on_ece_only(self) -> None: + rows = [r for r in self._rows() if r.id != "p3"] + pipe = self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertIsNotNone(scaler) + self.assertIn("reached calibration: 4/4", report) + self.assertNotIn("denominator incomplete", report) + self.assertIn("(calibrated, in-sample)", report) + # Expected outcome computed independently of the report text: refit T on + # the same calibration set and apply the 0.10 bar ourselves. + from application.utils.librarian.calibration.temperature import ( + expected_calibration_error, + fit_temperature, + ) + + logit_sets, labels = harness.calibration_set(rows, audits) + scaler_check = fit_temperature(logit_sets, labels) + ece_check = expected_calibration_error( + [scaler_check.confidence(s) for s in logit_sets], labels + ) + self.assertAlmostEqual(scaler.temperature, scaler_check.temperature, places=6) + self.assertEqual(status, 0 if ece_check < 0.10 else 1) + + def _well_calibrated_pipe(self) -> "CountingPipeline": + # Coin-flip shortlists on a balanced set: ECE is 0 at any T, so only + # coverage can decide these two tests. + return CountingPipeline( + { + "alpha": [("616-305", 0.0), ("999-999", 0.0)], + "alpha2": [("616-305", 0.0), ("999-999", 0.0)], + "beta": [("111-111", 0.0), ("222-222", 0.0)], + "beta2": [("111-111", 0.0), ("222-222", 0.0)], + } + ) + + def test_passing_ece_with_partial_coverage_still_fails(self) -> None: + rows, pipe = self._rows(), self._well_calibrated_pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, _ = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1, "the #1166 case: ECE passes, rows are missing") + self.assertIn("gate ECE<0.10: FAIL", report) + self.assertNotIn("PASS", report) + self.assertIn("ECE alone met the bar", report) + + def test_passing_ece_with_full_coverage_passes(self) -> None: + rows = [r for r in self._rows() if r.id != "p3"] + pipe = self._well_calibrated_pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, _ = harness.report_calibration(rows, audits) + self.assertEqual(status, 0) + self.assertIn("gate ECE<0.10: PASS", out.getvalue()) + + def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row( + "d2", "hard_negative", "beta", [], reason_code="BELOW_THRESHOLD" + ), + _golden_row("d3", "positive", "never-audited", ["616-305"]), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 20.0), ("999-999", 0.0)], + "beta": [("111-111", 1.0), ("222-222", 0.99)], + } + ) + audits = harness.live_audits(rows[:2], pipe, pipe) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, audits, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0, "still informational") + self.assertIn("1 of 3 rows with an expected decision had no audit", report) + self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report) + + def test_degenerate_set_still_reports_coverage(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "empty2", []), + ] + pipe = CountingPipeline( + {"alpha": [("616-305", 5.0)]} + ) # the other two are empty + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNone(scaler) + self.assertIn("reached calibration: 1/3", report) + self.assertIn("2 of 3 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate did not run)", report) + + def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row("d2", "positive", "alpha2", ["616-305"]), + ] + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0) + self.assertIn("2 of 2 rows with an expected decision had no audit", report) + self.assertIn( + "none of the 2 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) + + def test_selection_without_expected_decisions_keeps_the_original_line(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [_golden_row("e1", "explicit", "delta", ["616-305"])] + rows[0] = rows[0].model_copy( + update={"expected": rows[0].expected.model_copy(update={"decision": None})} + ) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + self.assertEqual(status, 0) + self.assertIn( + "no rows with an expected decision in this selection", out.getvalue() + ) + + if __name__ == "__main__": unittest.main() diff --git a/docs/gsoc_2026_module_c/runbook.md b/docs/gsoc_2026_module_c/runbook.md index b666eb494..fd0774eac 100644 --- a/docs/gsoc_2026_module_c/runbook.md +++ b/docs/gsoc_2026_module_c/runbook.md @@ -44,8 +44,9 @@ pass and one fitted `T` โ€” the pipeline is built once per run, not once per report. **C.3 is the only *additional* gate a live run adds.** A failed or skipped ECE -gate returns non-zero, so a live run cannot pass without calibration having -actually run, and C.4 stays informational. The hermetic gates still apply on top: +gate returns non-zero, and so does a run where eligible rows never reached the +fit, so a live run cannot pass without calibration having actually run over the +whole selection. C.4 stays informational. The hermetic gates still apply on top: a failed C.0.5 explicit slice, or a C.0 boundary that rejects the whole dataset, fails the run in either mode (ยง5). @@ -201,6 +202,12 @@ A degenerate calibration set โ€” all top-1 labels the same class โ€” cannot iden `T`. The harness fails rather than skipping, so this is reported, not hidden. Widen the slice selection so both the positive and hard_negative slices are in. +**C.3 fails with "gate denominator incomplete"** +Eligible rows never reached the fit, so the `reached calibration: used/eligible` +line shows `used` below `eligible`. Widening the selection will not help. Find +the rows with no audit or an empty reranked shortlist and fix why they came back +empty. + --- ## 6. Known limitations diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 34b945ebc..0d7f60bd8 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -327,6 +327,14 @@ def calibration_set( return logit_sets, labels +def calibration_eligible(rows: List[GoldenDatasetRow]) -> int: + """Rows the C.3 fit is *supposed* to draw on: the positive + hard_negative + slices of the selection. ``calibration_set`` may return fewer (no audit, or an + empty reranked shortlist); the gate counts against this number so a drop is + never silent (#1166, same class as the explicit-gate fix in #1140).""" + return sum(1 for r in rows if r.slice.value in ("positive", "hard_negative")) + + def report_calibration( rows: List[GoldenDatasetRow], audits: Dict[str, Any], @@ -336,12 +344,13 @@ def report_calibration( Reads the shared ``audits`` from ``live_audits``, the same shortlists ``report_retrieval_recall`` scores, so the positive slice is not retrieved and reranked a second time just to calibrate on it. Confidence is the top-1 mass - of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on - ECE < 0.10. + of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL. PASS + needs ECE < 0.10 and every eligible row to have reached the fit (#1166). - Returns ``(status, scaler)``. Status is 1 on a failed gate, and also on a - degenerate calibration set, so a live run can never exit 0 without the gate - actually having run. The fitted scaler is handed back (``None`` when the set + Returns ``(status, scaler)``. Status is 1 on a failed gate, on eligible rows + missing from the fit, and on a degenerate calibration set, so a live run can + never exit 0 without the gate actually having run over the whole selection. + The fitted scaler is handed back (``None`` when the set was degenerate) so the C.4 report thresholds on this same ``T`` instead of fitting its own. """ @@ -352,6 +361,16 @@ def report_calibration( ) logit_sets, labels = calibration_set(rows, audits) + # Coverage first, so the degenerate path below also says how many of the + # eligible rows reached the fit instead of only "need both outcomes". + eligible = calibration_eligible(rows) + used = len(labels) + print(f"calibration (C.3): reached calibration: {used}/{eligible}") + if used < eligible: + print( + f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " + "reach the fit (no audit or empty reranked shortlist)" + ) if len(set(labels)) < 2: # Degenerate calibration set: single-class labels, or nothing left after @@ -371,12 +390,23 @@ def report_calibration( conf_cal = [scaler.confidence(s) for s in logit_sets] ece_raw = expected_calibration_error(conf_raw, labels) ece_cal = expected_calibration_error(conf_cal, labels) - gate_ok = ece_cal < 0.10 + # The gate counts against every eligible row in the selection. Rows that + # never reached the fit (no audit, empty shortlist) shrink the denominator + # silently otherwise, and a hollowed-out set can still print PASS (#1166). + coverage_ok = used == eligible + ece_ok = ece_cal < 0.10 + gate_ok = ece_ok and coverage_ok print( - f"calibration (C.3, {len(labels)} rows): fitted T={scaler.temperature:.3f}; " - f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); " - f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}" + f"calibration (C.3, {used} rows): fitted T={scaler.temperature:.3f}; " + f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated, in-sample); " + f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}; " + f"reached calibration: {used}/{eligible}" ) + if not coverage_ok: + print( + "calibration (C.3): FAILED (gate denominator incomplete; " + f"ECE alone {'met' if ece_ok else 'missed'} the bar)" + ) return (0 if gate_ok else 1), scaler @@ -406,9 +436,23 @@ def report_decision_accuracy( from application.utils.librarian.decision_engine import decide from application.utils.librarian.schemas import Decision - graded = [r for r in rows if r.expected.decision is not None and r.id in audits] + selected = [r for r in rows if r.expected.decision is not None] + graded = [r for r in selected if r.id in audits] + if len(graded) < len(selected): + # Informational report, but a silent drop still hides a broken pipeline; + # say so even when nothing at all could be graded. + print( + f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows " + "with an expected decision had no audit and were not graded" + ) if not graded: - print("decision (C.4): no rows with an expected decision in this selection") + if selected: + print( + f"decision (C.4): none of the {len(selected)} rows with an expected " + "decision could be graded (no audits)" + ) + else: + print("decision (C.4): no rows with an expected decision in this selection") return 0 dec_match = reason_match = 0