From 46c2efd4c166a5f9706910dbdbde19c79141dd8d Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Wed, 21 Oct 2026 19:10:16 +0530 Subject: [PATCH 1/4] librarian: count rows that never reached the C.3 fit against the ECE gate (#1166) calibration_set() skipped rows with no audit or an empty reranked shortlist, and report_calibration() printed the survivors as if they were the selection, so a hollowed-out calibration set could print PASS and exit 0. Same class as the explicit-gate fix in #1140. - calibration_eligible(): positive + hard_negative rows in the selection - report_calibration(): prints "reached calibration: used/eligible", labels the ECE as in-sample, fails the gate when any eligible row was dropped and says how many; T is still returned so C.4 can report - report_decision_accuracy(): prints how many expected-decision rows had no audit instead of silently grading fewer - tests for all three --- .../tests/librarian/evaluate_harness_test.py | 82 +++++++++++++++++++ scripts/evaluate_librarian.py | 39 +++++++-- 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ae979a177..ea5f5185a 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -342,3 +342,85 @@ def test_total_boundary_rejection_fails_the_run(self) -> None: if __name__ == "__main__": unittest.main() + + +class CalibrationCoverageTest(unittest.TestCase): + """#1166: rows that never reach the C.3 fit must count against the gate.""" + + def _rows(self) -> list: + return [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "alpha2", ["616-305"]), + _golden_row("p3", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + _golden_row("n2", "hard_negative", "beta2", []), + _golden_row("e1", "explicit", "delta", ["616-305"]), + ] + + def _pipe(self) -> "CountingPipeline": + return CountingPipeline( + { + "alpha": [("616-305", 5.0), ("999-999", 0.1)], + "alpha2": [("616-305", 4.0), ("999-999", 0.2)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + "beta2": [("111-111", 2.0), ("222-222", 1.9)], + "delta": [("616-305", 1.0)], + # "empty" yields no candidates on purpose + } + ) + + def test_eligible_counts_only_the_two_calibration_slices(self) -> None: + self.assertEqual(harness.calibration_eligible(self._rows()), 5) + + def test_partial_drop_fails_the_gate_and_says_how_many(self) -> None: + rows, pipe = self._rows(), self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNotNone(scaler, "T is still fitted so C.4 can report") + self.assertIn("reached calibration: 4/5", report) + self.assertIn("1 of 5 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate denominator incomplete)", report) + + def test_full_coverage_gates_on_ece_only(self) -> None: + rows = [r for r in self._rows() if r.id != "p3"] + pipe = self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertIsNotNone(scaler) + self.assertIn("reached calibration: 4/4", report) + self.assertNotIn("denominator incomplete", report) + self.assertIn("(calibrated, in-sample)", report) + self.assertEqual(status, 0 if "gate ECE<0.10: PASS" in report else 1) + + def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row( + "d2", "hard_negative", "beta", [], reason_code="BELOW_THRESHOLD" + ), + _golden_row("d3", "positive", "never-audited", ["616-305"]), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 20.0), ("999-999", 0.0)], + "beta": [("111-111", 1.0), ("222-222", 0.99)], + } + ) + audits = harness.live_audits(rows[:2], pipe, pipe) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, audits, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0, "still informational") + self.assertIn("1 of 3 rows with an expected decision had no audit", report) + self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 34b945ebc..18d785567 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -327,6 +327,14 @@ def calibration_set( return logit_sets, labels +def calibration_eligible(rows: List[GoldenDatasetRow]) -> int: + """Rows the C.3 fit is *supposed* to draw on: the positive + hard_negative + slices of the selection. ``calibration_set`` may return fewer (no audit, or an + empty reranked shortlist); the gate counts against this number so a drop is + never silent (#1166, same class as the explicit-gate fix in #1140).""" + return sum(1 for r in rows if r.slice.value in ("positive", "hard_negative")) + + def report_calibration( rows: List[GoldenDatasetRow], audits: Dict[str, Any], @@ -371,12 +379,26 @@ def report_calibration( conf_cal = [scaler.confidence(s) for s in logit_sets] ece_raw = expected_calibration_error(conf_raw, labels) ece_cal = expected_calibration_error(conf_cal, labels) - gate_ok = ece_cal < 0.10 + # The gate counts against every eligible row in the selection. Rows that + # never reached the fit (no audit, empty shortlist) shrink the denominator + # silently otherwise, and a hollowed-out set can still print PASS (#1166). + eligible = calibration_eligible(rows) + used = len(labels) + coverage_ok = used == eligible + ece_ok = ece_cal < 0.10 + gate_ok = ece_ok and coverage_ok print( - f"calibration (C.3, {len(labels)} rows): fitted T={scaler.temperature:.3f}; " - f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); " - f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}" + f"calibration (C.3, {used} rows): fitted T={scaler.temperature:.3f}; " + f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated, in-sample); " + f"gate ECE<0.10: {'PASS' if ece_ok else 'FAIL'}; " + f"reached calibration: {used}/{eligible}" ) + if not coverage_ok: + print( + f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " + "reach the fit (no audit or empty reranked shortlist); FAILED " + "(gate denominator incomplete)" + ) return (0 if gate_ok else 1), scaler @@ -406,10 +428,17 @@ def report_decision_accuracy( from application.utils.librarian.decision_engine import decide from application.utils.librarian.schemas import Decision - graded = [r for r in rows if r.expected.decision is not None and r.id in audits] + selected = [r for r in rows if r.expected.decision is not None] + graded = [r for r in selected if r.id in audits] if not graded: print("decision (C.4): no rows with an expected decision in this selection") return 0 + if len(graded) < len(selected): + # Informational report, but a silent drop still hides a broken pipeline. + print( + f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows " + "with an expected decision had no audit and were not graded" + ) dec_match = reason_match = 0 link_total = link_correct = review_total = review_correct = 0 From 87353a07a0d00cd3d6fcf967ac1a445a814002af Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Wed, 21 Oct 2026 19:16:54 +0530 Subject: [PATCH 2/4] librarian: report C.3/C.4 coverage before the early returns; assert the ECE gate independently Review follow-ups on #1167: - report_calibration(): print "reached calibration: used/eligible" and the dropped-row count before the degenerate-set return, so a selection where most rows never reached the fit says so instead of only "need both outcomes". - report_decision_accuracy(): print the missing-audit count before the nothing-graded return, so "no rows with an expected decision" is never the only line when every selected row simply lacked an audit. - test: the full-coverage case now computes the expected gate outcome from the fixture logits and a refitted T instead of reading it back from the report. - tests for both early-return paths. --- .../tests/librarian/evaluate_harness_test.py | 52 ++++++++++++++++++- scripts/evaluate_librarian.py | 27 ++++++---- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ea5f5185a..9c9c645a8 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -395,7 +395,20 @@ def test_full_coverage_gates_on_ece_only(self) -> None: self.assertIn("reached calibration: 4/4", report) self.assertNotIn("denominator incomplete", report) self.assertIn("(calibrated, in-sample)", report) - self.assertEqual(status, 0 if "gate ECE<0.10: PASS" in report else 1) + # Expected outcome computed independently of the report text: refit T on + # the same calibration set and apply the 0.10 bar ourselves. + from application.utils.librarian.calibration.temperature import ( + expected_calibration_error, + fit_temperature, + ) + + logit_sets, labels = harness.calibration_set(rows, audits) + scaler_check = fit_temperature(logit_sets, labels) + ece_check = expected_calibration_error( + [scaler_check.confidence(s) for s in logit_sets], labels + ) + self.assertAlmostEqual(scaler.temperature, scaler_check.temperature, places=6) + self.assertEqual(status, 0 if ece_check < 0.10 else 1) def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: from application.utils.librarian.calibration.temperature import ( @@ -424,3 +437,40 @@ def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: self.assertEqual(status, 0, "still informational") self.assertIn("1 of 3 rows with an expected decision had no audit", report) self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report) + + def test_degenerate_set_still_reports_coverage(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "empty2", []), + ] + pipe = CountingPipeline( + {"alpha": [("616-305", 5.0)]} + ) # the other two are empty + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNone(scaler) + self.assertIn("reached calibration: 1/3", report) + self.assertIn("2 of 3 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate did not run)", report) + + def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row("d2", "positive", "alpha2", ["616-305"]), + ] + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0) + self.assertIn("2 of 2 rows with an expected decision had no audit", report) + self.assertIn("no rows with an expected decision", report) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 18d785567..ce5009f83 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -360,6 +360,16 @@ def report_calibration( ) logit_sets, labels = calibration_set(rows, audits) + # Coverage first, so the degenerate path below also says how many of the + # eligible rows reached the fit instead of only "need both outcomes". + eligible = calibration_eligible(rows) + used = len(labels) + print(f"calibration (C.3): reached calibration: {used}/{eligible}") + if used < eligible: + print( + f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " + "reach the fit (no audit or empty reranked shortlist)" + ) if len(set(labels)) < 2: # Degenerate calibration set: single-class labels, or nothing left after @@ -382,8 +392,6 @@ def report_calibration( # The gate counts against every eligible row in the selection. Rows that # never reached the fit (no audit, empty shortlist) shrink the denominator # silently otherwise, and a hollowed-out set can still print PASS (#1166). - eligible = calibration_eligible(rows) - used = len(labels) coverage_ok = used == eligible ece_ok = ece_cal < 0.10 gate_ok = ece_ok and coverage_ok @@ -394,11 +402,7 @@ def report_calibration( f"reached calibration: {used}/{eligible}" ) if not coverage_ok: - print( - f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " - "reach the fit (no audit or empty reranked shortlist); FAILED " - "(gate denominator incomplete)" - ) + print("calibration (C.3): FAILED (gate denominator incomplete)") return (0 if gate_ok else 1), scaler @@ -430,15 +434,16 @@ def report_decision_accuracy( selected = [r for r in rows if r.expected.decision is not None] graded = [r for r in selected if r.id in audits] - if not graded: - print("decision (C.4): no rows with an expected decision in this selection") - return 0 if len(graded) < len(selected): - # Informational report, but a silent drop still hides a broken pipeline. + # Informational report, but a silent drop still hides a broken pipeline; + # say so even when nothing at all could be graded. print( f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows " "with an expected decision had no audit and were not graded" ) + if not graded: + print("decision (C.4): no rows with an expected decision in this selection") + return 0 dec_match = reason_match = 0 link_total = link_correct = review_total = review_correct = 0 From 68080f894c7b119a63d236e1c72ede206a5360bf Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Wed, 21 Oct 2026 19:25:54 +0530 Subject: [PATCH 3/4] librarian: say that selected rows had no audit instead of 'no rows with an expected decision' when nothing could be graded (#1167 review) --- .../tests/librarian/evaluate_harness_test.py | 30 +++++++++++++++++-- scripts/evaluate_librarian.py | 8 ++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index 9c9c645a8..bdaa2b92b 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -279,7 +279,12 @@ def test_no_graded_rows_is_not_an_error(self) -> None: ) self.assertEqual(status, 0) - self.assertIn("no rows with an expected decision", out.getvalue()) + report = out.getvalue() + self.assertIn("1 of 1 rows with an expected decision had no audit", report) + self.assertIn( + "none of the 1 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) class BoundaryGateTest(unittest.TestCase): @@ -473,4 +478,25 @@ def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None: report = out.getvalue() self.assertEqual(status, 0) self.assertIn("2 of 2 rows with an expected decision had no audit", report) - self.assertIn("no rows with an expected decision", report) + self.assertIn( + "none of the 2 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) + + def test_selection_without_expected_decisions_keeps_the_original_line(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [_golden_row("e1", "explicit", "delta", ["616-305"])] + rows[0] = rows[0].model_copy( + update={"expected": rows[0].expected.model_copy(update={"decision": None})} + ) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + self.assertEqual(status, 0) + self.assertIn( + "no rows with an expected decision in this selection", out.getvalue() + ) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index ce5009f83..159b845f9 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -442,7 +442,13 @@ def report_decision_accuracy( "with an expected decision had no audit and were not graded" ) if not graded: - print("decision (C.4): no rows with an expected decision in this selection") + if selected: + print( + f"decision (C.4): none of the {len(selected)} rows with an expected " + "decision could be graded (no audits)" + ) + else: + print("decision (C.4): no rows with an expected decision in this selection") return 0 dec_match = reason_match = 0 From d4be948ab005e97cc5bae075cd7b4413805e59a5 Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Thu, 5 Nov 2026 16:39:25 +0530 Subject: [PATCH 4/4] librarian eval: test the #1166 case itself and stop printing PASS on it The tests never reached a passing ECE, so reverting the coverage check left them green. Add a fixture where ECE passes and only coverage decides the gate. The C.3 line now prints FAIL when coverage fails the gate, and the runbook and docstring cover the new failure. Move the new tests above the unittest.main guard so direct runs include them. --- .../tests/librarian/evaluate_harness_test.py | 42 ++++++++++++++++--- docs/gsoc_2026_module_c/runbook.md | 11 ++++- scripts/evaluate_librarian.py | 18 ++++---- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index bdaa2b92b..67c7cdd54 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -345,10 +345,6 @@ def test_total_boundary_rejection_fails_the_run(self) -> None: self.assertIn("FAILED (gates did not run)", out.getvalue()) -if __name__ == "__main__": - unittest.main() - - class CalibrationCoverageTest(unittest.TestCase): """#1166: rows that never reach the C.3 fit must count against the gate.""" @@ -387,7 +383,7 @@ def test_partial_drop_fails_the_gate_and_says_how_many(self) -> None: self.assertIsNotNone(scaler, "T is still fitted so C.4 can report") self.assertIn("reached calibration: 4/5", report) self.assertIn("1 of 5 eligible rows did not reach the fit", report) - self.assertIn("FAILED (gate denominator incomplete)", report) + self.assertIn("FAILED (gate denominator incomplete; ", report) def test_full_coverage_gates_on_ece_only(self) -> None: rows = [r for r in self._rows() if r.id != "p3"] @@ -415,6 +411,38 @@ def test_full_coverage_gates_on_ece_only(self) -> None: self.assertAlmostEqual(scaler.temperature, scaler_check.temperature, places=6) self.assertEqual(status, 0 if ece_check < 0.10 else 1) + def _well_calibrated_pipe(self) -> "CountingPipeline": + # Coin-flip shortlists on a balanced set: ECE is 0 at any T, so only + # coverage can decide these two tests. + return CountingPipeline( + { + "alpha": [("616-305", 0.0), ("999-999", 0.0)], + "alpha2": [("616-305", 0.0), ("999-999", 0.0)], + "beta": [("111-111", 0.0), ("222-222", 0.0)], + "beta2": [("111-111", 0.0), ("222-222", 0.0)], + } + ) + + def test_passing_ece_with_partial_coverage_still_fails(self) -> None: + rows, pipe = self._rows(), self._well_calibrated_pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, _ = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1, "the #1166 case: ECE passes, rows are missing") + self.assertIn("gate ECE<0.10: FAIL", report) + self.assertNotIn("PASS", report) + self.assertIn("ECE alone met the bar", report) + + def test_passing_ece_with_full_coverage_passes(self) -> None: + rows = [r for r in self._rows() if r.id != "p3"] + pipe = self._well_calibrated_pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, _ = harness.report_calibration(rows, audits) + self.assertEqual(status, 0) + self.assertIn("gate ECE<0.10: PASS", out.getvalue()) + def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: from application.utils.librarian.calibration.temperature import ( TemperatureScaler, @@ -500,3 +528,7 @@ def test_selection_without_expected_decisions_keeps_the_original_line(self) -> N self.assertIn( "no rows with an expected decision in this selection", out.getvalue() ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/gsoc_2026_module_c/runbook.md b/docs/gsoc_2026_module_c/runbook.md index b666eb494..fd0774eac 100644 --- a/docs/gsoc_2026_module_c/runbook.md +++ b/docs/gsoc_2026_module_c/runbook.md @@ -44,8 +44,9 @@ pass and one fitted `T` โ€” the pipeline is built once per run, not once per report. **C.3 is the only *additional* gate a live run adds.** A failed or skipped ECE -gate returns non-zero, so a live run cannot pass without calibration having -actually run, and C.4 stays informational. The hermetic gates still apply on top: +gate returns non-zero, and so does a run where eligible rows never reached the +fit, so a live run cannot pass without calibration having actually run over the +whole selection. C.4 stays informational. The hermetic gates still apply on top: a failed C.0.5 explicit slice, or a C.0 boundary that rejects the whole dataset, fails the run in either mode (ยง5). @@ -201,6 +202,12 @@ A degenerate calibration set โ€” all top-1 labels the same class โ€” cannot iden `T`. The harness fails rather than skipping, so this is reported, not hidden. Widen the slice selection so both the positive and hard_negative slices are in. +**C.3 fails with "gate denominator incomplete"** +Eligible rows never reached the fit, so the `reached calibration: used/eligible` +line shows `used` below `eligible`. Widening the selection will not help. Find +the rows with no audit or an empty reranked shortlist and fix why they came back +empty. + --- ## 6. Known limitations diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 159b845f9..0d7f60bd8 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -344,12 +344,13 @@ def report_calibration( Reads the shared ``audits`` from ``live_audits``, the same shortlists ``report_retrieval_recall`` scores, so the positive slice is not retrieved and reranked a second time just to calibrate on it. Confidence is the top-1 mass - of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL on - ECE < 0.10. + of softmax(logits / T); prints ECE at T=1 vs the fitted T and PASS/FAIL. PASS + needs ECE < 0.10 and every eligible row to have reached the fit (#1166). - Returns ``(status, scaler)``. Status is 1 on a failed gate, and also on a - degenerate calibration set, so a live run can never exit 0 without the gate - actually having run. The fitted scaler is handed back (``None`` when the set + Returns ``(status, scaler)``. Status is 1 on a failed gate, on eligible rows + missing from the fit, and on a degenerate calibration set, so a live run can + never exit 0 without the gate actually having run over the whole selection. + The fitted scaler is handed back (``None`` when the set was degenerate) so the C.4 report thresholds on this same ``T`` instead of fitting its own. """ @@ -398,11 +399,14 @@ def report_calibration( print( f"calibration (C.3, {used} rows): fitted T={scaler.temperature:.3f}; " f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated, in-sample); " - f"gate ECE<0.10: {'PASS' if ece_ok else 'FAIL'}; " + f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}; " f"reached calibration: {used}/{eligible}" ) if not coverage_ok: - print("calibration (C.3): FAILED (gate denominator incomplete)") + print( + "calibration (C.3): FAILED (gate denominator incomplete; " + f"ECE alone {'met' if ece_ok else 'missed'} the bar)" + ) return (0 if gate_ok else 1), scaler