From bfa3f6791dc47798a12bb3fe8fcfb2ced7ea1dd8 Mon Sep 17 00:00:00 2001 From: akshit9162 Date: Thu, 5 Nov 2026 20:32:28 +0530 Subject: [PATCH 1/2] fix(librarian): do not auto-link single-candidate shortlists C.3's confidence is the top-1 mass of softmax(logits / T), which is 1.0 by construction over a single logit, so a lone candidate cleared any threshold however poor the match. Route shortlists with fewer than two candidates to review (BELOW_THRESHOLD) through a shortlist_confidence helper shared by the pipeline and the C.4 evaluate report. decide(), the RFC schemas and the explicit-link path are unchanged. Fixes #1205 --- application/tests/librarian/pipeline_test.py | 31 ++++++++++++++++++- .../tests/librarian/safety_guard_test.py | 7 ++++- application/utils/librarian/pipeline.py | 17 +++++++++- scripts/evaluate_librarian.py | 3 +- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/application/tests/librarian/pipeline_test.py b/application/tests/librarian/pipeline_test.py index 9f013439b..bfc05de11 100644 --- a/application/tests/librarian/pipeline_test.py +++ b/application/tests/librarian/pipeline_test.py @@ -10,6 +10,7 @@ import unittest from datetime import datetime, timezone +from application.utils.librarian.calibration.temperature import TemperatureScaler from application.utils.librarian.pipeline import LibrarianPipeline from application.utils.librarian.schemas import ( CreCandidate, @@ -80,7 +81,12 @@ def confidence(self, logits): return self._conf -TOP = [CreCandidate(cre_id="616-305", score_rerank=1.5)] +# Two candidates: C.3's softmax confidence is only defined against a rival, so a +# one-candidate shortlist never auto-links (see test_single_candidate_*). +TOP = [ + CreCandidate(cre_id="616-305", score_rerank=1.5), + CreCandidate(cre_id="117-371", score_rerank=-2.0), +] def _pipeline(rows, reranked, conf): @@ -115,6 +121,29 @@ def test_empty_shortlist_reviews_no_candidates(self): self.assertIsInstance(env, ReviewItem) self.assertEqual(env.reason_code, ReasonCode.no_candidates) + def test_single_candidate_shortlist_reviews_below_threshold(self): + # softmax over one logit is 1.0 whatever the match quality, so a lone + # candidate must not auto-link however confident the scaler says it is. + lone = [CreCandidate(cre_id="616-305", score_rerank=-12.0)] + result = _pipeline([_row()], lone, 1.0).run(at=AT) + env = result.envelopes[0] + self.assertIsInstance(env, ReviewItem) + self.assertEqual(env.reason_code, ReasonCode.below_threshold) + self.assertEqual(result.stats.linked, 0) + + def test_single_candidate_with_the_real_scaler_reviews(self): + lone = [CreCandidate(cre_id="616-305", score_rerank=-12.0)] + pipeline = LibrarianPipeline( + _Source([_row()]), + _Retriever(), + _Reranker(lone), + TemperatureScaler(temperature=1.0), + threshold=0.8, + pipeline_run_id=RUN, + ) + result = pipeline.run(at=AT) + self.assertIsInstance(result.envelopes[0], ReviewItem) + def test_uncertain_row_is_skipped_at_boundary(self): result = _pipeline([_row(label="NOISE")], TOP, 0.95).run(at=AT) self.assertEqual(result.stats.skipped, 1) diff --git a/application/tests/librarian/safety_guard_test.py b/application/tests/librarian/safety_guard_test.py index f43a74fb1..4402578bf 100644 --- a/application/tests/librarian/safety_guard_test.py +++ b/application/tests/librarian/safety_guard_test.py @@ -70,7 +70,12 @@ def retrieve(self, text): class _Reranker: def rerank(self, text, audit): return audit.model_copy( - update={"reranked": [CreCandidate(cre_id="616-305", score_rerank=9.0)]} + update={ + "reranked": [ + CreCandidate(cre_id="616-305", score_rerank=9.0), + CreCandidate(cre_id="117-371", score_rerank=-2.0), + ] + } ) diff --git a/application/utils/librarian/pipeline.py b/application/utils/librarian/pipeline.py index e3a117f18..edccb7f5f 100644 --- a/application/utils/librarian/pipeline.py +++ b/application/utils/librarian/pipeline.py @@ -105,6 +105,21 @@ class Scaler(Protocol): def confidence(self, logits: Sequence[float]) -> float: ... +def shortlist_confidence(scaler: Scaler, logits: Sequence[float]) -> float: + """The confidence C.4 thresholds on for one reranked shortlist. + + C.3's confidence is the top-1 mass of ``softmax(logits / T)``, which only + means something relative to rival candidates: over a one-candidate shortlist + it is 1.0 by construction, however poor that match is, and would clear any + threshold. With fewer than two candidates there is nothing to calibrate + against, so the confidence is 0.0 and the chunk goes to human review (an + empty shortlist is still reported as NO_CANDIDATES by ``decide``). + """ + if len(logits) < 2: + return 0.0 + return scaler.confidence(logits) + + @dataclass(frozen=True) class RunStats: """Counts for one pipeline run. @@ -236,7 +251,7 @@ def run(self, *, at: datetime) -> RunResult: reranked = [c for c in audit.reranked if c.score_rerank is not None] logits = [float(c.score_rerank) for c in reranked] cre_ids = [c.cre_id for c in reranked] - confidence = self._scaler.confidence(logits) if logits else 0.0 + confidence = shortlist_confidence(self._scaler, logits) verdict = self._safety_guard.evaluate(section) result = decide( diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 34b945ebc..d9592a245 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -404,6 +404,7 @@ def report_decision_accuracy( itself is the Week 7 experiment, so hard-gating it now would be premature. """ from application.utils.librarian.decision_engine import decide + from application.utils.librarian.pipeline import shortlist_confidence from application.utils.librarian.schemas import Decision graded = [r for r in rows if r.expected.decision is not None and r.id in audits] @@ -418,7 +419,7 @@ def report_decision_accuracy( reranked = [c for c in audit.reranked if c.score_rerank is not None] logits = [float(c.score_rerank) for c in reranked] cre_ids = [c.cre_id for c in reranked] - confidence = scaler.confidence(logits) if logits else 0.0 + confidence = shortlist_confidence(scaler, logits) result = decide(confidence, cre_ids, threshold=threshold) matched = result.decision == row.expected.decision if matched: From f98be9556117069cc8920b5db35ecf9e5af58842 Mon Sep 17 00:00:00 2001 From: akshit9162 Date: Thu, 5 Nov 2026 20:44:05 +0530 Subject: [PATCH 2/2] fix(librarian): keep lone candidates out at tau = 0 and out of C.3 Address review on #1206: - A 0.0 confidence still clears a threshold of 0, so the lone-candidate rule now lives in decide_shortlist(), which runs decide() first (empty shortlists and safety flags keep their precedence) and then refuses to link a one-candidate shortlist at any threshold. The C.4 report uses the same helper. - calibration_set() skips shortlists with fewer than two scored candidates: their softmax is 1.0 whatever T is, so they distorted the temperature fit and ECE while C.4 never links them. --- .../tests/librarian/evaluate_harness_test.py | 45 ++++++++++++++--- application/tests/librarian/pipeline_test.py | 26 +++++++++- application/utils/librarian/pipeline.py | 49 +++++++++++++------ scripts/evaluate_librarian.py | 10 ++-- 4 files changed, 103 insertions(+), 27 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ae979a177..8eb319c23 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -125,7 +125,10 @@ def test_pipeline_runs_once_per_row_not_once_per_report(self) -> None: _golden_row("n1", "hard_negative", "beta", []), ] pipe = CountingPipeline( - {"alpha": [("616-305", 4.0)], "beta": [("111-111", 3.0)]} + { + "alpha": [("616-305", 4.0), ("999-999", 0.1)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + } ) audits = harness.live_audits(rows, pipe, pipe) @@ -153,10 +156,10 @@ def test_draws_only_the_two_calibration_slices(self) -> None: ] pipe = CountingPipeline( { - "alpha": [("616-305", 4.0)], - "beta": [("111-111", 3.0)], - "gamma": [("616-305", 2.0)], - "delta": [("616-305", 1.0)], + "alpha": [("616-305", 4.0), ("999-999", 0.1)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + "gamma": [("616-305", 2.0), ("999-999", 0.1)], + "delta": [("616-305", 1.0), ("999-999", 0.1)], } ) audits = harness.live_audits(rows, pipe, pipe) @@ -174,7 +177,10 @@ def test_skips_rows_with_no_audit_and_empty_shortlists(self) -> None: _golden_row("n1", "hard_negative", "beta", []), ] pipe = CountingPipeline( - {"alpha": [("616-305", 4.0)], "beta": [("111-111", 3.0)]} + { + "alpha": [("616-305", 4.0), ("999-999", 0.1)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + } ) # "empty" yields no candidates; p3 is never audited at all. audits = harness.live_audits(rows, pipe, pipe) @@ -183,6 +189,28 @@ def test_skips_rows_with_no_audit_and_empty_shortlists(self) -> None: self.assertEqual(len(logit_sets), 2) self.assertEqual(len(labels), 2) + def test_skips_single_candidate_shortlists(self) -> None: + # A one-candidate softmax is 1.0 whatever T is, and C.4 never links such + # a row, so it must not enter the fit or the ECE. + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "lone", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 4.0), ("999-999", 0.1)], + "lone": [("616-305", 4.0)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + } + ) + audits = harness.live_audits(rows, pipe, pipe) + + logit_sets, labels = harness.calibration_set(rows, audits) + + self.assertEqual(logit_sets, [[4.0, 0.1], [3.0, 2.9]]) + self.assertEqual(labels, [1.0, 0.0]) + class ReportCalibrationTest(unittest.TestCase): def test_returns_status_and_fitted_scaler(self) -> None: @@ -215,7 +243,10 @@ def test_degenerate_set_fails_and_yields_no_scaler(self) -> None: _golden_row("p2", "positive", "alpha2", ["616-305"]), ] pipe = CountingPipeline( - {"alpha": [("616-305", 5.0)], "alpha2": [("616-305", 4.0)]} + { + "alpha": [("616-305", 5.0), ("999-999", 0.1)], + "alpha2": [("616-305", 4.0), ("999-999", 0.2)], + } ) audits = harness.live_audits(rows, pipe, pipe) diff --git a/application/tests/librarian/pipeline_test.py b/application/tests/librarian/pipeline_test.py index bfc05de11..d58c60b3c 100644 --- a/application/tests/librarian/pipeline_test.py +++ b/application/tests/librarian/pipeline_test.py @@ -11,9 +11,10 @@ from datetime import datetime, timezone from application.utils.librarian.calibration.temperature import TemperatureScaler -from application.utils.librarian.pipeline import LibrarianPipeline +from application.utils.librarian.pipeline import LibrarianPipeline, decide_shortlist from application.utils.librarian.schemas import ( CreCandidate, + Decision, KnowledgeQueueItem, LinkProposal, ReasonCode, @@ -144,6 +145,29 @@ def test_single_candidate_with_the_real_scaler_reviews(self): result = pipeline.run(at=AT) self.assertIsInstance(result.envelopes[0], ReviewItem) + def test_single_candidate_reviews_even_at_a_zero_threshold(self): + # At tau = 0 a confidence of 0.0 would still clear the bar, so the + # lone-candidate rule cannot rest on the confidence alone. + lone = [CreCandidate(cre_id="616-305", score_rerank=-12.0)] + pipeline = LibrarianPipeline( + _Source([_row()]), + _Retriever(), + _Reranker(lone), + _Scaler(1.0), + threshold=0.0, + pipeline_run_id=RUN, + ) + env = pipeline.run(at=AT).envelopes[0] + self.assertIsInstance(env, ReviewItem) + self.assertEqual(env.reason_code, ReasonCode.below_threshold) + + def test_single_candidate_keeps_safety_reason_precedence(self): + result = decide_shortlist( + _Scaler(1.0), [-12.0], ["616-305"], threshold=0.0, adversarial=True + ) + self.assertEqual(result.decision, Decision.review) + self.assertEqual(result.reason_code, ReasonCode.adversarial_flag) + def test_uncertain_row_is_skipped_at_boundary(self): result = _pipeline([_row(label="NOISE")], TOP, 0.95).run(at=AT) self.assertEqual(result.stats.skipped, 1) diff --git a/application/utils/librarian/pipeline.py b/application/utils/librarian/pipeline.py index edccb7f5f..5b622e96d 100644 --- a/application/utils/librarian/pipeline.py +++ b/application/utils/librarian/pipeline.py @@ -29,13 +29,15 @@ from enum import Enum from typing import Any, Dict, Iterable, List, Optional, Protocol, Sequence, Union -from application.utils.librarian.decision_engine import decide +from application.utils.librarian.decision_engine import DecisionResult, decide from application.utils.librarian.emitter import emit from application.utils.librarian.safety_guard import NullSafetyGuard, SafetyGuard from application.utils.librarian.schemas import ( + Decision, KnowledgeQueueItem, LinkProposal, RetrievalAudit, + ReasonCode, ReviewItem, ) from application.utils.librarian.section_validator import ( @@ -105,19 +107,38 @@ class Scaler(Protocol): def confidence(self, logits: Sequence[float]) -> float: ... -def shortlist_confidence(scaler: Scaler, logits: Sequence[float]) -> float: - """The confidence C.4 thresholds on for one reranked shortlist. +def decide_shortlist( + scaler: Scaler, + logits: Sequence[float], + cre_ids: Sequence[str], + *, + threshold: float, + adversarial: bool = False, + update_ambiguous: bool = False, +) -> DecisionResult: + """Score one reranked shortlist with C.3 and decide it with C.4. C.3's confidence is the top-1 mass of ``softmax(logits / T)``, which only means something relative to rival candidates: over a one-candidate shortlist - it is 1.0 by construction, however poor that match is, and would clear any - threshold. With fewer than two candidates there is nothing to calibrate - against, so the confidence is 0.0 and the chunk goes to human review (an - empty shortlist is still reported as NO_CANDIDATES by ``decide``). + it is 1.0 by construction, however poor that match is. With fewer than two + candidates there is nothing to calibrate against, so the confidence is 0.0 + and a lone candidate never auto-links, whatever the threshold (τ = 0 + included). ``decide`` still runs first, so an empty shortlist reports + NO_CANDIDATES and the safety flags keep their precedence. """ - if len(logits) < 2: - return 0.0 - return scaler.confidence(logits) + confidence = scaler.confidence(logits) if len(logits) >= 2 else 0.0 + result = decide( + confidence, + cre_ids, + threshold=threshold, + adversarial=adversarial, + update_ambiguous=update_ambiguous, + ) + if result.decision == Decision.linked and len(logits) < 2: + return DecisionResult( + Decision.review, confidence, result.cre_ids, ReasonCode.below_threshold + ) + return result @dataclass(frozen=True) @@ -211,7 +232,7 @@ def __init__( *, threshold: float, pipeline_run_id: str, - safety_guard: Optional[SafetyGuard] = None + safety_guard: Optional[SafetyGuard] = None, ) -> None: self._source = source self._retriever = retriever @@ -251,11 +272,11 @@ def run(self, *, at: datetime) -> RunResult: reranked = [c for c in audit.reranked if c.score_rerank is not None] logits = [float(c.score_rerank) for c in reranked] cre_ids = [c.cre_id for c in reranked] - confidence = shortlist_confidence(self._scaler, logits) verdict = self._safety_guard.evaluate(section) - result = decide( - confidence, + result = decide_shortlist( + self._scaler, + logits, cre_ids, threshold=self._threshold, adversarial=verdict.adversarial, diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index d9592a245..a7ee7ed7c 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -319,7 +319,9 @@ def calibration_set( if audit is None: continue reranked = [c for c in audit.reranked if c.score_rerank is not None] - if not reranked: + # A one-candidate softmax is 1.0 whatever T is, and C.4 never links such + # a row, so it carries no signal for fitting T or measuring ECE. + if len(reranked) < 2: continue expected = set(row.expected.cre_ids or []) logit_sets.append([float(c.score_rerank) for c in reranked]) @@ -403,8 +405,7 @@ def report_decision_accuracy( reason codes that depend on them lag until that lands; and tuning the threshold itself is the Week 7 experiment, so hard-gating it now would be premature. """ - from application.utils.librarian.decision_engine import decide - from application.utils.librarian.pipeline import shortlist_confidence + from application.utils.librarian.pipeline import decide_shortlist from application.utils.librarian.schemas import Decision graded = [r for r in rows if r.expected.decision is not None and r.id in audits] @@ -419,8 +420,7 @@ def report_decision_accuracy( reranked = [c for c in audit.reranked if c.score_rerank is not None] logits = [float(c.score_rerank) for c in reranked] cre_ids = [c.cre_id for c in reranked] - confidence = shortlist_confidence(scaler, logits) - result = decide(confidence, cre_ids, threshold=threshold) + result = decide_shortlist(scaler, logits, cre_ids, threshold=threshold) matched = result.decision == row.expected.decision if matched: dec_match += 1