From f11f2219943b4fe635a0568aefb2df7c8da08592 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Fri, 11 Sep 2026 00:56:33 -0700 Subject: [PATCH 1/6] feat(libsy): add an up-front capability gate to the escalation route The escalation route judges trouble after it has happened, and a hand-off mid-task makes the strong tier redo the work, so for tasks that were never going to fit the weak tier the cheapest moment to escalate is the first request. A task-level rule in the trajectory-judge prompt cannot set that split reliably: worded as "spans several modules" or "the hardest minority" it latched almost every multi-file task in practice, because the judge has no reference distribution to calibrate against. `escalation.gate = { base_threshold }` runs the packaged capability forecaster once per session, on the first request, through the escalation judge's target, and applies capability mode's threshold policy to its p_solve. Below the threshold the session latches to the capable tier before the efficient tier is called; otherwise the request proceeds and the trajectory judge takes over. Unusable verdicts and failed gate calls fall open to the efficient tier. `threshold_step` and `prompt` mirror capability mode. Adds EscalationGateConfig with validation, the gate construction next to the capability classifier so both share one policy, four route tests and a validation test, schema and guide documentation, and a changelog entry. Signed-off-by: Lin Jia --- CHANGELOG.md | 6 + crates/libsy/src/algorithms/escalation.rs | 192 ++++++++++++++++++ crates/libsy/src/algorithms/llm_class.rs | 43 +++- .../libsy/src/algorithms/util/escalation.rs | 54 +++++ docs/reference/toml_schema.md | 4 + .../escalation_router_routing.md | 29 +++ 6 files changed, 327 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6adadd03..06656c81e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Escalation up-front capability gate** — `escalation.gate = { base_threshold }` + judges the first request of a session with the packaged capability forecaster + and latches to the strong tier when `p_solve` is below the threshold, before + the weak tier is called. Sets the turn-one split with a number instead of a + task-level rule in the judge prompt, which latched almost every multi-file + task in practice. The trajectory judge takes over afterwards. - **Per-target `reasoning_effort`** — a target can force the reasoning effort of every request it serves, replacing the caller's value (`reasoning.effort` on the Responses wire, `reasoning_effort` on Chat Completions), so a strong diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 47523dc13..406a4b83d 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -10,6 +10,7 @@ use switchyard_protocol::{ AggLlmResponse, LlmClientError, LlmResponse, Message, ModelId, Request, Response, Role, }; +use super::llm_class; use super::util::classifier_contract::ClassifierContractConfig; use super::util::decisive; use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; @@ -21,6 +22,8 @@ use crate::{LibsyError, Result}; /// Session-state key holding the consecutive-escalate streak. const STREAK_KEY: &str = "escalation_streak"; +/// Session-state key recording that the up-front capability gate has run for this session. +const GATE_KEY: &str = "escalation_gate_done"; fn streak(state: &State) -> u32 { match state.extra.get(STREAK_KEY) { @@ -44,6 +47,8 @@ fn assistant_message(response: &AggLlmResponse) -> Message { /// not pay for a second model call. struct EscalationClassifier { judge: JudgeClassifier, + /// Optional capability forecast run once per session before the first efficient call. + gate: Option>>, capable: ModelId, efficient: ModelId, /// Consecutive escalate verdicts required to latch. @@ -60,7 +65,19 @@ pub(super) fn build_classifier( max_output_tokens: u64, ) -> Result>> { let confirmations = config.confirmations; + let gate = match &config.gate { + Some(gate) => Some(llm_class::build_capability_gate( + judge_target.clone(), + efficient_target, + capable_target, + gate, + contract_config.response_format_type(), + max_output_tokens, + )?), + None => None, + }; let classifier: Arc> = Arc::new(EscalationClassifier { + gate, judge: escalation::build_judge( judge_target, capable_target.clone(), @@ -99,6 +116,44 @@ impl Classifier for EscalationClassifier { return Ok((decisive(&self.capable), None)); } + // The gate forecasts once per session, from the task framing alone, before the efficient + // tier spends anything. A verdict below the threshold latches immediately; anything else + // (including an unusable verdict or a failed call) falls open to the efficient tier and + // leaves the trajectory judge in charge. + if let Some(gate) = &self.gate + && !matches!(state.extra.get(GATE_KEY), Some(StateValue::Count(_))) + { + state + .extra + .insert(GATE_KEY.to_string(), StateValue::Count(1)); + let mut gate_request = request.clone(); + match gate.score(state, &mut gate_request, Some(driver)).await { + Ok((classification, _)) => { + let to_capable = classification + .argmax(false)? + .is_some_and(|score| score.target == self.capable); + if to_capable { + state.extra.insert( + STREAK_KEY.to_string(), + StateValue::Count(self.confirmations), + ); + tracing::info!( + target = %self.capable, + "escalation gate latched the session to the capable tier" + ); + driver.set_evidence(serde_json::json!({ + "source": "escalation", + "verdict": "gate", + })); + return Ok((decisive(&self.capable), None)); + } + } + Err(error) => { + tracing::warn!(%error, "escalation gate failed; serving the efficient tier"); + } + } + } + // Call efficient model and buffer the response so the judge can read it. // // If the efficient model exceeds its context window, fall through to capable. This call @@ -204,6 +259,7 @@ mod tests { use super::*; use crate::algorithms::llm_class::{LlmClassifierConfig, LlmTaskClassifier}; use crate::algorithms::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; + use crate::algorithms::util::escalation::EscalationGateConfig; use crate::core::testing::{Serve, reply, test_drive}; /// A queue of replies, drained in order. @@ -390,6 +446,142 @@ mod tests { Ok(()) } + /// Builds a router with the up-front capability gate at the given threshold. + fn gated_router(base_threshold: f64) -> Result> { + Ok(Arc::new(LlmTaskClassifier::new( + LlmClassifierConfig::Escalation { + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + gate: Some(EscalationGateConfig { + base_threshold, + threshold_step: 0.0, + prompt: None, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }, + )?)) + } + + const GATE_LOW: &str = r#"{"crux":"protocol framing","primary_rule":"LIM-2","capability_boundary":"unsupported","p_solve":0.2}"#; + const GATE_HIGH: &str = r#"{"crux":"local change","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#; + + #[tokio::test] + async fn gate_latches_before_the_efficient_tier_is_called() -> Result<()> { + // Only the gate verdict is queued for the judge and only one model reply exists: had the + // efficient tier been called first, capable would have received "unexpected call". + let judge = Queue::new([GATE_LOW]); + let model = Queue::new(["capable answer"]); + + let (selected_model, response) = test_drive( + gated_router(0.5)?, + classify_session_request(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "capable"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable answer".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn gate_passes_then_trajectory_judge_decides() -> Result<()> { + let judge = Queue::new([GATE_HIGH, r#"{"escalate":false,"reason":"progressing"}"#]); + let model = Queue::new(["efficient answer"]); + + let (selected_model, response) = test_drive( + gated_router(0.5)?, + classify_session_request(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "efficient"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("efficient answer".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn gate_runs_once_per_session() -> Result<()> { + // Second request: the judge queue holds only an escalate verdict. If the gate ran again + // it would consume that verdict (invalid for the gate, so fail-open) and the trajectory + // judge would then read "unexpected call" and keep efficient. + let judge = Queue::new([ + GATE_HIGH, + r#"{"escalate":false,"reason":"progressing"}"#, + r#"{"escalate":true,"reason":"stuck"}"#, + ]); + let model = Queue::new(["efficient t1", "efficient t2", "capable t2"]); + let router = gated_router(0.5)?; + let request = classify_session_request(); + + let (first, _) = test_drive( + router.clone(), + request.clone(), + queued(Arc::clone(&model), Arc::clone(&judge)), + ) + .await?; + let (second, _) = test_drive(router, request, queued(model, judge)).await?; + + assert_eq!(first, "efficient"); + assert_eq!(second, "capable"); + Ok(()) + } + + #[tokio::test] + async fn unusable_gate_verdict_falls_open_to_efficient() -> Result<()> { + let judge = Queue::new([ + "not a verdict", + r#"{"escalate":false,"reason":"progressing"}"#, + ]); + let model = Queue::new(["efficient answer"]); + + let (selected_model, _) = test_drive( + gated_router(0.5)?, + classify_session_request(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "efficient"); + Ok(()) + } + + #[test] + fn gate_threshold_is_validated() { + let build = |base_threshold: f64| { + LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + gate: Some(EscalationGateConfig { + base_threshold, + threshold_step: 0.0, + prompt: None, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }) + }; + assert!(build(0.4).is_ok()); + assert!(build(1.5).is_err()); + } + #[tokio::test] async fn falls_back_to_capable_when_efficient_overflows() -> Result<()> { let serve = |target: ModelId, _request: Request| async move { diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index e247566eb..8eb21a690 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -18,7 +18,7 @@ use super::util::affinity::{AffinityRouter, ClassifyTrigger}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, }; -use super::util::escalation::EscalationJudgeConfig; +use super::util::escalation::{EscalationGateConfig, EscalationJudgeConfig}; use super::util::llm_judge::{ ClassifierInput, JsonSchemaDecoder, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, @@ -803,6 +803,47 @@ impl LlmTaskClassifier { } } +/// Builds the up-front capability gate an escalation route runs once per session: the packaged +/// capability forecaster, judged through the escalation judge's target, mapped to a tier by the +/// same threshold policy capability mode uses. The gate reads the task framing only +/// (`recent_turn_window` unset), so it works before any trajectory exists. +pub(super) fn build_capability_gate( + judge_target: ModelId, + efficient_target: &ModelId, + capable_target: &ModelId, + gate: &EscalationGateConfig, + response_format_type: ClassifierResponseFormat, + max_output_tokens: u64, +) -> Result>> { + let mut contract_config = + ClassifierContractConfig::default().with_response_format_type(response_format_type); + if let Some(prompt) = &gate.prompt { + contract_config = contract_config.with_prompt(prompt.clone()); + } + let contract = + ClassifierContract::from_config(&contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?; + let config = TaskClassifierConfig { + base_threshold: gate.base_threshold, + threshold_step: gate.threshold_step, + ..TaskClassifierConfig::default() + }; + Ok(Arc::new( + JudgeClassifier::new( + StructuredJudge::new( + TaskInput { + recent_turn_window: None, + }, + contract, + SerdeDecoder::new(), + JudgeRuntimeConfig::new(max_output_tokens)?, + ), + judge_target, + TaskClassifierPolicy::new(efficient_target.clone(), capable_target.clone(), &config), + ) + .with_evidence(capability_evidence), + )) +} + #[async_trait] impl Classifier for TaskClassifier { async fn score( diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index fb093f52a..7163a3aa2 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -60,6 +60,56 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// Optional up-front capability gate. When set, the first request of a session is judged + /// from the task framing alone with the packaged capability forecaster, and a solve + /// probability below the threshold latches the session to the capable tier before the + /// efficient tier has spent anything. The trajectory judge takes over afterwards. + pub gate: Option, +} + +/// Numeric threshold for the escalation route's up-front capability gate. +/// +/// Prose in the trajectory-judge prompt cannot set a split reliably: a task-level bar worded +/// as "spans several modules" or "the hardest minority" latches almost every multi-file task. +/// The gate reuses the capability classifier's forecast (`p_solve`) and threshold policy so the +/// operator dials the split with a number, the same way capability mode does. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EscalationGateConfig { + /// Lowest solve probability that keeps the session on the efficient tier. In `[0, 1]`. + pub base_threshold: f64, + /// Added once for uncertain or unmatched verdicts and twice for unsupported verdicts, as in + /// capability mode. `base_threshold + 2 * threshold_step` must be at most `1`. + #[serde(default)] + pub threshold_step: f64, + /// Replaces the packaged capability-classifier prompt for the gate call only. + #[serde(default)] + pub prompt: Option, +} + +impl EscalationGateConfig { + fn validate(&self) -> Result<()> { + let reject = |message: String| Err(LibsyError::AlgorithmError { message }); + if !(0.0..=1.0).contains(&self.base_threshold) { + return reject(format!( + "gate.base_threshold must be between 0 and 1, got {}", + self.base_threshold + )); + } + if !self.threshold_step.is_finite() || self.threshold_step < 0.0 { + return reject(format!( + "gate.threshold_step must be finite and non-negative, got {}", + self.threshold_step + )); + } + let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step; + if unsupported_threshold > 1.0 { + return reject(format!( + "gate.base_threshold + 2 * gate.threshold_step must be at most 1, got {unsupported_threshold}" + )); + } + Ok(()) + } } impl EscalationJudgeConfig { @@ -69,6 +119,9 @@ impl EscalationJudgeConfig { if self.confirmations == 0 { return reject("confirmations must be at least 1".to_string()); } + if let Some(gate) = &self.gate { + gate.validate()?; + } if self.recent_turn_window == 0 { return reject("recent_turn_window must be at least 1".to_string()); } @@ -88,6 +141,7 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + gate: None, } } } diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 92733bd57..4de49262a 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -215,6 +215,10 @@ Escalation mode serves the weak target first and judges the completed turn. See | `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a session ID. | | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | +| `escalation.gate` | No | unset | Up-front capability gate: judges the first request of a session from the task framing with the packaged capability forecaster and latches to `strong_target` when `p_solve` is below the threshold, before `weak_target` is called. Needs a session ID. | +| `escalation.gate.base_threshold` | Yes, when `gate` is set | — | Lowest solve probability that keeps the session on `weak_target`. In `[0, 1]`. | +| `escalation.gate.threshold_step` | No | `0.0` | Added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `escalation.gate.prompt` | No | packaged capability prompt | Replaces the capability-forecaster prompt for the gate call only. | Existing configurations that contain `escalation` but omit `mode` remain valid. diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..7b02b1035 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -90,6 +90,35 @@ A judge that times out, errors, or returns an unparseable verdict fails open: th turn serves the buffered weak reply and the existing streak is held rather than cleared. A judge failure never creates a strong-tier latch. +## Up-front capability gate + +The trajectory judge only sees trouble once it has happened, and a hand-off +mid-task makes the strong tier redo the work, so for tasks that were never going +to fit the weak tier the cheapest moment to escalate is the first request. +`escalation.gate` adds that moment as a numeric threshold: + +```toml +escalation = { confirmations = 1, gate = { base_threshold = 0.4 } } +``` + +On the first request of a session the route calls the judge with the packaged +capability-classifier prompt, reading the task framing alone, and applies the +capability threshold policy to its `p_solve` forecast. Below `base_threshold` +the session latches to `strong_target` immediately and the weak tier is never +called. At or above it the request proceeds as usual and the trajectory judge +takes over for the rest of the session. The gate runs once per session, so it +needs a session ID, like `confirmations` above `1`. An unusable gate verdict or +a failed gate call falls open to the weak tier. + +Why a number rather than a task-level rule in the judge prompt: in practice a +prose bar ("spans several modules", "the hardest minority of tasks") latches +almost every multi-file task, because the judge has no reference distribution +to calibrate against. A threshold on a forecast is what capability mode already +uses to set its split, and the gate reuses that policy unchanged, including +`threshold_step` for uncertain and unsupported verdicts and `prompt` to replace +the packaged forecaster prompt. The gate verdict is recorded in the route's +evidence as `{"source": "escalation", "verdict": "gate"}`. + ## Judge model compatibility The trajectory judge uses the same response contract and provider/model From 9945700d1f30d4d521ff36a2662f6ebb6ed1b472 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Fri, 11 Sep 2026 01:01:09 -0700 Subject: [PATCH 2/6] fix(py): initialise the new escalation gate field in the Python bindings Signed-off-by: Lin Jia --- crates/switchyard-py/src/libsy_bindings.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index cc7801a42..40658abfe 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -110,6 +110,8 @@ impl PyEscalationClassifierConfig { confirmations, recent_turn_window, window_message_chars, + // The up-front capability gate is not exposed to Python yet. + gate: None, }, max_output_tokens, }) From 2a9869f172f1b81eb9263df58dceb2e5ad824603 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Fri, 11 Sep 2026 05:10:11 -0700 Subject: [PATCH 3/6] feat(libsy): log the escalation gate's forecast and carry score and threshold in its evidence The gate's usefulness depends on where the threshold sits relative to the forecaster's p_solve distribution, and nothing surfaced that distribution: the capability evidence was overwritten by the gate's own marker and no log line carried the score. The gate now logs every verdict with the forecast evidence at info level and copies score and threshold into its latch evidence. Adds a read-back accessor on Driver for wrapping components. Signed-off-by: Lin Jia --- crates/libsy/src/algorithms/escalation.rs | 23 +++++++++++++++++++++-- crates/libsy/src/core/algorithm.rs | 6 ++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 406a4b83d..b242fc919 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; +use serde_json::Value; use switchyard_protocol::{ AggLlmResponse, LlmClientError, LlmResponse, Message, ModelId, Request, Response, Role, }; @@ -132,6 +133,14 @@ impl Classifier for EscalationClassifier { let to_capable = classification .argmax(false)? .is_some_and(|score| score.target == self.capable); + // The forecaster's evidence carries `score` (p_solve) and `threshold`; log it + // so an operator can read the forecast distribution and tune the threshold. + let forecast = driver.evidence().unwrap_or(Value::Null); + tracing::info!( + latched = to_capable, + forecast = %forecast, + "escalation gate verdict" + ); if to_capable { state.extra.insert( STREAK_KEY.to_string(), @@ -141,10 +150,20 @@ impl Classifier for EscalationClassifier { target = %self.capable, "escalation gate latched the session to the capable tier" ); - driver.set_evidence(serde_json::json!({ + let mut evidence = serde_json::json!({ "source": "escalation", "verdict": "gate", - })); + }); + if let (Some(evidence), Some(forecast)) = + (evidence.as_object_mut(), forecast.as_object()) + { + for key in ["score", "threshold"] { + if let Some(value) = forecast.get(key) { + evidence.insert(key.to_string(), value.clone()); + } + } + } + driver.set_evidence(evidence); return Ok((decisive(&self.capable), None)); } } diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 6e01c20c7..b5f0390f8 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -151,6 +151,12 @@ impl Driver { *self.evidence.lock() = Some(evidence); } + /// Returns a copy of the evidence recorded so far, so a component that wraps another can + /// read the inner decision back (for example to log a judge's score and threshold). + pub(crate) fn evidence(&self) -> Option { + self.evidence.lock().clone() + } + /// Supply fallback evidence without replacing a decision made earlier in the cascade. pub(crate) fn set_evidence_if_empty(&self, evidence: Value) { let mut current = self.evidence.lock(); From 1a68ca8d4e345da7ca0945b90ee167108c05f30b Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Fri, 11 Sep 2026 06:40:08 -0700 Subject: [PATCH 4/6] feat(libsy): let the escalation gate name its own judge target The gate forecast runs once per session and rewards a strong model; the trajectory judge runs on every weak turn and is where a cheap model belongs. With one classifier_target for both, a deployment either pays a frontier model per turn (1,500 verdicts per 30 sessions measured) or gates with a weak forecaster. gate.classifier_target names a separate target for the forecast, resolved and validated by the runner like classifier_target and included in the route's callable targets; unset, the gate keeps using the route's judge. Signed-off-by: Lin Jia --- CHANGELOG.md | 2 ++ crates/libsy/src/algorithms/escalation.rs | 9 ++++++++- crates/libsy/src/algorithms/llm_class.rs | 6 ++++++ .../libsy/src/algorithms/util/escalation.rs | 7 +++++++ crates/switchyard-py/src/libsy_bindings.rs | 1 + crates/switchyard-runner/src/algorithm.rs | 20 ++++++++++++++++++- crates/switchyard-runner/src/config.rs | 18 +++++++++++++++++ docs/reference/toml_schema.md | 1 + .../escalation_router_routing.md | 5 ++++- 9 files changed, 66 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06656c81e..7c1ed2946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). the weak tier is called. Sets the turn-one split with a number instead of a task-level rule in the judge prompt, which latched almost every multi-file task in practice. The trajectory judge takes over afterwards. + `gate.classifier_target` lets the once-per-session forecast use a stronger + model than the per-turn trajectory judge. - **Per-target `reasoning_effort`** — a target can force the reasoning effort of every request it serves, replacing the caller's value (`reasoning.effort` on the Responses wire, `reasoning_effort` on Chat Completions), so a strong diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index b242fc919..3c922a7f1 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -58,6 +58,7 @@ struct EscalationClassifier { /// Builds the escalation classifier used by the shared LLM classifier route shell. pub(super) fn build_classifier( + gate_judge_target: Option, judge_target: ModelId, efficient_target: &ModelId, capable_target: &ModelId, @@ -68,7 +69,7 @@ pub(super) fn build_classifier( let confirmations = config.confirmations; let gate = match &config.gate { Some(gate) => Some(llm_class::build_capability_gate( - judge_target.clone(), + gate_judge_target.unwrap_or_else(|| judge_target.clone()), efficient_target, capable_target, gate, @@ -362,6 +363,7 @@ mod tests { ..EscalationJudgeConfig::default() }, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + gate_judge_target: None, }, )?)) } @@ -418,6 +420,7 @@ mod tests { ..EscalationJudgeConfig::default() }, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + gate_judge_target: None, })?); test_drive(router, classify_request(), serve).await?; @@ -479,10 +482,12 @@ mod tests { base_threshold, threshold_step: 0.0, prompt: None, + classifier_target: None, }), ..EscalationJudgeConfig::default() }, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + gate_judge_target: None, }, )?)) } @@ -591,10 +596,12 @@ mod tests { base_threshold, threshold_step: 0.0, prompt: None, + classifier_target: None, }), ..EscalationJudgeConfig::default() }, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + gate_judge_target: None, }) }; assert!(build(0.4).is_ok()); diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 8eb21a690..9e39decad 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -558,6 +558,8 @@ pub enum LlmClassifierConfig { config: EscalationJudgeConfig, /// Maximum completion tokens available to the escalation verdict. max_output_tokens: u64, + /// Resolved `escalation.gate.classifier_target`; `None` uses `judge_target`. + gate_judge_target: Option, }, /// Routes among named targets using a user-supplied schema and policy. Custom { @@ -594,7 +596,9 @@ impl LlmTaskClassifier { contract, config, max_output_tokens, + gate_judge_target, } => Self::build_escalation( + gate_judge_target, judge_target, efficient_target, capable_target, @@ -741,6 +745,7 @@ impl LlmTaskClassifier { } fn build_escalation( + gate_judge_target: Option, judge_target: ModelId, efficient_target: ModelId, capable_target: ModelId, @@ -749,6 +754,7 @@ impl LlmTaskClassifier { max_output_tokens: u64, ) -> Result { let inner = escalation::build_classifier( + gate_judge_target, judge_target, &efficient_target, &capable_target, diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 7163a3aa2..5cd3f2f43 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -85,6 +85,13 @@ pub struct EscalationGateConfig { /// Replaces the packaged capability-classifier prompt for the gate call only. #[serde(default)] pub prompt: Option, + /// Target the gate forecast is called through, by target name. Defaults to the route's + /// `classifier_target`. The gate runs once per session and benefits from a strong forecaster, + /// while the trajectory judge runs on every weak turn and is better served by a cheap model; + /// naming them separately lets a deployment pay for each where it matters. Resolved by the + /// deployment loader, which is why it is a name here rather than a model id. + #[serde(default)] + pub classifier_target: Option, } impl EscalationGateConfig { diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 40658abfe..ab36f82af 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -220,6 +220,7 @@ impl PyLlmClassifierConfig { let config = config.bind(py).try_borrow()?; Ok(Self { inner: LlmClassifierConfig::Escalation { + gate_judge_target: None, judge_target: ModelId::new(judge_target), efficient_target: ModelId::new(efficient_target), capable_target: ModelId::new(capable_target), diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 21a8fa9a0..3e0efd38d 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -510,7 +510,17 @@ impl AlgorithmSpec { pub fn callable_target_names(&self) -> Vec<&str> { let mut names = self.routing_target_names(); match self { - Self::LlmClassifier { config, .. } => names.push(&config.classifier_target), + Self::LlmClassifier { config, .. } => { + names.push(&config.classifier_target); + if let Some(gate_target) = config + .escalation + .as_ref() + .and_then(|escalation| escalation.gate.as_ref()) + .and_then(|gate| gate.classifier_target.as_deref()) + { + names.push(gate_target); + } + } Self::Passthrough { subagents: Some(subagents), .. @@ -942,6 +952,13 @@ fn build_algorithm( let strong = resolve_target_model_id(route_name, &config.strong_target, targets)?; let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; + let gate_judge_target = config + .judge + .gate + .as_ref() + .and_then(|gate| gate.classifier_target.as_deref()) + .map(|name| resolve_target_model_id(route_name, name, targets)) + .transpose()?; LlmTaskClassifier::new(LlmClassifierConfig::Escalation { judge_target: classifier, efficient_target: weak, @@ -950,6 +967,7 @@ fn build_algorithm( .with_response_format_type(config.response_format_type), config: config.judge, max_output_tokens: config.max_output_tokens, + gate_judge_target, }) } LlmClassifierModeConfig::Custom(config) => { diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 867a8e53a..1b2634496 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1035,6 +1035,24 @@ new = ["send_message"] Ok(()) } + /// The gate may name its own judge target; it is resolved like `classifier_target` and an + /// unknown name is rejected at load time. + #[test] + fn an_escalation_gate_can_name_its_own_judge_target() -> RunnerResult<()> { + let gated = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 1, gate = { base_threshold = 0.4, classifier_target = \"strong\" } }", + ); + runner_from_toml(&gated)?; + + let unknown = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 1, gate = { base_threshold = 0.4, classifier_target = \"nobody\" } }", + ); + assert!(error_message(&unknown).contains("nobody")); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 4de49262a..45c737912 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -219,6 +219,7 @@ Escalation mode serves the weak target first and judges the completed turn. See | `escalation.gate.base_threshold` | Yes, when `gate` is set | — | Lowest solve probability that keeps the session on `weak_target`. In `[0, 1]`. | | `escalation.gate.threshold_step` | No | `0.0` | Added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | | `escalation.gate.prompt` | No | packaged capability prompt | Replaces the capability-forecaster prompt for the gate call only. | +| `escalation.gate.classifier_target` | No | the route's `classifier_target` | Target the once-per-session gate forecast is called through, so a strong forecaster can gate while a cheap model judges every weak turn. | Existing configurations that contain `escalation` but omit `mode` remain valid. diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 7b02b1035..982a67bad 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -116,7 +116,10 @@ almost every multi-file task, because the judge has no reference distribution to calibrate against. A threshold on a forecast is what capability mode already uses to set its split, and the gate reuses that policy unchanged, including `threshold_step` for uncertain and unsupported verdicts and `prompt` to replace -the packaged forecaster prompt. The gate verdict is recorded in the route's +the packaged forecaster prompt. `gate.classifier_target` lets the gate call a +different target from the per-turn judge: the forecast runs once per session +and rewards a strong model, while the trajectory judge runs on every weak turn +and is where a cheap model belongs. The gate verdict is recorded in the route's evidence as `{"source": "escalation", "verdict": "gate"}`. ## Judge model compatibility From 510b9a42d584607b66a450767ae4020a4cd31af4 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Fri, 11 Sep 2026 15:34:38 -0700 Subject: [PATCH 5/6] feat(libsy): let the capability forecaster answer on an ordinal confidence ladder Asked for a probability, judges cluster on a few round values and, given a stated cutoff, write a number just under it (0.35 against 0.40 in more than half of one run's verdicts). Asked for one rung of a fixed ladder they rank more reliably. verdict_scale = "ordinal" on capability routes and escalation.gate.verdict_scale switch the packaged prompt and schema to eight named rungs from surely to almost_surely_not; each rung maps to the midpoint of the frequency band it names, so base_threshold keeps selecting the split and a threshold between two rungs latches exactly the rungs below it. The rung is recorded in the evidence next to the score. Signed-off-by: Lin Jia --- CHANGELOG.md | 6 + crates/libsy/src/algorithms/escalation.rs | 37 ++++ crates/libsy/src/algorithms/llm_class.rs | 167 +++++++++++++++--- .../libsy/src/algorithms/util/escalation.rs | 5 + crates/libsy/src/lib.rs | 2 +- .../capability-classifier/prompt-ordinal.md | 69 ++++++++ .../capability-classifier/schema-ordinal.json | 52 ++++++ crates/switchyard-py/src/libsy_bindings.rs | 4 +- crates/switchyard-runner/src/algorithm.rs | 11 +- docs/reference/toml_schema.md | 2 + .../escalation_router_routing.md | 11 +- 11 files changed, 341 insertions(+), 25 deletions(-) create mode 100644 crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md create mode 100644 crates/libsy/src/prompts/capability-classifier/schema-ordinal.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c1ed2946..4b5f04dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). task in practice. The trajectory judge takes over afterwards. `gate.classifier_target` lets the once-per-session forecast use a stronger model than the per-turn trajectory judge. +- **Ordinal confidence scale for the capability forecaster** — `verdict_scale = + "ordinal"` on capability routes and `escalation.gate.verdict_scale` ask the + judge for one of eight named rungs instead of a probability; rungs map to + band midpoints so thresholds keep working. Models rank tasks more reliably + than they number them, and a stated numeric cutoff invites answers just + under it. - **Per-target `reasoning_effort`** — a target can force the reasoning effort of every request it serves, replacing the caller's value (`reasoning.effort` on the Responses wire, `reasoning_effort` on Chat Completions), so a strong diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 3c922a7f1..bb99765b1 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -277,6 +277,7 @@ mod tests { }; use super::*; + use crate::algorithms::llm_class::VerdictScale; use crate::algorithms::llm_class::{LlmClassifierConfig, LlmTaskClassifier}; use crate::algorithms::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use crate::algorithms::util::escalation::EscalationGateConfig; @@ -483,6 +484,7 @@ mod tests { threshold_step: 0.0, prompt: None, classifier_target: None, + verdict_scale: VerdictScale::default(), }), ..EscalationJudgeConfig::default() }, @@ -517,6 +519,40 @@ mod tests { Ok(()) } + #[tokio::test] + async fn gate_on_the_ordinal_scale_latches_below_the_threshold_rung() -> Result<()> { + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + gate: Some(EscalationGateConfig { + // Between `uncertain` (0.50) and `unlikely` (0.38): unlikely and below latch. + base_threshold: 0.45, + threshold_step: 0.0, + prompt: None, + classifier_target: None, + verdict_scale: VerdictScale::Ordinal, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + gate_judge_target: None, + })?); + let judge = Queue::new([ + r#"{"crux":"protocol framing","primary_rule":"LIM-2","capability_boundary":"unsupported","confidence":"unlikely"}"#, + ]); + let model = Queue::new(["capable answer"]); + + let (selected_model, _) = + test_drive(router, classify_session_request(), queued(model, judge)).await?; + + assert_eq!(selected_model, "capable"); + Ok(()) + } + #[tokio::test] async fn gate_passes_then_trajectory_judge_decides() -> Result<()> { let judge = Queue::new([GATE_HIGH, r#"{"escalate":false,"reason":"progressing"}"#]); @@ -597,6 +633,7 @@ mod tests { threshold_step: 0.0, prompt: None, classifier_target: None, + verdict_scale: VerdictScale::default(), }), ..EscalationJudgeConfig::default() }, diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 9e39decad..ac4c0465d 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -32,6 +32,55 @@ use switchyard_protocol::{Request, Response}; const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../prompts/capability-classifier/schema.json"); +const ORDINAL_PROMPT_TEMPLATE: &str = + include_str!("../prompts/capability-classifier/prompt-ordinal.md"); +const ORDINAL_SCHEMA_TEMPLATE: &str = + include_str!("../prompts/capability-classifier/schema-ordinal.json"); + +/// How the capability forecaster reports its verdict. +/// +/// Models produce ordinal judgements more reliably than calibrated numbers: asked for a +/// probability they cluster on a few round values and, given a stated cutoff, write a number +/// just under it. The ordinal scale asks for one rung of a fixed ladder and maps it to the +/// band's midpoint, so the threshold policy is unchanged and a threshold between two rungs +/// selects exactly the rungs below it. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VerdictScale { + /// `p_solve` as a number in `[0, 1]`. + #[default] + Probability, + /// `confidence` as one of eight named rungs, from `surely` to `almost_surely_not`. + Ordinal, +} + +impl VerdictScale { + fn templates(self) -> (&'static str, &'static str) { + match self { + Self::Probability => (PROMPT_TEMPLATE, SCHEMA_TEMPLATE), + Self::Ordinal => (ORDINAL_PROMPT_TEMPLATE, ORDINAL_SCHEMA_TEMPLATE), + } + } +} + +/// Ladder rungs and the midpoint of the natural-frequency band each one names. +const CONFIDENCE_RUNGS: [(&str, f64); 8] = [ + ("surely", 0.97), + ("extremely_likely", 0.90), + ("very_likely", 0.78), + ("likely", 0.62), + ("uncertain", 0.50), + ("unlikely", 0.38), + ("very_unlikely", 0.22), + ("almost_surely_not", 0.08), +]; + +fn rung_midpoint(rung: &str) -> Option { + CONFIDENCE_RUNGS + .iter() + .find(|(name, _)| *name == rung) + .map(|(_, midpoint)| *midpoint) +} /// Telemetry label for this algorithm's spans, metrics, and logs. const ALGORITHM_NAME: &str = "llm_task_classifier"; @@ -52,13 +101,28 @@ struct TaskClassifierVerdict { crux: String, primary_rule: String, capability_boundary: String, - p_solve: f64, + /// Probability scale: the forecast as a number. + #[serde(default)] + p_solve: Option, + /// Ordinal scale: the forecast as a ladder rung. Exactly one of the two is present. + #[serde(default)] + confidence: Option, } impl TaskClassifierVerdict { + /// The forecast as a number on either scale: `p_solve` itself, or the midpoint of the + /// band the reported rung names. + fn p_solve(&self) -> Option { + match (self.p_solve, self.confidence.as_deref()) { + (Some(p), None) => Some(p), + (None, Some(rung)) => rung_midpoint(rung), + _ => None, + } + } + /// Rejects malformed or internally inconsistent verdicts before policy evaluation. fn is_valid(&self) -> bool { - (0.0..=1.0).contains(&self.p_solve) + self.p_solve().is_some_and(|p| (0.0..=1.0).contains(&p)) && !self.crux.trim().is_empty() && matches!( ( @@ -239,9 +303,9 @@ impl JudgePolicy for TaskClassifierPolicy { let Some(threshold) = self.threshold(verdict) else { return Classification::Ambiguous(vec![]); }; - let target = if verdict.p_solve >= threshold - || (threshold - verdict.p_solve).abs() <= f64::EPSILON - { + // `is_valid` guarantees the forecast is present on exactly one scale. + let p_solve = verdict.p_solve().unwrap_or(0.0); + let target = if p_solve >= threshold || (threshold - p_solve).abs() <= f64::EPSILON { &self.efficient_target } else { &self.capable_target @@ -269,16 +333,23 @@ fn capability_evidence( "reason_code": "invalid_verdict", })); }; - Some(serde_json::json!({ + let mut evidence = serde_json::json!({ "source": "llm-classifier", - "score": verdict.p_solve, + "score": verdict.p_solve(), "threshold": threshold, - })) + }); + if let (Some(evidence), Some(rung)) = (evidence.as_object_mut(), verdict.confidence.as_deref()) + { + evidence.insert("confidence".to_string(), Value::String(rung.to_string())); + } + Some(evidence) } #[derive(Clone, Debug)] /// Settings that control capability classifier prompting and routing. pub struct TaskClassifierConfig { + /// Whether the forecaster reports a probability or a ladder rung. + pub verdict_scale: VerdictScale, /// Lowest solve probability that routes a supported task to the efficient target. pub base_threshold: f64, /// Amount added per capability-boundary step. @@ -322,6 +393,8 @@ struct TaskClassifierConfigWire { response_format_type: ClassifierResponseFormat, #[serde(default = "default_judge_max_output_tokens")] max_output_tokens: u64, + #[serde(default)] + verdict_scale: VerdictScale, } impl<'de> Deserialize<'de> for TaskClassifierConfig { @@ -336,6 +409,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { } contract = contract.with_response_format_type(wire.response_format_type); Ok(Self { + verdict_scale: wire.verdict_scale, base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, classify_trigger: wire.classify_trigger, @@ -354,6 +428,7 @@ const fn default_judge_max_output_tokens() -> u64 { impl Default for TaskClassifierConfig { fn default() -> Self { Self { + verdict_scale: VerdictScale::default(), base_threshold: 0.0, threshold_step: 0.0, classify_trigger: ClassifyTrigger::default(), @@ -622,7 +697,7 @@ impl LlmTaskClassifier { config: TaskClassifierConfig, ) -> Result { config.validate()?; - let contract = Self::load_capability_contract(&config.contract)?; + let contract = Self::load_capability_contract(&config.contract, config.verdict_scale)?; let targets = vec![efficient_target.clone(), capable_target.clone()]; let classify_trigger = config.classify_trigger; let message_hash_fallback = config.message_hash_fallback; @@ -771,9 +846,13 @@ impl LlmTaskClassifier { }) } - /// Loads the packaged capability-classifier contract. - fn load_capability_contract(config: &ClassifierContractConfig) -> Result { - ClassifierContract::from_config(config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE) + /// Loads the packaged capability-classifier contract for the configured verdict scale. + fn load_capability_contract( + config: &ClassifierContractConfig, + scale: VerdictScale, + ) -> Result { + let (prompt, schema) = scale.templates(); + ClassifierContract::from_config(config, prompt, schema) } /// Keeps affinity and fallback ordering identical across judge-backed modes. @@ -826,9 +905,10 @@ pub(super) fn build_capability_gate( if let Some(prompt) = &gate.prompt { contract_config = contract_config.with_prompt(prompt.clone()); } - let contract = - ClassifierContract::from_config(&contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?; + let (prompt, schema) = gate.verdict_scale.templates(); + let contract = ClassifierContract::from_config(&contract_config, prompt, schema)?; let config = TaskClassifierConfig { + verdict_scale: gate.verdict_scale, base_threshold: gate.base_threshold, threshold_step: gate.threshold_step, ..TaskClassifierConfig::default() @@ -908,6 +988,43 @@ mod tests { const TEST_THRESHOLD: f64 = 0.5; + #[test] + fn ordinal_verdicts_map_to_band_midpoints_and_probability_verdicts_pass_through() { + let ordinal: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"framing","primary_rule":"LIM-2","capability_boundary":"unsupported","confidence":"unlikely"}"#, + ) + .expect("ordinal verdict parses"); + assert!(ordinal.is_valid()); + assert_eq!(ordinal.p_solve(), Some(0.38)); + + let probability: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"local change","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + ) + .expect("probability verdict parses"); + assert_eq!(probability.p_solve(), Some(0.9)); + + // An unknown rung, or both scales at once, is not a usable verdict. + let unknown: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"x","primary_rule":"SUP-1","capability_boundary":"supported","confidence":"maybe"}"#, + ) + .expect("parses"); + assert!(!unknown.is_valid()); + let both: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"x","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.5,"confidence":"likely"}"#, + ) + .expect("parses"); + assert!(!both.is_valid()); + } + + #[test] + fn ordinal_scale_loads_the_ladder_prompt_and_schema() { + let (prompt, schema) = VerdictScale::Ordinal.templates(); + assert!(prompt.contains("almost_surely_not") && !prompt.contains("p_solve")); + assert!(schema.contains("\"confidence\"") && !schema.contains("p_solve")); + let (prompt, schema) = VerdictScale::Probability.templates(); + assert!(prompt.contains("p_solve") && schema.contains("p_solve")); + } + fn test_config(base_threshold: f64) -> TaskClassifierConfig { TaskClassifierConfig { base_threshold, @@ -928,7 +1045,8 @@ mod tests { crux: "test crux".to_string(), primary_rule: primary_rule.to_string(), capability_boundary: capability_boundary.to_string(), - p_solve, + p_solve: Some(p_solve), + confidence: None, } } @@ -1365,7 +1483,10 @@ mod tests { fn capability_judge(recent_turn_window: Option) -> Result { Ok(StructuredJudge::new( TaskInput { recent_turn_window }, - LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, + LlmTaskClassifier::load_capability_contract( + &ClassifierContractConfig::default(), + VerdictScale::Probability, + )?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, )) @@ -1740,8 +1861,10 @@ mod tests { /// rejecting every production verdict. #[test] fn every_schema_property_round_trips_through_the_judge_parser() -> Result<()> { - let contract = - LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?; + let contract = LlmTaskClassifier::load_capability_contract( + &ClassifierContractConfig::default(), + VerdictScale::Probability, + )?; let schema = contract.response_format(); let reply = schema_shaped_verdict(schema)?; let judge: CapabilityJudge = StructuredJudge::new( @@ -1756,14 +1879,16 @@ mod tests { let verdict = judge.parse(&text_response(None, reply))?; assert!(verdict.is_valid()); - assert!((0.0..=1.0).contains(&verdict.p_solve)); + assert!(verdict.p_solve().is_some_and(|p| (0.0..=1.0).contains(&p))); Ok(()) } #[test] fn packaged_prompt_keeps_the_schema_in_the_structured_request() -> Result<()> { - let contract = - LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?; + let contract = LlmTaskClassifier::load_capability_contract( + &ClassifierContractConfig::default(), + VerdictScale::Probability, + )?; let prompt = contract.system_prompt(); let schema_name = contract .response_format() diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 5cd3f2f43..12ebcdd8e 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,6 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; +use crate::algorithms::llm_class::VerdictScale; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; use crate::{LibsyError, Result}; @@ -92,6 +93,10 @@ pub struct EscalationGateConfig { /// deployment loader, which is why it is a name here rather than a model id. #[serde(default)] pub classifier_target: Option, + /// Whether the forecaster reports `p_solve` as a number (default) or `confidence` as one + /// of eight ladder rungs, mapped to band midpoints before the threshold is applied. + #[serde(default)] + pub verdict_scale: VerdictScale, } impl EscalationGateConfig { diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 7999f2428..6b320aa0f 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -19,7 +19,7 @@ pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger}; pub use algorithms::composite::{CompositeRouter, CompositeRouterConfig}; pub use algorithms::llm_class::{ CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier, - TaskClassifierConfig, + TaskClassifierConfig, VerdictScale, }; pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; diff --git a/crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md b/crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md new file mode 100644 index 000000000..9929d83b4 --- /dev/null +++ b/crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md @@ -0,0 +1,69 @@ +You are a task-level probability forecaster for a model router. You receive the +task's opening instruction and, when present, its latest user follow-up, plus +the qualitative capability card below. + +Forecast one binary event: + +SUCCESS means that the efficient agent completes the whole task correctly on +one fresh run under the actual harness, tools, and budget, as judged by the +final verifier. FAILURE means any other outcome. The two outcomes are +exhaustive. + +Use only evidence in the instruction and the capability card. Do not assume +hidden repository state, unmentioned tools, validators, documentation, access, +or future work habits. Do not invent empirical counts, success rates, or base +rates. The capability card is qualitative evidence, not a measured prior. + +# Assessment procedure + +1. State the crux: the hardest material requirement for whole-task success. +2. Select the one capability rule that best describes the crux. Use + primary_rule=none and capability_boundary=unmatched when no rule applies. + Rule ids are opaque labels. Do not infer a boundary from an id's spelling. +3. Privately identify the strongest instruction-visible reasons for SUCCESS + and FAILURE, then imagine the most likely concrete failure. +4. Privately consider material unknowns. Missing information should limit + extreme rungs, but it is not evidence that the answer must be uncertain. +5. Choose the rung last. It describes the chance of whole-task SUCCESS, not + confidence in this assessment, a route recommendation, or a cost judgment. + +Report the forecast as one rung of a fixed ladder rather than a number. Each +rung names a band of natural frequencies: over 100 comparable fresh runs, + +- surely: 95 or more succeed +- extremely_likely: 85 to 95 +- very_likely: 70 to 85 +- likely: 55 to 70 +- uncertain: 45 to 55 +- unlikely: 30 to 45 +- very_unlikely: 15 to 30 +- almost_surely_not: fewer than 15 + +Use the whole ladder when justified. Supported does not mean surely, and +unsupported does not mean almost_surely_not. The downstream routing threshold +is not part of this forecast. + +# Efficient-agent capability card + +The route verbs in this source card are inherited qualitative descriptions. +They do not ask you to output a route and do not assign a fixed probability to +any boundary. + +- SUP-1 [supported]: Route to the Efficient model when the task provides a complete output contract and a deterministic local validator that covers the material requirements. +- SUP-2 [supported]: Route to the Efficient model when all required inputs are available, the target environment can be inspected, and correctness can be verified end-to-end without inaccessible external state. +- SUP-3 [supported]: Route to the Efficient model when mathematical behavior, interfaces, shapes, data types, tolerances, and performance requirements are explicit and exercised by a representative harness. +- SUP-4 [supported]: Route to the Efficient model when the required mechanism is identified, the relevant search space is bounded, and the success condition is executable. Do not infer this rule merely from the task's technical domain. +- SUP-5 [supported]: Route to the Efficient model when reconstruction or behavioral reproduction is constrained by an executable reference, parser, format specification, or checker strong enough to distinguish correct from merely plausible output. +- UNC-1 [uncertain]: Treat the route as uncertain when multiple reasonable interpretations of preprocessing, representation, indexing, naming, or output placement would produce different results and neither the instructions nor a validator resolve the choice. +- UNC-2 [uncertain]: Treat the route as uncertain when success requires finding every relevant item across heterogeneous inputs or environment state, but the task does not define the search boundary or provide a completeness check. +- LIM-1 [unsupported]: Prefer the Capable model when correctness depends primarily on extracting precise information from noisy visual, temporal, or rendered media and no machine-checkable extraction or replay mechanism is available. +- LIM-2 [unsupported]: Prefer the Capable model when success depends on reproducing undocumented reference behavior, hidden intermediate state, or an unknown configuration, and small deviations fail despite satisfying the visible specification. + +# Output + +Return exactly one JSON object matching the response schema supplied with the +request. Do not include markdown or commentary. + +confidence must be one of the ladder rungs above, spelled exactly as listed. +Do not output a probability, recommended_route, abstain, counts, task totals, +empirical rates, or any other field. diff --git a/crates/libsy/src/prompts/capability-classifier/schema-ordinal.json b/crates/libsy/src/prompts/capability-classifier/schema-ordinal.json new file mode 100644 index 000000000..5d2a6099e --- /dev/null +++ b/crates/libsy/src/prompts/capability-classifier/schema-ordinal.json @@ -0,0 +1,52 @@ +{ + "type": "json_schema", + "json_schema": { + "name": "CapabilityClassifierOrdinalDecision", + "strict": true, + "schema": { + "type": "object", + "additionalProperties": false, + "required": [ + "crux", + "primary_rule", + "capability_boundary", + "confidence" + ], + "properties": { + "crux": {"type": "string", "minLength": 1}, + "primary_rule": { + "type": "string", + "enum": [ + "SUP-1", + "SUP-2", + "SUP-3", + "SUP-4", + "SUP-5", + "UNC-1", + "UNC-2", + "LIM-1", + "LIM-2", + "none" + ] + }, + "capability_boundary": { + "type": "string", + "enum": ["supported", "uncertain", "unsupported", "unmatched"] + }, + "confidence": { + "type": "string", + "enum": [ + "surely", + "extremely_likely", + "very_likely", + "likely", + "uncertain", + "unlikely", + "very_unlikely", + "almost_surely_not" + ] + } + } + } + } +} diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index ab36f82af..a009f7fd3 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -16,7 +16,7 @@ use switchyard_libsy::{ CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, StageRouter, StageRouterConfig, Step as RustStep, - StepStream, TaskClassifierConfig, ToolSemantics, + StepStream, TaskClassifierConfig, ToolSemantics, VerdictScale, }; use switchyard_protocol::{ LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, ModelId, @@ -283,6 +283,8 @@ impl PyTaskClassifierConfig { ) -> PyResult { Ok(Self { inner: TaskClassifierConfig { + // The ordinal confidence scale is not exposed to Python yet. + verdict_scale: VerdictScale::default(), base_threshold, threshold_step, classify_trigger: classify_trigger(session_affinity), diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 3e0efd38d..203ff2654 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -15,7 +15,7 @@ use libsy::{ CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, - ToolSemantics, + ToolSemantics, VerdictScale, }; use serde::Deserialize; use switchyard_protocol::ModelId; @@ -108,6 +108,7 @@ struct CapabilityClassifierRouteConfig { prompt: Option, response_format_type: ClassifierResponseFormat, max_output_tokens: u64, + verdict_scale: VerdictScale, } #[derive(Clone, Debug)] @@ -169,6 +170,10 @@ pub struct LlmClassifierRouteConfig { /// Most completion tokens the judge verdict may use. #[serde(default = "default_classifier_max_output_tokens")] pub max_output_tokens: u64, + /// Capability mode: whether the forecaster reports `p_solve` as a probability or + /// `confidence` as one rung of the fixed ladder (the gate has its own `gate.verdict_scale`). + #[serde(default)] + pub verdict_scale: VerdictScale, /// Escalation mode: how many escalate verdicts latch the session, and how /// much of the transcript the judge sees. pub escalation: Option, @@ -421,6 +426,7 @@ impl StageClassifierConfig { contract: classifier_contract(self.prompt.as_deref()) .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, + verdict_scale: VerdictScale::default(), } } } @@ -611,6 +617,7 @@ impl LlmClassifierRouteConfig { prompt, response_format_type, max_output_tokens, + verdict_scale, escalation, targets, default_target, @@ -665,6 +672,7 @@ impl LlmClassifierRouteConfig { prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, + verdict_scale: *verdict_scale, }, )) } @@ -940,6 +948,7 @@ fn build_algorithm( contract: classifier_contract(config.prompt.as_deref()) .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, + verdict_scale: config.verdict_scale, }; LlmTaskClassifier::new(LlmClassifierConfig::Capability { judge_target: classifier, diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 45c737912..1aeb8c830 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -203,6 +203,7 @@ Capability mode classifies before serving. See | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | +| `verdict_scale` | No | `probability` | `probability` asks for `p_solve`; `ordinal` asks for `confidence` as one of eight named rungs mapped to band midpoints, so `base_threshold` selects rungs. See `escalation.gate.verdict_scale` for the ladder. | Escalation mode serves the weak target first and judges the completed turn. See [Escalation-Router Routing](../routing_algorithms/escalation_router_routing.md). @@ -220,6 +221,7 @@ Escalation mode serves the weak target first and judges the completed turn. See | `escalation.gate.threshold_step` | No | `0.0` | Added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | | `escalation.gate.prompt` | No | packaged capability prompt | Replaces the capability-forecaster prompt for the gate call only. | | `escalation.gate.classifier_target` | No | the route's `classifier_target` | Target the once-per-session gate forecast is called through, so a strong forecaster can gate while a cheap model judges every weak turn. | +| `escalation.gate.verdict_scale` | No | `probability` | `probability` asks the forecaster for `p_solve` in `[0, 1]`; `ordinal` asks for `confidence` as one of eight rungs (`surely`, `extremely_likely`, `very_likely`, `likely`, `uncertain`, `unlikely`, `very_unlikely`, `almost_surely_not`), mapped to band midpoints (0.97, 0.90, 0.78, 0.62, 0.50, 0.38, 0.22, 0.08) before the threshold applies. | Existing configurations that contain `escalation` but omit `mode` remain valid. diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 982a67bad..3b8d1eb06 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -119,7 +119,16 @@ uses to set its split, and the gate reuses that policy unchanged, including the packaged forecaster prompt. `gate.classifier_target` lets the gate call a different target from the per-turn judge: the forecast runs once per session and rewards a strong model, while the trajectory judge runs on every weak turn -and is where a cheap model belongs. The gate verdict is recorded in the route's +and is where a cheap model belongs. + +`gate.verdict_scale = "ordinal"` asks the forecaster for one rung of a fixed +ladder (`surely` down to `almost_surely_not`) instead of a probability. Models +produce ordinal judgements more reliably than calibrated numbers: asked for a +probability they cluster on a few round values, and given a stated cutoff they +write a number just under it. Each rung maps to the midpoint of the frequency +band it names, so `base_threshold` still selects the split; a threshold of 0.45 +latches `unlikely` and below, 0.55 latches `uncertain` and below. The chosen +rung is recorded in the evidence next to the score. The gate verdict is recorded in the route's evidence as `{"source": "escalation", "verdict": "gate"}`. ## Judge model compatibility From 6d4538e8ea6708cd7e5601cf3871fea4d6786e43 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Fri, 11 Sep 2026 16:30:44 -0700 Subject: [PATCH 6/6] feat(libsy): let the escalation gate take its threshold as a ladder rung With verdict_scale = "ordinal" the model answers in words, but the operator still had to write base_threshold = 0.45 and know that it means "unlikely and below". gate.min_confidence names the lowest rung the efficient tier keeps; every rung below it latches. The threshold is derived halfway between adjacent rung midpoints, so an ordinal configuration carries no number in the prompt, the answer or the config. Exactly one of base_threshold and min_confidence must be set; unknown rungs are rejected at load time. Signed-off-by: Lin Jia --- CHANGELOG.md | 3 +- crates/libsy/src/algorithms/escalation.rs | 37 +++++++++++++++-- crates/libsy/src/algorithms/llm_class.rs | 25 +++++++++++- .../libsy/src/algorithms/util/escalation.rs | 40 ++++++++++++++++--- docs/reference/toml_schema.md | 3 +- .../escalation_router_routing.md | 13 ++++-- 6 files changed, 105 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5f04dd1..0b048f273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). judge for one of eight named rungs instead of a probability; rungs map to band midpoints so thresholds keep working. Models rank tasks more reliably than they number them, and a stated numeric cutoff invites answers just - under it. + under it. `escalation.gate.min_confidence` writes the gate's threshold as a + rung, so an ordinal configuration carries no numbers at all. - **Per-target `reasoning_effort`** — a target can force the reasoning effort of every request it serves, replacing the caller's value (`reasoning.effort` on the Responses wire, `reasoning_effort` on Chat Completions), so a strong diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index bb99765b1..214716a0c 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -480,7 +480,8 @@ mod tests { config: EscalationJudgeConfig { confirmations: 1, gate: Some(EscalationGateConfig { - base_threshold, + base_threshold: Some(base_threshold), + min_confidence: None, threshold_step: 0.0, prompt: None, classifier_target: None, @@ -529,8 +530,9 @@ mod tests { config: EscalationJudgeConfig { confirmations: 1, gate: Some(EscalationGateConfig { - // Between `uncertain` (0.50) and `unlikely` (0.38): unlikely and below latch. - base_threshold: 0.45, + // Keep `uncertain` and above on the efficient tier; `unlikely` and below latch. + base_threshold: None, + min_confidence: Some("uncertain".to_string()), threshold_step: 0.0, prompt: None, classifier_target: None, @@ -629,7 +631,8 @@ mod tests { contract: ClassifierContractConfig::default(), config: EscalationJudgeConfig { gate: Some(EscalationGateConfig { - base_threshold, + base_threshold: Some(base_threshold), + min_confidence: None, threshold_step: 0.0, prompt: None, classifier_target: None, @@ -643,6 +646,32 @@ mod tests { }; assert!(build(0.4).is_ok()); assert!(build(1.5).is_err()); + + let rung = |min_confidence: Option<&str>, base_threshold: Option| { + LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + gate: Some(EscalationGateConfig { + base_threshold, + min_confidence: min_confidence.map(str::to_string), + threshold_step: 0.0, + prompt: None, + classifier_target: None, + verdict_scale: VerdictScale::Ordinal, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + gate_judge_target: None, + }) + }; + assert!(rung(Some("uncertain"), None).is_ok()); + assert!(rung(Some("maybe"), None).is_err()); + assert!(rung(Some("uncertain"), Some(0.5)).is_err()); + assert!(rung(None, None).is_err()); } #[tokio::test] diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index ac4c0465d..15f3db7e4 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -81,6 +81,20 @@ fn rung_midpoint(rung: &str) -> Option { .find(|(name, _)| *name == rung) .map(|(_, midpoint)| *midpoint) } + +/// The numeric threshold that keeps `rung` and every rung above it on the efficient tier and +/// sends every rung below it to the capable tier: halfway between `rung`'s midpoint and the +/// next lower rung's (or halfway to zero for the lowest rung). `None` for an unknown rung. +pub(crate) fn rung_threshold(rung: &str) -> Option { + let index = CONFIDENCE_RUNGS + .iter() + .position(|(name, _)| *name == rung)?; + let lower = CONFIDENCE_RUNGS + .get(index + 1) + .map(|(_, midpoint)| *midpoint) + .unwrap_or(0.0); + Some((CONFIDENCE_RUNGS[index].1 + lower) / 2.0) +} /// Telemetry label for this algorithm's spans, metrics, and logs. const ALGORITHM_NAME: &str = "llm_task_classifier"; @@ -909,7 +923,7 @@ pub(super) fn build_capability_gate( let contract = ClassifierContract::from_config(&contract_config, prompt, schema)?; let config = TaskClassifierConfig { verdict_scale: gate.verdict_scale, - base_threshold: gate.base_threshold, + base_threshold: gate.threshold()?, threshold_step: gate.threshold_step, ..TaskClassifierConfig::default() }; @@ -1016,6 +1030,15 @@ mod tests { assert!(!both.is_valid()); } + #[test] + fn rung_thresholds_sit_between_adjacent_rungs() { + // Keeping `uncertain` (0.50) and above means latching `unlikely` (0.38) and below. + let threshold = rung_threshold("uncertain").expect("known rung"); + assert!(threshold < 0.50 && threshold > 0.38); + assert!(rung_threshold("almost_surely_not").is_some_and(|t| t > 0.0 && t < 0.08)); + assert!(rung_threshold("maybe").is_none()); + } + #[test] fn ordinal_scale_loads_the_ladder_prompt_and_schema() { let (prompt, schema) = VerdictScale::Ordinal.templates(); diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 12ebcdd8e..1bf8ad0de 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,7 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; -use crate::algorithms::llm_class::VerdictScale; +use crate::algorithms::llm_class::{self, VerdictScale}; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; use crate::{LibsyError, Result}; @@ -78,7 +78,14 @@ pub struct EscalationJudgeConfig { #[serde(deny_unknown_fields)] pub struct EscalationGateConfig { /// Lowest solve probability that keeps the session on the efficient tier. In `[0, 1]`. - pub base_threshold: f64, + /// Exactly one of `base_threshold` and `min_confidence` must be set. + #[serde(default)] + pub base_threshold: Option, + /// The threshold as a ladder rung: the efficient tier keeps this rung and every rung above + /// it, and every rung below latches to the capable tier. Pairs naturally with + /// `verdict_scale = "ordinal"`, so an ordinal configuration carries no numbers at all. + #[serde(default)] + pub min_confidence: Option, /// Added once for uncertain or unmatched verdicts and twice for unsupported verdicts, as in /// capability mode. `base_threshold + 2 * threshold_step` must be at most `1`. #[serde(default)] @@ -100,12 +107,33 @@ pub struct EscalationGateConfig { } impl EscalationGateConfig { + /// The numeric threshold the gate applies, from whichever form the operator wrote. + pub(crate) fn threshold(&self) -> Result { + let reject = |message: String| Err(LibsyError::AlgorithmError { message }); + match (self.base_threshold, self.min_confidence.as_deref()) { + (Some(threshold), None) => Ok(threshold), + (None, Some(rung)) => llm_class::rung_threshold(rung).ok_or_else(|| { + LibsyError::AlgorithmError { + message: format!( + "gate.min_confidence must be a ladder rung (surely, extremely_likely, very_likely, likely, uncertain, unlikely, very_unlikely, almost_surely_not), got {rung:?}" + ), + } + }), + (Some(_), Some(_)) => reject( + "gate.base_threshold and gate.min_confidence cannot both be set".to_string(), + ), + (None, None) => reject( + "gate needs base_threshold or min_confidence".to_string(), + ), + } + } + fn validate(&self) -> Result<()> { let reject = |message: String| Err(LibsyError::AlgorithmError { message }); - if !(0.0..=1.0).contains(&self.base_threshold) { + let threshold = self.threshold()?; + if !(0.0..=1.0).contains(&threshold) { return reject(format!( - "gate.base_threshold must be between 0 and 1, got {}", - self.base_threshold + "gate.base_threshold must be between 0 and 1, got {threshold}" )); } if !self.threshold_step.is_finite() || self.threshold_step < 0.0 { @@ -114,7 +142,7 @@ impl EscalationGateConfig { self.threshold_step )); } - let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step; + let unsupported_threshold = threshold + 2.0 * self.threshold_step; if unsupported_threshold > 1.0 { return reject(format!( "gate.base_threshold + 2 * gate.threshold_step must be at most 1, got {unsupported_threshold}" diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 1aeb8c830..9022d8a7e 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -217,7 +217,8 @@ Escalation mode serves the weak target first and judges the completed turn. See | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | | `escalation.gate` | No | unset | Up-front capability gate: judges the first request of a session from the task framing with the packaged capability forecaster and latches to `strong_target` when `p_solve` is below the threshold, before `weak_target` is called. Needs a session ID. | -| `escalation.gate.base_threshold` | Yes, when `gate` is set | — | Lowest solve probability that keeps the session on `weak_target`. In `[0, 1]`. | +| `escalation.gate.base_threshold` | One of the two, when `gate` is set | — | Lowest solve probability that keeps the session on `weak_target`. In `[0, 1]`. | +| `escalation.gate.min_confidence` | One of the two, when `gate` is set | — | The threshold as a ladder rung: `weak_target` keeps this rung and every rung above it, every rung below latches to `strong_target`. Use with `verdict_scale = "ordinal"` for a configuration with no numbers. | | `escalation.gate.threshold_step` | No | `0.0` | Added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | | `escalation.gate.prompt` | No | packaged capability prompt | Replaces the capability-forecaster prompt for the gate call only. | | `escalation.gate.classifier_target` | No | the route's `classifier_target` | Target the once-per-session gate forecast is called through, so a strong forecaster can gate while a cheap model judges every weak turn. | diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 3b8d1eb06..e1779abd0 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -126,9 +126,16 @@ ladder (`surely` down to `almost_surely_not`) instead of a probability. Models produce ordinal judgements more reliably than calibrated numbers: asked for a probability they cluster on a few round values, and given a stated cutoff they write a number just under it. Each rung maps to the midpoint of the frequency -band it names, so `base_threshold` still selects the split; a threshold of 0.45 -latches `unlikely` and below, 0.55 latches `uncertain` and below. The chosen -rung is recorded in the evidence next to the score. The gate verdict is recorded in the route's +band it names, so `base_threshold` still selects the split. Better, write the +threshold as a rung too: + +```toml +escalation = { confirmations = 1, gate = { verdict_scale = "ordinal", min_confidence = "uncertain" } } +``` + +keeps `uncertain` and everything above it on the weak tier and latches +`unlikely` and below, with no number anywhere in the prompt, the answer or the +configuration. The chosen rung is recorded in the evidence next to the score. The gate verdict is recorded in the route's evidence as `{"source": "escalation", "verdict": "gate"}`. ## Judge model compatibility