switchyard_libsy/algorithms/llm_class/
decision.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Select a candidate by applying a cutoff to a decision model's choice score.
5
6use std::collections::{BTreeMap, HashSet};
7
8use async_trait::async_trait;
9use serde_json::{Value, json};
10use switchyard_protocol::{
11    Category, ChoiceOption, DecisionKind, DecisionQuestion, DecisionRequest, DecisionValue,
12    ModelId, Request, Response,
13};
14
15use crate::algorithms::llm_class::TaskInput;
16use crate::algorithms::util::llm_judge::{libsy_error_reason, report_fail_open};
17use crate::algorithms::util::robustness::safe_error_summary;
18use crate::{Classification, Classifier, Driver, LibsyError, Result, Score, State};
19
20/// Instructions, evidence, and cutoff policy for a decision judge.
21///
22/// Candidate labels keep model names out of the generated context. Evidence must use
23/// the same labels, with unknown outcomes left unknown. Only the first runtime
24/// capable and efficient targets are compared; extra candidates do not add routes.
25#[derive(Clone, Debug)]
26pub struct DecisionJudgeConfig {
27    /// Route capable only when its choice score is strictly above this cutoff.
28    /// This score is not a calibrated solve probability. Choose a cutoff from evaluations.
29    pub cutoff: f64,
30    /// Replaces the packaged instructions. Without custom criteria, preserve
31    /// the meaning of `advantage` and `no_advantage`.
32    pub instructions: Option<Value>,
33    /// Two choice descriptions keyed by candidate label. Requires custom instructions.
34    pub criteria: Option<BTreeMap<String, Value>>,
35    /// Maps anonymous labels used in evidence (e.g. `"a"`) to runtime model IDs.
36    /// Every compared target needs a unique label; extra candidates provide context.
37    pub candidates: BTreeMap<String, ModelId>,
38    /// JSON passed unchanged to the judge, such as candidate descriptions, reference
39    /// cases, outcome/cost summaries, and selection notes. The judge interprets it
40    /// using the instructions; the router neither reads its fields nor derives statistics.
41    pub evidence: Value,
42}
43
44impl DecisionJudgeConfig {
45    pub(super) fn validate(&self) -> Result<()> {
46        if !(0.0..=1.0).contains(&self.cutoff) {
47            return Err(LibsyError::AlgorithmError {
48                message: "decision cutoff must be between 0 and 1".into(),
49            });
50        }
51        if self.candidates.values().collect::<HashSet<_>>().len() != self.candidates.len() {
52            return Err(LibsyError::AlgorithmError {
53                message: "decision candidates must map to distinct targets".into(),
54            });
55        }
56        if let Some(criteria) = &self.criteria
57            && (self.instructions.is_none()
58                || criteria.len() != 2
59                || !criteria.keys().eq(self.candidates.keys()))
60        {
61            return Err(LibsyError::AlgorithmError {
62                message: "custom decision criteria require instructions and exactly two matching candidates".into(),
63            });
64        }
65        Ok(())
66    }
67
68    fn candidate(&self, model: &ModelId) -> Result<&str> {
69        self.candidates
70            .iter()
71            .find(|(_, target)| *target == model)
72            .map(|(id, _)| id.as_str())
73            .ok_or_else(|| LibsyError::AlgorithmError {
74                message: format!("decision candidate is missing for target {model}"),
75            })
76    }
77}
78
79pub(super) struct DecisionClassifier {
80    config: DecisionJudgeConfig,
81    input: TaskInput,
82    fail_open: bool,
83    question: DecisionQuestion,
84}
85
86impl DecisionClassifier {
87    pub(super) fn new(
88        mut config: DecisionJudgeConfig,
89        mut input: TaskInput,
90        fail_open: bool,
91    ) -> Result<Self> {
92        input.recent_turn_window.get_or_insert(10);
93        let instructions = match config.instructions.take() {
94            Some(instructions) => instructions,
95            None => serde_json::from_str(include_str!(
96                "../../prompts/capability-classifier/relative_advantage.json"
97            ))
98            .map_err(|error| LibsyError::external("loading decision judge instructions", error))?,
99        };
100        let options = match &config.criteria {
101            Some(criteria) => criteria
102                .iter()
103                .map(|(id, description)| ChoiceOption {
104                    id: id.clone(),
105                    description: Some(description.clone()),
106                })
107                .collect(),
108            None => [
109                ("advantage", "The capable candidate succeeds AND the efficient candidate fails."),
110                ("no_advantage", "The efficient candidate succeeds OR the capable candidate fails, including shared failure."),
111            ]
112            .into_iter()
113            .map(|(id, description)| ChoiceOption {
114                id: id.into(),
115                description: Some(json!(description)),
116            })
117            .collect(),
118        };
119        let question = DecisionQuestion {
120            instructions,
121            kind: DecisionKind::Choice { options },
122        };
123        Ok(Self {
124            config,
125            input,
126            fail_open,
127            question,
128        })
129    }
130}
131
132#[async_trait]
133impl Classifier for DecisionClassifier {
134    async fn score(
135        &self,
136        _state: &mut State,
137        request: &mut Request,
138        driver: &Driver,
139    ) -> Result<(Classification, Option<Response>)> {
140        let judge = driver.first_model_for(&Category::Judge)?;
141        let capable = driver.first_model_for(&Category::Capable)?;
142        let efficient = driver.first_model_for(&Category::Efficient)?;
143        let score_option = if self.config.criteria.is_some() {
144            self.config.candidate(capable)?
145        } else {
146            "advantage"
147        };
148        let decision = DecisionRequest {
149            model: None,
150            context: json!({
151                "task": self.input.messages(request),
152                "session": request.metadata.as_ref().and_then(|m| m.session_id.as_deref()),
153                "candidates": self.config.candidates.keys().collect::<Vec<_>>(),
154                "comparison": {
155                    "capable": self.config.candidate(capable)?,
156                    "efficient": self.config.candidate(efficient)?,
157                },
158                "evidence": self.config.evidence,
159            }),
160            questions: BTreeMap::from([("route".into(), self.question.clone())]),
161        };
162        let response = match driver.call_decision(decision, judge.clone()).await {
163            Ok(response) => response,
164            Err(error) if self.fail_open => {
165                return Ok(unavailable(
166                    driver,
167                    judge,
168                    safe_error_summary(&error),
169                    libsy_error_reason(&error),
170                ));
171            }
172            Err(error) => return Err(error),
173        };
174        // The provider's selected option can differ from the application's cutoff.
175        // Only the requested event's score is used; confidence is a separate signal.
176        let score = response
177            .answers
178            .get("route")
179            .and_then(|answer| match &answer.value {
180                DecisionValue::Choice {
181                    probabilities: Some(probabilities),
182                    ..
183                } => probabilities.get(score_option).map(|p| p.0),
184                _ => None,
185            })
186            .filter(|p| (0.0..=1.0).contains(p));
187        let Some(score) = score else {
188            return Ok(unavailable(
189                driver,
190                judge,
191                format!("missing or invalid {score_option} score"),
192                "invalid_verdict",
193            ));
194        };
195        let (target, category) = if score > self.config.cutoff {
196            (capable, Category::Capable)
197        } else {
198            (efficient, Category::Efficient)
199        };
200        driver.set_evidence(json!({
201            "source": "decision_classifier",
202            "verdict": if self.config.criteria.is_some() { "candidate_choice" } else { "relative_advantage" },
203            "score": score,
204            "threshold": self.config.cutoff,
205        }));
206        Ok((
207            Classification::Scores(vec![Score {
208                target: target.clone(),
209                confidence: 1.0,
210                category: Some(category),
211            }]),
212            None,
213        ))
214    }
215}
216
217fn unavailable(
218    driver: &Driver,
219    judge: &ModelId,
220    error: String,
221    reason: &'static str,
222) -> (Classification, Option<Response>) {
223    report_fail_open(judge.as_str(), error, reason);
224    driver.set_evidence_if_empty(json!({"source": "fail_open", "reason_code": reason}));
225    (Classification::Ambiguous(vec![]), None)
226}
switchyard_libsy/algorithms/llm_class/decision.rs

switchyard_libsy/algorithms/llm_class/
decision.rs
