switchyard_libsy/algorithms/llm_class/
decision.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Compare two candidates by the chance that the capable candidate alone succeeds.
5
6use std::collections::{BTreeMap, HashSet};
7
8use async_trait::async_trait;
9use serde_json::{Value, json};
10use switchyard_protocol::{
11    Category, ChoiceOption, ContentBlock, DecisionKind, DecisionQuestion, DecisionRequest,
12    DecisionValue, ImageSource, ModelId, Request, Response,
13};
14
15use crate::algorithms::llm_class::TaskInput;
16use crate::algorithms::util::llm_judge::{libsy_error_reason, report_fail_open};
17use crate::algorithms::util::robustness::safe_error_summary;
18use crate::{Classification, Classifier, Driver, LibsyError, Result, Score, State};
19
20/// Evidence and policy for a relative-advantage decision judge.
21///
22/// Candidate labels keep model names out of the generated context. Evidence must use
23/// the same labels, with unknown outcomes left unknown. Only the first runtime
24/// capable and efficient targets are compared; extra candidates do not add routes.
25#[derive(Clone, Debug)]
26pub struct DecisionJudgeConfig {
27    /// Route capable only when its advantage score is strictly above this cutoff.
28    /// This score is not a calibrated solve probability. Choose a cutoff from evaluations.
29    pub cutoff: f64,
30    /// Replaces the packaged structured instructions; must keep the meaning of
31    /// `advantage` (capable succeeds and efficient fails) and `no_advantage`.
32    pub instructions: Option<Value>,
33    /// Maps anonymous labels used in evidence (e.g. `"a"`) to runtime model IDs.
34    /// Every compared target needs a unique label; extra candidates provide context.
35    pub candidates: BTreeMap<String, ModelId>,
36    /// JSON passed unchanged to the judge, such as candidate descriptions, reference
37    /// cases, outcome/cost summaries, and selection notes. The judge interprets it
38    /// using the instructions; the router neither reads its fields nor derives statistics.
39    pub evidence: Value,
40}
41
42impl DecisionJudgeConfig {
43    pub(super) fn validate(&self) -> Result<()> {
44        if !(0.0..=1.0).contains(&self.cutoff) {
45            return Err(LibsyError::AlgorithmError {
46                message: "decision cutoff must be between 0 and 1".into(),
47            });
48        }
49        if self.candidates.values().collect::<HashSet<_>>().len() != self.candidates.len() {
50            return Err(LibsyError::AlgorithmError {
51                message: "decision candidates must map to distinct targets".into(),
52            });
53        }
54        Ok(())
55    }
56
57    fn candidate(&self, model: &ModelId) -> Result<&str> {
58        self.candidates
59            .iter()
60            .find(|(_, target)| *target == model)
61            .map(|(id, _)| id.as_str())
62            .ok_or_else(|| LibsyError::AlgorithmError {
63                message: format!("decision candidate is missing for target {model}"),
64            })
65    }
66}
67
68pub(super) struct DecisionClassifier {
69    config: DecisionJudgeConfig,
70    input: TaskInput,
71    fail_open: bool,
72    question: DecisionQuestion,
73}
74
75impl DecisionClassifier {
76    pub(super) fn new(
77        mut config: DecisionJudgeConfig,
78        input: TaskInput,
79        fail_open: bool,
80    ) -> Result<Self> {
81        let instructions = match config.instructions.take() {
82            Some(instructions) => instructions,
83            None => serde_json::from_str(include_str!(
84                "../../prompts/capability-classifier/relative_advantage.json"
85            ))
86            .map_err(|error| LibsyError::external("loading decision judge instructions", error))?,
87        };
88        let question = DecisionQuestion {
89            instructions,
90            kind: DecisionKind::Choice {
91                options: [
92                    ("advantage", "The capable candidate succeeds AND the efficient candidate fails."),
93                    ("no_advantage", "The efficient candidate succeeds OR the capable candidate fails, including shared failure."),
94                ]
95                .into_iter()
96                .map(|(id, description)| ChoiceOption {
97                    id: id.into(),
98                    description: Some(json!(description)),
99                })
100                .collect(),
101            },
102        };
103        Ok(Self {
104            config,
105            input,
106            fail_open,
107            question,
108        })
109    }
110}
111
112#[async_trait]
113impl Classifier<State> for DecisionClassifier {
114    async fn score(
115        &self,
116        _state: &mut State,
117        request: &mut Request,
118        driver: &Driver,
119    ) -> Result<(Classification, Option<Response>)> {
120        let judge = driver.first_model_for(&Category::Judge)?;
121        let capable = driver.first_model_for(&Category::Capable)?;
122        let efficient = driver.first_model_for(&Category::Efficient)?;
123        let mut messages = self.input.messages(request);
124        let mut images = Vec::new();
125        for message in &mut messages {
126            extract_images(&mut message.content, &mut images);
127        }
128        let decision = DecisionRequest {
129            model: None,
130            images,
131            context: json!({
132                "task": messages,
133                "candidates": self.config.candidates.keys().collect::<Vec<_>>(),
134                "comparison": {
135                    "capable": self.config.candidate(capable)?,
136                    "efficient": self.config.candidate(efficient)?,
137                },
138                "evidence": self.config.evidence,
139            }),
140            questions: BTreeMap::from([("route".into(), self.question.clone())]),
141        };
142        let response = match driver.call_decision(decision, judge.clone()).await {
143            Ok(response) => response,
144            Err(error) if self.fail_open => {
145                return Ok(unavailable(
146                    driver,
147                    judge,
148                    safe_error_summary(&error),
149                    libsy_error_reason(&error),
150                ));
151            }
152            Err(error) => return Err(error),
153        };
154        // The provider's selected option can differ from the application's cutoff.
155        // Only the requested event's score is used; confidence is a separate signal.
156        let advantage = response
157            .answers
158            .get("route")
159            .and_then(|answer| match &answer.value {
160                DecisionValue::Choice {
161                    probabilities: Some(probabilities),
162                    ..
163                } => probabilities.get("advantage").map(|p| p.0),
164                _ => None,
165            })
166            .filter(|p| (0.0..=1.0).contains(p));
167        let Some(advantage) = advantage else {
168            return Ok(unavailable(
169                driver,
170                judge,
171                "missing or invalid advantage score".into(),
172                "invalid_verdict",
173            ));
174        };
175        let (target, category) = if advantage > self.config.cutoff {
176            (capable, Category::Capable)
177        } else {
178            (efficient, Category::Efficient)
179        };
180        driver.set_evidence(json!({
181            "source": "decision_classifier",
182            "verdict": "relative_advantage",
183            "score": advantage,
184            "threshold": self.config.cutoff,
185        }));
186        Ok((
187            Classification::Scores(vec![Score {
188                target: target.clone(),
189                confidence: 1.0,
190                category: Some(category),
191            }]),
192            None,
193        ))
194    }
195}
196
197fn extract_images(blocks: &mut [ContentBlock], images: &mut Vec<ImageSource>) {
198    for block in blocks {
199        match block {
200            ContentBlock::Image { source } => {
201                images.push(source.clone());
202                // Keep the image's position without tokenizing its encoded bytes.
203                *block = ContentBlock::Text {
204                    text: format!("[Image {}]", images.len()),
205                };
206            }
207            ContentBlock::ToolResult(result) => extract_images(&mut result.content, images),
208            _ => {}
209        }
210    }
211}
212
213fn unavailable(
214    driver: &Driver,
215    judge: &ModelId,
216    error: String,
217    reason: &'static str,
218) -> (Classification, Option<Response>) {
219    report_fail_open(judge.as_str(), error, reason);
220    driver.set_evidence_if_empty(json!({"source": "fail_open", "reason_code": reason}));
221    (Classification::Ambiguous(vec![]), None)
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227    use switchyard_protocol::ToolResult;
228
229    #[test]
230    fn extracts_tool_images_in_order() {
231        let image = |url: &str| ImageSource::Url {
232            url: url.into(),
233            detail: None,
234        };
235        let mut blocks = vec![
236            ContentBlock::Image {
237                source: image("https://example.test/first.png"),
238            },
239            ContentBlock::ToolResult(ToolResult {
240                tool_call_id: "capture".into(),
241                content: vec![ContentBlock::Image {
242                    source: image("https://example.test/second.png"),
243                }],
244                is_error: Some(false),
245            }),
246        ];
247        let mut images = Vec::new();
248        extract_images(&mut blocks, &mut images);
249        assert_eq!(
250            images,
251            vec![
252                image("https://example.test/first.png"),
253                image("https://example.test/second.png")
254            ]
255        );
256        let ContentBlock::ToolResult(result) = &blocks[1] else {
257            panic!("expected tool result")
258        };
259        assert_eq!(result.tool_call_id, "capture");
260        assert_eq!(
261            result.content,
262            vec![ContentBlock::Text {
263                text: "[Image 2]".into()
264            }]
265        );
266        assert_eq!(result.is_error, Some(false));
267    }
268}
switchyard_libsy/algorithms/llm_class/decision.rs

switchyard_libsy/algorithms/llm_class/
decision.rs
