switchyard_runner/
route.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! One configured algorithm and the clients that serve its targets.
5
6use std::collections::HashMap;
7use std::error::Error;
8use std::sync::Arc;
9
10use libsy::{Algorithm, LibsyError, RoutingOutcome, RuntimeModels};
11use serde_json::Value;
12use switchyard_llm_client::{AuxiliaryOperation, ClientRouter, RunObserver, TranslatingLlmClient};
13use switchyard_protocol::{
14    LlmClientError, ModelId, Request, Response, RoutedLlmClient, WireFormat,
15};
16use thiserror::Error;
17
18use crate::DecisionTarget;
19use crate::config::EligibleLanes;
20use crate::eligibility::{EligibilityPolicy, has_target_requirements};
21
22/// Capabilities declared for one route.
23///
24/// `GET /v1/models` includes `context_window`, `tool_calling`, and `vision` in each
25/// standard `data` entry, using `null` for unset values. It omits `reasoning`.
26/// The server rejects tool inputs, reasoning controls, or images when the
27/// corresponding declaration is explicitly `false` for the selected route.
28#[derive(Clone, Copy, Default)]
29pub struct ModelCapabilities {
30    pub context_window: Option<u32>,
31    pub tool_calling: Option<bool>,
32    /// Whether the routed model accepts reasoning controls, as declared in config.
33    pub reasoning: Option<bool>,
34    /// Whether the routed model accepts image input, as declared in config.
35    pub vision: Option<bool>,
36}
37
38/// Caller credential family required by a forwarded-auth route.
39#[derive(Clone, Copy, Debug, Eq, PartialEq)]
40pub enum CallerAuthKind {
41    Anthropic,
42    OpenAi,
43}
44
45impl CallerAuthKind {
46    /// Stable provider name used by the server compatibility API.
47    pub const fn as_str(self) -> &'static str {
48        match self {
49            Self::Anthropic => "anthropic",
50            Self::OpenAi => "openai",
51        }
52    }
53
54    const fn accepts(self, wire_format: WireFormat) -> bool {
55        matches!(
56            (self, wire_format),
57            (Self::Anthropic, WireFormat::AnthropicMessages)
58                | (
59                    Self::OpenAi,
60                    WireFormat::OpenAiChat | WireFormat::OpenAiResponses
61                )
62        )
63    }
64}
65
66/// Exact upstream model and client used for an auxiliary provider operation.
67#[derive(Clone)]
68pub struct AuxiliaryTarget {
69    pub model: ModelId,
70    pub client: Arc<TranslatingLlmClient>,
71}
72
73/// Error returned while loading or executing configured routes.
74#[derive(Debug, Error)]
75pub enum RunnerError {
76    #[error("{message}")]
77    Configuration {
78        message: String,
79        #[source]
80        source: Option<Box<dyn Error + Send + Sync>>,
81    },
82    #[error("unknown route model {0:?}")]
83    UnknownRouteModel(String),
84    #[error("caller format is incompatible with {} credentials", .0.as_str())]
85    IncompatibleCallerFormat(CallerAuthKind),
86    #[error("route has no compatible target for the auxiliary operation")]
87    AuxiliaryUnsupported,
88    #[error(transparent)]
89    Algorithm(#[from] LibsyError),
90    #[error(transparent)]
91    Client(#[from] LlmClientError),
92}
93
94impl RunnerError {
95    pub(crate) fn configuration(message: impl Into<String>) -> Self {
96        Self::Configuration {
97            message: message.into(),
98            source: None,
99        }
100    }
101
102    pub(crate) fn configuration_source(
103        message: impl Into<String>,
104        source: impl Error + Send + Sync + 'static,
105    ) -> Self {
106        Self::Configuration {
107            message: message.into(),
108            source: Some(Box::new(source)),
109        }
110    }
111}
112
113/// A configured algorithm and the per-target clients its calls resolve through.
114pub(crate) struct ExecutionLane {
115    algorithm: Arc<dyn Algorithm>,
116    // Resolves each offloaded call to the client configured for the target the algorithm
117    // selected. A route is a synthetic model with no upstream of its own, so this is a
118    // per-target lookup, never one client serving the whole route.
119    clients: ClientRouter,
120    anthropic_auxiliary_target: Option<AuxiliaryTarget>,
121    responses_auxiliary_target: Option<AuxiliaryTarget>,
122    models: Arc<RuntimeModels>,
123}
124
125impl ExecutionLane {
126    pub(crate) fn new(
127        algorithm: Arc<dyn Algorithm>,
128        clients: ClientRouter,
129        anthropic_auxiliary_target: Option<AuxiliaryTarget>,
130        responses_auxiliary_target: Option<AuxiliaryTarget>,
131        models: RuntimeModels,
132    ) -> Self {
133        Self {
134            algorithm,
135            clients,
136            anthropic_auxiliary_target,
137            responses_auxiliary_target,
138            models: Arc::new(models),
139        }
140    }
141}
142
143struct EligibilityExecution {
144    policy: EligibilityPolicy,
145    lanes: EligibleLanes,
146}
147
148/// A configured route with optional target eligibility checks.
149#[derive(Clone)]
150pub struct Route {
151    standard: Arc<ExecutionLane>,
152    eligibility: Option<Arc<EligibilityExecution>>,
153    targets: Arc<HashMap<ModelId, DecisionTarget>>,
154    caller_auth: Option<CallerAuthKind>,
155    capabilities: ModelCapabilities,
156    infra: Option<libsy::InfraPolicy>,
157}
158
159/// The selected model and untouched response produced by a route execution.
160pub struct RunOutput {
161    pub selected_model: ModelId,
162    pub response: Response,
163}
164
165impl Route {
166    /// Creates a fully configured execution route.
167    #[allow(clippy::too_many_arguments)]
168    pub fn new(
169        algorithm: Arc<dyn Algorithm>,
170        clients: ClientRouter,
171        caller_auth: Option<CallerAuthKind>,
172        capabilities: ModelCapabilities,
173        anthropic_auxiliary_target: Option<AuxiliaryTarget>,
174        responses_auxiliary_target: Option<AuxiliaryTarget>,
175        targets: Vec<DecisionTarget>,
176        models: RuntimeModels,
177    ) -> Self {
178        let models = models.with_provider_models(
179            targets
180                .iter()
181                .map(|target| (ModelId::from(target.target.as_str()), target.model.clone()))
182                .collect(),
183        );
184        Self::from_lane(
185            ExecutionLane::new(
186                algorithm,
187                clients,
188                anthropic_auxiliary_target,
189                responses_auxiliary_target,
190                models,
191            ),
192            caller_auth,
193            capabilities,
194            targets,
195        )
196    }
197
198    pub(crate) fn from_lane(
199        lane: ExecutionLane,
200        caller_auth: Option<CallerAuthKind>,
201        capabilities: ModelCapabilities,
202        targets: Vec<DecisionTarget>,
203    ) -> Self {
204        let targets = targets
205            .into_iter()
206            .map(|target| (ModelId::from(target.target.as_str()), target))
207            .collect();
208        Self {
209            standard: Arc::new(lane),
210            eligibility: None,
211            targets: Arc::new(targets),
212            caller_auth,
213            capabilities,
214            infra: None,
215        }
216    }
217
218    pub(crate) fn with_eligibility(
219        mut self,
220        policy: EligibilityPolicy,
221        lanes: EligibleLanes,
222    ) -> Self {
223        self.eligibility = Some(Arc::new(EligibilityExecution { policy, lanes }));
224        self
225    }
226
227    fn select_lane(&self, request: &Request) -> Result<Arc<ExecutionLane>, RunnerError> {
228        if let Some(eligibility) = &self.eligibility {
229            let required = eligibility.policy.admit(request)?;
230            let lane = if required.required_labels().is_empty() {
231                Arc::clone(&self.standard)
232            } else {
233                eligibility
234                    .lanes
235                    .select(required.required_labels(), &self.standard)?
236            };
237            tracing::debug!(
238                switchyard.route = request.llm_request.model.as_deref().unwrap_or_default(),
239                switchyard.algorithm = self.algorithm_name(),
240                eligibility.required_label_count = required.required_labels().len(),
241                "eligible targets selected"
242            );
243            return Ok(lane);
244        }
245        if has_target_requirements(request) {
246            return Err(LlmClientError::InvalidRequest {
247                message: "route does not accept target requirements".to_string(),
248            }
249            .into());
250        }
251        Ok(Arc::clone(&self.standard))
252    }
253
254    /// Returns the configured libsy algorithm name.
255    pub fn algorithm_name(&self) -> &str {
256        self.standard.algorithm.name()
257    }
258
259    /// Returns model-list capability metadata.
260    pub fn capabilities(&self) -> ModelCapabilities {
261        self.capabilities
262    }
263
264    /// Returns the forwarded caller credential family.
265    pub fn caller_auth(&self) -> Option<CallerAuthKind> {
266        self.caller_auth
267    }
268
269    /// Returns non-secret metadata for a configured LLM target, including judges.
270    pub fn target(&self, selected: &ModelId) -> Option<&DecisionTarget> {
271        self.targets.get(selected)
272    }
273
274    /// Resolves a selected model to this route's non-secret target metadata.
275    pub(crate) fn decision_target(&self, model: &ModelId) -> Option<DecisionTarget> {
276        self.target(model).cloned()
277    }
278
279    /// The provider model that serves `selected`, or `selected` itself when it names no
280    /// target of this route.
281    pub fn provider_model<'a>(&'a self, selected: &'a ModelId) -> &'a ModelId {
282        self.targets
283            .get(selected)
284            .map(|target| &target.model)
285            .unwrap_or(selected)
286    }
287
288    /// Returns the models grouped for the standard execution lane.
289    pub fn models(&self) -> &RuntimeModels {
290        &self.standard.models
291    }
292
293    /// Rejects a caller format incompatible with forwarded credentials.
294    pub fn check_caller_format(&self, input_format: WireFormat) -> Result<(), RunnerError> {
295        if let Some(kind) = self.caller_auth
296            && !kind.accepts(input_format)
297        {
298            return Err(RunnerError::IncompatibleCallerFormat(kind));
299        }
300        Ok(())
301    }
302
303    /// Executes the configured route without consuming or proxying streamed responses.
304    pub async fn execute(
305        &self,
306        request: Request,
307        observer: Option<RunObserver>,
308    ) -> Result<RunOutput, RunnerError> {
309        let lane = self.select_lane(&request)?;
310        let (selected_model, response) = switchyard_llm_client::run(
311            Arc::clone(&lane.algorithm),
312            lane.clients.clone(),
313            request,
314            Arc::clone(&lane.models),
315            observer,
316        )
317        .await?;
318        Ok(RunOutput {
319            selected_model,
320            response,
321        })
322    }
323
324    /// Completes routing-time calls without serving a post-routing completion.
325    pub async fn decide(&self, request: Request) -> Result<RoutingOutcome, RunnerError> {
326        let lane = self.select_lane(&request)?;
327        switchyard_llm_client::decide(
328            Arc::clone(&lane.algorithm),
329            lane.clients.clone(),
330            request,
331            Arc::clone(&lane.models),
332        )
333        .await
334        .map_err(RunnerError::from)
335    }
336
337    pub(crate) fn with_infra(mut self, infra: Option<libsy::InfraPolicy>) -> Self {
338        self.infra = infra;
339        self
340    }
341
342    /// Apply the optional queue policy to a completed task decision.
343    ///
344    /// The host must hold its in-flight accounting lock through this call and the
345    /// increment for the selected model. Ordinary `decide` leaves task routing unchanged.
346    pub fn apply_infra(
347        &self,
348        outcome: &mut RoutingOutcome,
349        signals: &switchyard_protocol::InfraSignals,
350    ) -> Result<bool, RunnerError> {
351        match &self.infra {
352            Some(policy) => {
353                let lane = self.select_lane(&outcome.request)?;
354                if outcome.selected_model_ids.iter().any(|model| {
355                    !lane
356                        .models
357                        .models_for(&switchyard_protocol::Category::Any)
358                        .contains(model)
359                }) {
360                    return Err(LlmClientError::InvalidRequest {
361                        message: "target requirements changed after the routing decision".into(),
362                    }
363                    .into());
364                }
365                policy
366                    .apply(outcome, &lane.models, signals)
367                    .map_err(Into::into)
368            }
369            None => Ok(false),
370        }
371    }
372
373    /// Routes with a host-owned client for routing-time LLM calls, such as judges.
374    ///
375    /// The host owns transport, credentials, and target request settings. Configured
376    /// HTTP clients, target prompts, and decision clients are not used on this path.
377    /// Algorithm state is retained for each target setup. No post-routing completion
378    /// is dispatched. Target eligibility still applies to the host-owned path.
379    pub async fn decide_with_client(
380        &self,
381        request: Request,
382        client: Arc<dyn RoutedLlmClient>,
383    ) -> Result<RoutingOutcome, RunnerError> {
384        let lane = self.select_lane(&request)?;
385        switchyard_llm_client::decide(
386            Arc::clone(&lane.algorithm),
387            ClientRouter::single(client),
388            request,
389            Arc::clone(&lane.models),
390        )
391        .await
392        .map_err(RunnerError::from)
393    }
394
395    /// Executes a model-bearing provider operation through a compatible target.
396    pub async fn call_auxiliary(
397        &self,
398        request: Request,
399        operation: AuxiliaryOperation,
400    ) -> Result<Value, RunnerError> {
401        let lane = self.select_lane(&request)?;
402        let target = match operation {
403            AuxiliaryOperation::AnthropicCountTokens => &lane.anthropic_auxiliary_target,
404            AuxiliaryOperation::ResponsesInputTokens | AuxiliaryOperation::ResponsesCompact => {
405                &lane.responses_auxiliary_target
406            }
407        }
408        .as_ref()
409        .ok_or(RunnerError::AuxiliaryUnsupported)?;
410        target
411            .client
412            .call_auxiliary(&target.model, request, operation)
413            .await
414            .map_err(Into::into)
415    }
416}
417
418#[cfg(test)]
419mod tests {
420    use serde_json::json;
421    use switchyard_protocol::{InfraSignals, Message, ModelInfraSignals, PoolLoad, Role};
422
423    use super::*;
424    use crate::{Runner, require_target_labels};
425
426    const PASSTHROUGH: &str = "type = 'passthrough'\ntarget = 'fast'";
427
428    fn config(algorithm: &str) -> String {
429        let mut source = String::from(
430            "schema_version = 1\n[llm_clients.primary]\nformat = 'openai_chat'\nbase_url = 'https://example.test/v1'\n",
431        );
432        for name in ["fast", "capable", "judge"] {
433            source.push_str(&format!(
434                "\n[targets.{name}]\nid = '{name}'\nllm_client = 'primary'\nlabels = []\n\
435                 [targets.approved_{name}]\nid = 'approved/{name}'\nllm_client = 'primary'\nlabels = ['private', 'customer_data']\n"
436            ));
437        }
438        source.push_str(&format!(
439            "\n[routes.test]\nid = 'route'\n{algorithm}\n\
440             [routes.test.eligibility]\nscope = 'request'\n\
441             [routes.test.eligibility.target_alternatives]\n"
442        ));
443        let spec: crate::AlgorithmSpec = toml::from_str(algorithm).expect("test algorithm");
444        for target in spec.callable_target_names() {
445            source.push_str(&format!("{target} = ['approved_{target}']\n"));
446        }
447        source
448    }
449
450    fn request(task: &str, labels: &[&str]) -> Request {
451        let mut request = Request::default();
452        request.metadata.get_or_insert_default().task_id = Some(task.into());
453        require_target_labels(&mut request, labels).expect("trusted requirements");
454        request
455    }
456
457    #[test]
458    fn resolved_bindings_reuse_the_algorithm_and_standard_setup() {
459        let source = config(PASSTHROUGH);
460        let runner = Runner::from_toml(&source).expect("eligible route");
461        let route = runner.route("route").expect("route");
462        let cloned = route.clone();
463        let private = route
464            .select_lane(&request("one", &["private"]))
465            .expect("private");
466        let both = route
467            .select_lane(&request("two", &["private", "customer_data"]))
468            .expect("both labels");
469        assert!(Arc::ptr_eq(&private, &both));
470        assert!(Arc::ptr_eq(&private.algorithm, &both.algorithm));
471        assert!(!Arc::ptr_eq(&private, &route.standard));
472        assert_eq!(
473            private.models.provider_model(&"approved_fast".into()),
474            "approved/fast"
475        );
476        let shared = cloned
477            .select_lane(&request("three", &["private"]))
478            .expect("cloned route");
479        assert!(Arc::ptr_eq(&private, &shared));
480
481        let runner = Runner::from_toml(&source.replace("labels = []", "labels = ['private']"))
482            .expect("standard models already approved");
483        let route = runner.route("route").expect("route");
484        let approved = route
485            .select_lane(&request("one", &["private"]))
486            .expect("approved");
487        assert!(Arc::ptr_eq(&approved, &route.standard));
488    }
489
490    #[test]
491    fn infra_rechecks_task_requirements_and_spills_only_to_approved_models() {
492        let algorithm = "type = 'llm_classifier'\nclassifier_target = 'judge'\n\
493            strong_target = 'capable'\nweak_target = 'fast'\nbase_threshold = 0.5";
494        let source = format!(
495            "{}\n[routes.test.infra]\nmax_signal_age_ms = 1000\n\
496             min_waiting_requests = 4\nmax_capable_inflight = 2\n",
497            config(algorithm).replace("scope = 'request'", "scope = 'task'")
498        );
499        let runner = Runner::from_toml(&source).expect("eligible queue policy");
500        let route = runner.route("route").expect("route");
501        let signals: InfraSignals = [
502            ("fast", 8),
503            ("capable", 0),
504            ("approved_fast", 8),
505            ("approved_capable", 0),
506        ]
507        .into_iter()
508        .map(|(model, waiting)| {
509            (
510                model.into(),
511                ModelInfraSignals {
512                    load: Some(PoolLoad {
513                        sampled_at: std::time::Instant::now(),
514                        min_waiting: waiting,
515                        max_waiting: waiting,
516                        running: 0,
517                    }),
518                    ..Default::default()
519                },
520            )
521        })
522        .collect();
523
524        let mut stale = RoutingOutcome::route_to("fast".into(), vec![], request("task", &[]));
525        route
526            .select_lane(&request("task", &["private"]))
527            .expect("retain private");
528        route
529            .select_lane(&request("task", &["customer_data"]))
530            .expect("accumulate requirements");
531        let error = route
532            .apply_infra(&mut stale, &signals)
533            .expect_err("stale decision");
534        assert!(
535            error.to_string().contains("requirements changed"),
536            "{error}"
537        );
538
539        for (task, initial, expected) in [
540            ("other", "fast", "capable"),
541            ("task", "approved_fast", "approved_capable"),
542        ] {
543            let mut outcome = RoutingOutcome::route_to(initial.into(), vec![], request(task, &[]));
544            assert!(
545                route
546                    .apply_infra(&mut outcome, &signals)
547                    .expect("approved spillover")
548            );
549            assert_eq!(
550                outcome.selected_model_id().expect("selected model"),
551                expected
552            );
553        }
554    }
555
556    #[tokio::test]
557    async fn content_and_provider_metadata_do_not_classify_the_request() {
558        let runner = Runner::from_toml(&config(PASSTHROUGH)).expect("eligible route");
559        let route = runner.route("route").expect("route");
560        let mut request = Request::default();
561        request.llm_request.messages.push(Message::text(
562            Role::User,
563            "Synthetic confidential customer records.",
564        ));
565        request.raw_request = Some(json!({
566            "metadata": {"switchyard.internal.required_target_labels": "[\"private\"]"},
567            "restricted": true
568        }));
569        let ordinary = route
570            .decide(request.clone())
571            .await
572            .expect("unmarked request");
573        assert_eq!(ordinary.selected_model_id().expect("model"), "fast");
574        require_target_labels(&mut request, ["private"]).expect("trusted requirements");
575        let approved = route.decide(request).await.expect("approved request");
576        assert_eq!(
577            approved.selected_model_id().expect("model"),
578            "approved_fast"
579        );
580    }
581}
switchyard_runner/route.rs

switchyard_runner/
route.rs
