switchyard_runner/
route.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! One configured algorithm and the clients that serve its targets.
5
6use std::error::Error;
7use std::sync::Arc;
8
9use libsy::{Algorithm, LibsyError, RoutingOutcome, RuntimeModels};
10use serde_json::Value;
11use switchyard_llm_client::{AuxiliaryOperation, ClientRouter, RunObserver, TranslatingLlmClient};
12use switchyard_protocol::{
13    LlmClientError, ModelId, Request, Response, RoutedLlmClient, WireFormat,
14};
15use thiserror::Error;
16
17use crate::DecisionTarget;
18
19/// Capabilities declared for one route.
20///
21/// `GET /v1/models` includes `context_window`, `tool_calling`, and `vision` in each
22/// standard `data` entry, using `null` for unset values. It omits `reasoning`.
23/// The server rejects tool inputs, reasoning controls, or images when the
24/// corresponding declaration is explicitly `false` for the selected route.
25#[derive(Clone, Copy, Default)]
26pub struct ModelCapabilities {
27    pub context_window: Option<u32>,
28    pub tool_calling: Option<bool>,
29    /// Whether the routed model accepts reasoning controls, as declared in config.
30    pub reasoning: Option<bool>,
31    /// Whether the routed model accepts image input, as declared in config.
32    pub vision: Option<bool>,
33}
34
35/// Caller credential family required by a forwarded-auth route.
36#[derive(Clone, Copy, Debug, Eq, PartialEq)]
37pub enum CallerAuthKind {
38    Anthropic,
39    OpenAi,
40}
41
42impl CallerAuthKind {
43    /// Stable provider name used by the server compatibility API.
44    pub const fn as_str(self) -> &'static str {
45        match self {
46            Self::Anthropic => "anthropic",
47            Self::OpenAi => "openai",
48        }
49    }
50
51    const fn accepts(self, wire_format: WireFormat) -> bool {
52        matches!(
53            (self, wire_format),
54            (Self::Anthropic, WireFormat::AnthropicMessages)
55                | (
56                    Self::OpenAi,
57                    WireFormat::OpenAiChat | WireFormat::OpenAiResponses
58                )
59        )
60    }
61}
62
63/// Exact upstream model and client used for an auxiliary provider operation.
64#[derive(Clone)]
65pub struct AuxiliaryTarget {
66    pub model: ModelId,
67    pub client: Arc<TranslatingLlmClient>,
68}
69
70/// Error returned while loading or executing configured routes.
71#[derive(Debug, Error)]
72pub enum RunnerError {
73    #[error("{message}")]
74    Configuration {
75        message: String,
76        #[source]
77        source: Option<Box<dyn Error + Send + Sync>>,
78    },
79    #[error("unknown route model {0:?}")]
80    UnknownRouteModel(String),
81    #[error("caller format is incompatible with {} credentials", .0.as_str())]
82    IncompatibleCallerFormat(CallerAuthKind),
83    #[error("route has no compatible target for the auxiliary operation")]
84    AuxiliaryUnsupported,
85    #[error(transparent)]
86    Algorithm(#[from] LibsyError),
87    #[error(transparent)]
88    Client(#[from] LlmClientError),
89}
90
91impl RunnerError {
92    pub(crate) fn configuration(message: impl Into<String>) -> Self {
93        Self::Configuration {
94            message: message.into(),
95            source: None,
96        }
97    }
98
99    pub(crate) fn configuration_source(
100        message: impl Into<String>,
101        source: impl Error + Send + Sync + 'static,
102    ) -> Self {
103        Self::Configuration {
104            message: message.into(),
105            source: Some(Box::new(source)),
106        }
107    }
108}
109
110/// A configured algorithm and the per-target clients its calls resolve through.
111pub struct Route {
112    algorithm: Arc<dyn Algorithm>,
113    // Resolves each offloaded call to the client configured for the target the algorithm
114    // selected. A route is a synthetic model with no upstream of its own, so this is a
115    // per-target lookup, never one client serving the whole route.
116    clients: ClientRouter,
117    caller_auth: Option<CallerAuthKind>,
118    capabilities: ModelCapabilities,
119    anthropic_auxiliary_target: Option<AuxiliaryTarget>,
120    responses_auxiliary_target: Option<AuxiliaryTarget>,
121    decision_targets: Vec<DecisionTarget>,
122    models: Arc<RuntimeModels>,
123    infra: Option<libsy::InfraPolicy>,
124}
125
126/// The selected model and untouched response produced by a route execution.
127pub struct RunOutput {
128    pub selected_model: ModelId,
129    pub response: Response,
130}
131
132impl Route {
133    /// Creates a fully configured execution route.
134    #[allow(clippy::too_many_arguments)]
135    pub fn new(
136        algorithm: Arc<dyn Algorithm>,
137        clients: ClientRouter,
138        caller_auth: Option<CallerAuthKind>,
139        capabilities: ModelCapabilities,
140        anthropic_auxiliary_target: Option<AuxiliaryTarget>,
141        responses_auxiliary_target: Option<AuxiliaryTarget>,
142        decision_targets: Vec<DecisionTarget>,
143        models: RuntimeModels,
144    ) -> Self {
145        Self {
146            algorithm,
147            clients,
148            caller_auth,
149            capabilities,
150            anthropic_auxiliary_target,
151            responses_auxiliary_target,
152            decision_targets,
153            models: Arc::new(models),
154            infra: None,
155        }
156    }
157
158    /// Returns the configured libsy algorithm name.
159    pub fn algorithm_name(&self) -> &str {
160        self.algorithm.name()
161    }
162
163    /// Returns model-list capability metadata.
164    pub fn capabilities(&self) -> ModelCapabilities {
165        self.capabilities
166    }
167
168    /// Returns the forwarded caller credential family.
169    pub fn caller_auth(&self) -> Option<CallerAuthKind> {
170        self.caller_auth
171    }
172
173    /// Resolves a selected model to this route's non-secret target metadata.
174    pub(crate) fn decision_target(&self, model: &ModelId) -> Option<DecisionTarget> {
175        self.decision_targets
176            .iter()
177            .find(|target| target.model == *model)
178            .cloned()
179    }
180
181    /// Returns the models grouped for one algorithm execution.
182    pub fn models(&self) -> &RuntimeModels {
183        &self.models
184    }
185
186    /// Rejects a caller format incompatible with forwarded credentials.
187    pub fn check_caller_format(&self, input_format: WireFormat) -> Result<(), RunnerError> {
188        if let Some(kind) = self.caller_auth
189            && !kind.accepts(input_format)
190        {
191            return Err(RunnerError::IncompatibleCallerFormat(kind));
192        }
193        Ok(())
194    }
195
196    /// Executes the configured route without consuming or proxying streamed responses.
197    pub async fn execute(
198        &self,
199        request: Request,
200        observer: Option<RunObserver>,
201    ) -> Result<RunOutput, RunnerError> {
202        let (selected_model, response) = switchyard_llm_client::run(
203            Arc::clone(&self.algorithm),
204            self.clients.clone(),
205            request,
206            Arc::clone(&self.models),
207            observer,
208        )
209        .await?;
210        Ok(RunOutput {
211            selected_model,
212            response,
213        })
214    }
215
216    /// Completes routing-time calls without serving a post-routing completion.
217    pub async fn decide(&self, request: Request) -> Result<RoutingOutcome, RunnerError> {
218        switchyard_llm_client::decide(
219            Arc::clone(&self.algorithm),
220            self.clients.clone(),
221            request,
222            Arc::clone(&self.models),
223        )
224        .await
225        .map_err(Into::into)
226    }
227
228    pub(crate) fn with_infra(mut self, infra: Option<libsy::InfraPolicy>) -> Self {
229        self.infra = infra;
230        self
231    }
232
233    /// Apply the optional queue policy to a completed task decision.
234    ///
235    /// The host must hold its in-flight accounting lock through this call and the
236    /// increment for the selected model. Ordinary `decide` leaves task routing unchanged.
237    pub fn apply_infra(
238        &self,
239        outcome: &mut RoutingOutcome,
240        signals: &switchyard_protocol::InfraSignals,
241    ) -> Result<bool, RunnerError> {
242        match &self.infra {
243            Some(policy) => policy
244                .apply(outcome, &self.models, signals)
245                .map_err(Into::into),
246            None => Ok(false),
247        }
248    }
249
250    /// Routes with a host-owned client for routing-time LLM calls, such as judges.
251    ///
252    /// The host owns transport, credentials, and target request settings. Configured
253    /// HTTP clients, target prompts, and decision clients are not used on this path.
254    /// The same algorithm instance retains its state across calls. No post-routing
255    /// completion is dispatched.
256    pub async fn decide_with_client(
257        &self,
258        request: Request,
259        client: Arc<dyn RoutedLlmClient>,
260    ) -> Result<RoutingOutcome, RunnerError> {
261        switchyard_llm_client::decide(
262            Arc::clone(&self.algorithm),
263            ClientRouter::single(client),
264            request,
265            Arc::clone(&self.models),
266        )
267        .await
268        .map_err(Into::into)
269    }
270
271    /// Executes a model-bearing provider operation through a compatible target.
272    pub async fn call_auxiliary(
273        &self,
274        request: Request,
275        operation: AuxiliaryOperation,
276    ) -> Result<Value, RunnerError> {
277        let target = match operation {
278            AuxiliaryOperation::AnthropicCountTokens => &self.anthropic_auxiliary_target,
279            AuxiliaryOperation::ResponsesInputTokens | AuxiliaryOperation::ResponsesCompact => {
280                &self.responses_auxiliary_target
281            }
282        }
283        .as_ref()
284        .ok_or(RunnerError::AuxiliaryUnsupported)?;
285        target
286            .client
287            .call_auxiliary(&target.model, request, operation)
288            .await
289            .map_err(Into::into)
290    }
291}
switchyard_runner/route.rs

switchyard_runner/
route.rs
