switchyard_runner/
runner.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Named route table and server-facing route metadata.
5
6use std::collections::BTreeMap;
7use std::path::Path;
8use std::sync::Arc;
9use std::time::Duration;
10
11use libsy::RoutingOutcome;
12use serde_json::Value;
13use switchyard_llm_client::TranslatingLlmClient;
14use switchyard_protocol::{ModelId, WireFormat};
15
16use crate::config;
17use crate::{ModelCapabilities, Route, RunnerConfig, RunnerError};
18
19/// Immutable named route table.
20pub struct Runner {
21    routes: Vec<(ModelId, Route)>,
22    fallback_base_url: Option<String>,
23    provider_api_keys: Vec<String>,
24    model_properties_probes: BTreeMap<ModelId, ModelPropertiesProbe>,
25}
26
27/// Borrowed model metadata returned while listing routes.
28pub struct ModelInfo<'a> {
29    pub id: &'a ModelId,
30    pub algorithm: &'a str,
31    pub capabilities: ModelCapabilities,
32}
33
34/// Fully resolved routing decision.
35pub struct DecisionDescription {
36    pub selected: DecisionTarget,
37    pub fallbacks: Vec<DecisionTarget>,
38    /// Metadata the algorithm attached to the routing outcome, if any.
39    pub metadata: Option<libsy::OutcomeMetadata>,
40}
41
42/// Non-secret configured target details returned by the decision endpoint.
43#[derive(Clone)]
44pub struct DecisionTarget {
45    pub target: String,
46    pub model: ModelId,
47    pub format: WireFormat,
48    pub base_url: String,
49    pub extra_body: BTreeMap<String, Value>,
50    /// Effort the target forces, so a caller that dispatches the call itself can apply it.
51    pub reasoning_effort: Option<String>,
52}
53
54pub(crate) struct ModelPropertiesProbe {
55    pub(crate) model: ModelId,
56    pub(crate) format: WireFormat,
57    pub(crate) client: Arc<TranslatingLlmClient>,
58}
59
60impl Runner {
61    /// Loads and validates a version-1 deployment TOML file.
62    pub fn load(path: impl AsRef<Path>) -> Result<Self, RunnerError> {
63        config::load_runner(path)
64    }
65
66    /// Loads and validates a version-1 deployment TOML document.
67    ///
68    /// Use [`Self::load`] when the deployment is stored in a file.
69    pub fn from_toml(source: &str) -> Result<Self, RunnerError> {
70        config::runner_from_toml(source)
71    }
72
73    /// Validates typed deployment configuration using the same construction as TOML.
74    pub fn from_config(config: RunnerConfig) -> Result<Self, RunnerError> {
75        Arc::new(config).build()
76    }
77
78    /// Builds a runner from named routes in caller-provided order.
79    /// Pre-condition: There must be at least one route.
80    pub fn new(routes: Vec<(ModelId, Route)>) -> Self {
81        Self {
82            routes,
83            fallback_base_url: None,
84            provider_api_keys: Vec::new(),
85            model_properties_probes: BTreeMap::new(),
86        }
87    }
88
89    /// Registers deployment-owned API keys for serving-surface output redaction.
90    /// TOML loading registers these automatically; programmatic hosts must supply them.
91    pub fn with_provider_api_keys(mut self, keys: Vec<String>) -> Self {
92        self.provider_api_keys = keys;
93        self
94    }
95
96    /// Returns deployment-owned secrets for serving-surface output redactors.
97    pub fn provider_api_keys(&self) -> &[String] {
98        &self.provider_api_keys
99    }
100
101    pub(crate) fn with_fallback_url(mut self, fallback_base_url: Option<String>) -> Self {
102        self.fallback_base_url = fallback_base_url;
103        self
104    }
105
106    pub(crate) fn with_model_properties_probes(
107        mut self,
108        probes: BTreeMap<ModelId, ModelPropertiesProbe>,
109    ) -> Self {
110        self.model_properties_probes = probes;
111        self
112    }
113
114    /// Returns the route registered for a model.
115    pub fn route(&self, model: &str) -> Option<&Route> {
116        self.routes
117            .iter()
118            .find(|(id, _)| id.as_str() == model)
119            .map(|(_, route)| route)
120    }
121
122    /// Iterates over configured routes in caller-provided order.
123    pub fn models(&self) -> impl Iterator<Item = ModelInfo<'_>> {
124        self.routes.iter().map(|(id, route)| ModelInfo {
125            id,
126            algorithm: route.algorithm_name(),
127            capabilities: route.capabilities(),
128        })
129    }
130
131    /// Reads one route's configured local backend properties without exposing its credentials.
132    pub async fn model_properties(&self, model: &str, timeout: Duration) -> Option<Value> {
133        let probe = self.model_properties_probes.get(model)?;
134        probe
135            .client
136            .get_model_properties(&probe.model, probe.format, timeout)
137            .await
138            .ok()
139    }
140
141    /// Returns the validated API root used for unmatched HTTP requests.
142    pub fn fallback_base_url(&self) -> Option<&str> {
143        self.fallback_base_url.as_deref()
144    }
145
146    /// Resolves an outcome to configured target names and non-secret client settings.
147    pub fn describe_decision(
148        &self,
149        model: &ModelId,
150        outcome: &RoutingOutcome,
151    ) -> Option<DecisionDescription> {
152        let route = self.route(model.as_str())?;
153        let resolve = |selected: &ModelId| {
154            let mut target = route.decision_target(selected)?;
155            let mut url = reqwest::Url::parse(&target.base_url).ok()?;
156            let query: Vec<_> = url
157                .query_pairs()
158                .filter(|(name, _)| !matches!(name.as_ref(), "key" | "api_key"))
159                .map(|(name, value)| (name.into_owned(), value.into_owned()))
160                .collect();
161            if query.len() != url.query_pairs().count() {
162                url.set_query(None);
163                if !query.is_empty() {
164                    url.query_pairs_mut().extend_pairs(query);
165                }
166                // Only the returned metadata changes; inference still needs its credentials.
167                target.base_url = url.into();
168            }
169            Some(target)
170        };
171        let mut model_ids = outcome.selected_model_ids.iter();
172        Some(DecisionDescription {
173            selected: resolve(model_ids.next()?)?,
174            fallbacks: model_ids.map(resolve).collect::<Option<Vec<_>>>()?,
175            metadata: outcome.metadata.clone(),
176        })
177    }
178}
switchyard_runner/runner.rs

switchyard_runner/
runner.rs
