1use std::collections::{BTreeMap, BTreeSet, HashMap};
7use std::fs;
8use std::path::Path;
9use std::sync::Arc;
10use std::time::Duration;
11
12use libsy::{Algorithm, RuntimeModels};
13use parking_lot::Mutex;
14use serde::de::DeserializeOwned;
15use serde::{Deserialize, Deserializer};
16use serde_json::Value;
17use switchyard_llm_client::{
18 AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
19 OpenAiDecisionClient, ResponsesReasoningPolicy, SystemOneClient, TranslatingLlmClient,
20};
21use switchyard_protocol::{
22 LlmClientError, ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat,
23};
24
25use crate::algorithm_factory::AlgorithmAndModels;
26use crate::eligibility::{EligibilityPolicy, validate_labels};
27use crate::route::ExecutionLane;
28use crate::runner::ModelPropertiesProbe;
29use crate::{
30 AlgorithmFactory, AlgorithmSpec, AuxiliaryTarget, CallerAuthKind, DecisionTarget,
31 ModelCapabilities, Route, Runner, RunnerError,
32};
33
34const SUPPORTED_SCHEMA_VERSION: u32 = 1;
35const MAX_CONFIGURED_RETRIES: u32 = 10;
36
37pub fn api_key_variables(source: &str) -> Result<BTreeSet<String>, RunnerError> {
40 let config: DeploymentConfig = toml::from_str(source).map_err(|_| {
41 RunnerError::configuration(
42 "Could not parse the server config to read credential references.",
43 )
44 })?;
45 if config.schema_version != SUPPORTED_SCHEMA_VERSION {
46 return Err(RunnerError::configuration(
47 "Unsupported server config schema version.",
48 ));
49 }
50 let mut variables: BTreeSet<_> = config
51 .llm_clients
52 .values()
53 .filter_map(|client| client.api_key_env.clone())
54 .collect();
55 for client in config.decision_clients.values() {
56 variables.insert(client.api_key_env.clone());
57 }
58 if variables
59 .iter()
60 .any(|name| name.trim().is_empty() || name.contains(['\0', '=']))
61 {
62 return Err(RunnerError::configuration(
63 "Invalid credential environment variable name.",
64 ));
65 }
66 Ok(variables)
67}
68
69#[derive(Clone, Debug, PartialEq, Eq)]
71pub struct RouteTargets {
72 pub name: String,
74 pub id: ModelId,
76 pub kind: &'static str,
78 pub answer_targets: Vec<ConfiguredTarget>,
81 pub judge_targets: Vec<ConfiguredTarget>,
83}
84
85#[derive(Clone, Debug, PartialEq, Eq)]
87pub struct ConfiguredTarget {
88 pub name: String,
89 pub model: Option<TargetModel>,
93}
94
95#[derive(Clone, Debug, PartialEq, Eq)]
97pub struct TargetModel {
98 pub client: String,
99 pub id: ModelId,
100}
101
102pub fn route_targets(source: &str) -> Result<Vec<RouteTargets>, RunnerError> {
105 let config: DeploymentConfig = toml::from_str(source).map_err(|error| {
106 RunnerError::configuration_source(format!("failed to parse TOML: {error}"), error)
107 })?;
108 config.check_schema_version()?;
109 Ok(config
110 .routes
111 .iter()
112 .map(|(name, route)| RouteTargets {
113 name: name.clone(),
114 id: route.id.clone(),
115 kind: (&route.algorithm).into(),
116 answer_targets: config.configured_targets(route.routing_target_names()),
117 judge_targets: config.configured_targets(route.algorithm.judge_target_names()),
118 })
119 .collect())
120}
121
122type RunnerResult<T> = Result<T, RunnerError>;
123
124pub(crate) fn load_runner(path: impl AsRef<Path>) -> RunnerResult<Runner> {
125 let path = path.as_ref();
126 let source = fs::read_to_string(path).map_err(|error| {
127 RunnerError::configuration_source(
128 format!("failed to read server config {}: {error}", path.display()),
129 error,
130 )
131 })?;
132 runner_from_toml(&source).map_err(|error| {
133 RunnerError::configuration_source(
134 format!("invalid server config {}: {error}", path.display()),
135 error,
136 )
137 })
138}
139
140pub(crate) fn runner_from_toml(source: &str) -> RunnerResult<Runner> {
141 AlgorithmFactory::from_toml(source)?.config.build()
142}
143
144#[derive(Debug, Deserialize)]
145#[serde(deny_unknown_fields)]
146pub(crate) struct DeploymentConfig {
147 schema_version: u32,
148 fallback_client: Option<String>,
149 #[serde(default)]
150 llm_clients: BTreeMap<String, LlmClientConfig>,
151 targets: BTreeMap<String, TargetConfig>,
152 #[serde(default)]
153 decision_clients: BTreeMap<String, DecisionClientConfig>,
154 #[serde(default)]
155 decision_targets: BTreeMap<String, DecisionTargetConfig>,
156 routes: BTreeMap<String, RouteConfig>,
157}
158
159#[derive(Debug)]
160pub(crate) struct RouteConfig {
161 pub(crate) id: ModelId,
162 context_window: Option<u32>,
163 tool_calling: Option<bool>,
164 reasoning: Option<bool>,
165 vision: Option<bool>,
166 algorithm: AlgorithmSpec,
167 infra: Option<libsy::InfraPolicy>,
168 eligibility: Option<TargetEligibilityConfig>,
169}
170
171pub(crate) struct RouteSetup<'a> {
173 pub(crate) route_name: &'a str,
174 pub(crate) config: &'a RouteConfig,
175 pub(crate) algorithm: Arc<dyn Algorithm>,
176 pub(crate) models: RuntimeModels,
177}
178
179#[derive(Debug, Deserialize)]
180#[serde(deny_unknown_fields)]
181struct TargetEligibilityConfig {
182 #[serde(default)]
183 require_explicit_requirements: bool,
184 #[serde(default)]
185 target_alternatives: BTreeMap<String, Vec<String>>,
186 #[serde(default)]
187 scope: RequirementScope,
188}
189
190#[derive(Debug, Default, Deserialize)]
191#[serde(rename_all = "snake_case")]
192enum RequirementScope {
193 #[default]
194 Request,
195 Task,
196}
197
198struct TargetPromptPolicy {
199 prompts: HashMap<ModelId, String>,
200 routing_answer_target: Option<ModelId>,
201}
202
203struct ResolvedLlmTarget<'a> {
205 name: &'a str,
206 config: &'a TargetConfig,
207}
208
209type LaneTargets<'a> = BTreeMap<&'a str, ResolvedLlmTarget<'a>>;
210type TargetOverrides<'a> = BTreeMap<&'a str, &'a str>;
211type LaneDecisions<'a> = BTreeMap<&'a str, ResolvedDecisionTarget<'a>>;
212
213fn bound_target<'a>(name: &'a str, overrides: &TargetOverrides<'a>) -> &'a str {
215 overrides.get(name).copied().unwrap_or(name)
216}
217
218fn callable_targets(algorithm: &AlgorithmSpec) -> BTreeSet<&str> {
219 algorithm.callable_target_names().into_iter().collect()
220}
221
222fn check_random_targets(
223 algorithm: &AlgorithmSpec,
224 targets: &LaneTargets<'_>,
225 has_overrides: bool,
226) -> RunnerResult<()> {
227 if matches!(algorithm, AlgorithmSpec::Random { .. }) && has_overrides {
228 let mut models = BTreeSet::new();
229 for name in algorithm.routing_target_names() {
230 let target = targets[name].name;
231 if !models.insert(target) {
232 return Err(LlmClientError::InvalidRequest {
233 message: format!(
234 "random route alternatives map multiple pool entries to target {target}; use distinct targets"
235 ),
236 }
237 .into());
238 }
239 }
240 }
241 Ok(())
242}
243
244fn build_resolved_algorithm<'a>(
246 route_name: &str,
247 algorithm: &AlgorithmSpec,
248 targets: &LaneTargets<'_>,
249 decisions: impl Iterator<Item = (&'a str, &'a ModelId)>,
250) -> RunnerResult<AlgorithmAndModels> {
251 let model_ids = targets
252 .iter()
253 .map(|(name, target)| ((*name).to_owned(), ModelId::from(target.name)))
254 .chain(decisions.map(|(name, model)| (name.to_owned(), model.clone())))
255 .collect();
256 let (algorithm, models) = algorithm
257 .build_with_runtime_models(route_name, &model_ids)
258 .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
259 let provider_models = targets
260 .values()
261 .map(|target| (ModelId::from(target.name), target.config.id.clone()))
262 .collect();
263 Ok((algorithm, models.with_provider_models(provider_models)))
264}
265
266pub(crate) type TargetBindings = Vec<(String, String)>;
267pub(crate) const MAX_ELIGIBLE_LANES: usize = 128;
268
269pub(crate) struct EligibleLanes {
271 deployment: Arc<DeploymentConfig>,
272 route_name: String,
273 clients: Arc<BTreeMap<String, Arc<TranslatingLlmClient>>>,
274 decision_clients: Arc<BTreeMap<String, Arc<dyn RoutedDecisionClient>>>,
275 lanes: Mutex<BTreeMap<TargetBindings, Arc<ExecutionLane>>>,
276}
277
278impl EligibleLanes {
279 pub(crate) fn select(
280 &self,
281 requirements: &BTreeSet<String>,
282 standard: &Arc<ExecutionLane>,
283 ) -> RunnerResult<Arc<ExecutionLane>> {
284 let Some(key) = self
285 .deployment
286 .select_eligible_targets(&self.route_name, requirements)?
287 else {
288 return Ok(Arc::clone(standard));
289 };
290 let mut lanes = self.lanes.lock();
291 if let Some(lane) = lanes.get(&key) {
292 return Ok(Arc::clone(lane));
293 }
294 if lanes.len() >= MAX_ELIGIBLE_LANES {
295 return Err(LlmClientError::TemporarilyUnavailable.into());
296 }
297 let algorithm = self
298 .deployment
299 .bound_algorithm_spec(&self.route_name, &key)?;
300 let overrides = key
301 .iter()
302 .map(|(name, target)| (name.as_str(), target.as_str()))
303 .collect();
304 let targets = self
305 .deployment
306 .resolve_lane_targets(&algorithm, &overrides)?;
307 check_random_targets(
308 &algorithm,
309 &targets,
310 key.iter().any(|(name, target)| name != target),
311 )?;
312 let decisions = self.deployment.resolve_decision_targets(
313 &self.route_name,
314 &algorithm,
315 &overrides,
316 &self.decision_clients,
317 )?;
318 let (lane, _) = self.deployment.build_lane(
320 &self.route_name,
321 &algorithm,
322 &targets,
323 &decisions,
324 &self.clients,
325 None,
326 )?;
327 let lane = Arc::new(lane);
328 lanes.insert(key, Arc::clone(&lane));
329 Ok(lane)
330 }
331}
332
333struct DecisionTargetMapping<'a> {
334 target_name: &'a str,
335 model_id: &'a ModelId,
336}
337
338#[derive(Clone, Copy)]
339struct ResolvedDecisionTarget<'a> {
340 model: &'a ModelId,
341 client: &'a Arc<dyn RoutedDecisionClient>,
342}
343
344impl<'de> Deserialize<'de> for RouteConfig {
345 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
346 where
347 D: Deserializer<'de>,
348 {
349 let mut table = toml::Table::deserialize(deserializer)?;
350 let id = take_required(&mut table, "id")?;
351 let context_window = take_optional(&mut table, "context_window")?;
352 let tool_calling = take_optional(&mut table, "tool_calling")?;
353 let reasoning = take_optional(&mut table, "reasoning")?;
354 let vision = take_optional(&mut table, "vision")?;
355 let infra = take_optional(&mut table, "infra")?;
356 let eligibility = take_optional(&mut table, "eligibility")?;
357 let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table))
358 .map_err(serde::de::Error::custom)?;
359 Ok(Self {
360 id,
361 context_window,
362 tool_calling,
363 reasoning,
364 vision,
365 algorithm,
366 infra,
367 eligibility,
368 })
369 }
370}
371
372fn take_required<T, E>(table: &mut toml::Table, name: &'static str) -> Result<T, E>
373where
374 T: DeserializeOwned,
375 E: serde::de::Error,
376{
377 let value = table.remove(name).ok_or_else(|| E::missing_field(name))?;
378 T::deserialize(value).map_err(E::custom)
379}
380
381fn take_optional<T, E>(table: &mut toml::Table, name: &'static str) -> Result<Option<T>, E>
382where
383 T: DeserializeOwned,
384 E: serde::de::Error,
385{
386 table
387 .remove(name)
388 .map(|value| T::deserialize(value).map_err(E::custom))
389 .transpose()
390}
391
392impl RouteConfig {
393 fn capabilities(&self) -> ModelCapabilities {
394 ModelCapabilities {
395 context_window: self.context_window,
396 tool_calling: self.tool_calling,
397 reasoning: self.reasoning,
398 vision: self.vision,
399 }
400 }
401
402 fn routing_target_names(&self) -> Vec<&str> {
403 self.algorithm.routing_target_names()
404 }
405
406 fn callable_target_names(&self) -> Vec<&str> {
407 self.algorithm.callable_target_names()
408 }
409}
410
411impl DeploymentConfig {
412 pub(crate) fn infer_decision_judges(&mut self) {
415 for route in self.routes.values_mut() {
416 route
417 .algorithm
418 .infer_decision_judges(|name| self.decision_targets.contains_key(name));
419 }
420 }
421
422 pub(crate) fn requires_target_admission(&self) -> bool {
423 self.routes
424 .values()
425 .any(|route| route.eligibility.is_some())
426 }
427
428 pub(crate) fn eligibility_policy(&self, route_name: &str) -> Option<EligibilityPolicy> {
429 let config = self.routes.get(route_name)?.eligibility.as_ref()?;
430 Some(EligibilityPolicy::new(
431 config.require_explicit_requirements,
432 matches!(config.scope, RequirementScope::Task),
433 ))
434 }
435
436 pub(crate) fn route_target_models(
437 &self,
438 route_name: &str,
439 ) -> RunnerResult<BTreeMap<ModelId, TargetModel>> {
440 let route = self
441 .routes
442 .get(route_name)
443 .ok_or_else(|| RunnerError::configuration(format!("unknown route {route_name}")))?;
444 route
445 .callable_target_names()
446 .into_iter()
447 .chain(
448 route
449 .eligibility
450 .iter()
451 .flat_map(|config| config.target_alternatives.values().flatten())
452 .map(String::as_str),
453 )
454 .map(|name| {
455 if let Some(target) = self.decision_targets.get(name) {
456 Ok((
457 target.id.clone(),
458 TargetModel {
459 client: target.decision_client.clone(),
460 id: target.id.clone(),
461 },
462 ))
463 } else {
464 let target = self.targets.get(name).ok_or_else(|| {
465 RunnerError::configuration(format!(
466 "route {route_name} references unknown target {name}"
467 ))
468 })?;
469 Ok((
470 ModelId::from(name),
471 TargetModel {
472 client: target.llm_client.clone(),
473 id: target.id.clone(),
474 },
475 ))
476 }
477 })
478 .collect()
479 }
480
481 pub(crate) fn select_eligible_targets(
484 &self,
485 route_name: &str,
486 requirements: &BTreeSet<String>,
487 ) -> RunnerResult<Option<TargetBindings>> {
488 let route = self
489 .routes
490 .get(route_name)
491 .ok_or_else(|| RunnerError::configuration(format!("unknown route {route_name}")))?;
492 let config = route
493 .eligibility
494 .as_ref()
495 .ok_or_else(|| RunnerError::configuration("route has no eligibility configuration"))?;
496 let callable = callable_targets(&route.algorithm);
497 let bindings: Vec<_> = callable
498 .iter()
499 .filter_map(|name| {
500 std::iter::once(*name)
501 .chain(
502 config
503 .target_alternatives
504 .get(*name)
505 .into_iter()
506 .flatten()
507 .map(String::as_str),
508 )
509 .find(|target| {
510 self.target_labels(target)
511 .is_some_and(|labels| requirements.is_subset(labels))
512 })
513 .map(|target| (*name, target))
514 })
515 .collect();
516 if bindings.len() == callable.len() && bindings.iter().all(|(name, target)| name == target)
517 {
518 return Ok(None);
519 }
520 Ok(Some(
521 bindings
522 .into_iter()
523 .map(|(name, target)| (name.to_owned(), target.to_owned()))
524 .collect(),
525 ))
526 }
527
528 fn bound_algorithm_spec(
529 &self,
530 route_name: &str,
531 bindings: &TargetBindings,
532 ) -> RunnerResult<AlgorithmSpec> {
533 let route = self
534 .routes
535 .get(route_name)
536 .ok_or_else(|| RunnerError::configuration(format!("unknown route {route_name}")))?;
537 let allowed = bindings.iter().map(|(name, _)| name.as_str()).collect();
538 route.algorithm.restricted_to(&allowed).map_err(|error| {
539 LlmClientError::InvalidRequest {
540 message: error.to_string(),
541 }
542 .into()
543 })
544 }
545
546 pub(crate) fn build_bound_algorithm(
547 &self,
548 route_name: &str,
549 bindings: &TargetBindings,
550 ) -> RunnerResult<AlgorithmAndModels> {
551 let algorithm = self.bound_algorithm_spec(route_name, bindings)?;
552 let overrides = bindings
553 .iter()
554 .map(|(name, target)| (name.as_str(), target.as_str()))
555 .collect();
556 let targets = self.resolve_lane_targets(&algorithm, &overrides)?;
557 check_random_targets(
558 &algorithm,
559 &targets,
560 bindings.iter().any(|(name, target)| name != target),
561 )?;
562 let decisions = self
563 .resolve_decision_models(route_name, &algorithm, &overrides)?
564 .into_iter()
565 .map(|mapping| (mapping.target_name, mapping.model_id));
566 let (algorithm, models) =
567 build_resolved_algorithm(route_name, &algorithm, &targets, decisions)?;
568 Ok((algorithm, models.with_target_restriction()))
569 }
570
571 fn resolve_decision_models<'a>(
572 &'a self,
573 route_name: &str,
574 algorithm: &'a AlgorithmSpec,
575 overrides: &TargetOverrides<'_>,
576 ) -> RunnerResult<Vec<DecisionTargetMapping<'a>>> {
577 algorithm
578 .decision_target_names()
579 .into_iter()
580 .map(|name| {
581 let bound = bound_target(name, overrides);
582 let target = self.decision_targets.get(bound).ok_or_else(|| {
583 RunnerError::configuration(format!(
584 "route {route_name} requires unknown decision target {bound}"
585 ))
586 })?;
587 Ok(DecisionTargetMapping {
588 target_name: name,
589 model_id: &target.id,
590 })
591 })
592 .collect()
593 }
594
595 pub(crate) fn validate_decision_models(&self, route_name: &str) -> RunnerResult<()> {
597 let route = self
598 .routes
599 .get(route_name)
600 .ok_or_else(|| RunnerError::configuration(format!("unknown route {route_name}")))?;
601 let mut clients = HashMap::new();
602 for name in route.algorithm.decision_target_names() {
603 let alternatives = route
604 .eligibility
605 .as_ref()
606 .and_then(|config| config.target_alternatives.get(name));
607 for candidate in
608 std::iter::once(name).chain(alternatives.into_iter().flatten().map(String::as_str))
609 {
610 let target = self.decision_targets.get(candidate).ok_or_else(|| {
611 RunnerError::configuration(format!(
612 "route {route_name} requires unknown decision target {candidate}"
613 ))
614 })?;
615 if clients
616 .insert(&target.id, &target.decision_client)
617 .is_some_and(|client| client != &target.decision_client)
618 {
619 return Err(RunnerError::configuration(format!(
620 "route {route_name} maps decision model {} to different clients",
621 target.id
622 )));
623 }
624 }
625 }
626 Ok(())
627 }
628
629 fn target_labels(&self, name: &str) -> Option<&BTreeSet<String>> {
630 self.targets
631 .get(name)
632 .map(|target| &target.labels)
633 .or_else(|| self.decision_targets.get(name).map(|target| &target.labels))
634 }
635
636 fn check_schema_version(&self) -> RunnerResult<()> {
637 if self.schema_version == SUPPORTED_SCHEMA_VERSION {
638 return Ok(());
639 }
640 Err(RunnerError::configuration(format!(
641 "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}",
642 self.schema_version
643 )))
644 }
645
646 fn configured_targets(&self, names: Vec<&str>) -> Vec<ConfiguredTarget> {
649 names
650 .into_iter()
651 .map(|name| ConfiguredTarget {
652 name: name.to_string(),
653 model: self
654 .targets
655 .get(name)
656 .map(|target| (&target.llm_client, &target.id))
657 .or_else(|| {
658 self.decision_targets
659 .get(name)
660 .map(|target| (&target.decision_client, &target.id))
661 })
662 .map(|(client, id)| TargetModel {
663 client: client.clone(),
664 id: id.clone(),
665 }),
666 })
667 .collect()
668 }
669
670 fn decision_target(&self, name: &str) -> Option<DecisionTarget> {
671 let target = self.targets.get(name)?;
672 let client = self.llm_clients.get(&target.llm_client)?;
673 Some(DecisionTarget {
674 target: name.to_string(),
675 model: target.id.clone(),
676 format: client.format.wire_format(),
677 base_url: client.base_url.as_str().to_string(),
678 extra_body: target.extra_body.clone(),
679 reasoning_effort: target.reasoning_effort.clone(),
680 })
681 }
682
683 pub(crate) fn validate(&self) -> RunnerResult<()> {
685 self.check_schema_version()?;
686
687 let mut route_names_by_id = HashMap::new();
688 for (route_name, config) in &self.routes {
689 validate_value("route name", route_name)?;
690 validate_value(&format!("route {route_name} id"), &config.id)?;
691 if let Some(first_route_name) =
692 route_names_by_id.insert(config.id.as_str(), route_name.as_str())
693 {
694 return Err(RunnerError::configuration(format!(
695 "routes {first_route_name} and {route_name} both use id {}; route ids must be unique",
696 config.id
697 )));
698 }
699 }
700
701 for (name, target) in &self.targets {
702 validate_labels(&target.labels)
703 .map_err(|error| RunnerError::configuration(error.to_string()))?;
704 validate_value("target name", name)?;
705 validate_value(&format!("target {name} id"), &target.id)?;
706 if !self.llm_clients.contains_key(&target.llm_client) {
707 return Err(RunnerError::configuration(format!(
708 "target {name} references unknown llm client {}",
709 target.llm_client
710 )));
711 }
712 }
713 let mut decision_approvals = HashMap::new();
714 for (name, target) in &self.decision_targets {
715 validate_labels(&target.labels)
716 .map_err(|error| RunnerError::configuration(error.to_string()))?;
717 if decision_approvals
718 .insert((&target.decision_client, &target.id), &target.labels)
719 .is_some_and(|approved| approved != &target.labels)
720 {
721 return Err(RunnerError::configuration(format!(
722 "decision target {name} shares a model and client with an alias but disagrees on approval labels"
723 )));
724 }
725 validate_value("decision target name", name)?;
726 validate_value(&format!("decision target {name} id"), &target.id)?;
727 if self.targets.contains_key(name) {
728 return Err(RunnerError::configuration(format!(
729 "target {name} is defined as both an LLM and decision target"
730 )));
731 }
732 if self.targets.contains_key(target.id.as_str()) {
734 return Err(RunnerError::configuration(format!(
735 "decision target {name} id {} is also an LLM target name; rename one of them",
736 target.id
737 )));
738 }
739 if !self.decision_clients.contains_key(&target.decision_client) {
740 return Err(RunnerError::configuration(format!(
741 "decision target {name} references unknown decision client {}",
742 target.decision_client
743 )));
744 }
745 }
746 for (route_name, config) in &self.routes {
747 if let Some(eligibility) = &config.eligibility {
748 self.validate_eligibility(route_name, &config.algorithm, eligibility)?;
749 }
750 }
751 Ok(())
752 }
753
754 pub(crate) fn build_algorithms(&self) -> impl Iterator<Item = RunnerResult<RouteSetup<'_>>> {
756 let targets = self.build_targets();
757 self.routes.iter().map(move |(route_name, config)| {
758 let decision_targets = config.algorithm.decision_target_names();
759 for name in config.callable_target_names() {
760 let (exists, kind) = if decision_targets.contains(&name) {
761 (self.decision_targets.contains_key(name), "decision")
762 } else {
763 (self.targets.contains_key(name), "LLM")
764 };
765 if !exists {
766 return Err(RunnerError::configuration(format!(
767 "route references unknown target {name}; route {route_name} requires target kind {kind}"
768 )));
769 }
770 }
771 for name in config.routing_target_names() {
772 if !self.targets.contains_key(name) {
773 return Err(RunnerError::configuration(format!(
774 "route {route_name} completion target {name} must be an LLM target"
775 )));
776 }
777 }
778 if config.context_window == Some(0) {
779 return Err(RunnerError::configuration(format!(
780 "route {route_name} context_window must be greater than zero"
781 )));
782 }
783 let (algorithm, models) = config
784 .algorithm
785 .build_with_runtime_models(route_name, &targets)
786 .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
787 let models = models.with_provider_models(
788 config
789 .callable_target_names()
790 .into_iter()
791 .filter_map(|name| {
792 self.targets
793 .get(name)
794 .map(|target| (ModelId::from(name), target.id.clone()))
795 })
796 .collect(),
797 );
798 if let Some(infra) = &config.infra {
799 if !matches!(&config.algorithm, AlgorithmSpec::LlmClassifier { config }
800 if matches!(config.mode, None | Some(crate::ClassifierMode::Capability))
801 && config.escalation.is_none())
802 {
803 return Err(RunnerError::configuration(format!(
804 "route {route_name}: infra requires capability llm_classifier without escalation"
805 )));
806 }
807 infra.validate().map_err(|error| {
808 RunnerError::configuration_source(format!("route {route_name}: {error}"), error)
809 })?;
810 }
811 Ok(RouteSetup {
812 route_name: route_name.as_str(),
813 config,
814 algorithm,
815 models,
816 })
817 })
818 }
819
820 pub(crate) fn validate_target_approvals(&self) -> RunnerResult<()> {
821 let mut approvals = HashMap::new();
822 for (target_name, target) in &self.targets {
823 if let Some((first_name, labels)) = approvals.insert(
824 (&target.llm_client, &target.id),
825 (target_name, &target.labels),
826 ) && labels != &target.labels
827 {
828 return Err(RunnerError::configuration(format!(
829 "targets {first_name} and {target_name} share a model and llm client but disagree on approval labels"
830 )));
831 }
832 }
833 Ok(())
834 }
835
836 fn build(self: Arc<Self>) -> RunnerResult<Runner> {
837 self.validate()?;
838 self.validate_target_approvals()?;
839
840 let mut provider_api_keys = Vec::new();
841 let clients = Arc::new(self.build_clients(&mut provider_api_keys)?);
842 let decision_clients = Arc::new(self.build_decision_clients(&mut provider_api_keys)?);
843 let fallback_base_url = self.fallback_base_url()?;
844 let mut routes = Vec::with_capacity(self.routes.len());
845 let mut model_properties_probes = BTreeMap::new();
846 for entry in self.build_algorithms() {
847 let RouteSetup {
848 route_name,
849 config,
850 algorithm,
851 models,
852 } = entry?;
853 let capabilities = config.capabilities();
854 if let AlgorithmSpec::Vgr { config: vgr } = &config.algorithm
855 && let Some(probe) =
856 self.build_model_properties_probe(&vgr.efficient_target, &clients)
857 {
858 model_properties_probes.insert(config.id.clone(), probe);
859 }
860 let no_overrides = TargetOverrides::new();
861 let standard_targets = self.resolve_lane_targets(&config.algorithm, &no_overrides)?;
862 let decisions = self.resolve_decision_targets(
863 route_name,
864 &config.algorithm,
865 &no_overrides,
866 &decision_clients,
867 )?;
868 let (standard, caller_auth) = self.build_lane(
869 route_name,
870 &config.algorithm,
871 &standard_targets,
872 &decisions,
873 &clients,
874 Some((algorithm, models)),
875 )?;
876 let mut target_names = callable_targets(&config.algorithm);
878 if let Some(eligibility) = &config.eligibility {
879 target_names.extend(
880 eligibility
881 .target_alternatives
882 .values()
883 .flatten()
884 .map(String::as_str),
885 );
886 }
887 let route_targets = target_names
888 .into_iter()
889 .filter_map(|name| self.decision_target(name))
890 .collect();
891 let mut route = Route::from_lane(standard, caller_auth, capabilities, route_targets)
892 .with_infra(config.infra.clone());
893 if let Some(eligibility) = &config.eligibility {
894 let policy = EligibilityPolicy::new(
895 eligibility.require_explicit_requirements,
896 matches!(eligibility.scope, RequirementScope::Task),
897 );
898 route = route.with_eligibility(
899 policy,
900 EligibleLanes {
901 deployment: Arc::clone(&self),
902 route_name: route_name.to_owned(),
903 clients: Arc::clone(&clients),
904 decision_clients: Arc::clone(&decision_clients),
905 lanes: Mutex::new(BTreeMap::new()),
906 },
907 );
908 }
909 routes.push((config.id.clone(), route));
910 }
911 let runner = Runner::new(routes)
912 .with_fallback_url(fallback_base_url)
913 .with_provider_api_keys(provider_api_keys)
914 .with_model_properties_probes(model_properties_probes);
915 Ok(runner)
916 }
917
918 fn validate_eligibility(
920 &self,
921 route_name: &str,
922 algorithm: &AlgorithmSpec,
923 config: &TargetEligibilityConfig,
924 ) -> RunnerResult<()> {
925 if !algorithm.supports_target_restrictions() {
926 return Err(RunnerError::configuration(format!(
927 "route {route_name} cannot use eligibility with prefill_router"
928 )));
929 }
930 let callable = callable_targets(algorithm);
931 for (source, alternatives) in &config.target_alternatives {
932 if !callable.contains(source.as_str()) {
933 return Err(RunnerError::configuration(format!(
934 "route {route_name} target_alternatives source {source} is not used by the algorithm"
935 )));
936 }
937 if alternatives.is_empty() {
938 return Err(RunnerError::configuration(
939 "target alternatives must not be empty",
940 ));
941 }
942 for target in alternatives {
943 let same_kind = (self.targets.contains_key(source)
944 && self.targets.contains_key(target))
945 || (self.decision_targets.contains_key(source)
946 && self.decision_targets.contains_key(target));
947 if !same_kind {
948 return Err(RunnerError::configuration(format!(
949 "route {route_name} alternative {target} must exist and match the target kind of {source}"
950 )));
951 }
952 }
953 }
954 for name in callable.into_iter().chain(
955 config
956 .target_alternatives
957 .values()
958 .flatten()
959 .map(String::as_str),
960 ) {
961 if let Some(target) = self.targets.get(name)
962 && self
963 .llm_clients
964 .get(&target.llm_client)
965 .is_some_and(|client| client.forward_auth)
966 {
967 return Err(RunnerError::configuration(format!(
968 "route {route_name} cannot use eligibility with forwarded caller authentication"
969 )));
970 }
971 }
972 Ok(())
973 }
974
975 fn resolve_lane_targets<'a>(
976 &'a self,
977 algorithm: &'a AlgorithmSpec,
978 overrides: &TargetOverrides<'_>,
979 ) -> RunnerResult<LaneTargets<'a>> {
980 let decisions = algorithm.decision_target_names();
981 callable_targets(algorithm)
982 .into_iter()
983 .filter(|name| !decisions.contains(name))
984 .map(|name| {
985 let bound = bound_target(name, overrides);
986 let (resolved_name, config) =
987 self.targets.get_key_value(bound).ok_or_else(|| {
988 RunnerError::configuration(format!(
989 "route references unknown target {bound}"
990 ))
991 })?;
992 Ok((
993 name,
994 ResolvedLlmTarget {
995 name: resolved_name.as_str(),
996 config,
997 },
998 ))
999 })
1000 .collect()
1001 }
1002
1003 fn resolve_decision_targets<'a>(
1004 &'a self,
1005 route_name: &str,
1006 algorithm: &'a AlgorithmSpec,
1007 overrides: &TargetOverrides<'_>,
1008 clients: &'a BTreeMap<String, Arc<dyn RoutedDecisionClient>>,
1009 ) -> RunnerResult<LaneDecisions<'a>> {
1010 algorithm
1011 .decision_target_names()
1012 .into_iter()
1013 .map(|name| {
1014 let bound = bound_target(name, overrides);
1015 let target = self.decision_targets.get(bound).ok_or_else(|| {
1016 RunnerError::configuration(format!(
1017 "route {route_name} requires unknown decision target {bound}"
1018 ))
1019 })?;
1020 let client = clients.get(&target.decision_client).ok_or_else(|| {
1021 RunnerError::configuration(format!(
1022 "decision target {bound} references unknown decision client {}",
1023 target.decision_client
1024 ))
1025 })?;
1026 Ok((
1027 name,
1028 ResolvedDecisionTarget {
1029 model: &target.id,
1030 client,
1031 },
1032 ))
1033 })
1034 .collect()
1035 }
1036
1037 fn build_lane(
1038 &self,
1039 route_name: &str,
1040 route: &AlgorithmSpec,
1041 targets: &LaneTargets<'_>,
1042 decisions: &LaneDecisions<'_>,
1043 clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
1044 built: Option<AlgorithmAndModels>,
1045 ) -> RunnerResult<(ExecutionLane, Option<CallerAuthKind>)> {
1046 let (algorithm, mut models) = if let Some(built) = built {
1047 built
1048 } else {
1049 build_resolved_algorithm(
1050 route_name,
1051 route,
1052 targets,
1053 decisions.iter().map(|(name, target)| (*name, target.model)),
1054 )?
1055 };
1056 let (route_clients, caller_auth) =
1057 self.build_route_clients(route_name, route, targets, decisions, clients)?;
1058 let anthropic_auxiliary_target =
1059 self.build_anthropic_auxiliary_target(route, targets, clients);
1060 let responses_auxiliary_target =
1061 self.build_responses_auxiliary_target(route, targets, clients);
1062 if self
1063 .routes
1064 .get(route_name)
1065 .is_some_and(|config| config.eligibility.is_some())
1066 {
1067 models = models.with_target_restriction();
1068 }
1069 Ok((
1070 ExecutionLane::new(
1071 algorithm,
1072 route_clients,
1073 anthropic_auxiliary_target,
1074 responses_auxiliary_target,
1075 models,
1076 ),
1077 caller_auth,
1078 ))
1079 }
1080
1081 fn build_clients(
1082 &self,
1083 provider_api_keys: &mut Vec<String>,
1084 ) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {
1085 let mut models_by_client = self
1086 .llm_clients
1087 .keys()
1088 .map(|name| (name.clone(), Vec::new()))
1089 .collect::<BTreeMap<String, Vec<ModelConfig>>>();
1090
1091 for (name, client_config) in &self.llm_clients {
1092 validate_value("llm client name", name)?;
1093 if client_config.responses_reasoning.is_some()
1094 && !matches!(client_config.format, ClientFormat::OpenAiResponses)
1095 {
1096 return Err(RunnerError::configuration(format!(
1097 "llm client {name} responses_reasoning is only valid for openai_responses"
1098 )));
1099 }
1100 let backend = build_backend(
1101 name,
1102 client_config,
1103 &BTreeMap::new(),
1104 &BTreeSet::new(),
1105 None,
1106 )?;
1107 let (Backend::OpenAiChat(config)
1108 | Backend::OpenAiResponses(config)
1109 | Backend::Anthropic(config)) = backend;
1110 if let Some(key) = config.api_key {
1111 provider_api_keys.push(key);
1112 }
1113 }
1114 for (target_name, target) in &self.targets {
1115 let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
1116 RunnerError::configuration(format!(
1117 "target {target_name} references unknown llm client {}",
1118 target.llm_client
1119 ))
1120 })?;
1121 let model_configs = models_by_client
1122 .get_mut(&target.llm_client)
1123 .ok_or_else(|| {
1124 RunnerError::configuration("validated llm client was not initialized")
1125 })?;
1126 if let Some(effort) = &target.reasoning_effort {
1127 if !target.reasoning_effort_map.is_empty() {
1128 return Err(RunnerError::configuration(format!(
1129 "target {target_name} cannot set both reasoning_effort and reasoning_effort_map"
1130 )));
1131 }
1132 if effort.trim().is_empty() {
1133 return Err(RunnerError::configuration(format!(
1134 "target {target_name} reasoning_effort must not be empty"
1135 )));
1136 }
1137 if matches!(client_config.format, ClientFormat::AnthropicMessages)
1139 && !ANTHROPIC_EFFORTS.contains(&effort.as_str())
1140 {
1141 return Err(RunnerError::configuration(format!(
1142 "target {target_name} reasoning_effort {effort:?} is not an anthropic_messages effort; use one of {}",
1143 ANTHROPIC_EFFORTS.join(", ")
1144 )));
1145 }
1146 }
1147 for (requested, mapped) in &target.reasoning_effort_map {
1148 if requested.trim().is_empty() || mapped.trim().is_empty() {
1149 return Err(RunnerError::configuration(format!(
1150 "target {target_name} reasoning_effort_map keys and values must not be empty"
1151 )));
1152 }
1153 if matches!(client_config.format, ClientFormat::AnthropicMessages)
1154 && !ANTHROPIC_EFFORTS.contains(&mapped.as_str())
1155 {
1156 return Err(RunnerError::configuration(format!(
1157 "target {target_name} reasoning_effort_map value {mapped:?} is not an anthropic_messages effort. Use one of {}",
1158 ANTHROPIC_EFFORTS.join(", ")
1159 )));
1160 }
1161 }
1162 model_configs.push(
1163 ModelConfig::new(
1164 target_name.as_str(),
1165 build_backend(
1166 &target.llm_client,
1167 client_config,
1168 &target.extra_body,
1169 &target.omit_body_fields,
1170 target.reasoning_effort.clone(),
1171 )?,
1172 None,
1173 )
1174 .with_reasoning_effort_map(target.reasoning_effort_map.clone())
1175 .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default())
1176 .with_provider_model(target.id.clone()),
1177 );
1178 }
1179
1180 let mut clients = BTreeMap::new();
1181 for (name, model_configs) in models_by_client {
1182 let mut client = TranslatingLlmClient::new(&model_configs)
1183 .map_err(|error| RunnerError::configuration(error.to_string()))?;
1184 if !self.llm_clients[&name].fallback_on_rate_limit {
1185 client = client.without_rate_limit_cooldown();
1186 }
1187 clients.insert(name, Arc::new(client));
1188 }
1189 Ok(clients)
1190 }
1191
1192 fn build_decision_clients(
1193 &self,
1194 provider_api_keys: &mut Vec<String>,
1195 ) -> RunnerResult<BTreeMap<String, Arc<dyn RoutedDecisionClient>>> {
1196 self.decision_clients
1197 .iter()
1198 .map(|(name, config)| {
1199 validate_value("decision client name", name)?;
1200 let DecisionClientConfig {
1201 format,
1202 endpoint,
1203 api_key_env,
1204 timeout_ms,
1205 } = config;
1206 if *timeout_ms == 0 {
1207 return Err(RunnerError::configuration(format!(
1208 "decision client {name} timeout_ms must be at least 1"
1209 )));
1210 }
1211 let api_key = read_api_key(&format!("decision client {name}"), api_key_env)?;
1212 let timeout = Duration::from_millis(*timeout_ms);
1213 let client = match format {
1214 DecisionFormat::SystemOne => {
1215 SystemOneClient::new(endpoint.0.clone(), api_key.clone(), timeout)
1216 .map(|client| Arc::new(client) as Arc<dyn RoutedDecisionClient>)
1217 }
1218 DecisionFormat::OpenaiDecisions => {
1219 OpenAiDecisionClient::new(endpoint.0.clone(), api_key.clone(), timeout)
1220 .map(|client| Arc::new(client) as Arc<dyn RoutedDecisionClient>)
1221 }
1222 }
1223 .map_err(|error| RunnerError::configuration(error.to_string()))?;
1224 provider_api_keys.push(api_key);
1225 Ok((name.clone(), client))
1226 })
1227 .collect()
1228 }
1229
1230 fn build_targets(&self) -> BTreeMap<String, ModelId> {
1234 self.targets
1235 .keys()
1236 .map(|name| (name.clone(), ModelId::from(name.as_str())))
1237 .chain(
1238 self.decision_targets
1239 .iter()
1240 .map(|(name, config)| (name.clone(), config.id.clone())),
1241 )
1242 .collect()
1243 }
1244
1245 fn build_route_clients(
1249 &self,
1250 route_name: &str,
1251 route: &AlgorithmSpec,
1252 targets: &LaneTargets<'_>,
1253 decisions: &LaneDecisions<'_>,
1254 clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
1255 ) -> RunnerResult<(ClientRouter, Option<CallerAuthKind>)> {
1256 let TargetPromptPolicy {
1257 prompts,
1258 routing_answer_target,
1259 } = self.build_route_target_prompts(route_name, route, targets)?;
1260 let mut by_model = HashMap::new();
1261 let mut caller_auth = None;
1262 let mut has_mixed_families = false;
1263 let mut forwarding_origins = BTreeSet::new();
1264 let mut decisions_by_model = HashMap::new();
1265 for name in route.callable_target_names() {
1266 if let Some(target) = decisions.get(name) {
1267 if let Some(previous) =
1268 decisions_by_model.insert(target.model.clone(), Arc::clone(target.client))
1269 && !Arc::ptr_eq(&previous, target.client)
1270 {
1271 return Err(RunnerError::configuration(format!(
1272 "route {route_name} maps decision model {} to different clients",
1273 target.model
1274 )));
1275 }
1276 continue;
1277 }
1278 let target = targets.get(name).ok_or_else(|| {
1279 RunnerError::configuration(format!("route references unknown target {name}"))
1280 })?;
1281 let client = clients.get(&target.config.llm_client).ok_or_else(|| {
1282 RunnerError::configuration(format!("target {name} has no constructed llm client"))
1283 })?;
1284 let client_config =
1285 self.llm_clients
1286 .get(&target.config.llm_client)
1287 .ok_or_else(|| {
1288 RunnerError::configuration(format!(
1289 "target {name} references unknown llm client {}",
1290 target.config.llm_client
1291 ))
1292 })?;
1293 if client_config.forward_auth {
1294 let target_auth = client_config.format.caller_auth_kind();
1295 has_mixed_families |= caller_auth.is_some_and(|kind| kind != target_auth);
1296 caller_auth = Some(target_auth);
1297 forwarding_origins.insert(client_config.base_url.0.origin().ascii_serialization());
1298 }
1299 let client: Arc<dyn RoutedLlmClient> = client.clone();
1300 by_model.insert(ModelId::from(target.name), client);
1301 }
1302 if has_mixed_families {
1308 if forwarding_origins.len() > 1 {
1309 let origins = Vec::from_iter(forwarding_origins).join(", ");
1310 return Err(RunnerError::configuration(format!(
1311 "route {route_name} cannot forward both Anthropic and OpenAI caller credentials to different origins ({origins}); point all of its forwarding clients at one origin (same scheme, host, and port), such as an LLM gateway, or set api_key_env instead of forward_auth on one provider's clients"
1312 )));
1313 }
1314 caller_auth = Some(CallerAuthKind::OpenAi);
1315 }
1316 let completion_targets = route
1317 .routing_target_names()
1318 .into_iter()
1319 .map(|name| ModelId::from(targets[name].name))
1320 .collect::<Vec<_>>();
1321 let router = ClientRouter::new_with_decision_clients(
1322 by_model,
1323 decisions_by_model,
1324 prompts,
1325 routing_answer_target,
1326 &completion_targets,
1327 )
1328 .without_rate_limit_fallback(
1329 route
1330 .routing_target_names()
1331 .into_iter()
1332 .filter_map(|name| {
1333 let target = &targets[name];
1334 (!self.llm_clients[&target.config.llm_client].fallback_on_rate_limit)
1335 .then(|| ModelId::from(target.name))
1336 })
1337 .collect(),
1338 );
1339 Ok((router, caller_auth))
1340 }
1341
1342 fn build_route_target_prompts(
1344 &self,
1345 route_name: &str,
1346 route: &AlgorithmSpec,
1347 targets: &LaneTargets<'_>,
1348 ) -> RunnerResult<TargetPromptPolicy> {
1349 let mut prompts = HashMap::new();
1350 for name in route.routing_target_names() {
1351 let target = targets.get(name).ok_or_else(|| {
1352 RunnerError::configuration(format!("route references unknown target {name}"))
1353 })?;
1354 if let Some(prompt) = &target.config.system_prompt {
1355 prompts.insert(ModelId::from(target.name), prompt.clone());
1356 }
1357 }
1358 let mut policy = TargetPromptPolicy {
1359 prompts,
1360 routing_answer_target: None,
1361 };
1362 let Some((response_name, dependency_name)) = route.routing_response_and_dependency() else {
1363 return Ok(policy);
1364 };
1365 let response = &targets[response_name];
1366 if !policy.prompts.contains_key(response.name) {
1367 return Ok(policy);
1368 }
1369 if std::iter::once(dependency_name)
1370 .chain(route.judge_target_names())
1371 .any(|name| {
1372 targets
1373 .get(name)
1374 .is_some_and(|dependency| dependency.name == response.name)
1375 })
1376 {
1377 return Err(RunnerError::configuration(format!(
1378 "route {route_name} cannot apply system_prompt to target {response_name}: it is also the routing-only target"
1379 )));
1380 }
1381 policy.routing_answer_target = Some(ModelId::from(response.name));
1382 Ok(policy)
1383 }
1384
1385 fn fallback_base_url(&self) -> RunnerResult<Option<String>> {
1386 let Some(name) = &self.fallback_client else {
1387 return Ok(None);
1388 };
1389 let config = self.llm_clients.get(name).ok_or_else(|| {
1390 RunnerError::configuration(format!(
1391 "fallback_client references unknown llm client {name}"
1392 ))
1393 })?;
1394 Ok(Some(config.base_url.as_str().to_string()))
1395 }
1396
1397 fn build_model_properties_probe(
1398 &self,
1399 efficient_target_name: &str,
1400 clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
1401 ) -> Option<ModelPropertiesProbe> {
1402 let efficient_target = self.targets.get(efficient_target_name)?;
1403 let client_config = self.llm_clients.get(&efficient_target.llm_client)?;
1404 if !matches!(
1405 client_config.context_discovery,
1406 Some(ContextDiscovery::LlamaCpp)
1407 ) {
1408 return None;
1409 }
1410 let client = clients.get(&efficient_target.llm_client)?;
1411 Some(ModelPropertiesProbe {
1412 model: ModelId::from(efficient_target_name),
1413 format: client_config.format.wire_format(),
1414 client: Arc::clone(client),
1415 })
1416 }
1417
1418 fn build_anthropic_auxiliary_target(
1419 &self,
1420 route: &AlgorithmSpec,
1421 targets: &LaneTargets<'_>,
1422 clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
1423 ) -> Option<AuxiliaryTarget> {
1424 route
1425 .routing_target_names()
1426 .into_iter()
1427 .enumerate()
1428 .filter_map(|(index, name)| {
1429 let config = targets.get(name)?;
1430 let target = self.build_auxiliary_target(
1431 config.name,
1432 config.config,
1433 clients,
1434 AuxiliaryOperation::AnthropicCountTokens,
1435 )?;
1436 Some((
1437 count_tokens_priority(config.name, &config.config.id),
1438 index,
1439 target,
1440 ))
1441 })
1442 .min_by_key(|(priority, index, _)| (*priority, *index))
1443 .map(|(_, _, target)| target)
1444 }
1445
1446 fn build_responses_auxiliary_target(
1447 &self,
1448 route: &AlgorithmSpec,
1449 targets: &LaneTargets<'_>,
1450 clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
1451 ) -> Option<AuxiliaryTarget> {
1452 route.routing_target_names().into_iter().find_map(|name| {
1453 let target = targets.get(name)?;
1454 self.build_auxiliary_target(
1455 target.name,
1456 target.config,
1457 clients,
1458 AuxiliaryOperation::ResponsesInputTokens,
1459 )
1460 })
1461 }
1462
1463 fn build_auxiliary_target(
1464 &self,
1465 name: &str,
1466 target: &TargetConfig,
1467 clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
1468 operation: AuxiliaryOperation,
1469 ) -> Option<AuxiliaryTarget> {
1470 let client = clients.get(&target.llm_client)?;
1471 let model = ModelId::from(name);
1472 client
1473 .supports_auxiliary(&model, operation)
1474 .then(|| AuxiliaryTarget {
1475 model,
1476 client: client.clone(),
1477 })
1478 }
1479}
1480
1481const ANTHROPIC_EFFORTS: [&str; 5] = ["low", "medium", "high", "xhigh", "max"];
1483
1484fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
1485 let target_name = target_name.to_ascii_lowercase();
1486 let model_id = model_id.to_ascii_lowercase();
1487 ["opus", "sonnet", "haiku"]
1488 .iter()
1489 .position(|hint| target_name.contains(hint) || model_id.contains(hint))
1490 .unwrap_or(3)
1491}
1492
1493#[derive(Clone, Debug)]
1498struct HttpBaseUrl(reqwest::Url);
1499
1500impl HttpBaseUrl {
1501 fn as_str(&self) -> &str {
1502 self.0.as_str()
1503 }
1504}
1505
1506impl<'de> Deserialize<'de> for HttpBaseUrl {
1507 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
1508 where
1509 D: serde::Deserializer<'de>,
1510 {
1511 let raw = String::deserialize(deserializer)?;
1512 let url = reqwest::Url::parse(raw.trim()).map_err(|error| {
1513 serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}"))
1514 })?;
1515 if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
1516 return Err(serde::de::Error::custom(
1517 "base_url must be an absolute HTTP(S) URL",
1518 ));
1519 }
1520 Ok(Self(url))
1521 }
1522}
1523
1524#[derive(Debug, Deserialize)]
1525#[serde(deny_unknown_fields)]
1526struct LlmClientConfig {
1527 format: ClientFormat,
1528 base_url: HttpBaseUrl,
1529 context_discovery: Option<ContextDiscovery>,
1531 api_key_env: Option<String>,
1532 #[serde(default)]
1533 forward_auth: bool,
1534 #[serde(default)]
1535 extra_headers: BTreeMap<String, String>,
1536 #[serde(default = "default_max_retries")]
1537 max_retries: u32,
1538 #[serde(default = "default_fallback_on_rate_limit")]
1539 fallback_on_rate_limit: bool,
1540 #[serde(default = "default_failure_cooldown_ms")]
1542 failure_cooldown_ms: u64,
1543 timeout_ms: Option<u64>,
1545 responses_reasoning: Option<ResponsesReasoningPolicy>,
1546}
1547
1548#[derive(Debug, Deserialize)]
1549#[serde(rename_all = "snake_case")]
1550enum ContextDiscovery {
1551 LlamaCpp,
1552}
1553
1554#[derive(Debug, Deserialize)]
1555#[serde(deny_unknown_fields)]
1556struct DecisionClientConfig {
1557 format: DecisionFormat,
1558 endpoint: HttpBaseUrl,
1559 api_key_env: String,
1560 timeout_ms: u64,
1561}
1562
1563#[derive(Debug, Deserialize)]
1564#[serde(rename_all = "snake_case")]
1565enum DecisionFormat {
1566 SystemOne,
1567 OpenaiDecisions,
1568}
1569
1570#[derive(Debug, Deserialize)]
1571#[serde(deny_unknown_fields)]
1572struct DecisionTargetConfig {
1573 #[serde(default)]
1574 labels: BTreeSet<String>,
1575 id: ModelId,
1576 decision_client: String,
1577}
1578
1579#[derive(Debug, Deserialize)]
1580#[serde(deny_unknown_fields)]
1581struct TargetConfig {
1582 #[serde(default)]
1583 labels: BTreeSet<String>,
1584 id: ModelId,
1585 llm_client: String,
1586 #[serde(default)]
1587 extra_body: BTreeMap<String, Value>,
1588 #[serde(default)]
1591 omit_body_fields: BTreeSet<String>,
1592 system_prompt: Option<String>,
1593 reasoning_effort: Option<String>,
1595 #[serde(default)]
1596 reasoning_effort_map: BTreeMap<String, String>,
1597}
1598
1599#[derive(Clone, Copy, Debug, Deserialize)]
1600enum ClientFormat {
1601 #[serde(rename = "openai_chat")]
1602 OpenAiChat,
1603 #[serde(rename = "openai_responses")]
1604 OpenAiResponses,
1605 #[serde(rename = "anthropic_messages")]
1606 AnthropicMessages,
1607}
1608
1609impl ClientFormat {
1610 const fn wire_format(self) -> WireFormat {
1611 match self {
1612 Self::OpenAiChat => WireFormat::OpenAiChat,
1613 Self::OpenAiResponses => WireFormat::OpenAiResponses,
1614 Self::AnthropicMessages => WireFormat::AnthropicMessages,
1615 }
1616 }
1617
1618 const fn caller_auth_kind(self) -> CallerAuthKind {
1619 match self {
1620 Self::AnthropicMessages => CallerAuthKind::Anthropic,
1621 Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi,
1622 }
1623 }
1624}
1625
1626fn build_backend(
1627 client_name: &str,
1628 config: &LlmClientConfig,
1629 extra_body: &BTreeMap<String, Value>,
1630 omit_body_fields: &BTreeSet<String>,
1631 reasoning_effort: Option<String>,
1632) -> RunnerResult<Backend> {
1633 if config.max_retries > MAX_CONFIGURED_RETRIES {
1634 return Err(RunnerError::configuration(format!(
1635 "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"
1636 )));
1637 }
1638 if config.timeout_ms == Some(0) {
1639 return Err(RunnerError::configuration(format!(
1640 "llm client {client_name} timeout_ms must be at least 1"
1641 )));
1642 }
1643 if config.forward_auth && config.api_key_env.is_some() {
1644 return Err(RunnerError::configuration(format!(
1645 "llm client {client_name} cannot set both forward_auth and api_key_env"
1646 )));
1647 }
1648 let api_key = config
1649 .api_key_env
1650 .as_deref()
1651 .map(|variable| read_api_key(&format!("llm client {client_name}"), variable))
1652 .transpose()?;
1653 let http = HttpBackendConfig {
1654 base_url: config.base_url.as_str().to_string(),
1655 api_key,
1656 forward_auth: config.forward_auth,
1657 extra_headers: config.extra_headers.clone(),
1658 extra_body: extra_body.clone(),
1659 omit_body_fields: omit_body_fields.clone(),
1660 reasoning_effort,
1661 max_retries: config.max_retries,
1662 failure_cooldown: Duration::from_millis(config.failure_cooldown_ms),
1663 timeout: config.timeout_ms.map(Duration::from_millis),
1664 };
1665 let backend = match config.format {
1666 ClientFormat::OpenAiChat => Backend::OpenAiChat(http),
1667 ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http),
1668 ClientFormat::AnthropicMessages => Backend::Anthropic(http),
1669 };
1670 Ok(backend)
1671}
1672
1673fn read_api_key(client: &str, variable: &str) -> RunnerResult<String> {
1674 if variable.trim().is_empty() {
1675 return Err(RunnerError::configuration(format!(
1676 "{client} api_key_env must not be empty"
1677 )));
1678 }
1679 if variable.contains(['\0', '=']) {
1681 return Err(RunnerError::configuration(format!(
1682 "{client} api_key_env must not contain '=' or NUL"
1683 )));
1684 }
1685 let api_key = std::env::var(variable).map_err(|error| {
1686 let reason = match error {
1688 std::env::VarError::NotPresent => "environment variable not found",
1689 std::env::VarError::NotUnicode(_) => "environment variable is not valid Unicode",
1690 };
1691 RunnerError::configuration(format!(
1692 "{client} could not read api_key_env {variable}: {reason}"
1693 ))
1694 })?;
1695 if api_key.trim().is_empty() {
1696 return Err(RunnerError::configuration(format!(
1697 "{client} api_key_env {variable} is empty"
1698 )));
1699 }
1700 Ok(api_key)
1701}
1702
1703const fn default_max_retries() -> u32 {
1705 DEFAULT_MAX_RETRIES
1706}
1707
1708const fn default_fallback_on_rate_limit() -> bool {
1709 true
1710}
1711
1712const fn default_failure_cooldown_ms() -> u64 {
1713 5000
1714}
1715
1716fn validate_value(label: &str, value: &str) -> RunnerResult<()> {
1717 if value.trim().is_empty() || value.trim() != value {
1718 return Err(RunnerError::configuration(format!(
1719 "{label} must be non-empty and have no surrounding whitespace"
1720 )));
1721 }
1722 Ok(())
1723}
1724
1725#[cfg(test)]
1726mod tests {
1727 use super::*;
1728
1729 #[test]
1730 fn route_presentation_fields_are_split_from_the_algorithm() {
1731 let route: RouteConfig = toml::from_str(
1732 r#"
1733type = "random"
1734id = "switchyard/random"
1735context_window = 128000
1736tool_calling = true
1737reasoning = false
1738targets = ["fast", "strong"]
1739weights = [1.0, 2.0]
1740seed = 7
1741"#,
1742 )
1743 .expect("route should deserialize");
1744
1745 assert_eq!(route.id, "switchyard/random");
1746 assert_eq!(route.context_window, Some(128_000));
1747 assert_eq!(route.tool_calling, Some(true));
1748 assert_eq!(route.reasoning, Some(false));
1749 assert_eq!(route.algorithm.routing_target_names(), ["fast", "strong"]);
1750 }
1751
1752 #[test]
1753 fn route_algorithm_unknown_fields_are_rejected() {
1754 let error = toml::from_str::<RouteConfig>(
1755 r#"
1756type = "passthrough"
1757id = "switchyard/fast"
1758target = "fast"
1759bogus = true
1760"#,
1761 )
1762 .expect_err("unknown route keys must be rejected");
1763
1764 assert!(error.to_string().contains("bogus"), "{error}");
1765 }
1766
1767 #[test]
1768 fn deployment_errors_keep_one_path_context() {
1769 let path = Path::new("/definitely/missing/switchyard-routes.toml");
1770 let error = match load_runner(path) {
1771 Ok(_) => panic!("missing config should fail"),
1772 Err(error) => error,
1773 };
1774 let message = error.to_string();
1775
1776 assert_eq!(message.matches(&path.display().to_string()).count(), 1);
1777 assert!(message.starts_with("failed to read server config"));
1778 }
1779}
1780#[cfg(test)]
1781mod deployment_tests {
1782 use super::*;
1783 use crate::require_target_labels;
1784 use serde_json::json;
1785 use switchyard_protocol::{Category, Request};
1786
1787 const VALID_CONFIG: &str = r#"
1788schema_version = 1
1789
1790[llm_clients.primary]
1791format = "openai_chat"
1792base_url = "https://example.test/v1"
1793
1794[llm_clients.responses]
1795format = "openai_responses"
1796base_url = "https://example.test/v1"
1797
1798[llm_clients.anthropic]
1799format = "anthropic_messages"
1800base_url = "https://example.test"
1801
1802[targets.classifier]
1803id = "classifier/model"
1804llm_client = "primary"
1805
1806[targets.strong]
1807id = "strong/model"
1808llm_client = "responses"
1809
1810[targets.weak]
1811id = "weak/model"
1812llm_client = "anthropic"
1813
1814[routes.noop]
1815id = "switchyard/noop"
1816type = "noop"
1817
1818[routes.random]
1819id = "switchyard/random"
1820type = "random"
1821targets = ["strong", "weak"]
1822
1823[routes.classifier]
1824id = "switchyard/classifier"
1825type = "llm_classifier"
1826classifier_target = "classifier"
1827strong_target = "strong"
1828weak_target = "weak"
1829base_threshold = 0.5
1830
1831[routes.passthrough]
1832id = "switchyard/passthrough"
1833type = "passthrough"
1834target = "weak"
1835
1836[routes.plan_execute]
1837id = "switchyard/plan-execute"
1838type = "stage_router"
1839capable_target = "strong"
1840efficient_target = "weak"
1841picker = "capable_first"
1842confidence_threshold = 0.0
1843efficient_hold_turns = -1
1844capable_system_prompt = "Inspect the task and plan before modifying files."
1845"#;
1846
1847 fn decision_eligibility_config() -> String {
1848 let config = VALID_CONFIG
1849 .replace(
1850 "id = \"strong/model\"",
1851 "id = \"strong/model\"\nlabels = ['private']",
1852 )
1853 .replace(
1854 "id = \"weak/model\"",
1855 "id = \"weak/model\"\nlabels = ['private']",
1856 );
1857 format!(
1858 r#"{config}
1859
1860[decision_clients.privacy]
1861format = "system_one"
1862endpoint = "https://example.test/v1/systemone"
1863api_key_env = "PATH"
1864timeout_ms = 5000
1865
1866[decision_targets.standard]
1867id = "decision/standard"
1868decision_client = "privacy"
1869labels = ["private"]
1870
1871[routes.typed]
1872id = "switchyard/typed"
1873type = "llm_classifier"
1874mode = "capability"
1875classifier_target = "standard"
1876strong_target = "strong"
1877weak_target = "weak"
1878
1879[routes.typed.decision]
1880cutoff = 0.4
1881
1882[routes.typed.eligibility]
1883"#
1884 )
1885 }
1886
1887 fn eligible_pool_config() -> String {
1888 let config = VALID_CONFIG
1889 .replace("id = \"strong/model\"", "id = \"strong/model\"\nlabels = ['private']")
1890 .replace(
1891 "type = \"passthrough\"\ntarget = \"weak\"",
1892 "type = \"random\"\ntargets = [\"weak\", \"strong\"]\nweights = [3.0, 1.0]\nseed = 7",
1893 );
1894 format!(
1895 r#"{config}
1896
1897[routes.passthrough.eligibility]
1898"#
1899 )
1900 }
1901
1902 #[tokio::test]
1903 async fn eligibility_requires_approved_roles_and_decision_targets() -> RunnerResult<()> {
1904 let configured = decision_eligibility_config();
1905 let runner = Runner::from_toml(&configured)?;
1906 let route = runner
1907 .route("switchyard/typed")
1908 .expect("typed eligible route should exist");
1909 assert_eq!(
1910 route.models().models_for(&Category::Judge),
1911 [ModelId::from("decision/standard")]
1912 );
1913
1914 for (config, expected) in [
1915 (
1916 configured.replace(
1917 "decision_client = \"privacy\"\nlabels = [\"private\"]",
1918 "decision_client = \"privacy\"",
1919 ),
1920 "required target standard",
1921 ),
1922 (
1923 configured.replace(
1924 "id = \"weak/model\"\nlabels = ['private']",
1925 "id = \"weak/model\"",
1926 ),
1927 "required target weak",
1928 ),
1929 ] {
1930 let runner = Runner::from_toml(&config)?;
1931 let mut request = Request::default();
1932 require_target_labels(&mut request, ["private"])?;
1933 let message = runner
1934 .route("switchyard/typed")
1935 .expect("typed route")
1936 .decide(request)
1937 .await
1938 .err()
1939 .expect("unapproved role must fail before model I/O")
1940 .to_string();
1941 assert!(
1942 message.contains(expected),
1943 "expected {expected:?}, got {message}"
1944 );
1945 }
1946 Ok(())
1947 }
1948
1949 #[tokio::test]
1950 async fn label_requirements_select_only_approved_targets() -> RunnerResult<()> {
1951 let configured = eligible_pool_config();
1952 let runner = Runner::from_toml(&configured)?;
1953 let route = runner
1954 .route("switchyard/passthrough")
1955 .expect("eligible route should exist");
1956
1957 let mut request = Request::default();
1958 require_target_labels(&mut request, ["private"])?;
1959 let outcome = route.decide(request).await?;
1960 assert_eq!(outcome.selected_model_id()?, "strong");
1961 assert_eq!(
1962 &runner
1963 .describe_decision(&ModelId::from("switchyard/passthrough"), &outcome)
1964 .expect("selected target metadata")
1965 .selected
1966 .target,
1967 outcome.selected_model_id()?.as_str()
1968 );
1969 for (config, route_id, expected) in [
1970 (
1971 configured.replace("labels = ['private']", "labels = []"),
1972 "switchyard/passthrough",
1973 "without an eligible target",
1974 ),
1975 (
1976 format!("{configured}\n[routes.plan_execute.eligibility]\n"),
1977 "switchyard/plan-execute",
1978 "required target weak",
1979 ),
1980 ] {
1981 let runner = Runner::from_toml(&config)?;
1982 let mut request = Request::default();
1983 require_target_labels(&mut request, ["private"])?;
1984 let error = runner
1985 .route(route_id)
1986 .expect("configured route")
1987 .decide(request)
1988 .await
1989 .err()
1990 .expect("no eligible setup");
1991 assert!(error.to_string().contains(expected), "{error}");
1992 }
1993 Ok(())
1994 }
1995
1996 #[tokio::test]
1997 async fn ordinary_routes_reject_label_requirements() -> RunnerResult<()> {
1998 let runner = Runner::from_toml(VALID_CONFIG)?;
1999 let ordinary = runner.route("switchyard/random").expect("ordinary route");
2000 for labels in [&[][..], &["private"][..]] {
2001 let mut request = Request::default();
2002 require_target_labels(&mut request, labels)?;
2003 assert!(ordinary.decide(request).await.is_err());
2004 }
2005 Ok(())
2006 }
2007
2008 fn alternative_config(algorithm: &str) -> String {
2009 let globals = VALID_CONFIG.split("[routes.noop]").next().expect("globals");
2010 let mut config = globals.to_string();
2011 for name in ["weak", "strong", "classifier"] {
2012 config.push_str(&format!(
2013 "\n[targets.approved_{name}]\nid = 'approved/{name}'\nllm_client = 'primary'\nlabels = ['private']\n"
2014 ));
2015 }
2016 config.push_str(&format!(
2017 "\n[routes.test]\nid = 'route'\n{algorithm}\n[routes.test.eligibility.target_alternatives]\n"
2018 ));
2019 let spec: AlgorithmSpec = toml::from_str(algorithm).expect("test algorithm");
2020 for name in spec
2021 .callable_target_names()
2022 .into_iter()
2023 .collect::<BTreeSet<_>>()
2024 {
2025 config.push_str(&format!("{name} = ['approved_{name}']\n"));
2026 }
2027 config
2028 }
2029
2030 #[tokio::test]
2032 async fn alternatives_preserve_router_roles() -> RunnerResult<()> {
2033 for (algorithm, initial) in [
2034 ("type = 'passthrough'\ntarget = 'weak'", "weak"),
2035 (
2036 "type = 'stage_router'\ncapable_target = 'strong'\nefficient_target = 'weak'\npicker = 'capable_first'\nconfidence_threshold = 0.0",
2037 "strong",
2038 ),
2039 ] {
2040 let runner = Runner::from_toml(&alternative_config(algorithm))?;
2041 let route = runner.route("route").expect("route");
2042 let standard = route.decide(Request::default()).await?;
2043 assert_eq!(standard.selected_model_id()?.as_str(), initial);
2044 let mut request = Request::default();
2045 require_target_labels(&mut request, ["private"])?;
2046 let outcome = route.decide(request).await?;
2047 assert_eq!(outcome.selected_model_id()?, &format!("approved_{initial}"));
2048 assert!(
2049 outcome
2050 .selected_model_ids
2051 .iter()
2052 .all(|model| model.starts_with("approved_"))
2053 );
2054 let description = runner
2055 .describe_decision(&ModelId::from("route"), &outcome)
2056 .expect("mapped metadata");
2057 assert_eq!(description.selected.target, format!("approved_{initial}"));
2058 }
2059 Ok(())
2060 }
2061
2062 #[tokio::test]
2064 async fn alternatives_reject_invalid_bindings() -> RunnerResult<()> {
2065 let config = alternative_config(
2066 "type = 'auto'\ncapable_target = 'strong'\nefficient_target = 'weak'",
2067 );
2068 for (invalid, expected) in [
2069 (
2070 format!("{config}typo = ['approved_weak']\n"),
2071 "not used by the algorithm",
2072 ),
2073 (
2074 config.replace("weak = ['approved_weak']", "weak = ['missing']"),
2075 "must exist and match the target kind",
2076 ),
2077 (
2078 config.replace("weak = ['approved_weak']", "weak = []"),
2079 "target alternatives must not be empty",
2080 ),
2081 (
2082 config.replace(
2083 "[llm_clients.primary]",
2084 "[llm_clients.primary]\nforward_auth = true",
2085 ),
2086 "forwarded caller authentication",
2087 ),
2088 ] {
2089 let message = error_message(&invalid);
2090 assert!(
2091 message.contains(expected),
2092 "expected {expected:?}, got {message}"
2093 );
2094 }
2095 let random =
2096 alternative_config("type = 'random'\ntargets = ['weak', 'strong']\nweights = [0, 1]")
2097 .replace("strong = ['approved_strong']", "strong = ['approved_weak']");
2098 for (source, expected) in [
2099 (
2100 config.replace("weak = ['approved_weak']", "weak = ['strong']"),
2101 "required target weak",
2102 ),
2103 (random, "multiple pool entries to target approved_weak"),
2104 ] {
2105 let runner = Runner::from_toml(&source)?;
2106 let mut request = Request::default();
2107 require_target_labels(&mut request, ["private"])?;
2108 let error = runner
2109 .route("route")
2110 .expect("configured route")
2111 .decide(request)
2112 .await
2113 .err()
2114 .expect("invalid eligible bindings");
2115 assert!(error.to_string().contains(expected), "{error}");
2116 }
2117 Runner::from_toml(&config.replace("id = 'approved/weak'", "id = 'weak/model'"))?;
2118 let shared_model =
2119 alternative_config("type = 'random'\ntargets = ['weak', 'strong']\nweights = [0, 1]")
2120 .replace("id = 'approved/strong'", "id = 'approved/weak'");
2121 let runner = Runner::from_toml(&shared_model)?;
2122 let mut request = Request::default();
2123 require_target_labels(&mut request, ["private"])?;
2124 assert_eq!(
2125 runner
2126 .route("route")
2127 .expect("route")
2128 .decide(request)
2129 .await?
2130 .selected_model_id()?,
2131 "approved_strong"
2132 );
2133 let wrong_kind = format!(
2134 "{}\n[routes.typed.eligibility.target_alternatives]\nstandard = ['weak']\n",
2135 decision_eligibility_config()
2136 );
2137 assert!(error_message(&wrong_kind).contains("must exist and match the target kind"));
2138 Ok(())
2139 }
2140
2141 #[tokio::test]
2143 async fn alternatives_reject_prompted_vgr_secondary_judge_alias() -> RunnerResult<()> {
2144 let source = alternative_config(
2145 "type = 'vgr'\nefficient_target = 'weak'\ncapable_target = 'strong'\njudge_target = 'strong'\ncapable_judge_target = 'classifier'",
2146 )
2147 .replace(
2148 "classifier = ['approved_classifier']",
2149 "classifier = ['approved_weak']",
2150 )
2151 .replace(
2152 "[targets.approved_weak]",
2153 "[targets.approved_weak]\nsystem_prompt = 'Answer the user.'",
2154 );
2155 let runner = Runner::from_toml(&source)?;
2156 let mut request = Request::default();
2157 require_target_labels(&mut request, ["private"])?;
2158 let error = runner
2159 .route("route")
2160 .expect("VGR route")
2161 .decide(request)
2162 .await
2163 .err()
2164 .expect("secondary judge alias must fail before model I/O");
2165 assert!(matches!(error, RunnerError::Configuration { .. }));
2166 assert!(error.to_string().contains("routing-only target"), "{error}");
2167 Ok(())
2168 }
2169
2170 #[test]
2171 fn factory_decision_ids_identify_one_client_across_alternatives() -> RunnerResult<()> {
2172 let source = format!(
2173 r#"{}
2174[decision_clients.other]
2175format = "system_one"
2176endpoint = "https://private.example.test/v1/decisions"
2177api_key_env = "PATH"
2178timeout_ms = 5000
2179
2180[decision_targets.alternative]
2181id = "decision/alternative"
2182decision_client = "other"
2183labels = ["private"]
2184
2185[routes.typed.eligibility.target_alternatives]
2186standard = ["alternative"]
2187"#,
2188 decision_eligibility_config()
2189 );
2190 AlgorithmFactory::from_toml(&source)?.build_configured()?;
2191
2192 let shared_id = source.replace("decision/alternative", "decision/standard");
2193 let ambiguous = shared_id.replace(
2194 "decision_client = \"privacy\"\nlabels = [\"private\"]",
2195 "decision_client = \"privacy\"",
2196 );
2197 Runner::from_toml(&ambiguous)?;
2198 let later_alternative = format!(
2199 r#"{}
2200[decision_targets.later]
2201id = "decision/standard"
2202decision_client = "other"
2203labels = ["private", "customer_data"]
2204"#,
2205 source.replace("[\"alternative\"]", "[\"alternative\", \"later\"]")
2206 );
2207 for source in [&ambiguous, &later_alternative] {
2208 let error = AlgorithmFactory::from_toml(source)?
2209 .build_configured()
2210 .err()
2211 .expect("ambiguous endpoint must fail during construction");
2212 assert!(
2213 error
2214 .to_string()
2215 .contains("maps decision model decision/standard to different clients"),
2216 "{error}"
2217 );
2218 }
2219
2220 let same_client = shared_id.replace(
2221 "decision_client = \"other\"",
2222 "decision_client = \"privacy\"",
2223 );
2224 AlgorithmFactory::from_toml(&same_client)?.build_configured()?;
2225 let separate_routes = format!(
2226 r#"{}
2227[routes.other]
2228id = "switchyard/other"
2229type = "llm_classifier"
2230classifier_target = "alternative"
2231strong_target = "strong"
2232weak_target = "weak"
2233"#,
2234 shared_id.replace("standard = [\"alternative\"]", "")
2235 );
2236 AlgorithmFactory::from_toml(&separate_routes)?.build_configured()?;
2237 Ok(())
2238 }
2239
2240 #[tokio::test]
2242 async fn alternatives_reject_ambiguous_composite_decision_clients() -> RunnerResult<()> {
2243 let source = decision_eligibility_config().replace(
2244 "decision_client = \"privacy\"\nlabels = [\"private\"]",
2245 "decision_client = \"privacy\"",
2246 );
2247 let source = format!(
2248 r#"{source}
2249[decision_clients.other]
2250format = "system_one"
2251endpoint = "http://127.0.0.1:1/decisions"
2252api_key_env = "PATH"
2253timeout_ms = 1
2254
2255[decision_targets.secondary]
2256id = "decision/secondary"
2257decision_client = "privacy"
2258
2259[decision_targets.approved_primary]
2260id = "decision/shared"
2261decision_client = "privacy"
2262labels = ["private"]
2263
2264[decision_targets.approved_secondary]
2265id = "decision/shared"
2266decision_client = "other"
2267labels = ["private"]
2268
2269[routes.ambiguous]
2270id = "switchyard/ambiguous"
2271type = "composite"
2272classifier = {{ target = "standard", classify_trigger = "user_turn", decision = {{ cutoff = 0.4 }} }}
2273stage = {{ capable_target = "strong", efficient_target = "weak", confidence_threshold = 0.5 }}
2274efficient = {{ classifier = {{ target = "secondary", decision = {{ cutoff = 0.4 }} }} }}
2275
2276[routes.ambiguous.eligibility.target_alternatives]
2277standard = ["approved_primary"]
2278secondary = ["approved_secondary"]
2279"#
2280 );
2281 let runner = Runner::from_toml(&source)?;
2282 let mut request = Request::default();
2283 require_target_labels(&mut request, ["private"])?;
2284 let factory_error = AlgorithmFactory::from_toml(&source)?
2285 .build_configured()
2286 .err()
2287 .expect("ambiguous factory decisions must fail during construction");
2288 let error = runner
2289 .route("switchyard/ambiguous")
2290 .expect("composite route")
2291 .decide(request)
2292 .await
2293 .err()
2294 .expect("ambiguous decision clients must fail before model I/O");
2295 for error in [error, factory_error] {
2296 assert!(matches!(error, RunnerError::Configuration { .. }));
2297 assert!(
2298 error
2299 .to_string()
2300 .contains("maps decision model decision/shared to different clients"),
2301 "{error}"
2302 );
2303 }
2304 let standard_alias = source.replace(
2305 "id = \"decision/secondary\"\ndecision_client = \"privacy\"",
2306 "id = \"decision/standard\"\ndecision_client = \"other\"",
2307 );
2308 let ungoverned = standard_alias
2309 .split_once("[routes.ambiguous.eligibility.target_alternatives]")
2310 .expect("alternatives fixture")
2311 .0;
2312 for source in [standard_alias.as_str(), ungoverned] {
2313 let error = AlgorithmFactory::from_toml(source)?
2314 .build_configured()
2315 .err()
2316 .expect("ambiguous standard decisions must fail during construction");
2317 assert!(
2318 error
2319 .to_string()
2320 .contains("maps decision model decision/standard to different clients"),
2321 "{error}"
2322 );
2323 }
2324 Ok(())
2325 }
2326
2327 #[test]
2328 fn eligibility_rejects_unknown_fields_and_invalid_scope() {
2329 let configured = eligible_pool_config();
2330 for (invalid, expected) in [
2331 (
2332 configured.replace("labels = ['private']", "label = ['private']"),
2333 "unknown field `label`",
2334 ),
2335 (
2336 configured.replace(
2337 "[routes.passthrough.eligibility]",
2338 "[routes.passthrough.eligible]",
2339 ),
2340 "unknown field `eligible`",
2341 ),
2342 (
2343 decision_eligibility_config().replace(
2344 "[decision_targets.standard]",
2345 "[decision_targets.standard]\nlabel = ['private']",
2346 ),
2347 "unknown field `label`",
2348 ),
2349 (
2350 format!("{configured}\nrequire_explicit_requirement = true\n"),
2351 "unknown field `require_explicit_requirement`",
2352 ),
2353 (
2354 format!("{configured}\nscope = 'tenant'\n"),
2355 "unknown variant `tenant`",
2356 ),
2357 ] {
2358 let message = error_message(&invalid);
2359 assert!(
2360 message.contains(expected),
2361 "expected {expected:?}, got {message}"
2362 );
2363 }
2364 }
2365
2366 #[test]
2367 fn eligibility_rejects_conflicting_approvals_and_forwarded_auth() {
2368 let configured = eligible_pool_config();
2369 for (invalid, expected) in [
2370 (
2371 format!(
2372 "{configured}\n[targets.alias]\nid = \"strong/model\"\nllm_client = \"responses\"\n"
2373 ),
2374 "disagree on approval labels",
2375 ),
2376 (
2377 format!(
2378 "{}\n[decision_targets.alias]\nid = \"decision/standard\"\ndecision_client = \"privacy\"\n",
2379 decision_eligibility_config()
2380 ),
2381 "disagrees on approval labels",
2382 ),
2383 (
2384 configured.replace(
2385 "[llm_clients.responses]\nformat = \"openai_responses\"",
2386 "[llm_clients.responses]\nformat = \"openai_responses\"\nforward_auth = true",
2387 ),
2388 "cannot use eligibility with forwarded caller authentication",
2389 ),
2390 (
2391 alternative_config(
2392 "type = 'prefill_router'\ntargets = ['weak']\ncheckpoint = 'unused'",
2393 ),
2394 "cannot use eligibility with prefill_router",
2395 ),
2396 ] {
2397 let message = error_message(&invalid);
2398 assert!(
2399 message.contains(expected),
2400 "expected {expected:?}, got {message}"
2401 );
2402 }
2403 }
2404
2405 #[test]
2406 fn public_runner_from_toml_builds_a_deployment() -> RunnerResult<()> {
2407 let config = format!(
2408 "{VALID_CONFIG}\n[routes.classifier.infra]\n\
2409 max_signal_age_ms = 1000\nmin_waiting_requests = 4\nmax_capable_inflight = 2"
2410 );
2411 let runner = Runner::from_toml(&config)?;
2412 assert!(
2413 Runner::from_toml(
2414 &config.replace("min_waiting_requests = 4", "min_waiting_requests = 0")
2415 )
2416 .is_err()
2417 );
2418
2419 let classifier = runner
2420 .route("switchyard/classifier")
2421 .expect("classifier route should exist");
2422 let models = classifier.models();
2423 assert_eq!(
2424 models.models_for(&Category::Judge),
2425 [ModelId::from("classifier")]
2426 );
2427 assert_eq!(
2428 models.models_for(&Category::Efficient),
2429 [ModelId::from("weak")]
2430 );
2431 assert_eq!(
2432 models.models_for(&Category::Capable),
2433 [ModelId::from("strong")]
2434 );
2435 assert_eq!(
2436 models.models_for(&Category::Any),
2437 [ModelId::from("weak"), ModelId::from("strong")]
2438 );
2439 let plan_execute = runner
2440 .route("switchyard/plan-execute")
2441 .expect("plan-execute route should exist");
2442 assert_eq!(
2443 plan_execute.models().models_for(&Category::Capable),
2444 [ModelId::from("strong")]
2445 );
2446 assert_eq!(
2447 plan_execute.models().models_for(&Category::Efficient),
2448 [ModelId::from("weak")]
2449 );
2450 assert!(runner.route("switchyard/passthrough").is_some());
2451 Ok(())
2452 }
2453
2454 #[test]
2456 fn removed_plan_execute_type_is_rejected() {
2457 let route = r#"
2458type = "plan_execute"
2459capable_target = "strong"
2460efficient_target = "weak"
2461"#;
2462 let config = format!("{VALID_CONFIG}\n[routes.obsolete]\nid = \"obsolete\"\n{route}");
2463 assert!(error_message(&config).contains("unknown variant `plan_execute`"));
2464
2465 let error = serde_json::from_str::<AlgorithmSpec>(
2466 r#"{"type":"plan_execute","capable_target":"strong","efficient_target":"weak"}"#,
2467 )
2468 .expect_err("the removed algorithm must be rejected");
2469 assert!(error.to_string().contains("unknown variant `plan_execute`"));
2470 }
2471
2472 #[test]
2475 fn stage_rejects_obsolete_plan_execute_settings() {
2476 let stage = json!({
2477 "type": "stage_router", "capable_target": "strong", "efficient_target": "weak",
2478 "picker": "capable_first", "confidence_threshold": 0.0,
2479 "efficient_hold_turns": -1, "capable_system_prompt": "Plan before editing.",
2480 });
2481 assert!(serde_json::from_str::<AlgorithmSpec>(&stage.to_string()).is_ok());
2482 for (field, value, toml_value) in [
2483 ("planning_prompt", json!("plan"), r#""plan""#),
2484 ("handoff_prompt", json!("execute"), r#""execute""#),
2485 ("planner_reasoning_as_text", json!(true), "true"),
2486 ("execution_trigger", json!("edit"), r#""edit""#),
2487 (
2488 "judge",
2489 json!({"target": "classifier"}),
2490 r#"{ target = "classifier" }"#,
2491 ),
2492 ] {
2493 let mut invalid = stage.clone();
2494 invalid[field] = value;
2495 let error = serde_json::from_str::<AlgorithmSpec>(&invalid.to_string())
2496 .expect_err("obsolete fields must be rejected");
2497 assert!(error.to_string().contains(field), "{error}");
2498 let error = error_message(&format!("{VALID_CONFIG}\n{field} = {toml_value}\n"));
2499 assert!(error.contains(field), "{error}");
2500 }
2501 }
2502
2503 #[test]
2504 fn duplicate_route_ids_are_rejected() {
2505 let config = format!(
2506 r#"{VALID_CONFIG}
2507
2508[routes.duplicate]
2509id = "switchyard/passthrough"
2510type = "passthrough"
2511target = "strong"
2512"#
2513 );
2514
2515 let error = error_message(&config);
2516
2517 assert!(
2518 error.contains(
2519 "routes duplicate and passthrough both use id switchyard/passthrough; route ids must be unique"
2520 ),
2521 "{error}"
2522 );
2523 }
2524
2525 fn error_message(toml: &str) -> String {
2526 match runner_from_toml(toml) {
2527 Ok(_) => "configuration unexpectedly succeeded".to_string(),
2528 Err(error) => error.to_string(),
2529 }
2530 }
2531
2532 fn vgr_config() -> String {
2533 format!(
2534 r#"{VALID_CONFIG}
2535
2536[routes.vgr]
2537id = "switchyard/vgr"
2538type = "vgr"
2539efficient_target = "weak"
2540capable_target = "strong"
2541judge_target = "classifier"
2542mode = "active"
2543active_approval = "prospective-validation-and-canary-approved"
2544task_typing = false
2545"#
2546 )
2547 }
2548
2549 #[test]
2550 fn vgr_compaction_is_opt_in_from_toml() -> RunnerResult<()> {
2551 for setting in ["", "compact_handoff = false", "compact_handoff = true"] {
2552 let source = format!("{}\nagentic_handoff = true\n{setting}\n", vgr_config());
2553 let parsed: DeploymentConfig = toml::from_str(&source).expect("valid VGR config");
2554 let AlgorithmSpec::Vgr { config } = &parsed.routes["vgr"].algorithm else {
2555 panic!("expected VGR route");
2556 };
2557 assert_eq!(config.compact_handoff, setting.ends_with("= true"));
2558 assert!(runner_from_toml(&source)?.route("switchyard/vgr").is_some());
2559 }
2560 Ok(())
2561 }
2562
2563 #[test]
2564 fn vgr_route_builds_and_rejects_unsafe_configuration() -> RunnerResult<()> {
2565 let runner = runner_from_toml(&vgr_config())?;
2566 assert!(runner.route("switchyard/vgr").is_some());
2567
2568 let compact = format!("{}\ncompact_handoff = true\n", vgr_config());
2569 assert!(error_message(&compact).contains("compact_handoff requires agentic_handoff"));
2570 let compact = format!("{compact}\nagentic_handoff = true\n");
2571 assert!(
2572 runner_from_toml(&compact)?
2573 .route("switchyard/vgr")
2574 .is_some()
2575 );
2576
2577 let identical_ids = vgr_config().replace(
2578 "efficient_target = \"weak\"",
2579 "efficient_target = \"strong\"",
2580 );
2581 assert!(error_message(&identical_ids).contains("distinct efficient and capable model IDs"));
2582
2583 let bad_approval =
2584 vgr_config().replace("prospective-validation-and-canary-approved", "approved");
2585 assert!(error_message(&bad_approval).contains("approval attestation"));
2586
2587 let missing_target = vgr_config().replace(
2588 "efficient_target = \"weak\"",
2589 "efficient_target = \"missing\"",
2590 );
2591 assert!(error_message(&missing_target).contains("unknown target missing"));
2592
2593 let zero_cooldown = format!("{}breaker_cooldown_seconds = 0\n", vgr_config());
2594 assert!(
2595 error_message(&zero_cooldown).contains("breaker_cooldown_seconds must be a positive")
2596 );
2597 Ok(())
2598 }
2599
2600 #[test]
2601 fn vgr_local_server_example_builds_and_rejects_unknown_discovery() -> RunnerResult<()> {
2602 let example = include_str!("../../../examples/vgr-local-servers.toml");
2603 let runner = runner_from_toml(example)?;
2604 assert!(runner.route("switchyard/vgr").is_some());
2605 assert!(runner.route("switchyard/only-efficient").is_some());
2606 let parsed: DeploymentConfig = toml::from_str(example).expect("valid VGR example");
2607 assert!(matches!(
2608 &parsed.routes["auxiliary"].algorithm,
2609 AlgorithmSpec::Passthrough { target, .. } if target == "efficient"
2610 ));
2611 let invalid = example.replace(
2612 "context_discovery = \"llama_cpp\"",
2613 "context_discovery = \"guess\"",
2614 );
2615 assert!(error_message(&invalid).contains("unknown variant `guess`"));
2616 Ok(())
2617 }
2618
2619 fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String {
2620 let mut configured = config.to_string();
2621 configured.push_str(&format!("\n[routes.{route}.subagents]\n"));
2622 configured.push_str(
2623 r#"type = "llm_classifier"
2624mode = "custom"
2625models = { judge = ["classifier"], capable = ["strong"], efficient = ["weak"], any = ["strong", "weak"] }
2626default_target = "efficient"
2627prompt = "Select a target for this delegated task."
2628response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["capable","efficient"]}},"required":["target"],"additionalProperties":false}'
2629policy = { type = "target_selector", selector = "/target" }
2630classify_trigger = "new_session""#,
2631 );
2632 configured.push_str(extra);
2633 configured
2634 }
2635
2636 #[test]
2637 fn subagent_models_stay_separate_from_the_parent_tiers() -> RunnerResult<()> {
2638 let runner = runner_from_toml(&with_subagent_passthrough(&stage_config(), "stage"))?;
2642 let models = runner
2643 .route("switchyard/stage")
2644 .expect("stage route should exist")
2645 .models();
2646
2647 assert_eq!(
2648 models.subagent_models_for(&Category::Any),
2649 [ModelId::from("strong")]
2650 );
2651 assert_eq!(
2652 models.models_for(&Category::Any),
2653 [ModelId::from("strong"), ModelId::from("weak")]
2654 );
2655 Ok(())
2656 }
2657
2658 fn with_subagent_passthrough(config: &str, route: &str) -> String {
2659 format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"")
2660 }
2661
2662 fn stage_config() -> String {
2663 format!(
2664 r#"{VALID_CONFIG}
2665[targets.stage_judge]
2666id = "stage-judge/model"
2667llm_client = "primary"
2668
2669[routes.stage]
2670id = "switchyard/stage"
2671type = "stage_router"
2672capable_target = "strong"
2673efficient_target = "weak"
2674picker = "efficient_first"
2675confidence_threshold = 1.0
2676capable_hold_turns = 2
2677
2678[routes.stage.tool_semantics]
2679observe = ["lookup_customer"]
2680mutate = ["send_payment"]
2681plan = ["create_workflow"]
2682new = ["send_message"]
2683
2684[routes.stage.classifier]
2685target = "stage_judge"
2686base_threshold = 0.5
2687"#
2688 )
2689 }
2690
2691 fn composite_config() -> String {
2692 format!(
2693 r#"{VALID_CONFIG}
2694[targets.tier_judge]
2695id = "tier-judge/model"
2696llm_client = "primary"
2697
2698[routes.composed]
2699id = "switchyard/hier"
2700type = "composite"
2701
2702[routes.composed.classifier]
2703target = "tier_judge"
2704base_threshold = 0.5
2705classify_trigger = "user_turn"
2706
2707[routes.composed.stage]
2708capable_target = "strong"
2709efficient_target = "weak"
2710confidence_threshold = 0.5
2711capable_hold_turns = 2
2712
2713[routes.composed.stage.tool_semantics]
2714new = ["send_message"]
2715"#
2716 )
2717 }
2718
2719 #[test]
2720 fn stage_and_composite_validate_hold_turns() {
2721 let composite =
2722 composite_config().replace("capable_hold_turns = 2", "capable_hold_turns = 0");
2723 for source in [
2724 stage_config(),
2725 stage_config().replace("efficient_first", "capable_first"),
2726 composite_config(),
2727 format!("{composite}\n[routes.composed.efficient]\ncapable_hold_turns = 2"),
2728 format!("{composite}\n[routes.composed.capable]\ncapable_hold_turns = 2"),
2729 ] {
2730 for field in ["capable_hold_turns", "efficient_hold_turns"] {
2731 for (turns, valid) in [
2732 (-2_i64, false),
2733 (-1, true),
2734 (0, true),
2735 (3, true),
2736 (i64::from(u32::MAX), true),
2737 (i64::from(u32::MAX) + 1, false),
2738 ] {
2739 let configured =
2740 source.replace("capable_hold_turns = 2", &format!("{field} = {turns}"));
2741 let result = AlgorithmFactory::from_toml(&configured)
2742 .and_then(|factory| factory.build());
2743 assert_eq!(result.is_ok(), valid, "{field} = {turns}");
2744 if let Err(error) = result {
2745 assert!(error.to_string().contains(field), "{error}");
2746 }
2747 }
2748 }
2749 }
2750 }
2751
2752 #[test]
2753 fn composite_decision_judge_reuses_capability_validation() {
2754 let source = composite_config().replace(
2755 "[routes.composed.classifier]\ntarget = \"tier_judge\"\nbase_threshold = 0.5",
2756 "[routes.composed.classifier]\ntarget = \"tier_judge\"",
2757 ) + r#"
2758[routes.composed.classifier.decision]
2759cutoff = 0.4
2760instructions = "candidate comparison"
2761"#;
2762 for (source, valid) in [
2763 (source.clone(), true),
2764 (
2765 source.replace(
2766 "target = \"tier_judge\"",
2767 "target = \"tier_judge\"\nbase_threshold = 0.5",
2768 ),
2769 false,
2770 ),
2771 (source.replace("cutoff = 0.4", "cutoff = 1.1"), false),
2772 ] {
2773 let deployment: DeploymentConfig = toml::from_str(&source).unwrap();
2774 let spec = &deployment.routes["composed"].algorithm;
2775 assert_eq!(spec.classifier_targets().next().unwrap().0, "tier_judge");
2776 assert_eq!(
2777 spec.build_with_runtime_models("composed", &deployment.build_targets())
2778 .is_ok(),
2779 valid
2780 );
2781 }
2782 }
2783
2784 #[test]
2785 fn decision_target_judges_use_default_decision_settings() -> RunnerResult<()> {
2786 let source = format!(
2787 r#"{VALID_CONFIG}
2788[decision_clients.decisions]
2789format = "openai_decisions"
2790endpoint = "https://example.test/v1/decisions"
2791api_key_env = "PATH"
2792timeout_ms = 5000
2793
2794[decision_targets.judge]
2795id = "decision/judge"
2796decision_client = "decisions"
2797
2798[routes.judged]
2799id = "switchyard/judged"
2800type = "llm_classifier"
2801classifier_target = "judge"
2802strong_target = "strong"
2803weak_target = "weak"
2804
2805[routes.staged]
2806id = "switchyard/staged"
2807type = "stage_router"
2808capable_target = "strong"
2809efficient_target = "weak"
2810picker = "efficient_first"
2811confidence_threshold = 1.0
2812
2813[routes.staged.classifier]
2814target = "judge"
2815
2816[routes.composed]
2817id = "switchyard/composed"
2818type = "composite"
2819
2820[routes.composed.classifier]
2821target = "judge"
2822classify_trigger = "user_turn"
2823
2824[routes.composed.stage]
2825capable_target = "strong"
2826efficient_target = "weak"
2827
2828[routes.composed.efficient.classifier]
2829target = "judge"
2830"#
2831 );
2832 let factory = AlgorithmFactory::from_toml(&source)?;
2833 let routes = &factory.config.routes;
2834 for route in ["judged", "staged", "composed"] {
2835 let judges: Vec<_> = routes[route].algorithm.classifier_targets().collect();
2836 assert!(!judges.is_empty(), "{route}");
2837 assert!(
2838 judges
2839 .iter()
2840 .all(|(target, decision)| *target == "judge" && decision.is_some()),
2841 "{route}"
2842 );
2843 }
2844 assert!(
2846 routes["classifier"]
2847 .algorithm
2848 .classifier_targets()
2849 .all(|(_, d)| d.is_none())
2850 );
2851 factory.build()?;
2852 runner_from_toml(&source)?;
2853 Ok(())
2854 }
2855
2856 #[test]
2857 fn composite_custom_judge_builds_and_rejects_capability_settings() -> RunnerResult<()> {
2858 let source = composite_config().replace(
2859 "base_threshold = 0.5\nclassify_trigger",
2860 r#"mode = "custom"
2861prompt = 'Select efficient or capable.'
2862response_schema = '{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["efficient","capable"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}'
2863policy = { type = "target_selector", selector = "/decision/target" }
2864classify_trigger"#,
2865 );
2866 runner_from_toml(&source)?;
2867 runner_from_toml(
2868 &source.replace("mode = \"custom\"", "mode = \"custom\"\nfail_open = true"),
2869 )?;
2870 for (setting, expected) in [
2871 (
2872 "base_threshold = 0.5",
2873 "composite custom classifier cannot use capability thresholds",
2874 ),
2875 (
2876 "fail_open = false",
2877 "composite custom classifiers always fail open",
2878 ),
2879 ] {
2880 let invalid = source.replace(
2881 "mode = \"custom\"",
2882 &format!("mode = \"custom\"\n{setting}"),
2883 );
2884 assert!(error_message(&invalid).contains(expected));
2885 }
2886 Ok(())
2887 }
2888
2889 #[test]
2890 fn composite_route_builds_and_claims_both_tiers_and_its_judge() -> RunnerResult<()> {
2891 let runner = runner_from_toml(&composite_config())?;
2892 assert!(
2893 runner
2894 .models()
2895 .any(|model| model.id.as_str() == "switchyard/hier")
2896 );
2897 Ok(())
2898 }
2899
2900 #[test]
2901 fn builds_all_supported_algorithm_types() -> RunnerResult<()> {
2902 let state = runner_from_toml(VALID_CONFIG)?;
2903 assert_eq!(
2905 state
2906 .models()
2907 .map(|model| model.id.as_str())
2908 .collect::<Vec<_>>(),
2909 [
2910 "switchyard/classifier",
2911 "switchyard/noop",
2912 "switchyard/passthrough",
2913 "switchyard/plan-execute",
2914 "switchyard/random",
2915 ]
2916 );
2917 Ok(())
2918 }
2919
2920 #[test]
2921 fn stage_rejects_ambiguous_or_non_additive_tool_semantics() {
2922 for (configured, expected) in [
2923 (
2924 stage_config().replace(
2925 "mutate = [\"send_payment\"]",
2926 "mutate = [\"LOOKUP_CUSTOMER\"]",
2927 ),
2928 "appears in both tool_semantics.observe and tool_semantics.mutate",
2929 ),
2930 (
2931 stage_config().replace("new = [\"send_message\"]", "new = [\"Read\"]"),
2932 "already has built-in semantics and cannot be reclassified",
2933 ),
2934 (
2935 stage_config().replace("observe = [\"lookup_customer\"]", "observe = [\" \"]"),
2936 "contains an empty tool name",
2937 ),
2938 ] {
2939 let message = error_message(&configured);
2940 assert!(
2941 message.contains(expected),
2942 "expected {expected:?} in error, got: {message}"
2943 );
2944 }
2945 }
2946
2947 #[test]
2948 fn passthrough_and_stage_accept_subagent_routing() -> RunnerResult<()> {
2949 let stage = stage_config();
2950 let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", "");
2951 let parsed: DeploymentConfig = toml::from_str(&stage_with_classifier).map_err(|error| {
2952 RunnerError::configuration(format!("failed to parse stage config: {error}"))
2953 })?;
2954 let Some(stage_route) = parsed.routes.get("stage") else {
2955 return Err(RunnerError::configuration("stage route is missing"));
2956 };
2957 let callable_targets = stage_route.callable_target_names();
2958 for expected in ["strong", "weak", "stage_judge", "classifier"] {
2959 assert!(callable_targets.contains(&expected));
2960 }
2961
2962 for configured in [
2963 with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""),
2964 with_subagent_passthrough(VALID_CONFIG, "passthrough"),
2965 stage_with_classifier,
2966 with_subagent_passthrough(&stage, "stage"),
2967 ] {
2968 runner_from_toml(&configured)?;
2969 }
2970 Ok(())
2971 }
2972
2973 #[test]
2975 fn rejects_prompted_answer_target_used_as_routing_only_target() {
2976 let configured = VALID_CONFIG
2977 .replace(
2978 "classifier_target = \"classifier\"",
2979 "classifier_target = \"weak\"",
2980 )
2981 .replace(
2982 "id = \"weak/model\"\nllm_client = \"anthropic\"",
2983 "id = \"weak/model\"\nllm_client = \"anthropic\"\nsystem_prompt = \"answer prompt\"",
2984 )
2985 .replace(
2986 "base_threshold = 0.5",
2987 "base_threshold = 0.5\nescalation = { confirmations = 1 }",
2988 );
2989
2990 assert_eq!(
2991 error_message(&configured),
2992 "route classifier cannot apply system_prompt to target weak: it is also the routing-only target"
2993 );
2994 }
2995
2996 #[test]
2997 fn rejects_invalid_unreferenced_llm_client() {
2998 for (settings, expected) in [
2999 (
3000 "base_url = \"not a url\"",
3001 "base_url must be an absolute HTTP(S) URL",
3002 ),
3003 (
3004 "base_url = \"https://example.test\"\ntimeout_ms = 0",
3005 "timeout_ms must be at least 1",
3006 ),
3007 ] {
3008 let invalid = format!(
3009 "{VALID_CONFIG}\n[llm_clients.unused]\nformat = \"openai_chat\"\n{settings}"
3010 );
3011 let message = error_message(&invalid);
3012 assert!(message.contains(expected), "unexpected error: {message}");
3013 }
3014 }
3015
3016 #[test]
3017 fn an_escalation_table_switches_the_classifier_route_to_escalation() -> RunnerResult<()> {
3018 let escalating = VALID_CONFIG.replace(
3022 "base_threshold = 0.5",
3023 "base_threshold = 0.5\nescalation = { confirmations = 2 }",
3024 );
3025 runner_from_toml(&escalating)?;
3026
3027 let reversible = VALID_CONFIG.replace(
3028 "base_threshold = 0.5",
3029 "base_threshold = 0.5\nescalation = { confirmations = 2, deescalation = { strong_min_calls = 3, confirmations = 2, strong_max_calls = 6, weak_cooldown_calls = 8 } }",
3030 );
3031 runner_from_toml(&reversible)?;
3032
3033 let starved = VALID_CONFIG.replace(
3036 "base_threshold = 0.5",
3037 "base_threshold = 0.5\nescalation = { confirmations = 0 }",
3038 );
3039 assert!(error_message(&starved).contains("confirmations must be at least 1"));
3040
3041 Ok(())
3042 }
3043
3044 #[test]
3045 fn unmatched_steps_is_a_capability_setting() -> RunnerResult<()> {
3046 runner_from_toml(&VALID_CONFIG.replace(
3048 "base_threshold = 0.5",
3049 "base_threshold = 0.5\nunmatched_steps = 2",
3050 ))?;
3051 assert!(
3052 error_message(&VALID_CONFIG.replace(
3053 "base_threshold = 0.5",
3054 "base_threshold = 0.5\nunmatched_steps = 3",
3055 ))
3056 .contains("unmatched_steps must be 0, 1, or 2")
3057 );
3058
3059 assert!(
3063 error_message(&VALID_CONFIG.replace(
3064 "base_threshold = 0.5",
3065 "base_threshold = 0.5\nunmatched_steps = 2\nescalation = { confirmations = 2 }",
3066 ))
3067 .contains("mode escalation cannot use capability routing settings")
3068 );
3069 Ok(())
3070 }
3071
3072 #[test]
3073 fn a_target_reasoning_effort_parses_on_every_client_format() -> RunnerResult<()> {
3074 let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";
3075 let weak = "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\"";
3076 assert!(VALID_CONFIG.contains(strong) && VALID_CONFIG.contains(weak));
3077
3078 let forced = VALID_CONFIG.replace(strong, &format!("{strong}\nreasoning_effort = \"max\""));
3079 runner_from_toml(&forced)?;
3080
3081 let blank = VALID_CONFIG.replace(strong, &format!("{strong}\nreasoning_effort = \" \""));
3082 assert!(error_message(&blank).contains("reasoning_effort must not be empty"));
3083
3084 let anthropic = VALID_CONFIG.replace(weak, &format!("{weak}\nreasoning_effort = \"high\""));
3085 runner_from_toml(&anthropic)?;
3086
3087 let minimal =
3088 VALID_CONFIG.replace(weak, &format!("{weak}\nreasoning_effort = \"minimal\""));
3089 assert!(
3090 error_message(&minimal).contains("is not an anthropic_messages effort"),
3091 "{}",
3092 error_message(&minimal)
3093 );
3094 Ok(())
3095 }
3096
3097 #[test]
3098 fn target_reasoning_effort_map_checks_the_mapped_value() -> RunnerResult<()> {
3099 let weak = "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\"";
3100 let config = |setting| VALID_CONFIG.replace(weak, &format!("{weak}\n{setting}"));
3101 runner_from_toml(&config(r#"reasoning_effort_map = { minimal = "low" }"#))?;
3102
3103 for (setting, expected) in [
3104 (
3105 "reasoning_effort_map = { minimal = \"low\" }\nreasoning_effort = \"high\"",
3106 "cannot set both reasoning_effort and reasoning_effort_map",
3107 ),
3108 (
3109 r#"reasoning_effort_map = { minimal = "" }"#,
3110 "reasoning_effort_map keys and values must not be empty",
3111 ),
3112 (
3113 r#"reasoning_effort_map = { low = "minimal" }"#,
3114 "reasoning_effort_map value \"minimal\" is not an anthropic_messages effort",
3115 ),
3116 ] {
3117 let message = error_message(&config(setting));
3118 assert!(message.contains(expected), "{message}");
3119 }
3120 Ok(())
3121 }
3122
3123 #[test]
3124 fn decision_target_ids_must_not_match_llm_target_names() {
3125 let configured = format!(
3126 "{VALID_CONFIG}\n[decision_clients.decisions]\nformat = \"system_one\"\nendpoint = \"https://example.test/decisions\"\napi_key_env = \"PATH\"\ntimeout_ms = 5000\n\n[decision_targets.judge]\nid = \"strong\"\ndecision_client = \"decisions\"\n"
3127 );
3128 let message = error_message(&configured);
3129 assert!(
3130 message.contains("decision target judge id strong is also an LLM target name"),
3131 "{message}"
3132 );
3133 }
3134
3135 #[test]
3136 fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> {
3137 let capability = VALID_CONFIG.replace(
3138 "base_threshold = 0.5",
3139 "base_threshold = 0.5\nmax_output_tokens = 512",
3140 );
3141 runner_from_toml(&capability)?;
3142
3143 let escalation = VALID_CONFIG.replace(
3144 "base_threshold = 0.5",
3145 "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",
3146 );
3147 runner_from_toml(&escalation)?;
3148 Ok(())
3149 }
3150
3151 #[test]
3152 fn classifier_prompts_are_configurable_in_both_modes() -> RunnerResult<()> {
3153 let capability = VALID_CONFIG.replace(
3154 "base_threshold = 0.5",
3155 "base_threshold = 0.5\nprompt = \"custom capability rubric\"",
3156 );
3157 runner_from_toml(&capability)?;
3158
3159 let escalation = VALID_CONFIG.replace(
3160 "base_threshold = 0.5",
3161 "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }",
3162 );
3163 runner_from_toml(&escalation)?;
3164
3165 let empty = VALID_CONFIG.replace(
3166 "base_threshold = 0.5",
3167 "base_threshold = 0.5\nprompt = \" \"",
3168 );
3169 assert!(error_message(&empty).contains("classifier prompt must not be empty"));
3170
3171 let schema_placeholder = VALID_CONFIG.replace(
3172 "base_threshold = 0.5",
3173 "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"",
3174 );
3175 assert!(
3176 error_message(&schema_placeholder)
3177 .contains("Switchyard supplies the schema automatically")
3178 );
3179 Ok(())
3180 }
3181
3182 #[test]
3183 fn mode_custom_rejects_capability_fields() {
3184 let mixed = VALID_CONFIG.replace(
3185 "base_threshold = 0.5",
3186 "mode = \"custom\"\nbase_threshold = 0.5",
3187 );
3188
3189 assert!(
3190 error_message(&mixed)
3191 .contains("mode custom cannot use capability or escalation fields")
3192 );
3193 }
3194
3195 #[test]
3196 fn stage_router_rejects_an_unknown_field() {
3197 let config = stage_config().replace(
3198 "picker = \"efficient_first\"",
3199 "picker = \"efficient_first\"\nmagic = true",
3200 );
3201 assert!(error_message(&config).contains("unknown field"));
3202 }
3203
3204 #[test]
3205 fn auto_route_builds_a_stage_router_with_no_extra_fields() -> RunnerResult<()> {
3206 let config = format!(
3207 r#"{VALID_CONFIG}
3208[routes.auto]
3209id = "switchyard/auto"
3210type = "auto"
3211capable_target = "strong"
3212efficient_target = "weak"
3213"#
3214 );
3215 let runner = runner_from_toml(&config)?;
3216 assert!(runner.route("switchyard/auto").is_some());
3217 Ok(())
3218 }
3219
3220 #[test]
3221 fn composite_stage_block_rejects_an_unknown_field() {
3222 let config = composite_config().replace(
3223 "confidence_threshold = 0.5",
3224 "confidence_threshold = 0.5\nmagic = true",
3225 );
3226 assert!(error_message(&config).contains("unknown field"));
3227 }
3228
3229 #[test]
3230 fn rejects_unknown_fields_and_algorithm_types() {
3231 let unknown_field =
3232 VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");
3233 assert!(error_message(&unknown_field).contains("unknown field"));
3234
3235 let nested_completion_cap = VALID_CONFIG.replace(
3236 "base_threshold = 0.5",
3237 "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",
3238 );
3239 assert!(error_message(&nested_completion_cap).contains("unknown field"));
3240
3241 let unknown_classifier_field = VALID_CONFIG.replace(
3242 "base_threshold = 0.5",
3243 "base_threshold = 0.5\nclassifier_magic = true",
3244 );
3245 assert!(error_message(&unknown_classifier_field).contains("unknown field"));
3246
3247 let target_capability = VALID_CONFIG.replace(
3248 "llm_client = \"responses\"",
3249 "llm_client = \"responses\"\ncontext_window = 1000000",
3250 );
3251 assert!(error_message(&target_capability).contains("unknown field `context_window`"));
3252
3253 let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");
3254 assert!(error_message(&unknown_algorithm).contains("unknown variant"));
3255 }
3256
3257 #[test]
3258 fn rejects_unknown_stage_classifier_fields() {
3259 let config = format!(
3261 r#"{VALID_CONFIG}
3262
3263[routes.stage]
3264id = "switchyard/stage"
3265type = "stage_router"
3266capable_target = "strong"
3267efficient_target = "weak"
3268picker = "efficient_first"
3269confidence_threshold = 1.0
3270
3271[routes.stage.classifier]
3272target = "classifier"
3273base_threshold = 0.5
3274classifier_magic = true
3275"#
3276 );
3277
3278 let error = error_message(&config);
3279 assert!(
3280 error.contains("unknown field `classifier_magic`"),
3281 "{error}"
3282 );
3283 }
3284
3285 #[test]
3286 fn rejects_invalid_references_and_parameters() {
3287 let cases = [
3288 (
3289 VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""),
3290 "unknown llm client missing",
3291 ),
3292 (
3293 VALID_CONFIG.replace(
3294 "targets = [\"strong\", \"weak\"]",
3295 "targets = [\"strong\", \"missing\"]",
3296 ),
3297 "unknown target missing",
3298 ),
3299 (
3300 VALID_CONFIG.replace(
3301 "targets = [\"strong\", \"weak\"]",
3302 "targets = [\"strong\", \"weak\"]\nweights = [0, 0]",
3303 ),
3304 "at least one weight must be positive",
3305 ),
3306 (
3307 VALID_CONFIG.replace(
3308 "targets = [\"strong\", \"weak\"]",
3309 "targets = [\"strong\", \"strong\"]",
3310 ),
3311 "targets must be unique, strong is repeated",
3312 ),
3313 (
3314 VALID_CONFIG.replace(
3315 "targets = [\"strong\", \"weak\"]",
3316 "targets = [\"strong\", \"weak\"]\nweights = [1]",
3317 ),
3318 "expected 2 weights, got 1",
3319 ),
3320 (
3321 VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),
3322 "base_threshold must be between 0 and 1",
3323 ),
3324 (
3325 VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""),
3326 "route references unknown target",
3327 ),
3328 (
3329 VALID_CONFIG.replace(
3330 "classifier_target = \"classifier\"",
3331 "classifier_target = \"\"",
3332 ),
3333 "route references unknown target",
3334 ),
3335 (
3336 VALID_CONFIG.replace(
3337 "classifier_target = \"classifier\"",
3338 "classifier_target = \" \"",
3339 ),
3340 "route references unknown target",
3341 ),
3342 (
3343 VALID_CONFIG.replace(
3344 "base_threshold = 0.5",
3345 "base_threshold = 0.5\nthreshold_step = -0.1",
3346 ),
3347 "threshold_step must be finite and greater than or equal to 0",
3348 ),
3349 (
3350 VALID_CONFIG.replace(
3351 "base_threshold = 0.5",
3352 "base_threshold = 0.8\nthreshold_step = 0.11",
3353 ),
3354 "base_threshold + 2 * threshold_step must be at most 1",
3355 ),
3356 (
3357 VALID_CONFIG.replace(
3358 "base_threshold = 0.5",
3359 "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",
3360 ),
3361 "max_output_tokens must be at least 1",
3362 ),
3363 (
3364 VALID_CONFIG.replace(
3365 "base_threshold = 0.5",
3366 "base_threshold = 0.5\nmessage_hash_fallback = true",
3367 ),
3368 "message_hash_fallback requires classify_trigger = new_session",
3369 ),
3370 (
3371 with_subagent_llm_classifier(
3372 VALID_CONFIG,
3373 "passthrough",
3374 "\nmessage_hash_fallback = true",
3375 ),
3376 "cannot use message_hash_fallback",
3377 ),
3378 (
3379 with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "")
3380 .replace("mode = \"custom\"", "mode = \"capability\""),
3381 "mode capability cannot use custom classifier fields",
3382 ),
3383 (
3384 VALID_CONFIG.replace(
3385 "base_threshold = 0.5",
3386 "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"",
3387 ),
3388 "mode escalation cannot use classify_trigger",
3389 ),
3390 (
3391 VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"),
3392 "unsupported schema_version 2",
3393 ),
3394 (
3395 VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),
3396 "target name must be non-empty and have no surrounding whitespace",
3397 ),
3398 (
3399 VALID_CONFIG.replace(
3400 "targets = [\"strong\", \"weak\"]",
3401 "targets = [\"strong\", \"weak\"]\ncontext_window = 0",
3402 ),
3403 "route random context_window must be greater than zero",
3404 ),
3405 ];
3406
3407 for (toml, expected) in cases {
3408 let error = AlgorithmFactory::from_toml(&toml)
3409 .and_then(|factory| factory.build())
3410 .err()
3411 .expect("invalid configuration should fail")
3412 .to_string();
3413 assert!(
3414 error.contains(expected),
3415 "expected error containing {expected}, got {error}"
3416 );
3417 }
3418 }
3419
3420 #[test]
3421 fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> {
3422 const SAME_MODEL_TWO_ROUTES: &str = r#"
3427schema_version = 1
3428
3429[llm_clients.primary]
3430format = "openai_chat"
3431base_url = "https://example.test/v1"
3432
3433[targets.fast]
3434id = "gpt-4o"
3435llm_client = "primary"
3436
3437[targets.smart]
3438id = "gpt-4o"
3439llm_client = "primary"
3440
3441[routes.fast]
3442id = "switchyard/fast"
3443type = "passthrough"
3444target = "fast"
3445
3446[routes.smart]
3447id = "switchyard/smart"
3448type = "passthrough"
3449target = "smart"
3450"#;
3451 let state = runner_from_toml(SAME_MODEL_TWO_ROUTES)?;
3452 assert_eq!(
3453 state
3454 .models()
3455 .map(|model| model.id.as_str())
3456 .collect::<Vec<_>>(),
3457 ["switchyard/fast", "switchyard/smart"]
3458 );
3459 Ok(())
3460 }
3461
3462 #[test]
3463 fn accepts_same_model_id_on_different_llm_clients() -> RunnerResult<()> {
3464 const CROSS_PROVIDER: &str = r#"
3466schema_version = 1
3467
3468[llm_clients.openai]
3469format = "openai_chat"
3470base_url = "https://example.test/v1"
3471
3472[llm_clients.azure]
3473format = "openai_chat"
3474base_url = "https://azure.test/v1"
3475
3476[targets.openai]
3477id = "gpt-4o"
3478llm_client = "openai"
3479
3480[targets.azure]
3481id = "gpt-4o"
3482llm_client = "azure"
3483
3484[routes.openai]
3485id = "switchyard/openai-gpt4o"
3486type = "passthrough"
3487target = "openai"
3488
3489[routes.azure]
3490id = "switchyard/azure-gpt4o"
3491type = "passthrough"
3492target = "azure"
3493"#;
3494 runner_from_toml(CROSS_PROVIDER)?;
3495 Ok(())
3496 }
3497
3498 #[test]
3499 fn accepts_relative_weights_and_seed() -> RunnerResult<()> {
3500 let weighted = VALID_CONFIG.replace(
3501 "targets = [\"strong\", \"weak\"]",
3502 "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42",
3503 );
3504 runner_from_toml(&weighted)?;
3505 Ok(())
3506 }
3507
3508 #[test]
3509 fn accepts_new_session_trigger_with_message_hash_fallback() -> RunnerResult<()> {
3510 let configured = VALID_CONFIG.replace(
3511 "base_threshold = 0.5",
3512 "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true",
3513 );
3514 runner_from_toml(&configured)?;
3515 Ok(())
3516 }
3517
3518 #[test]
3519 fn target_extra_body_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> {
3520 let configured = VALID_CONFIG.replacen(
3521 "llm_client = \"primary\"",
3522 "llm_client = \"primary\"\n\
3523 extra_body = { service_tier = \"priority\", \
3524 chat_template_kwargs = { enable_thinking = false } }",
3525 1,
3526 );
3527 let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {
3528 RunnerError::configuration(format!("failed to parse config: {error}"))
3529 })?;
3530 let Some(target) = config.targets.get("classifier") else {
3531 return Err(RunnerError::configuration("classifier target is missing"));
3532 };
3533 let Some(client) = config.llm_clients.get("primary") else {
3534 return Err(RunnerError::configuration("primary llm client is missing"));
3535 };
3536 let backend = build_backend(
3537 "primary",
3538 client,
3539 &target.extra_body,
3540 &target.omit_body_fields,
3541 None,
3542 )?;
3543
3544 assert_eq!(
3545 backend.extra_body().get("service_tier"),
3546 Some(&json!("priority"))
3547 );
3548 assert_eq!(
3549 backend
3550 .extra_body()
3551 .get("chat_template_kwargs")
3552 .and_then(|value| value.get("enable_thinking")),
3553 Some(&json!(false))
3554 );
3555 Ok(())
3556 }
3557
3558 #[test]
3559 fn target_omit_body_fields_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> {
3560 let configured = VALID_CONFIG.replacen(
3561 "llm_client = \"primary\"",
3562 "llm_client = \"primary\"\n\
3563 omit_body_fields = [\"max_output_tokens\"]",
3564 1,
3565 );
3566 let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {
3567 RunnerError::configuration(format!("failed to parse config: {error}"))
3568 })?;
3569 let Some(target) = config.targets.get("classifier") else {
3570 return Err(RunnerError::configuration("classifier target is missing"));
3571 };
3572 let Some(client) = config.llm_clients.get("primary") else {
3573 return Err(RunnerError::configuration("primary llm client is missing"));
3574 };
3575 let backend = build_backend(
3576 "primary",
3577 client,
3578 &target.extra_body,
3579 &target.omit_body_fields,
3580 None,
3581 )?;
3582
3583 assert!(backend.omit_body_fields().contains("max_output_tokens"));
3584 Ok(())
3585 }
3586
3587 #[test]
3588 fn client_deadline_defaults_and_rejects_zero() -> RunnerResult<()> {
3589 for (setting, expected) in [
3590 ("", None),
3591 ("timeout_ms = 1500", Some(1500)),
3592 ("timeout_ms = 0", Some(0)),
3593 ] {
3594 let source = format!(
3595 "format = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\n{setting}"
3596 );
3597 let config: LlmClientConfig = toml::from_str(&source).expect("valid deadline config");
3598 let backend = build_backend("test", &config, &BTreeMap::new(), &BTreeSet::new(), None);
3599 if expected == Some(0) {
3600 assert!(backend.is_err());
3601 } else {
3602 assert_eq!(backend?.timeout(), expected.map(Duration::from_millis));
3603 }
3604 }
3605 Ok(())
3606 }
3607
3608 #[test]
3609 fn retry_budget_defaults_and_accepts_an_override() -> RunnerResult<()> {
3610 let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| {
3611 RunnerError::configuration(format!("failed to parse default config: {error}"))
3612 })?;
3613 let Some(primary) = default.llm_clients.get("primary") else {
3614 return Err(RunnerError::configuration("primary llm client is missing"));
3615 };
3616 assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);
3617
3618 let explicit = VALID_CONFIG.replacen(
3619 "base_url = \"https://example.test/v1\"",
3620 "base_url = \"https://example.test/v1\"\nmax_retries = 0",
3621 1,
3622 );
3623 let config: DeploymentConfig = toml::from_str(&explicit).map_err(|error| {
3624 RunnerError::configuration(format!("failed to parse explicit retry config: {error}"))
3625 })?;
3626 let Some(primary) = config.llm_clients.get("primary") else {
3627 return Err(RunnerError::configuration("primary llm client is missing"));
3628 };
3629 assert_eq!(primary.max_retries, 0);
3630
3631 let maximum = VALID_CONFIG.replacen(
3632 "base_url = \"https://example.test/v1\"",
3633 &format!(
3634 "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"
3635 ),
3636 1,
3637 );
3638 let config: DeploymentConfig = toml::from_str(&maximum).map_err(|error| {
3639 RunnerError::configuration(format!("failed to parse maximum retry config: {error}"))
3640 })?;
3641 let Some(primary) = config.llm_clients.get("primary") else {
3642 return Err(RunnerError::configuration("primary llm client is missing"));
3643 };
3644 assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);
3645 Ok(())
3646 }
3647
3648 #[test]
3649 fn responses_reasoning_defaults_and_accepts_drop() -> RunnerResult<()> {
3650 let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| {
3651 RunnerError::configuration(format!("failed to parse config: {error}"))
3652 })?;
3653 let responses = default
3654 .llm_clients
3655 .get("responses")
3656 .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?;
3657 assert_eq!(responses.responses_reasoning, None);
3658
3659 let configured = VALID_CONFIG.replacen(
3660 "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"",
3661 "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"",
3662 1,
3663 );
3664 let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {
3665 RunnerError::configuration(format!("failed to parse config: {error}"))
3666 })?;
3667 let responses = config
3668 .llm_clients
3669 .get("responses")
3670 .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?;
3671 assert_eq!(
3672 responses.responses_reasoning,
3673 Some(ResponsesReasoningPolicy::Drop)
3674 );
3675 runner_from_toml(&configured)?;
3676 Ok(())
3677 }
3678
3679 #[test]
3680 fn responses_reasoning_rejects_other_formats() {
3681 let invalid = VALID_CONFIG.replacen(
3682 "format = \"openai_chat\"",
3683 "format = \"openai_chat\"\nresponses_reasoning = \"drop\"",
3684 1,
3685 );
3686 assert!(
3687 error_message(&invalid)
3688 .contains("responses_reasoning is only valid for openai_responses")
3689 );
3690 }
3691
3692 #[test]
3693 fn responses_reasoning_rejects_unreferenced_other_format() {
3694 let invalid = format!(
3695 "{VALID_CONFIG}\n[llm_clients.unused]\nformat = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"\n"
3696 );
3697 assert!(
3698 error_message(&invalid)
3699 .contains("responses_reasoning is only valid for openai_responses")
3700 );
3701 }
3702
3703 #[test]
3704 fn rejects_headers_that_switchyard_sets() {
3705 let cases = [
3706 (
3707 "base_url = \"https://example.test/v1\"",
3708 "base_url = \"https://example.test/v1\"\n\
3709 extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }",
3710 "AUTHORIZATION",
3711 ),
3712 (
3713 "base_url = \"https://example.test\"",
3714 "base_url = \"https://example.test\"\n\
3715 extra_headers = { \"X-Api-Key\" = \"custom-key\" }",
3716 "X-Api-Key",
3717 ),
3718 (
3719 "base_url = \"https://example.test\"",
3720 "base_url = \"https://example.test\"\n\
3721 extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }",
3722 "ANTHROPIC-VERSION",
3723 ),
3724 ];
3725
3726 for (original, replacement, header) in cases {
3727 let configured = VALID_CONFIG.replacen(original, replacement, 1);
3728 let error = error_message(&configured);
3729 assert!(
3730 error.contains(&format!("extra_headers cannot set {header:?}")),
3731 "expected {header} to be rejected, got: {error}"
3732 );
3733 }
3734 }
3735
3736 #[test]
3737 fn accepts_additional_headers() -> RunnerResult<()> {
3738 let configured = VALID_CONFIG.replacen(
3739 "base_url = \"https://example.test/v1\"",
3740 "base_url = \"https://example.test/v1\"\n\
3741 extra_headers = { X-Inference-Priority = \"batch\" }",
3742 1,
3743 );
3744
3745 runner_from_toml(&configured)?;
3746 Ok(())
3747 }
3748
3749 #[test]
3750 fn retry_budget_rejects_negative_values() {
3751 let invalid = VALID_CONFIG.replacen(
3752 "base_url = \"https://example.test/v1\"",
3753 "base_url = \"https://example.test/v1\"\nmax_retries = -1",
3754 1,
3755 );
3756 assert!(error_message(&invalid).contains("max_retries"));
3757 }
3758
3759 #[test]
3760 fn retry_budget_rejects_excessive_values() {
3761 let invalid = VALID_CONFIG.replacen(
3762 "base_url = \"https://example.test/v1\"",
3763 "base_url = \"https://example.test/v1\"\nmax_retries = 11",
3764 1,
3765 );
3766 assert!(
3767 error_message(&invalid).contains("llm client primary max_retries must be at most 10")
3768 );
3769 }
3770
3771 #[test]
3772 fn api_key_environment_reference_is_validated() {
3773 let missing = VALID_CONFIG.replacen(
3774 "base_url = \"https://example.test/v1\"",
3775 "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"",
3776 1,
3777 );
3778 assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET"));
3779
3780 const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY";
3781 unsafe {
3782 std::env::set_var(EMPTY_KEY_ENV, "");
3784 }
3785 let empty = VALID_CONFIG.replacen(
3786 "base_url = \"https://example.test/v1\"",
3787 &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""),
3788 1,
3789 );
3790 let message = error_message(&empty);
3791 unsafe {
3792 std::env::remove_var(EMPTY_KEY_ENV);
3793 }
3794 assert!(message.contains("is empty"));
3795
3796 let with_equals = VALID_CONFIG.replacen(
3799 "base_url = \"https://example.test/v1\"",
3800 "base_url = \"https://example.test/v1\"\napi_key_env = \"NVIDIA_API_KEY=DUMMY_SECRET_DO_NOT_LOG\"",
3801 1,
3802 );
3803 let message = error_message(&with_equals);
3804 assert!(message.contains("llm client primary api_key_env must not contain '=' or NUL"));
3805 assert!(!message.contains("DUMMY_SECRET_DO_NOT_LOG"), "{message}");
3806 }
3807
3808 #[cfg(any(unix, windows))]
3810 #[test]
3811 fn invalid_unicode_api_key_is_not_printed() {
3812 const RAW_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_RAW_KEY";
3813 #[cfg(unix)]
3814 let raw = {
3815 use std::os::unix::ffi::OsStringExt;
3816 let mut bytes = b"DUMMY_SECRET_DO_NOT_LOG".to_vec();
3817 bytes.push(0xff);
3818 std::ffi::OsString::from_vec(bytes)
3819 };
3820 #[cfg(windows)]
3821 let raw = {
3822 use std::os::windows::ffi::OsStringExt;
3823 let mut words: Vec<_> = "DUMMY_SECRET_DO_NOT_LOG".encode_utf16().collect();
3824 words.push(0xd800);
3825 std::ffi::OsString::from_wide(&words)
3826 };
3827 unsafe {
3828 std::env::set_var(RAW_KEY_ENV, raw);
3830 }
3831 let result = read_api_key("llm client fixture", RAW_KEY_ENV);
3832 unsafe {
3833 std::env::remove_var(RAW_KEY_ENV);
3834 }
3835 let error = result.expect_err("invalid Unicode credential").to_string();
3836 assert!(error.contains("not valid Unicode"), "{error}");
3837 assert!(!error.contains("DUMMY_SECRET_DO_NOT_LOG"), "{error}");
3838 }
3839
3840 #[test]
3841 fn forward_auth_rejects_conflicting_credentials() {
3842 let competing_auth = VALID_CONFIG.replacen(
3843 "base_url = \"https://example.test/v1\"",
3844 "base_url = \"https://example.test/v1\"\n\
3845 forward_auth = true\n\
3846 api_key_env = \"UNUSED_TEST_KEY\"",
3847 1,
3848 );
3849 assert!(
3850 error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env")
3851 );
3852
3853 let static_auth = VALID_CONFIG.replacen(
3854 "base_url = \"https://example.test\"",
3855 "base_url = \"https://example.test\"\n\
3856 forward_auth = true\n\
3857 extra_headers = { Authorization = \"static-value\" }",
3858 1,
3859 );
3860 assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\""));
3861
3862 let static_beta = static_auth.replace("Authorization", "anthropic-beta");
3863 assert!(
3864 error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"")
3865 );
3866
3867 for header in ["chatgpt-account-id", "x-openai-fedramp"] {
3868 let static_context = VALID_CONFIG.replacen(
3869 "base_url = \"https://example.test/v1\"",
3870 &format!(
3871 "base_url = \"https://example.test/v1\"\n\
3872 forward_auth = true\n\
3873 extra_headers = {{ \"{header}\" = \"static-value\" }}"
3874 ),
3875 1,
3876 );
3877 assert!(
3878 error_message(&static_context)
3879 .contains(&format!("extra_headers cannot set \"{header}\""))
3880 );
3881 }
3882 }
3883
3884 fn mixed_forwarding_config(messages_url: &str) -> String {
3887 format!(
3888 r#"
3889schema_version = 1
3890
3891[llm_clients.responses]
3892format = "openai_responses"
3893base_url = "https://gateway.example.test/v1"
3894forward_auth = true
3895
3896[llm_clients.messages]
3897format = "anthropic_messages"
3898base_url = "{messages_url}"
3899forward_auth = true
3900
3901[targets]
3902gpt = {{ id = "gpt/model", llm_client = "responses" }}
3903claude = {{ id = "claude/model", llm_client = "messages" }}
3904
3905[routes.mixed]
3906id = "switchyard/mixed"
3907type = "random"
3908targets = ["gpt", "claude"]
3909
3910[routes.claude]
3911id = "switchyard/claude"
3912type = "passthrough"
3913target = "claude"
3914"#
3915 )
3916 }
3917
3918 #[test]
3921 fn forwarding_route_mixes_formats_only_on_one_host() -> RunnerResult<()> {
3922 let runner = runner_from_toml(&mixed_forwarding_config("https://gateway.example.test"))?;
3925 let caller_auth = |id| runner.route(id).and_then(Route::caller_auth);
3926 assert_eq!(
3927 caller_auth("switchyard/mixed"),
3928 Some(CallerAuthKind::OpenAi)
3929 );
3930 assert_eq!(
3931 caller_auth("switchyard/claude"),
3932 Some(CallerAuthKind::Anthropic)
3933 );
3934
3935 for other in [
3937 "https://api.anthropic.test",
3938 "https://gateway.example.test:8443",
3939 "http://gateway.example.test",
3940 ] {
3941 let error = error_message(&mixed_forwarding_config(other));
3942 assert!(
3943 error.contains("different origins") && error.contains(other),
3944 "{error}"
3945 );
3946 }
3947 Ok(())
3948 }
3949
3950 const ADVISOR_CONFIG: &str = r#"
3951schema_version = 1
3952
3953[llm_clients.anthropic]
3954format = "anthropic_messages"
3955base_url = "https://example.test"
3956
3957[targets.executor]
3958id = "executor/model"
3959llm_client = "anthropic"
3960
3961[targets.advisor]
3962id = "advisor/model"
3963llm_client = "anthropic"
3964
3965[routes.gated]
3966id = "switchyard/advisor"
3967type = "advisor"
3968executor_target = "executor"
3969advisor_target = "advisor"
3970"#;
3971
3972 #[test]
3973 fn advisor_route_parses_with_defaults_and_builds() -> RunnerResult<()> {
3974 let state = runner_from_toml(ADVISOR_CONFIG)?;
3975 let route = state
3976 .route("switchyard/advisor")
3977 .expect("advisor route should exist");
3978 let models = route.models();
3979 assert_eq!(
3982 models.models_for(&Category::Efficient),
3983 [ModelId::from("executor")]
3984 );
3985 assert_eq!(
3986 models.models_for(&Category::Any),
3987 [ModelId::from("executor")]
3988 );
3989 assert_eq!(
3990 models.models_for(&Category::Judge),
3991 [ModelId::from("advisor")]
3992 );
3993 assert_eq!(
3994 state
3995 .models()
3996 .map(|model| model.id.as_str())
3997 .collect::<Vec<_>>(),
3998 ["switchyard/advisor"]
3999 );
4000 Ok(())
4001 }
4002
4003 #[test]
4004 fn advisor_route_accepts_every_gate_knob() -> RunnerResult<()> {
4005 let tuned = ADVISOR_CONFIG.replace(
4006 "advisor_target = \"advisor\"",
4007 concat!(
4008 "advisor_target = \"advisor\"\n",
4009 "reviewer_system_prompt = \"review it\"\n",
4010 "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",
4011 "gate_trigger = \"pattern\"\n",
4012 "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",
4013 "max_reviews = 2\n",
4014 "gate_stall_turns = 40\n",
4015 "gate_min_tool_results = 1\n",
4016 "advisor_max_tokens = 1024\n",
4017 "advisor_temperature = 0.0\n",
4018 "transcript_max_chars = 100000\n",
4019 "fail_open = false\n",
4020 "context_window = 200000\n",
4021 "tool_calling = true\n",
4022 "reasoning = true",
4023 ),
4024 );
4025 runner_from_toml(&tuned)?;
4026 Ok(())
4027 }
4028
4029 #[test]
4030 fn advisor_route_rejects_unknown_keys() {
4031 let invalid = ADVISOR_CONFIG.replace(
4032 "advisor_target = \"advisor\"",
4033 "advisor_target = \"advisor\"\nbogus_field = 1",
4034 );
4035 assert!(error_message(&invalid).contains("bogus_field"));
4036 }
4037
4038 #[test]
4039 fn advisor_route_requires_both_targets() {
4040 let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");
4041 assert!(error_message(&missing).contains("advisor_target"));
4042 }
4043
4044 #[test]
4045 fn advisor_route_rejects_unknown_target() {
4046 let invalid = ADVISOR_CONFIG.replace(
4047 "advisor_target = \"advisor\"",
4048 "advisor_target = \"missing\"",
4049 );
4050 assert!(error_message(&invalid).contains("missing"));
4051 }
4052
4053 #[test]
4054 fn advisor_route_rejects_invalid_pattern() {
4055 let invalid = ADVISOR_CONFIG.replace(
4056 "advisor_target = \"advisor\"",
4057 "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",
4058 );
4059 assert!(error_message(&invalid).contains("not a valid regex"));
4060 }
4061
4062 #[test]
4063 fn advisor_route_rejects_pattern_without_pattern_trigger() {
4064 let invalid = ADVISOR_CONFIG.replace(
4065 "advisor_target = \"advisor\"",
4066 "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",
4067 );
4068 assert!(
4069 error_message(&invalid)
4070 .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")
4071 );
4072 }
4073
4074 #[test]
4075 fn advisor_route_pattern_trigger_requires_pattern() {
4076 let invalid = ADVISOR_CONFIG.replace(
4077 "advisor_target = \"advisor\"",
4078 "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",
4079 );
4080 assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));
4081 }
4082
4083 #[test]
4084 fn advisor_route_rejects_zero_max_reviews() {
4085 let invalid = ADVISOR_CONFIG.replace(
4086 "advisor_target = \"advisor\"",
4087 "advisor_target = \"advisor\"\nmax_reviews = 0",
4088 );
4089 assert!(error_message(&invalid).contains("max_reviews must be at least 1"));
4090 }
4091}