1use std::collections::{HashMap, HashSet};
22use std::sync::Arc;
23use std::time::Instant;
24
25use futures::{StreamExt, TryStreamExt, stream};
26use http::StatusCode;
27use parking_lot::Mutex;
28use serde_json::{Value, json};
29use switchyard_libsy::{
30 Algorithm, AnswerSummary, Call, CallDecision, LibsyError, OutcomeMetadata, Result,
31 RoutingOutcome, RuntimeModels, drive,
32};
33use switchyard_protocol::{
34 AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,
35 ModelId, Request, Response, ResponseAccumulator, RoutedDecisionClient, RoutedLlmClient,
36 RoutingFallbackReason, Usage, WireFormat,
37};
38use switchyard_translation::prepare_request_for_target;
39
40use crate::observation::{ModelCallObservation, RunObservation, RunObserver};
41use crate::{metrics, observability};
42
43pub async fn run(
57 algorithm: Arc<dyn Algorithm>,
58 clients: ClientRouter,
59 request: Request,
60 models: Arc<RuntimeModels>,
61 observer: Option<RunObserver>,
62) -> Result<(ModelId, Response)> {
63 let algorithm_name = algorithm.name().to_string();
64 let run_started = Instant::now();
65 let routing_clients = clients.clone();
66 let routing_observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new())));
69 let outcome = match clients.stored_state_owner(&request) {
70 Some(owner) => Ok(continue_on(owner, &algorithm_name, request)),
71 None => {
72 drive(Arc::clone(&algorithm), request, models, {
73 let routing_observations = routing_observations.clone();
74 move |call| serve(routing_clients.clone(), call, routing_observations.clone())
75 })
76 .await
77 }
78 };
79 let answered_model = outcome
80 .as_ref()
81 .ok()
82 .and_then(|outcome| outcome.response.as_ref())
83 .and_then(Response::served_model);
84 emit_routing_observations(&observer, &routing_observations, answered_model);
85 let outcome = outcome?;
86 let overhead = run_started.elapsed();
87 metrics::record_routing_overhead(&algorithm_name, overhead);
88
89 let selected_model_id = outcome.selected_model_id()?.clone();
90 if let Some(observer) = &observer
91 && let Some(metadata) = outcome.metadata
92 {
93 observer(RunObservation::Outcome(metadata));
94 }
95 let result = if let Some(response) = outcome.response {
96 let result = clients.remember_state_owner(&outcome.request, response);
97 let served_model = result
98 .as_ref()
99 .ok()
100 .and_then(Response::served_model)
101 .unwrap_or(&selected_model_id);
102 let served_model = clients.provider_model(served_model).clone();
103 metrics::observe_routed_request(&algorithm_name, &served_model, None, result)
104 } else {
105 let observe = |observation| {
106 if let Some(observer) = &observer {
107 observer(RunObservation::AnswerCall(observation));
108 }
109 };
110 call_first_available(
111 &clients,
112 &algorithm_name,
113 &outcome.request,
114 outcome.fallback_request.as_ref(),
115 &outcome.candidate_requests,
116 &outcome.selected_model_ids,
117 &observe,
118 )
119 .await
120 .and_then(|(request, response)| clients.remember_state_owner(request, response))
121 };
122 if let Some(observer) = &observer {
123 observer(RunObservation::RoutingOverhead(overhead));
124 }
125 let mut response = result?;
126 if algorithm.tracks_answers()
127 && let Some(session) = session_id(&outcome.request)
128 {
129 response = report_answer(algorithm, session, response).await;
130 }
131 Ok((selected_model_id, response))
132}
133
134fn session_id(request: &Request) -> Option<String> {
135 let session = request.metadata.as_ref()?.session_id.as_deref()?;
136 (!session.is_empty()).then(|| session.to_string())
137}
138
139async fn report_answer(
142 algorithm: Arc<dyn Algorithm>,
143 session: String,
144 mut response: Response,
145) -> Response {
146 let stream = match response.llm_response {
147 LlmResponse::Agg(aggregate) => {
148 algorithm
149 .observe_answer(&session, AnswerSummary::of(&aggregate))
150 .await;
151 response.llm_response = LlmResponse::Agg(aggregate);
152 return response;
153 }
154 LlmResponse::Stream(stream) => stream,
155 };
156 algorithm
157 .observe_answer(&session, AnswerSummary { final_reply: false })
158 .await;
159 let report = (algorithm, session, Some(ResponseAccumulator::new()));
160 response.llm_response = LlmResponse::Stream(Box::pin(stream::unfold(
161 (stream, report),
162 |(mut stream, (algorithm, session, mut accumulator))| async move {
163 let item = stream.next().await?;
164 let failed = match &item {
165 Err(_) => true,
166 Ok(event) => event
167 .normalized()
168 .iter()
169 .any(|chunk| matches!(chunk, LlmResponseChunk::StreamError { .. })),
170 };
171 if failed {
172 accumulator = None;
173 algorithm
174 .observe_answer(&session, AnswerSummary { final_reply: false })
175 .await;
176 }
177 let mut stopped = false;
178 if let (Ok(event), Some(answer)) = (&item, accumulator.as_mut()) {
179 for chunk in event.normalized() {
180 stopped |= matches!(chunk, LlmResponseChunk::MessageStop { .. });
181 answer.push(chunk.clone());
182 }
183 }
184 if stopped && let Some(answer) = accumulator.take() {
185 let answer = AnswerSummary::of(&answer.finish());
186 algorithm.observe_answer(&session, answer).await;
187 }
188 Some((item, (stream, (algorithm, session, accumulator))))
189 },
190 )));
191 response
192}
193
194pub async fn decide(
199 algorithm: Arc<dyn Algorithm>,
200 clients: ClientRouter,
201 request: Request,
202 models: Arc<RuntimeModels>,
203) -> Result<RoutingOutcome> {
204 let routing_clients = clients.clone();
205 let mut outcome = match clients.stored_state_owner(&request) {
206 Some(owner) => continue_on(owner, algorithm.name(), request),
207 None => {
208 drive(algorithm, request, models, move |call| {
209 serve(routing_clients.clone(), call, None)
210 })
211 .await?
212 }
213 };
214 let selected_model_id = outcome.selected_model_id()?.clone();
215 outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
216 outcome.response = outcome
217 .response
218 .map(|response| clients.remember_state_owner(&outcome.request, response))
219 .transpose()?;
220 Ok(outcome)
221}
222
223#[tracing::instrument(
224 name = "libsy.decision_client_call",
225 skip_all,
226 fields(algorithm = %call.algorithm, selected_model = %call.model, outcome = tracing::field::Empty),
227)]
228async fn serve_decision(
229 clients: &ClientRouter,
230 call: CallDecision,
231 observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,
232) -> Result<()> {
233 let client = clients.route_decision(&call.model);
234 let model = call.model.clone();
235 let request = call.request.clone();
236 call.respond(async {
237 let started = Instant::now();
238 let result = async { client?.call(request).await }.await;
239 tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });
240 if let Some(observations) = observations {
241 observations
242 .lock()
243 .push(RunObservation::DecisionCall(ModelCallObservation {
244 selected_model: model.clone(),
245 is_success: result.is_ok(),
246 duration: started.elapsed(),
247 usage: result.as_ref().ok().map(|response| response.usage.clone()),
248 }));
249 }
250 result.map_err(|error| LibsyError::client_call(model, error))
251 })
252 .await
253}
254
255fn emit_routing_observations(
257 observer: &Option<RunObserver>,
258 observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,
259 answered_model: Option<&ModelId>,
260) {
261 let (Some(observer), Some(observations)) = (observer, observations) else {
262 return;
263 };
264 let mut answer_observed = false;
265 for observation in observations.lock().drain(..) {
266 observer(match observation {
267 RunObservation::LlmCall(call)
268 if !answer_observed && answered_model == Some(&call.selected_model) =>
269 {
270 answer_observed = true;
271 RunObservation::AnswerCall(call)
272 }
273 observation => observation,
274 });
275 }
276}
277
278async fn serve(
283 clients: ClientRouter,
284 call: Call,
285 observations: Option<Arc<Mutex<Vec<RunObservation>>>>,
286) -> Result<()> {
287 let call = match call {
288 Call::Model(call) => *call,
289 Call::Decision(call) => return serve_decision(&clients, *call, &observations).await,
290 };
291 let observe = |observation| {
292 if let Some(observations) = &observations {
293 observations
294 .lock()
295 .push(RunObservation::LlmCall(observation));
296 }
297 };
298 let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
299 let request = clients.prepare_routing_request(call.request.clone(), &target);
300 let algorithm = call.algorithm.clone();
301 let candidates = call.models.len();
302 call.respond(call_one(
303 &clients,
304 &target,
305 request,
306 &algorithm,
307 &observe,
308 0,
309 candidates,
310 CallPhase::Routing,
311 ))
312 .await
313}
314
315async fn call_first_available<'a>(
317 clients: &ClientRouter,
318 algorithm: &str,
319 request: &'a Request,
320 fallback_request: Option<&'a Request>,
321 candidate_requests: &'a HashMap<ModelId, Request>,
322 models: &[ModelId],
323 observe: &(dyn Fn(ModelCallObservation) + Send + Sync),
324) -> Result<(&'a Request, Response)> {
325 for (index, target) in models.iter().enumerate() {
326 let request = candidate_requests
327 .get(target)
328 .unwrap_or(match fallback_request {
329 Some(fallback) if index > 0 => fallback,
330 _ => request,
331 });
332 let prepared = clients.prepare_completion_request(request.clone(), target);
333 match call_one(
334 clients,
335 target,
336 prepared,
337 algorithm,
338 observe,
339 index,
340 models.len(),
341 CallPhase::Completion,
342 )
343 .await
344 {
345 Ok(response) => return Ok((request, response)),
346 Err(error) if index + 1 == models.len() => return Err(error),
347 Err(error)
348 if clients.rate_limit_fallback_disabled.contains(target)
349 && matches!(&error, LibsyError::ClientCall {
350 source: LlmClientError::UpstreamHttp { status, .. }, ..
351 } if *status == StatusCode::TOO_MANY_REQUESTS) =>
352 {
353 return Err(error);
354 }
355 Err(error) => match fallback_reason(&error, fallback_request.is_none()) {
356 Some(reason) => tracing::info!(
357 from = %target,
358 to = %models[index + 1],
359 reason = reason.as_str(),
360 "model call failed; trying next candidate"
361 ),
362 None => return Err(error),
363 },
364 }
365 }
366 Err(LibsyError::NoTargets)
367}
368
369#[derive(Clone, Copy)]
374enum CallPhase {
375 Routing,
376 Completion,
377}
378
379impl CallPhase {
380 const fn as_str(self) -> &'static str {
381 match self {
382 Self::Routing => "routing",
383 Self::Completion => "completion",
384 }
385 }
386}
387
388#[allow(clippy::too_many_arguments)]
390#[tracing::instrument(
391 target = "libsy",
392 name = "libsy.client_call",
393 skip_all,
394 fields(
395 algorithm = algorithm,
396 switchyard.algorithm = algorithm,
397 switchyard.candidate = index + 1,
398 switchyard.candidate_count = count,
399 switchyard.call_phase = phase.as_str(),
400 selected_model = %clients.provider_model(model_id),
401 selected_target = %model_id,
402 otel.kind = "client",
403 otel.name = %format_args!("chat {}", clients.provider_model(model_id)),
404 openinference.span.kind = "LLM",
405 gen_ai.operation.name = "chat",
406 gen_ai.request.model = %clients.provider_model(model_id),
407 gen_ai.request.stream = tracing::field::Empty,
408 gen_ai.request.temperature = tracing::field::Empty,
409 gen_ai.request.top_p = tracing::field::Empty,
410 gen_ai.request.top_k = tracing::field::Empty,
411 gen_ai.request.max_tokens = tracing::field::Empty,
412 gen_ai.request.reasoning.level = tracing::field::Empty,
413 gen_ai.output.type = tracing::field::Empty,
414 gen_ai.conversation.id = tracing::field::Empty,
415 server.address = tracing::field::Empty,
416 server.port = tracing::field::Empty,
417 gen_ai.response.id = tracing::field::Empty,
418 gen_ai.response.model = tracing::field::Empty,
419 gen_ai.usage.input_tokens = tracing::field::Empty,
420 gen_ai.usage.output_tokens = tracing::field::Empty,
421 gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
422 gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
423 gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty,
424 gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
425 outcome = tracing::field::Empty,
426 otel.status_code = tracing::field::Empty,
427 error.type = tracing::field::Empty,
428 error = tracing::field::Empty,
429 )
430)]
431async fn call_one(
432 clients: &ClientRouter,
433 model_id: &ModelId,
434 request: Request,
435 algorithm: &str,
436 observe: &(dyn Fn(ModelCallObservation) + Send + Sync),
437 index: usize,
439 count: usize,
441 phase: CallPhase,
442) -> Result<Response> {
443 let is_routing = matches!(phase, CallPhase::Routing);
444 let span = tracing::Span::current();
445 observability::record_gen_ai_request(&span, &request.llm_request);
446 if let Some(session_id) = request
447 .metadata
448 .as_ref()
449 .and_then(|metadata| metadata.session_id.as_deref())
450 {
451 span.record("gen_ai.conversation.id", session_id);
452 }
453 let client = clients.route(model_id);
456 let started = Instant::now();
457 let mut routing_usage = None;
458 let result = async {
459 let response = async { client?.call(request).await }
460 .await
461 .map_err(|source| LibsyError::client_call(model_id.clone(), source));
462 let mut response = observability::observe_client_call(response)?;
464 if is_routing && let LlmResponse::Stream(chunks) = response.llm_response {
465 let (chunks, usage) = buffer_routing_stream(chunks, &response.upstream_headers)
466 .await
467 .map_err(|source| LibsyError::client_call(model_id.clone(), source))?;
468 routing_usage = usage;
469 response.llm_response = LlmResponse::Stream(chunks);
470 }
471 Ok(response)
472 }
473 .await;
474 let duration = started.elapsed();
475
476 let result = result.map(|mut response| {
477 response.set_served_model(model_id);
478 response
479 });
480 let result = if is_routing {
481 result
482 } else {
483 metrics::observe_routed_request(
484 algorithm,
485 clients.provider_model(model_id),
486 Some(duration),
487 result,
488 )
489 };
490 observe(ModelCallObservation {
491 selected_model: model_id.clone(),
492 is_success: result.is_ok(),
493 duration,
494 usage: result
495 .as_ref()
496 .ok()
497 .and_then(|response| response.llm_response.as_agg())
498 .map(|response| response.usage.clone())
499 .or(routing_usage),
500 });
501 result
502}
503
504async fn buffer_routing_stream(
506 mut chunks: LlmResponseStream,
507 upstream_headers: &http::HeaderMap,
508) -> std::result::Result<(LlmResponseStream, Option<Usage>), LlmClientError> {
509 let mut events = Vec::new();
510 let mut usage = None;
511 while let Some(event) = chunks.try_next().await? {
512 for chunk in event.normalized() {
513 match chunk {
514 LlmResponseChunk::Usage(latest) => usage = Some(latest.clone()),
515 LlmResponseChunk::DecodeError { message } => {
516 return Err(LlmClientError::ResponseTranslation(message.clone()));
517 }
518 LlmResponseChunk::StreamError { message } => {
519 return Err(LlmClientError::UpstreamHttp {
520 status: StatusCode::BAD_GATEWAY,
521 body: message.clone(),
522 headers: Box::new(upstream_headers.clone()),
523 });
524 }
525 _ => {}
526 }
527 }
528 events.push(event);
529 }
530 Ok((stream::iter(events.into_iter().map(Ok)).boxed(), usage))
531}
532
533fn fallback_reason(
535 error: &LibsyError,
536 allow_policy_fallback: bool,
537) -> Option<RoutingFallbackReason> {
538 let LibsyError::ClientCall { source, .. } = error else {
539 return None;
540 };
541 match source {
542 LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),
543 LlmClientError::Transport { .. } | LlmClientError::TemporarilyUnavailable => {
544 Some(RoutingFallbackReason::Unavailable)
545 }
546 LlmClientError::UpstreamHttp { status, body, .. } => {
547 let policy_denial = serde_json::from_str::<Value>(body).is_ok_and(|value| {
548 value["error"]["code"].as_str() == Some("content_policy_violation")
549 });
550 if policy_denial && !allow_policy_fallback {
551 return None;
552 }
553 ((*status == StatusCode::BAD_REQUEST && policy_denial)
554 || matches!(
555 *status,
556 StatusCode::FORBIDDEN
557 | StatusCode::REQUEST_TIMEOUT
558 | StatusCode::TOO_MANY_REQUESTS
559 )
560 || status.is_server_error())
561 .then_some(RoutingFallbackReason::Unavailable)
562 }
563 _ => None,
564 }
565}
566
567fn continue_on(owner: StateOwner, algorithm: &str, mut request: Request) -> RoutingOutcome {
569 tracing::debug!(
570 target = %owner.model,
571 trigger = owner.trigger,
572 "routing Responses continuation to the model that holds its state"
573 );
574 if let Some(history) = owner.history {
575 let mut messages = Vec::with_capacity(history.len + request.llm_request.messages.len());
576 history.extend(&mut messages);
577 messages.append(&mut request.llm_request.messages);
578 request.llm_request.messages = messages;
579 }
580 let mut outcome = RoutingOutcome::route_to(owner.model, Vec::new(), request);
581 outcome.metadata = Some(OutcomeMetadata::new(
582 algorithm.to_string(),
583 Some(json!({"source": "stored_state", "trigger": owner.trigger})),
584 ));
585 outcome
586}
587
588struct StateOwner {
590 model: ModelId,
591 trigger: &'static str,
592 history: Option<Arc<MessageHistory>>,
593}
594
595struct MessageHistory {
596 parent: Option<Arc<Self>>,
597 segment: Arc<[Message]>,
598 len: usize,
599}
600
601impl MessageHistory {
602 fn new(parent: Option<Arc<Self>>, segment: Arc<[Message]>) -> Self {
603 let len = parent.as_ref().map_or(0, |history| history.len) + segment.len();
604 Self {
605 parent,
606 segment,
607 len,
608 }
609 }
610
611 fn extend(&self, messages: &mut Vec<Message>) {
612 let mut segments = Vec::new();
613 let mut history = Some(self);
614 while let Some(node) = history {
615 segments.push(node.segment.as_ref());
616 history = node.parent.as_deref();
617 }
618 for segment in segments.into_iter().rev() {
619 messages.extend_from_slice(segment);
620 }
621 }
622}
623
624#[derive(Clone)]
625struct StoredState {
626 model: ModelId,
627 history: Option<Arc<MessageHistory>>,
629}
630
631#[derive(Clone)]
632struct CanonicalInput {
633 parent: Option<Arc<MessageHistory>>,
634 messages: Arc<[Message]>,
635}
636
637#[derive(Default)]
642struct StateOwners {
643 by_id: HashMap<String, StoredState>,
644}
645
646const MAX_STATE_OWNERS: usize = 65_536;
647
648impl StateOwners {
649 fn owner(&self, id: &str) -> Option<&StoredState> {
650 self.by_id.get(id)
651 }
652
653 fn remember(
654 &mut self,
655 response_id: Option<&str>,
656 conversation_id: Option<&str>,
657 model: &ModelId,
658 history: Option<Arc<MessageHistory>>,
659 ) -> std::result::Result<(), LlmClientError> {
660 let materialized = history.is_some();
661 let ids = [
662 response_id,
663 conversation_id.filter(|id| Some(*id) != response_id),
664 ];
665 let mut new_ids = 0;
666 for id in ids.into_iter().flatten() {
667 match self.by_id.get(id) {
668 Some(owner) if owner.model != *model && !materialized => {
669 return Err(LlmClientError::ResponseStateConflict);
670 }
671 None => new_ids += 1,
672 _ => {}
673 }
674 }
675 if self.by_id.len() + new_ids > MAX_STATE_OWNERS {
676 return Err(LlmClientError::ResponseStateLimitExceeded {
677 limit: MAX_STATE_OWNERS,
678 });
679 }
680 let state = StoredState {
681 model: model.clone(),
682 history,
683 };
684 if let Some(id) = response_id {
685 if materialized {
686 self.by_id.insert(id.to_owned(), state.clone());
687 } else {
688 self.by_id
689 .entry(id.to_owned())
690 .or_insert_with(|| state.clone());
691 }
692 }
693 if let Some(id) = conversation_id {
694 self.by_id.entry(id.to_owned()).or_insert(state);
695 }
696 Ok(())
697 }
698}
699
700fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {
702 let conversation = fields.get("conversation")?;
703 conversation
704 .as_str()
705 .or_else(|| conversation.get("id").and_then(Value::as_str))
706}
707
708#[derive(Clone)]
716pub struct ClientRouter {
717 inner: Arc<ClientRouting>,
718 rate_limit_fallback_disabled: Arc<HashSet<ModelId>>,
719}
720
721struct ClientRouting {
722 routing: Routing,
723 decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
724 target_prompts: HashMap<ModelId, String>,
725 routing_answer_target: Option<ModelId>,
726 state_owners: Mutex<StateOwners>,
728 track_provider_state: bool,
730}
731
732enum Routing {
733 Single(Arc<dyn RoutedLlmClient>),
735 ByModel(HashMap<ModelId, Arc<dyn RoutedLlmClient>>),
737}
738
739impl ClientRouter {
740 pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
742 Self::new_with_target_prompts(by_model, HashMap::new(), None)
743 }
744
745 pub fn new_with_target_prompts(
754 by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
755 target_prompts: HashMap<ModelId, String>,
756 routing_answer_target: Option<ModelId>,
757 ) -> Self {
758 let completion_targets = by_model.keys().cloned().collect::<Vec<_>>();
759 Self::new_with_completion_targets(
760 by_model,
761 target_prompts,
762 routing_answer_target,
763 &completion_targets,
764 )
765 }
766
767 pub fn new_with_completion_targets(
775 by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
776 target_prompts: HashMap<ModelId, String>,
777 routing_answer_target: Option<ModelId>,
778 completion_targets: &[ModelId],
779 ) -> Self {
780 Self::new_with_decision_clients(
781 by_model,
782 HashMap::new(),
783 target_prompts,
784 routing_answer_target,
785 completion_targets,
786 )
787 }
788
789 pub fn new_with_decision_clients(
792 by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
793 decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
794 target_prompts: HashMap<ModelId, String>,
795 routing_answer_target: Option<ModelId>,
796 completion_targets: &[ModelId],
797 ) -> Self {
798 let mut clients = completion_targets
799 .iter()
800 .filter_map(|model| by_model.get(model));
801 let spans_providers = clients
802 .next()
803 .is_some_and(|first| clients.any(|client| !Arc::ptr_eq(first, client)));
804 Self {
805 rate_limit_fallback_disabled: Arc::default(),
806 inner: Arc::new(ClientRouting {
807 routing: Routing::ByModel(by_model),
808 decision_clients,
809 target_prompts,
810 routing_answer_target,
811 state_owners: Mutex::default(),
812 track_provider_state: spans_providers,
813 }),
814 }
815 }
816
817 pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
823 Self::single_with_decision_clients(client, HashMap::new())
824 }
825
826 pub fn single_with_decision_clients(
828 client: Arc<dyn RoutedLlmClient>,
829 decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
830 ) -> Self {
831 Self {
832 rate_limit_fallback_disabled: Arc::default(),
833 inner: Arc::new(ClientRouting {
834 routing: Routing::Single(client),
835 decision_clients,
836 target_prompts: HashMap::new(),
837 routing_answer_target: None,
838 state_owners: Mutex::default(),
839 track_provider_state: false,
840 }),
841 }
842 }
843
844 pub fn without_rate_limit_fallback(mut self, models: HashSet<ModelId>) -> Self {
850 self.rate_limit_fallback_disabled = Arc::new(models);
851 self
852 }
853
854 fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
856 self.route(target)
857 .map_or(target, |client| client.provider_model(target))
858 }
859
860 pub fn route(
865 &self,
866 model: &ModelId,
867 ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
868 match &self.inner.routing {
869 Routing::Single(client) => Ok(client),
870 Routing::ByModel(by_model) => {
871 by_model
872 .get(model)
873 .ok_or_else(|| LlmClientError::Configuration {
874 message: format!("no llm client is configured for model {model:?}"),
875 })
876 }
877 }
878 }
879
880 fn route_decision(
882 &self,
883 model: &ModelId,
884 ) -> std::result::Result<&Arc<dyn RoutedDecisionClient>, LlmClientError> {
885 self.inner
886 .decision_clients
887 .get(model)
888 .ok_or_else(|| LlmClientError::Configuration {
889 message: format!("no decision client is configured for model {model:?}"),
890 })
891 }
892
893 fn stored_state_owner(&self, request: &Request) -> Option<StateOwner> {
895 let fields = &request.llm_request.extensions.fields;
896 let (trigger, id) = match fields.get("previous_response_id").and_then(Value::as_str) {
897 Some(id) => ("previous_response_id", id),
898 None => ("conversation", conversation_id(fields)?),
899 };
900 let state = self.inner.state_owners.lock().owner(id)?.clone();
901 Some(StateOwner {
902 model: state.model,
903 trigger,
904 history: state.history,
905 })
906 }
907
908 fn canonical_input(&self, request: &Request) -> CanonicalInput {
909 let parent = request
910 .llm_request
911 .extensions
912 .fields
913 .get("previous_response_id")
914 .and_then(Value::as_str)
915 .and_then(|id| self.inner.state_owners.lock().owner(id)?.history.clone());
916 let parent_len = parent.as_ref().map_or(0, |history| history.len);
917 let (parent, messages) = match request.llm_request.messages.get(parent_len..) {
918 Some(messages) => (parent, messages.to_vec()),
919 None => (None, request.llm_request.messages.clone()),
920 };
921 CanonicalInput {
922 parent,
923 messages: Arc::from(messages),
924 }
925 }
926
927 fn remember_state_owner(&self, request: &Request, mut response: Response) -> Result<Response> {
929 let Some(model) = response.served_model().cloned() else {
930 return Ok(response);
931 };
932 let fields = &request.llm_request.extensions.fields;
933 let store = fields.get("store").and_then(Value::as_bool) != Some(false);
934 let conversation = conversation_id(fields).map(str::to_owned);
935 let responses_format = WireFormat::OpenAiResponses.into();
936 let responses_request = request
937 .llm_request
938 .preservation
939 .requests
940 .contains_key(&responses_format)
941 || fields.contains_key("previous_response_id")
942 || fields.contains_key("conversation");
943 if !responses_request && !self.inner.track_provider_state {
944 return Ok(response);
945 }
946 let canonical_input = responses_request.then(|| self.canonical_input(request));
947 response.llm_response = match response.llm_response {
948 LlmResponse::Agg(agg) => {
949 if let Some(body) = agg.preservation.responses.get(&responses_format) {
950 if self.inner.track_provider_state {
951 self.remember_response(body, &model, store, conversation.as_deref())
952 .map_err(|error| LibsyError::client_call(model.clone(), error))?;
953 }
954 } else if let Some(input) = &canonical_input {
955 self.remember_canonical_response(&agg, &model, store, input)
956 .map_err(|error| LibsyError::client_call(model.clone(), error))?;
957 }
958 LlmResponse::Agg(agg)
959 }
960 LlmResponse::Stream(stream) => {
961 let router = self.clone();
962 LlmResponse::Stream(Box::pin(futures_util::stream::try_unfold(
963 (
964 stream,
965 router,
966 model,
967 conversation,
968 canonical_input,
969 ResponseAccumulator::new(),
970 false,
971 true,
972 ),
973 move |(
974 mut stream,
975 router,
976 model,
977 conversation,
978 canonical_input,
979 mut accumulator,
980 mut native_responses,
981 mut valid,
982 )| async move {
983 let Some(event) = stream.next().await else {
984 if let Some(input) = &canonical_input
985 && !native_responses
986 && valid
987 {
988 router.remember_canonical_response(
989 &accumulator.finish(),
990 &model,
991 store,
992 input,
993 )?;
994 }
995 return Ok(None);
996 };
997 let event = event?;
998 if let Some(preserved) = event.preservation()
999 && preserved.source().as_str() == WireFormat::OpenAiResponses.as_str()
1000 {
1001 native_responses = true;
1002 if router.inner.track_provider_state
1003 && let Some(body) = preserved.raw().get("response")
1004 {
1005 router.remember_response(
1006 body,
1007 &model,
1008 store,
1009 conversation.as_deref(),
1010 )?;
1011 }
1012 }
1013 if responses_request {
1014 for chunk in event.normalized() {
1015 if matches!(
1016 chunk,
1017 LlmResponseChunk::DecodeError { .. }
1018 | LlmResponseChunk::StreamError { .. }
1019 ) {
1020 valid = false;
1021 } else {
1022 accumulator.push(chunk.clone());
1023 }
1024 }
1025 }
1026 Ok(Some((
1027 event,
1028 (
1029 stream,
1030 router,
1031 model,
1032 conversation,
1033 canonical_input,
1034 accumulator,
1035 native_responses,
1036 valid,
1037 ),
1038 )))
1039 },
1040 )))
1041 }
1042 };
1043 Ok(response)
1044 }
1045
1046 fn remember_response(
1049 &self,
1050 body: &Value,
1051 model: &ModelId,
1052 store: bool,
1053 conversation: Option<&str>,
1054 ) -> std::result::Result<(), LlmClientError> {
1055 let mut owners = self.inner.state_owners.lock();
1056 let response_id = body
1057 .get("id")
1058 .and_then(Value::as_str)
1059 .filter(|_| body.get("store").and_then(Value::as_bool).unwrap_or(store));
1060 let conversation = body.as_object().and_then(conversation_id).or(conversation);
1061 if let Err(error) = owners.remember(response_id, conversation, model, None) {
1062 if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
1063 tracing::warn!(%error, "Responses state owner capacity reached; state was not retained");
1064 } else {
1065 tracing::error!(%error, "could not record Responses state owner");
1066 return Err(error);
1067 }
1068 }
1069 Ok(())
1070 }
1071
1072 fn remember_canonical_response(
1073 &self,
1074 response: &AggLlmResponse,
1075 model: &ModelId,
1076 store: bool,
1077 input: &CanonicalInput,
1078 ) -> std::result::Result<(), LlmClientError> {
1079 let response_id = response.id.as_deref().filter(|_| store);
1080 if response_id.is_none() {
1081 return Ok(());
1082 }
1083 let mut segment = input.messages.to_vec();
1084 segment.extend(
1085 response
1086 .outputs
1087 .iter()
1088 .map(|output| Message::new(output.role, output.content.clone())),
1089 );
1090 let history = Arc::new(MessageHistory::new(
1091 input.parent.clone(),
1092 Arc::from(segment),
1093 ));
1094 let result =
1095 self.inner
1096 .state_owners
1097 .lock()
1098 .remember(response_id, None, model, Some(history));
1099 if let Err(error) = result {
1100 if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
1101 tracing::warn!(%error, "cross-format Responses state capacity reached; history was not retained");
1102 return Ok(());
1103 }
1104 tracing::error!(%error, "could not record cross-format Responses state");
1105 return Err(error);
1106 }
1107 Ok(())
1108 }
1109
1110 fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request {
1112 let prompt = self.inner.target_prompts.get(target).map(String::as_str);
1113 prepare_request_for_target(&mut request.llm_request, target, prompt);
1114 request
1115 }
1116
1117 fn prepare_routing_request(&self, mut request: Request, target: &ModelId) -> Request {
1119 let prompt = if self.inner.routing_answer_target.as_ref() == Some(target) {
1120 self.inner.target_prompts.get(target).map(String::as_str)
1121 } else {
1122 None
1123 };
1124 prepare_request_for_target(&mut request.llm_request, target, prompt);
1125 request
1126 }
1127}
1128
1129impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
1130 fn from_iter<I: IntoIterator<Item = (ModelId, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
1131 Self::new(iter.into_iter().collect())
1132 }
1133}
1134
1135#[cfg(test)]
1136mod tests {
1137 use std::collections::{BTreeMap, BTreeSet};
1138
1139 use super::*;
1140 use async_trait::async_trait;
1141 use futures::StreamExt;
1142 use http::StatusCode;
1143 use switchyard_libsy::{Driver, RoutingOutcome};
1144 use switchyard_protocol::{
1145 ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Role,
1146 ToolResult, completion_text, text_request, text_response,
1147 };
1148 use wiremock::matchers::method;
1149 use wiremock::{Mock, MockServer, ResponseTemplate};
1150
1151 use crate::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
1152 use switchyard_protocol::Category;
1153
1154 struct CandidateAlgorithm {}
1155
1156 struct AnsweredAlgorithm {
1157 model: ModelId,
1158 }
1159
1160 #[async_trait]
1161 impl Algorithm for CandidateAlgorithm {
1162 fn name(&self) -> &str {
1163 "candidate_test"
1164 }
1165
1166 async fn route(
1167 self: Arc<Self>,
1168 driver: Driver,
1169 request: Request,
1170 ) -> Result<RoutingOutcome> {
1171 let models = driver.models_for(&Category::Any);
1172 let selected_model = models.first().cloned().ok_or(LibsyError::NoTargets)?;
1173 Ok(RoutingOutcome::route_to(
1174 selected_model,
1175 models.iter().skip(1).cloned().collect(),
1176 request,
1177 ))
1178 }
1179 }
1180
1181 #[async_trait]
1182 impl Algorithm for AnsweredAlgorithm {
1183 fn name(&self) -> &str {
1184 "answered_test"
1185 }
1186
1187 async fn route(
1188 self: Arc<Self>,
1189 driver: Driver,
1190 request: Request,
1191 ) -> Result<RoutingOutcome> {
1192 let response = driver
1193 .call_model(request.clone(), vec![self.model.clone()])
1194 .await?;
1195 Ok(RoutingOutcome::answered(
1196 self.model.clone(),
1197 request,
1198 response,
1199 ))
1200 }
1201 }
1202
1203 struct RewritingAlgorithm {}
1205
1206 #[async_trait]
1207 impl Algorithm for RewritingAlgorithm {
1208 fn name(&self) -> &str {
1209 "rewriting_test"
1210 }
1211
1212 async fn route(
1213 self: Arc<Self>,
1214 _driver: Driver,
1215 request: Request,
1216 ) -> Result<RoutingOutcome> {
1217 let mut rewritten = request.clone();
1218 rewritten
1219 .llm_request
1220 .messages
1221 .push(Message::text(Role::User, "for weak only"));
1222 let mut outcome =
1223 RoutingOutcome::route_to("weak".into(), vec!["strong".into()], rewritten);
1224 outcome.fallback_request = Some(request);
1225 Ok(outcome)
1226 }
1227 }
1228
1229 #[derive(Clone, Copy)]
1230 enum FirstOutcome {
1231 ContextWindow,
1232 ContentPolicy(StatusCode),
1233 Unauthorized,
1234 StreamSuccess,
1235 MidStreamError,
1236 }
1237
1238 struct CandidateClient {
1239 calls: Mutex<Vec<ModelId>>,
1240 requests: Mutex<Vec<Request>>,
1241 first: FirstOutcome,
1242 }
1243
1244 #[async_trait]
1245 impl RoutedLlmClient for CandidateClient {
1246 async fn call(&self, request: Request) -> std::result::Result<Response, LlmClientError> {
1247 let model = request.model_id().unwrap_or_default();
1248 self.calls.lock().push(model.clone());
1249 self.requests.lock().push(request);
1250 if model == "weak" {
1251 return match self.first {
1252 FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded {
1253 model,
1254 message: "too long".to_string(),
1255 }),
1256 FirstOutcome::ContentPolicy(status) => Err(LlmClientError::UpstreamHttp {
1257 status,
1258 body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),
1259 headers: Box::default(),
1260 }),
1261 FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {
1262 status: StatusCode::UNAUTHORIZED,
1263 body: "unauthorized".to_string(),
1264 headers: Box::default(),
1265 }),
1266 FirstOutcome::StreamSuccess => Ok(stream_response(vec![
1267 LlmResponseChunk::TextDelta {
1268 index: 0,
1269 text: "streamed".to_string(),
1270 },
1271 LlmResponseChunk::MessageStop {
1272 reason: Some("stop".to_string()),
1273 },
1274 ])),
1275 FirstOutcome::MidStreamError => Ok(stream_response(vec![
1276 LlmResponseChunk::TextDelta {
1277 index: 0,
1278 text: "partial".to_string(),
1279 },
1280 LlmResponseChunk::StreamError {
1281 message: "stream failed".to_string(),
1282 },
1283 LlmResponseChunk::MessageStop { reason: Some("stop".into()) },
1284 ])),
1285 };
1286 }
1287 Ok(Response {
1288 llm_response: LlmResponse::Agg(AggLlmResponse {
1289 id: Some("candidate_response".into()),
1290 ..text_response(Some(model.to_string()), model)
1291 }),
1292 metadata: None,
1293 upstream_headers: http::HeaderMap::new(),
1294 })
1295 }
1296 }
1297
1298 fn stream_response(chunks: Vec<LlmResponseChunk>) -> Response {
1299 Response {
1300 llm_response: LlmResponse::Stream(
1301 futures::stream::iter(
1302 chunks
1303 .into_iter()
1304 .map(|chunk| Ok(LlmResponseStreamEvent::from(chunk))),
1305 )
1306 .boxed(),
1307 ),
1308 metadata: None,
1309 upstream_headers: http::HeaderMap::new(),
1310 }
1311 }
1312
1313 fn request() -> Request {
1314 Request {
1315 llm_request: text_request(Some("auto".to_string()), "hello".to_string()),
1316 raw_request: None,
1317 metadata: None,
1318 }
1319 }
1320
1321 #[tokio::test]
1322 async fn system_one_serves_decisions_and_returns_failures_to_the_classifier()
1323 -> std::result::Result<(), Box<dyn std::error::Error>> {
1324 use std::time::Duration;
1325 use switchyard_libsy::{
1326 CapabilityJudgeConfig, DecisionJudgeConfig, LlmClassifierConfig, LlmTaskClassifier,
1327 TaskClassifierConfig,
1328 };
1329 use switchyard_protocol::{
1330 BooleanEstimate, DecisionRequest, DecisionValue, Probability, ScoreValue,
1331 };
1332 use wiremock::matchers::{body_partial_json, header, path};
1333
1334 let server = MockServer::start().await;
1335 let client = Arc::new(crate::SystemOneClient::new(
1336 format!("{}/v1/systemone", server.uri()).parse()?,
1337 "test-key".into(),
1338 Duration::from_secs(2),
1339 )?);
1340 assert!(matches!(
1341 crate::SystemOneClient::new(
1342 format!("{}/v1/systemone", server.uri()).parse()?,
1343 "invalid\nkey".into(),
1344 Duration::from_secs(2),
1345 ),
1346 Err(LlmClientError::Configuration { .. })
1347 ));
1348 let mixed: DecisionRequest = serde_json::from_value(json!({
1349 "model": "judge", "input": [Message::text(Role::User, "A simple task")],
1350 "questions": {
1351 "boolean": {"instructions": "Is it simple?", "kind": {"type": "boolean", "data": {
1352 "true_description": ["Simple"], "false_description": null
1353 }}},
1354 "route": {"instructions": "Choose", "kind": {"type": "choice", "data": {
1355 "options": [{"id": "capable", "description": "A wins"}, {"id": "efficient", "description": "B wins"}]
1356 }}},
1357 "score": {"instructions": "Rate difficulty", "kind": {"type": "score", "data": {
1358 "levels": ["Easy", {"description": "Hard"}]
1359 }}}
1360 }
1361 }))?;
1362 let mut body = json!({
1363 "model": "jev-1.13.0", "usage": {"input_tokens": 42, "output_tokens": 3},
1364 "answers": {
1365 "boolean": {"type": "noul", "noul": 0.9},
1366 "route": {"type": "choice", "choice": "efficient", "confidence": 0.7,
1367 "probabilities": {"capable": 0.2, "efficient": 0.8}},
1368 "score": {"type": "score", "score": 0.6, "confidence": 0.3,
1369 "legend": {"0": "Easy", "1": {"description": "Hard"}},
1370 "probabilities": {"1": 0.6, "0": 0.4}}
1371 }
1372 });
1373 Mock::given(method("POST"))
1374 .and(path("/v1/systemone"))
1375 .and(header("authorization", "Bearer test-key"))
1376 .and(body_partial_json(json!({
1377 "model": "judge", "state": mixed.input,
1378 "questions": {
1379 "boolean": {"type": "noul", "criteria": {"true": ["Simple"]}},
1380 "route": {"type": "choice", "instructions": "Choose",
1381 "criteria": {"capable": "A wins", "efficient": "B wins"}},
1382 "score": {"type": "score", "criteria": ["Easy", {"description": "Hard"}]}
1383 }
1384 })))
1385 .respond_with(ResponseTemplate::new(200).set_body_json(&body))
1386 .expect(1)
1387 .mount(&server)
1388 .await;
1389 let response = client.call(mixed.clone()).await?;
1390 assert_eq!(response.model.as_deref(), Some("jev-1.13.0"));
1391 assert_eq!(response.id, None);
1392 assert_eq!(response.usage.input_tokens, Some(42));
1393 assert_eq!(response.usage.total_tokens, None);
1394 assert_eq!(
1395 response.answers["boolean"].value,
1396 DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(Probability(0.9)))
1397 );
1398 assert_eq!(
1399 response.answers["score"].value,
1400 DecisionValue::Score {
1401 value: ScoreValue(0.6),
1402 probabilities: Some(vec![Probability(0.4), Probability(0.6)]),
1403 }
1404 );
1405 assert_eq!(
1406 response.answers["route"].provider_confidence.map(|c| c.0),
1407 Some(0.7)
1408 );
1409 server.verify().await;
1410
1411 let mut partial = body.clone();
1412 partial["answers"]
1413 .as_object_mut()
1414 .unwrap()
1415 .remove("boolean");
1416 let mut invalid_responses = vec![json!({"answers": {}}), partial];
1417 for (id, answer) in [
1418 ("unknown-question", json!({"type": "noul", "noul": 0.9})),
1419 ("boolean", json!({"type": "score", "score": 0.0})),
1420 (
1421 "route",
1422 json!({"type": "choice", "choice": "unknown-option",
1423 "probabilities": {"capable": 0.2, "efficient": 0.8}}),
1424 ),
1425 ("score", json!({"type": "score", "score": -0.1})),
1426 (
1427 "score",
1428 json!({"type": "score", "score": 1.1,
1429 "probabilities": {"0": 0.4, "1": 0.6}}),
1430 ),
1431 ] {
1432 let mut invalid = body.clone();
1433 invalid["answers"][id] = answer;
1434 if id == "unknown-question" {
1435 invalid["answers"]
1436 .as_object_mut()
1437 .unwrap()
1438 .remove("boolean");
1439 }
1440 invalid_responses.push(invalid);
1441 }
1442 for template in invalid_responses
1443 .into_iter()
1444 .map(|body| ResponseTemplate::new(200).set_body_json(body))
1445 .chain([ResponseTemplate::new(200).set_body_string("invalid JSON")])
1446 {
1447 server.reset().await;
1448 Mock::given(method("POST"))
1449 .respond_with(template)
1450 .expect(1)
1451 .mount(&server)
1452 .await;
1453 assert!(matches!(
1454 client.call(mixed.clone()).await,
1455 Err(LlmClientError::ResponseTranslation(_) | LlmClientError::InvalidResponse { .. })
1456 ));
1457 server.verify().await;
1458 }
1459
1460 body["answers"].as_object_mut().unwrap().remove("boolean");
1461 body["answers"].as_object_mut().unwrap().remove("score");
1462 let models = Arc::new(RuntimeModels::new(HashMap::from([
1463 (Category::Judge, vec!["judge".into()]),
1464 (Category::Capable, vec!["capable".into()]),
1465 (Category::Efficient, vec!["efficient".into()]),
1466 (Category::Any, vec!["efficient".into(), "capable".into()]),
1467 ])));
1468 let algorithm = || -> Result<Arc<dyn Algorithm>> {
1469 Ok(Arc::new(LlmTaskClassifier::new(
1470 LlmClassifierConfig::Capability {
1471 config: TaskClassifierConfig {
1472 judge: CapabilityJudgeConfig::Decision(DecisionJudgeConfig {
1473 cutoff: Some(0.4),
1474 ..DecisionJudgeConfig::default()
1475 }),
1476 ..TaskClassifierConfig::default()
1477 },
1478 },
1479 )?))
1480 };
1481 for (template, expected, is_success) in [
1482 (
1483 ResponseTemplate::new(200).set_body_json(&body),
1484 "efficient",
1485 true,
1486 ),
1487 (
1488 ResponseTemplate::new(503).set_body_string("unavailable"),
1489 "capable",
1490 false,
1491 ),
1492 ] {
1493 server.reset().await;
1494 Mock::given(method("POST"))
1495 .and(path("/v1/systemone"))
1496 .respond_with(template)
1497 .expect(2)
1498 .mount(&server)
1499 .await;
1500 let llm = Arc::new(CandidateClient {
1501 calls: Mutex::default(),
1502 requests: Mutex::default(),
1503 first: FirstOutcome::Unauthorized,
1504 });
1505 let clients = ClientRouter::single_with_decision_clients(
1506 llm.clone(),
1507 HashMap::from([(
1508 ModelId::from("judge"),
1509 client.clone() as Arc<dyn RoutedDecisionClient>,
1510 )]),
1511 );
1512 let outcome = decide(algorithm()?, clients.clone(), request(), models.clone()).await?;
1513 assert_eq!(outcome.selected_model_id()?, expected);
1514 assert!(llm.calls.lock().is_empty());
1515 let events = Arc::new(Mutex::new(Vec::new()));
1516 let captured = events.clone();
1517 let (selected, _) = run(
1518 algorithm()?,
1519 clients,
1520 request(),
1521 models.clone(),
1522 Some(Arc::new(move |event| captured.lock().push(event))),
1523 )
1524 .await?;
1525 assert_eq!(selected, expected);
1526 assert_eq!(&*llm.calls.lock(), &[ModelId::from(expected)]);
1527 {
1528 let events = events.lock();
1529 let RunObservation::DecisionCall(call) = &events[0] else {
1530 panic!("missing decision observation")
1531 };
1532 assert_eq!(call.selected_model, "judge");
1533 assert_eq!(call.is_success, is_success);
1534 assert_eq!(
1535 call.usage.as_ref().and_then(|u| u.input_tokens),
1536 is_success.then_some(42)
1537 );
1538 assert!(
1539 events
1540 .iter()
1541 .any(|event| matches!(event, RunObservation::AnswerCall(_)))
1542 );
1543 }
1544 server.verify().await;
1545 }
1546 Ok(())
1547 }
1548
1549 fn instruction_text(request: &Request) -> Vec<&str> {
1550 request
1551 .llm_request
1552 .instructions
1553 .iter()
1554 .flat_map(|instruction| &instruction.content)
1555 .filter_map(|block| match block {
1556 ContentBlock::Text { text } => Some(text.as_str()),
1557 _ => None,
1558 })
1559 .collect()
1560 }
1561
1562 async fn run_candidates(
1563 first: FirstOutcome,
1564 ) -> (Arc<CandidateClient>, Result<(ModelId, Response)>) {
1565 let client = Arc::new(CandidateClient {
1566 calls: Mutex::new(Vec::new()),
1567 requests: Mutex::new(Vec::new()),
1568 first,
1569 });
1570 let algorithm = Arc::new(CandidateAlgorithm {});
1571 let models = to_category_map(&["weak", "strong"]);
1572 let result = run(
1573 algorithm,
1574 ClientRouter::single(client.clone()),
1575 request(),
1576 models,
1577 None,
1578 )
1579 .await;
1580 (client, result)
1581 }
1582
1583 fn to_category_map(names: &[&str]) -> Arc<RuntimeModels> {
1584 Arc::new(RuntimeModels::new(
1585 [(
1586 Category::Any,
1587 names.iter().map(|name| ModelId::from(*name)).collect(),
1588 )]
1589 .into(),
1590 ))
1591 }
1592
1593 #[tokio::test]
1594 async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> {
1595 let client = Arc::new(CandidateClient {
1596 calls: Mutex::new(Vec::new()),
1597 requests: Mutex::new(Vec::new()),
1598 first: FirstOutcome::StreamSuccess,
1599 });
1600 let observations = Arc::new(Mutex::new(Vec::new()));
1601 let observed = Arc::clone(&observations);
1602 let observer: RunObserver = Arc::new(move |event| observed.lock().push(event));
1603
1604 let (selected, response) = run(
1605 Arc::new(AnsweredAlgorithm {
1606 model: "weak".into(),
1607 }),
1608 ClientRouter::single(client.clone()),
1609 request(),
1610 Arc::new(RuntimeModels::default()),
1611 Some(observer),
1612 )
1613 .await?;
1614
1615 assert_eq!(selected, "weak");
1616 assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
1617 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
1618 let observations = observations.lock();
1619 assert!(matches!(observations[0], RunObservation::AnswerCall(_)));
1620 assert!(matches!(
1621 observations[1],
1622 RunObservation::Outcome(ref metadata) if metadata.algorithm == "answered_test"
1623 ));
1624 assert!(matches!(
1625 observations[2],
1626 RunObservation::RoutingOverhead(_)
1627 ));
1628 assert_eq!(observations.len(), 3);
1629 Ok(())
1630 }
1631
1632 #[tokio::test]
1633 async fn served_answer_is_reported_before_the_client_sees_it_end() -> Result<()> {
1634 #[derive(Default)]
1635 struct ReportingAlgorithm {
1636 answers: Mutex<Vec<(String, bool)>>,
1637 tracking: bool,
1638 }
1639
1640 #[async_trait]
1641 impl Algorithm for ReportingAlgorithm {
1642 fn name(&self) -> &str {
1643 "reporting_test"
1644 }
1645
1646 async fn route(
1647 self: Arc<Self>,
1648 _driver: Driver,
1649 request: Request,
1650 ) -> Result<RoutingOutcome> {
1651 Ok(RoutingOutcome::route_to("weak".into(), Vec::new(), request))
1652 }
1653
1654 fn tracks_answers(&self) -> bool {
1655 self.tracking
1656 }
1657
1658 async fn observe_answer(&self, session_id: &str, answer: AnswerSummary) {
1659 self.answers
1660 .lock()
1661 .push((session_id.to_string(), answer.final_reply));
1662 }
1663 }
1664
1665 assert!(!CandidateAlgorithm {}.tracks_answers());
1666 for (tracking, first, final_reply, cancel) in [
1667 (false, FirstOutcome::StreamSuccess, true, false),
1668 (true, FirstOutcome::StreamSuccess, true, false),
1669 (true, FirstOutcome::MidStreamError, false, false),
1670 (true, FirstOutcome::StreamSuccess, false, true),
1671 ] {
1672 let algorithm = Arc::new(ReportingAlgorithm {
1673 tracking,
1674 ..Default::default()
1675 });
1676 let client = Arc::new(CandidateClient {
1677 calls: Mutex::new(Vec::new()),
1678 requests: Mutex::new(Vec::new()),
1679 first,
1680 });
1681 let request = Request {
1682 metadata: Some(switchyard_protocol::Metadata {
1683 session_id: Some("session-1".to_string()),
1684 ..Default::default()
1685 }),
1686 ..request()
1687 };
1688 let (_, response) = run(
1689 algorithm.clone(),
1690 ClientRouter::single(client),
1691 request,
1692 to_category_map(&["weak"]),
1693 None,
1694 )
1695 .await?;
1696 let LlmResponse::Stream(mut stream) = response.llm_response else {
1697 panic!("expected a streamed answer");
1698 };
1699 if !tracking {
1700 assert!(algorithm.answers.lock().is_empty());
1701 while let Some(item) = stream.next().await {
1702 assert!(item.is_ok());
1703 }
1704 assert!(algorithm.answers.lock().is_empty());
1705 continue;
1706 }
1707 if cancel {
1708 stream.next().await;
1709 drop(stream);
1710 assert_eq!(
1711 *algorithm.answers.lock(),
1712 [("session-1".to_string(), false)]
1713 );
1714 continue;
1715 }
1716 while let Some(item) = stream.next().await {
1717 let stops = item.as_ref().is_ok_and(|event| {
1718 event
1719 .normalized()
1720 .iter()
1721 .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }))
1722 });
1723 if stops {
1724 assert_eq!(algorithm.answers.lock().len(), 2);
1725 }
1726 }
1727 assert_eq!(
1728 *algorithm.answers.lock(),
1729 if final_reply {
1730 vec![
1731 ("session-1".to_string(), false),
1732 ("session-1".to_string(), true),
1733 ]
1734 } else {
1735 vec![
1736 ("session-1".to_string(), false),
1737 ("session-1".to_string(), false),
1738 ]
1739 }
1740 );
1741 }
1742 Ok(())
1743 }
1744
1745 #[test]
1746 fn answer_observation_keeps_call_order() {
1747 let pending = Some(Arc::new(Mutex::new(
1748 ["answer", "judge"]
1749 .map(|model| ModelCallObservation {
1750 selected_model: model.into(),
1751 is_success: true,
1752 duration: std::time::Duration::ZERO,
1753 usage: None,
1754 })
1755 .map(RunObservation::LlmCall)
1756 .into(),
1757 )));
1758 let emitted = Arc::new(Mutex::new(Vec::new()));
1759 let captured = Arc::clone(&emitted);
1760 let observer: RunObserver = Arc::new(move |event| captured.lock().push(event));
1761 let answer = ModelId::from("answer");
1762
1763 emit_routing_observations(&Some(observer), &pending, Some(&answer));
1764
1765 assert!(matches!(
1766 &emitted.lock()[..],
1767 [RunObservation::AnswerCall(answer), RunObservation::LlmCall(judge)]
1768 if answer.selected_model == "answer" && judge.selected_model == "judge"
1769 ));
1770 }
1771
1772 #[tokio::test]
1773 async fn responses_state_tracking_uses_upstream_storage_and_format() -> Result<()> {
1774 for (responses, store, conversation) in [
1775 (true, true, None),
1776 (true, false, None),
1777 (true, false, Some(json!("conv_plain"))),
1778 (true, false, Some(json!({"id": "conv_object"}))),
1779 (false, true, None),
1780 ] {
1781 for stream in [false, true] {
1782 let server = MockServer::start().await;
1783 Mock::given(method("POST"))
1784 .respond_with(move |request: &wiremock::Request| {
1785 let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1786 let response = if responses {
1787 json!({
1788 "id": "resp_seed", "object": "response", "model": "weak",
1789 "status": "completed", "output": [], "store": body["store"],
1790 "conversation": body["conversation"],
1791 })
1792 } else if stream {
1793 json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "delta": {"content": "ok"}, "finish_reason": "stop"}]})
1794 } else {
1795 json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}]})
1796 };
1797 if stream {
1798 let event = if responses { json!({"type": "response.completed", "response": response}) } else { response };
1799 ResponseTemplate::new(200)
1800 .insert_header("content-type", "text/event-stream")
1801 .set_body_string(if responses {
1802 format!("event: response.completed\ndata: {event}\n\n")
1803 } else {
1804 format!("data: {event}\n\ndata: [DONE]\n\n")
1805 })
1806 } else {
1807 ResponseTemplate::new(200).set_body_json(response)
1808 }
1809 })
1810 .mount(&server).await;
1811 let mut clients = HashMap::new();
1812 for model in ["weak", "strong"] {
1813 let config = HttpBackendConfig {
1814 base_url: server.uri(),
1815 api_key: None,
1816 forward_auth: false,
1817 extra_headers: BTreeMap::new(),
1818 extra_body: BTreeMap::from([("store".to_string(), json!(store))]),
1819 omit_body_fields: BTreeSet::new(),
1820 reasoning_effort: None,
1821 max_retries: 0,
1822 failure_cooldown: std::time::Duration::ZERO,
1823 timeout: None,
1824 };
1825 let backend = if responses {
1826 Backend::OpenAiResponses(config)
1827 } else {
1828 Backend::OpenAiChat(config)
1829 };
1830 let client: Arc<dyn RoutedLlmClient> = Arc::new(
1831 TranslatingLlmClient::new(&[ModelConfig::new(model, backend, None)])
1832 .map_err(|error| LibsyError::external("building test client", error))?,
1833 );
1834 clients.insert(ModelId::from(model), client);
1835 }
1836 let clients = ClientRouter::new(clients);
1837 let mut seed = request();
1838 seed.llm_request.stream = stream;
1839 if let Some(conversation) = &conversation {
1840 seed.llm_request
1841 .extensions
1842 .fields
1843 .insert("conversation".to_string(), conversation.clone());
1844 }
1845 let (_, response) = run(
1846 Arc::new(switchyard_libsy::Passthrough),
1847 clients.clone(),
1848 seed,
1849 to_category_map(&["weak"]),
1850 None,
1851 )
1852 .await?;
1853 response
1854 .llm_response
1855 .into_agg()
1856 .await
1857 .map_err(|error| LibsyError::client_call("weak", error))?;
1858 let mut continuations = vec![(
1859 "previous_response_id",
1860 json!("resp_seed"),
1861 responses && store,
1862 )];
1863 if let Some(conversation) = conversation.clone() {
1864 continuations.push(("conversation", conversation, true));
1865 }
1866 for (field, id, pinned) in continuations {
1867 let mut follow = request();
1868 follow
1869 .llm_request
1870 .extensions
1871 .fields
1872 .insert(field.to_string(), id);
1873 let outcome = decide(
1874 Arc::new(switchyard_libsy::Passthrough),
1875 clients.clone(),
1876 follow,
1877 to_category_map(&["strong"]),
1878 )
1879 .await?;
1880 assert_eq!(
1881 outcome.selected_model_id()?.as_str(),
1882 if pinned { "weak" } else { "strong" },
1883 "responses={responses}, store={store}, stream={stream}, field={field}"
1884 );
1885 }
1886 let requests = server.received_requests().await.expect("request recording");
1887 assert_eq!(requests.len(), 1);
1888 assert_eq!(
1889 serde_json::from_slice::<Value>(&requests[0].body).expect("request JSON")["store"],
1890 store
1891 );
1892 }
1893 }
1894 Ok(())
1895 }
1896
1897 #[tokio::test]
1898 async fn responses_stored_tool_continuation_materializes_for_anthropic() -> Result<()> {
1899 let server = MockServer::start().await;
1900 Mock::given(method("POST"))
1901 .respond_with(|request: &wiremock::Request| {
1902 let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1903 let blocks = body["messages"]
1904 .as_array()
1905 .into_iter()
1906 .flatten()
1907 .filter_map(|message| message["content"].as_array())
1908 .flatten()
1909 .collect::<Vec<_>>();
1910 let has_result = blocks.iter().any(|block| block["type"] == "tool_result");
1911 let has_call = blocks.iter().any(|block| block["type"] == "tool_use");
1912 let has_thanks = blocks
1913 .iter()
1914 .any(|block| block["type"] == "text" && block["text"] == "thanks")
1915 || body["messages"].as_array().is_some_and(|messages| {
1916 messages
1917 .iter()
1918 .any(|message| message["content"] == "thanks")
1919 });
1920 if has_result && !has_call {
1921 return ResponseTemplate::new(400).set_body_json(json!({
1922 "type": "error",
1923 "error": {
1924 "type": "invalid_request_error",
1925 "message": "tool_result has no corresponding tool_use"
1926 }
1927 }));
1928 }
1929 let response = if has_thanks {
1930 json!({
1931 "id": "msg_third", "type": "message", "role": "assistant",
1932 "model": "weak", "content": [{"type": "text", "text": "done"}],
1933 "stop_reason": "end_turn", "usage": {"input_tokens": 6, "output_tokens": 1}
1934 })
1935 } else if has_result {
1936 json!({
1937 "id": "msg_follow", "type": "message", "role": "assistant",
1938 "model": "weak", "content": [{"type": "text", "text": "sunny"}],
1939 "stop_reason": "end_turn", "usage": {"input_tokens": 4, "output_tokens": 1}
1940 })
1941 } else {
1942 json!({
1943 "id": "msg_seed", "type": "message", "role": "assistant",
1944 "model": "weak",
1945 "content": [{
1946 "type": "tool_use", "id": "toolu_weather", "name": "get_weather",
1947 "input": {"city": "Paris"}
1948 }],
1949 "stop_reason": "tool_use", "usage": {"input_tokens": 3, "output_tokens": 1}
1950 })
1951 };
1952 ResponseTemplate::new(200).set_body_json(response)
1953 })
1954 .mount(&server)
1955 .await;
1956
1957 let client: Arc<dyn RoutedLlmClient> = Arc::new(
1958 TranslatingLlmClient::new(&[ModelConfig::new(
1959 "weak",
1960 Backend::Anthropic(HttpBackendConfig {
1961 base_url: server.uri(),
1962 api_key: None,
1963 forward_auth: false,
1964 extra_headers: BTreeMap::new(),
1965 extra_body: BTreeMap::new(),
1966 omit_body_fields: BTreeSet::new(),
1967 reasoning_effort: None,
1968 max_retries: 0,
1969 failure_cooldown: std::time::Duration::ZERO,
1970 timeout: None,
1971 }),
1972 None,
1973 )])
1974 .map_err(|error| LibsyError::external("building test client", error))?,
1975 );
1976 let clients = ClientRouter::new(HashMap::from([(ModelId::from("weak"), client)]));
1977 let models = to_category_map(&["weak"]);
1978
1979 let seed = Request {
1980 llm_request: switchyard_translation::decode_request(
1981 WireFormat::OpenAiResponses,
1982 &json!({
1983 "model": "route",
1984 "input": "Call get_weather for Paris",
1985 "tools": [{
1986 "type": "function", "name": "get_weather",
1987 "parameters": {"type": "object"}
1988 }],
1989 "store": true
1990 }),
1991 )
1992 .map_err(|error| LibsyError::external("decoding seed request", error))?,
1993 raw_request: None,
1994 metadata: None,
1995 };
1996 let (_, response) = run(
1997 Arc::new(switchyard_libsy::Passthrough),
1998 clients.clone(),
1999 seed,
2000 models.clone(),
2001 None,
2002 )
2003 .await?;
2004 assert_eq!(
2005 response
2006 .llm_response
2007 .as_agg()
2008 .and_then(|agg| agg.id.as_deref()),
2009 Some("msg_seed")
2010 );
2011
2012 let follow = Request {
2013 llm_request: switchyard_translation::decode_request(
2014 WireFormat::OpenAiResponses,
2015 &json!({
2016 "model": "route",
2017 "previous_response_id": "msg_seed",
2018 "input": [{
2019 "type": "function_call_output",
2020 "call_id": "toolu_weather",
2021 "output": "Paris is sunny"
2022 }],
2023 "tools": [{
2024 "type": "function", "name": "get_weather",
2025 "parameters": {"type": "object"}
2026 }],
2027 "store": true
2028 }),
2029 )
2030 .map_err(|error| LibsyError::external("decoding continuation request", error))?,
2031 raw_request: None,
2032 metadata: None,
2033 };
2034 let (_, response) = run(
2035 Arc::new(switchyard_libsy::Passthrough),
2036 clients.clone(),
2037 follow,
2038 models.clone(),
2039 None,
2040 )
2041 .await?;
2042 assert_eq!(
2043 completion_text(
2044 response
2045 .llm_response
2046 .as_agg()
2047 .expect("buffered Anthropic response"),
2048 ),
2049 "sunny"
2050 );
2051
2052 let third = Request {
2053 llm_request: switchyard_translation::decode_request(
2054 WireFormat::OpenAiResponses,
2055 &json!({
2056 "model": "route",
2057 "previous_response_id": "msg_follow",
2058 "input": "thanks",
2059 "store": true
2060 }),
2061 )
2062 .map_err(|error| LibsyError::external("decoding chained request", error))?,
2063 raw_request: None,
2064 metadata: None,
2065 };
2066 let history = clients
2067 .stored_state_owner(&third)
2068 .and_then(|owner| owner.history)
2069 .expect("chained canonical history");
2070 assert_eq!(history.len, 4);
2071 assert_eq!(history.segment.len(), 2);
2072 assert!(history.parent.is_some());
2073 let (_, response) = run(
2074 Arc::new(switchyard_libsy::Passthrough),
2075 clients,
2076 third,
2077 models,
2078 None,
2079 )
2080 .await?;
2081 assert_eq!(
2082 completion_text(
2083 response
2084 .llm_response
2085 .as_agg()
2086 .expect("buffered Anthropic response"),
2087 ),
2088 "done"
2089 );
2090
2091 let requests = server.received_requests().await.expect("request recording");
2092 assert_eq!(requests.len(), 3);
2093 let follow: Value = serde_json::from_slice(&requests[1].body)
2094 .map_err(|error| LibsyError::external("decoding captured request", error))?;
2095 assert_eq!(follow["messages"][1]["content"][0]["type"], "tool_use");
2096 assert_eq!(follow["messages"][2]["content"][0]["type"], "tool_result");
2097 let third: Value = serde_json::from_slice(&requests[2].body)
2098 .map_err(|error| LibsyError::external("decoding captured request", error))?;
2099 assert_eq!(third["messages"][1]["content"][0]["type"], "tool_use");
2100 assert_eq!(third["messages"][2]["content"][0]["type"], "tool_result");
2101 assert_eq!(third["messages"][3]["content"], "sunny");
2102 assert_eq!(third["messages"][4]["content"][0]["text"], "thanks");
2103 Ok(())
2104 }
2105
2106 #[tokio::test]
2107 async fn streamed_cross_format_state_is_recorded_only_after_completion() -> Result<()> {
2108 let client = Arc::new(CandidateClient {
2109 calls: Mutex::new(Vec::new()),
2110 requests: Mutex::new(Vec::new()),
2111 first: FirstOutcome::StreamSuccess,
2112 });
2113 let clients = ClientRouter::new(HashMap::from([(
2114 ModelId::from("weak"),
2115 client as Arc<dyn RoutedLlmClient>,
2116 )]));
2117 let mut seed = request();
2118 seed.llm_request.messages = vec![Message::text(Role::User, "inspect")];
2119 seed.llm_request
2120 .preservation
2121 .requests
2122 .insert(WireFormat::OpenAiResponses.into(), json!({}));
2123 let mut response = stream_response(vec![
2124 LlmResponseChunk::MessageStart {
2125 id: Some("msg_stream".to_string()),
2126 model: Some("weak".to_string()),
2127 },
2128 LlmResponseChunk::ToolCallDelta {
2129 index: 0,
2130 id: Some("toolu_stream".to_string()),
2131 name: Some("lookup".to_string()),
2132 arguments_delta: Some("{}".to_string()),
2133 },
2134 LlmResponseChunk::MessageStop {
2135 reason: Some("tool_use".to_string()),
2136 },
2137 ]);
2138 response.set_served_model(&ModelId::from("weak"));
2139 let response = clients.remember_state_owner(&seed, response)?;
2140
2141 let mut follow = request();
2142 follow
2143 .llm_request
2144 .extensions
2145 .fields
2146 .insert("previous_response_id".to_string(), json!("msg_stream"));
2147 follow.llm_request.messages = vec![Message::new(
2148 Role::Tool,
2149 vec![ContentBlock::ToolResult(ToolResult {
2150 tool_call_id: "toolu_stream".to_string(),
2151 content: vec![ContentBlock::Text {
2152 text: "ready".to_string(),
2153 }],
2154 is_error: None,
2155 })],
2156 )];
2157 assert!(clients.stored_state_owner(&follow).is_none());
2158 response
2159 .llm_response
2160 .into_agg()
2161 .await
2162 .map_err(|error| LibsyError::client_call("weak", error))?;
2163
2164 let outcome = continue_on(
2165 clients
2166 .stored_state_owner(&follow)
2167 .expect("completed stream state"),
2168 "passthrough",
2169 follow,
2170 );
2171 assert_eq!(outcome.request.llm_request.messages.len(), 3);
2172 assert!(matches!(
2173 outcome.request.llm_request.messages[1].content.as_slice(),
2174 [ContentBlock::ToolCall(call)] if call.id == "toolu_stream"
2175 ));
2176 assert!(matches!(
2177 outcome.request.llm_request.messages[2].content.as_slice(),
2178 [ContentBlock::ToolResult(result)] if result.tool_call_id == "toolu_stream"
2179 ));
2180 Ok(())
2181 }
2182
2183 #[test]
2184 fn cross_format_response_with_store_false_is_not_retained() -> Result<()> {
2185 let client = Arc::new(CandidateClient {
2186 calls: Mutex::new(Vec::new()),
2187 requests: Mutex::new(Vec::new()),
2188 first: FirstOutcome::StreamSuccess,
2189 });
2190 let clients = ClientRouter::new(HashMap::from([(
2191 ModelId::from("weak"),
2192 client as Arc<dyn RoutedLlmClient>,
2193 )]));
2194 let mut seed = request();
2195 seed.llm_request
2196 .extensions
2197 .fields
2198 .insert("store".to_string(), json!(false));
2199 seed.llm_request
2200 .preservation
2201 .requests
2202 .insert(WireFormat::OpenAiResponses.into(), json!({}));
2203 let mut agg = text_response(Some("weak".to_string()), "done".to_string());
2204 agg.id = Some("msg_not_stored".to_string());
2205 let mut response = Response {
2206 llm_response: LlmResponse::Agg(agg),
2207 metadata: None,
2208 upstream_headers: http::HeaderMap::new(),
2209 };
2210 response.set_served_model(&ModelId::from("weak"));
2211 clients.remember_state_owner(&seed, response)?;
2212
2213 let mut follow = request();
2214 follow
2215 .llm_request
2216 .extensions
2217 .fields
2218 .insert("previous_response_id".to_string(), json!("msg_not_stored"));
2219 assert!(clients.stored_state_owner(&follow).is_none());
2220 Ok(())
2221 }
2222
2223 #[tokio::test]
2224 async fn state_capacity_does_not_fail_completed_responses() -> Result<()> {
2225 let client = Arc::new(CandidateClient {
2226 calls: Mutex::new(Vec::new()),
2227 requests: Mutex::new(Vec::new()),
2228 first: FirstOutcome::StreamSuccess,
2229 });
2230 let clients = ClientRouter::new(HashMap::from([(
2231 ModelId::from("weak"),
2232 client as Arc<dyn RoutedLlmClient>,
2233 )]));
2234 let model = ModelId::from("weak");
2235 clients.inner.state_owners.lock().by_id = (0..MAX_STATE_OWNERS)
2236 .map(|id| {
2237 (
2238 format!("existing_{id}"),
2239 StoredState {
2240 model: model.clone(),
2241 history: None,
2242 },
2243 )
2244 })
2245 .collect();
2246
2247 let mut request = request();
2248 request
2249 .llm_request
2250 .preservation
2251 .requests
2252 .insert(WireFormat::OpenAiResponses.into(), json!({}));
2253 let mut agg = text_response(Some("weak".to_string()), "done".to_string());
2254 agg.id = Some("overflow_buffered".to_string());
2255 let mut buffered = Response {
2256 llm_response: LlmResponse::Agg(agg),
2257 metadata: None,
2258 upstream_headers: http::HeaderMap::new(),
2259 };
2260 buffered.set_served_model(&model);
2261 clients.remember_state_owner(&request, buffered)?;
2262
2263 let mut streamed = stream_response(vec![
2264 LlmResponseChunk::MessageStart {
2265 id: Some("overflow_streamed".to_string()),
2266 model: Some("weak".to_string()),
2267 },
2268 LlmResponseChunk::TextDelta {
2269 index: 0,
2270 text: "done".to_string(),
2271 },
2272 LlmResponseChunk::MessageStop {
2273 reason: Some("stop".to_string()),
2274 },
2275 ]);
2276 streamed.set_served_model(&model);
2277 clients
2278 .remember_state_owner(&request, streamed)?
2279 .llm_response
2280 .into_agg()
2281 .await
2282 .map_err(|error| LibsyError::client_call("weak", error))?;
2283
2284 assert_eq!(
2285 clients.inner.state_owners.lock().by_id.len(),
2286 MAX_STATE_OWNERS
2287 );
2288 Ok(())
2289 }
2290
2291 #[test]
2292 fn state_owners_preserve_existing_records_at_capacity()
2293 -> std::result::Result<(), LlmClientError> {
2294 let mut owners = StateOwners::default();
2295 let model = ModelId::from("model/a");
2296 for id in 0..MAX_STATE_OWNERS - 1 {
2297 owners.remember(Some(&format!("resp_{id}")), None, &model, None)?;
2298 }
2299 assert!(matches!(
2300 owners.remember(Some("new_response"), Some("new_conversation"), &model, None,),
2301 Err(LlmClientError::ResponseStateLimitExceeded {
2302 limit: MAX_STATE_OWNERS
2303 })
2304 ));
2305 assert!(owners.owner("new_response").is_none());
2306 assert!(owners.owner("new_conversation").is_none());
2307 owners.remember(Some("last_id"), Some("last_id"), &model, None)?;
2308 owners.remember(Some("last_id"), Some("resp_0"), &model, None)?;
2309 assert!(matches!(
2310 owners.remember(Some("over_limit"), None, &model, None),
2311 Err(LlmClientError::ResponseStateLimitExceeded { .. })
2312 ));
2313 assert!(matches!(
2314 owners.remember(Some("resp_0"), None, &ModelId::from("model/b"), None,),
2315 Err(LlmClientError::ResponseStateConflict)
2316 ));
2317 assert_eq!(
2318 owners.owner("resp_0").map(|state| &state.model),
2319 Some(&model)
2320 );
2321 assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
2322 Ok(())
2323 }
2324
2325 #[test]
2326 fn concurrent_state_owners_do_not_exceed_capacity() {
2327 let model = ModelId::from("model/a");
2328 let owners = Arc::new(Mutex::new(StateOwners {
2329 by_id: (0..MAX_STATE_OWNERS - 16)
2330 .map(|id| {
2331 (
2332 format!("existing_{id}"),
2333 StoredState {
2334 model: model.clone(),
2335 history: None,
2336 },
2337 )
2338 })
2339 .collect(),
2340 }));
2341 let barrier = Arc::new(std::sync::Barrier::new(32));
2342 let accepted = std::thread::scope(|scope| {
2343 let handles = (0..32)
2344 .map(|id| {
2345 let owners = owners.clone();
2346 let barrier = barrier.clone();
2347 let model = model.clone();
2348 scope.spawn(move || {
2349 barrier.wait();
2350 let result = owners.lock().remember(
2351 Some(&format!("resp_{id}")),
2352 Some(&format!("conv_{id}")),
2353 &model,
2354 None,
2355 );
2356 match result {
2357 Ok(()) => Some(id),
2358 Err(LlmClientError::ResponseStateLimitExceeded { .. }) => None,
2359 other => panic!("unexpected state tracking result: {other:?}"),
2360 }
2361 })
2362 })
2363 .collect::<Vec<_>>();
2364 handles
2365 .into_iter()
2366 .filter_map(|handle| handle.join().expect("request thread panicked"))
2367 .collect::<Vec<_>>()
2368 });
2369 let owners = owners.lock();
2370 assert_eq!(accepted.len(), 8);
2371 assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
2372 assert_eq!(
2373 owners.owner("existing_0").map(|state| &state.model),
2374 Some(&model)
2375 );
2376 for id in accepted {
2377 assert_eq!(
2378 owners
2379 .owner(&format!("resp_{id}"))
2380 .map(|state| &state.model),
2381 Some(&model)
2382 );
2383 assert_eq!(
2384 owners
2385 .owner(&format!("conv_{id}"))
2386 .map(|state| &state.model),
2387 Some(&model)
2388 );
2389 }
2390 }
2391
2392 #[tokio::test]
2393 async fn each_fallback_candidate_receives_only_its_own_prompt() -> Result<()> {
2394 let client = Arc::new(CandidateClient {
2395 calls: Mutex::new(Vec::new()),
2396 requests: Mutex::new(Vec::new()),
2397 first: FirstOutcome::ContextWindow,
2398 });
2399 let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
2400 let clients = ClientRouter::new_with_target_prompts(
2401 HashMap::from([
2402 (ModelId::from("weak"), Arc::clone(&routed_client)),
2403 (ModelId::from("strong"), routed_client),
2404 ]),
2405 HashMap::from([
2406 ("weak".into(), "weak prompt".to_string()),
2407 ("strong".into(), "strong prompt".to_string()),
2408 ]),
2409 None,
2410 );
2411
2412 run(
2413 Arc::new(CandidateAlgorithm {}),
2414 clients,
2415 request(),
2416 to_category_map(&["weak", "strong"]),
2417 None,
2418 )
2419 .await?;
2420
2421 let calls = client.requests.lock();
2422 assert_eq!(calls.len(), 2);
2423 assert_eq!(instruction_text(&calls[0]), ["weak prompt"]);
2424 assert_eq!(instruction_text(&calls[1]), ["strong prompt"]);
2425 Ok(())
2426 }
2427
2428 #[tokio::test]
2429 async fn decision_prompts_a_routing_response_target() -> Result<()> {
2430 let client = Arc::new(CandidateClient {
2431 calls: Mutex::new(Vec::new()),
2432 requests: Mutex::new(Vec::new()),
2433 first: FirstOutcome::StreamSuccess,
2434 });
2435 let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
2436 let clients = ClientRouter::new_with_target_prompts(
2437 HashMap::from([(ModelId::from("answer"), routed_client)]),
2438 HashMap::from([("answer".into(), "answer prompt".to_string())]),
2439 Some("answer".into()),
2440 );
2441
2442 decide(
2443 Arc::new(AnsweredAlgorithm {
2444 model: "answer".into(),
2445 }),
2446 clients,
2447 request(),
2448 Arc::new(RuntimeModels::default()),
2449 )
2450 .await?;
2451
2452 let calls = client.requests.lock();
2453 assert_eq!(calls.len(), 1);
2454 assert_eq!(instruction_text(&calls[0]), ["answer prompt"]);
2455 Ok(())
2456 }
2457
2458 #[tokio::test]
2459 async fn decision_prepares_only_the_selected_request() -> Result<()> {
2460 let client = Arc::new(CandidateClient {
2461 calls: Mutex::new(Vec::new()),
2462 requests: Mutex::new(Vec::new()),
2463 first: FirstOutcome::StreamSuccess,
2464 });
2465 let routed_client: Arc<dyn RoutedLlmClient> = client;
2466 let clients = ClientRouter::new_with_target_prompts(
2467 HashMap::from([
2468 (ModelId::from("weak"), Arc::clone(&routed_client)),
2469 (ModelId::from("strong"), routed_client),
2470 ]),
2471 HashMap::from([
2472 (ModelId::from("weak"), "weak prompt".to_string()),
2473 (ModelId::from("strong"), "strong prompt".to_string()),
2474 ]),
2475 None,
2476 );
2477
2478 let outcome = decide(
2479 Arc::new(CandidateAlgorithm {}),
2480 clients,
2481 request(),
2482 to_category_map(&["weak", "strong"]),
2483 )
2484 .await?;
2485
2486 assert_eq!(
2487 outcome.selected_model_ids,
2488 [ModelId::from("weak"), ModelId::from("strong")]
2489 );
2490 assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);
2491 Ok(())
2492 }
2493
2494 #[test]
2495 fn fallback_only_accepts_context_policy_and_unavailable_failures() {
2496 let error = |source| LibsyError::client_call("target", source);
2497 assert_eq!(
2498 fallback_reason(
2499 &error(LlmClientError::ContextWindowExceeded {
2500 model: "target".into(),
2501 message: "too long".to_string(),
2502 }),
2503 true
2504 ),
2505 Some(RoutingFallbackReason::ContextWindow)
2506 );
2507 for status in [
2508 StatusCode::FORBIDDEN,
2509 StatusCode::REQUEST_TIMEOUT,
2510 StatusCode::TOO_MANY_REQUESTS,
2511 StatusCode::INTERNAL_SERVER_ERROR,
2512 StatusCode::from_u16(599).expect("599 is a valid status"),
2513 ] {
2514 assert_eq!(
2515 fallback_reason(
2516 &error(LlmClientError::UpstreamHttp {
2517 status,
2518 body: "failed".to_string(),
2519 headers: Box::default(),
2520 }),
2521 true
2522 ),
2523 Some(RoutingFallbackReason::Unavailable)
2524 );
2525 }
2526 for (body, expected) in [
2527 (
2528 r#"{"error":{"code":"content_policy_violation"}}"#,
2529 Some(RoutingFallbackReason::Unavailable),
2530 ),
2531 (
2532 r#"{"error":{"code":"invalid_request_error"},"metadata":"content_policy_violation"}"#,
2533 None,
2534 ),
2535 (r#"{"error":{"message":"content_policy_violation"}}"#, None),
2536 ("content_policy_violation", None),
2537 ] {
2538 assert_eq!(
2539 fallback_reason(
2540 &error(LlmClientError::UpstreamHttp {
2541 status: StatusCode::BAD_REQUEST,
2542 body: body.to_string(),
2543 headers: Box::default(),
2544 }),
2545 true
2546 ),
2547 expected,
2548 "{body}"
2549 );
2550 }
2551 for status in [
2552 StatusCode::BAD_REQUEST,
2553 StatusCode::UNAUTHORIZED,
2554 StatusCode::NOT_FOUND,
2555 StatusCode::CONFLICT,
2556 StatusCode::from_u16(499).expect("499 is a valid status"),
2557 StatusCode::from_u16(600).expect("600 is a valid status"),
2558 ] {
2559 assert_eq!(
2560 fallback_reason(
2561 &error(LlmClientError::UpstreamHttp {
2562 status,
2563 body: "failed".to_string(),
2564 headers: Box::default(),
2565 }),
2566 true
2567 ),
2568 None
2569 );
2570 }
2571 }
2572
2573 #[tokio::test]
2574 async fn candidate_failures_follow_the_fallback_policy() -> Result<()> {
2575 let (client, result) = run_candidates(FirstOutcome::ContextWindow).await;
2577 let (_, response) = result?;
2578 assert_eq!(
2579 &*client.calls.lock(),
2580 &[ModelId::from("weak"), "strong".into()]
2581 );
2582 assert_eq!(
2583 response
2584 .llm_response
2585 .as_agg()
2586 .map(|response| response.model.as_deref()),
2587 Some(Some("strong"))
2588 );
2589 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2590
2591 let (client, result) =
2593 run_candidates(FirstOutcome::ContentPolicy(StatusCode::BAD_REQUEST)).await;
2594 let (_, response) = result?;
2595 assert_eq!(
2596 &*client.calls.lock(),
2597 &[ModelId::from("weak"), "strong".into()]
2598 );
2599 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2600
2601 let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;
2603 assert!(matches!(
2604 result,
2605 Err(LibsyError::ClientCall {
2606 source: LlmClientError::UpstreamHttp {
2607 status: StatusCode::UNAUTHORIZED,
2608 ..
2609 },
2610 ..
2611 })
2612 ));
2613 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2614 Ok(())
2615 }
2616
2617 #[tokio::test]
2618 async fn fallback_candidates_receive_the_fallback_request() -> Result<()> {
2619 let client = Arc::new(CandidateClient {
2620 calls: Mutex::new(Vec::new()),
2621 requests: Mutex::new(Vec::new()),
2622 first: FirstOutcome::ContextWindow,
2623 });
2624 let clients = ClientRouter::single(client.clone());
2625 let mut seed = request();
2626 seed.llm_request
2627 .preservation
2628 .requests
2629 .insert(WireFormat::OpenAiResponses.into(), json!({}));
2630 run(
2631 Arc::new(RewritingAlgorithm {}),
2632 clients.clone(),
2633 seed,
2634 to_category_map(&["weak", "strong"]),
2635 None,
2636 )
2637 .await?;
2638 let sent: Vec<_> = client
2639 .requests
2640 .lock()
2641 .iter()
2642 .map(|request| request.llm_request.messages.clone())
2643 .collect();
2644 assert_eq!(sent.len(), 2);
2645 assert_eq!(sent[0].len(), request().llm_request.messages.len() + 1);
2646 assert_eq!(sent[1], request().llm_request.messages);
2647 let mut follow = request();
2648 follow
2649 .llm_request
2650 .extensions
2651 .fields
2652 .insert("previous_response_id".into(), json!("candidate_response"));
2653 follow.llm_request.messages = vec![Message::text(Role::User, "continue")];
2654 let owner = clients
2655 .stored_state_owner(&follow)
2656 .expect("fallback Responses state");
2657 let continued = continue_on(owner, "test", follow);
2658 let mut expected = sent[1].clone();
2659 expected.push(Message::text(Role::Assistant, "strong"));
2660 expected.push(Message::text(Role::User, "continue"));
2661 assert_eq!(continued.request.llm_request.messages, expected);
2662 Ok(())
2663 }
2664
2665 #[tokio::test]
2666 async fn fallback_request_does_not_bypass_content_policy() -> Result<()> {
2667 for status in [StatusCode::BAD_REQUEST, StatusCode::FORBIDDEN] {
2668 let client = Arc::new(CandidateClient {
2669 calls: Mutex::new(Vec::new()),
2670 requests: Mutex::new(Vec::new()),
2671 first: FirstOutcome::ContentPolicy(status),
2672 });
2673 let result = run(
2674 Arc::new(RewritingAlgorithm {}),
2675 ClientRouter::single(client.clone()),
2676 request(),
2677 to_category_map(&["weak", "strong"]),
2678 None,
2679 )
2680 .await;
2681 assert!(
2682 matches!(result, Err(LibsyError::ClientCall { source: LlmClientError::UpstreamHttp { status: actual, .. }, .. }) if actual == status)
2683 );
2684 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2685 }
2686 Ok(())
2687 }
2688
2689 #[tokio::test]
2690 async fn retry_exhaustion_falls_back_and_optional_cooldown_skips_failed_backend() -> Result<()>
2691 {
2692 let server = MockServer::start().await;
2693 let calls = Arc::new(Mutex::new(Vec::new()));
2694 let observed_calls = Arc::clone(&calls);
2695 Mock::given(method("POST"))
2696 .respond_with(move |request: &wiremock::Request| {
2697 let body: serde_json::Value =
2698 serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
2699 let model = body["model"].as_str().unwrap_or_default().to_string();
2700 observed_calls.lock().push(model.clone());
2701 if model == "weak" {
2702 ResponseTemplate::new(503)
2703 .insert_header("retry-after", "0")
2704 .set_body_string("unavailable")
2705 } else {
2706 ResponseTemplate::new(200).set_body_json(serde_json::json!({
2707 "id": "answer",
2708 "model": "strong",
2709 "choices": [{
2710 "index": 0,
2711 "message": {"role": "assistant", "content": "ok"},
2712 "finish_reason": "stop"
2713 }],
2714 "usage": {}
2715 }))
2716 }
2717 })
2718 .mount(&server)
2719 .await;
2720
2721 let client_with_cooldown = |cooldown| {
2722 let backend = || {
2723 Backend::OpenAiChat(HttpBackendConfig {
2724 base_url: format!("{}/v1", server.uri()),
2725 api_key: None,
2726 forward_auth: false,
2727 extra_headers: BTreeMap::new(),
2728 extra_body: BTreeMap::new(),
2729 omit_body_fields: BTreeSet::new(),
2730 reasoning_effort: None,
2731 max_retries: 2,
2732 failure_cooldown: cooldown,
2733 timeout: None,
2734 })
2735 };
2736 TranslatingLlmClient::new(&[
2737 ModelConfig::new("weak", backend(), None),
2738 ModelConfig::new("strong", backend(), None),
2739 ])
2740 .map(Arc::new)
2741 .map_err(|error| LibsyError::external("building test client", error))
2742 };
2743 let call = async |client: Arc<TranslatingLlmClient>| {
2744 run(
2745 Arc::new(CandidateAlgorithm {}),
2746 ClientRouter::single(client),
2747 request(),
2748 to_category_map(&["weak", "strong"]),
2749 None,
2750 )
2751 .await?;
2752 Ok::<_, LibsyError>(std::mem::take(&mut *calls.lock()))
2753 };
2754
2755 let client = client_with_cooldown(std::time::Duration::ZERO)?;
2756 assert_eq!(
2757 call(client.clone()).await?,
2758 ["weak", "weak", "weak", "strong"]
2759 );
2760 assert_eq!(call(client).await?, ["weak", "weak", "weak", "strong"]);
2761
2762 let cooldown = std::time::Duration::from_secs(60);
2763 let client = client_with_cooldown(cooldown)?;
2764 assert_eq!(
2765 call(client.clone()).await?,
2766 ["weak", "weak", "weak", "strong"]
2767 );
2768 assert_eq!(call(client.clone()).await?, ["strong"]);
2769
2770 tokio::time::pause();
2771 tokio::time::advance(cooldown).await;
2772 tokio::time::resume();
2773
2774 assert_eq!(call(client).await?, ["weak", "weak", "weak", "strong"]);
2775 Ok(())
2776 }
2777
2778 #[tokio::test]
2779 async fn refused_connection_falls_back_to_the_next_candidate() -> Result<()> {
2780 let server = MockServer::start().await;
2781 Mock::given(method("POST"))
2782 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2783 "id": "answer",
2784 "model": "strong",
2785 "choices": [{
2786 "index": 0,
2787 "message": {"role": "assistant", "content": "ok"},
2788 "finish_reason": "stop"
2789 }],
2790 "usage": {}
2791 })))
2792 .mount(&server)
2793 .await;
2794
2795 let backend = |base_url: String| {
2796 Backend::OpenAiChat(HttpBackendConfig {
2797 base_url,
2798 api_key: None,
2799 forward_auth: false,
2800 extra_headers: BTreeMap::new(),
2801 extra_body: BTreeMap::new(),
2802 omit_body_fields: BTreeSet::new(),
2803 reasoning_effort: None,
2804 max_retries: 0,
2805 failure_cooldown: std::time::Duration::ZERO,
2806 timeout: None,
2807 })
2808 };
2809 let client = Arc::new(
2810 TranslatingLlmClient::new(&[
2811 ModelConfig::new("weak", backend("http://127.0.0.1:1/v1".to_string()), None),
2813 ModelConfig::new("strong", backend(format!("{}/v1", server.uri())), None),
2814 ])
2815 .map_err(|error| LibsyError::external("building test client", error))?,
2816 );
2817 let (_, response) = run(
2818 Arc::new(CandidateAlgorithm {}),
2819 ClientRouter::single(client),
2820 request(),
2821 to_category_map(&["weak", "strong"]),
2822 None,
2823 )
2824 .await?;
2825
2826 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2827 Ok(())
2828 }
2829
2830 #[tokio::test]
2831 async fn streams_are_outside_the_candidate_fallback_boundary() -> Result<()> {
2832 let (client, result) = run_candidates(FirstOutcome::StreamSuccess).await;
2834 let (_, response) = result?;
2835 assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
2836 let aggregate = response
2837 .llm_response
2838 .into_agg()
2839 .await
2840 .map_err(|source| LibsyError::client_call("weak", source))?;
2841 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2842 assert_eq!(completion_text(&aggregate), "streamed");
2843
2844 let (client, result) = run_candidates(FirstOutcome::MidStreamError).await;
2846 let (_, response) = result?;
2847 let error = response
2848 .llm_response
2849 .into_agg()
2850 .await
2851 .expect_err("the stream should fail during aggregation");
2852 assert!(error.to_string().contains("stream failed"));
2853 assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2854 Ok(())
2855 }
2856
2857 const OVERFLOW_SSE: &str = "data: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.\",\"object\":\"error\",\"param\":null,\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n";
2860
2861 const STRONG_SSE: &str = "data: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
2863
2864 async fn run_streaming_candidates(
2868 weak_sse: &'static str,
2869 ) -> (
2870 MockServer,
2871 Arc<Mutex<Vec<String>>>,
2872 Result<(ModelId, Response)>,
2873 ) {
2874 let server = MockServer::start().await;
2875 let calls = Arc::new(Mutex::new(Vec::new()));
2876 let observed_calls = Arc::clone(&calls);
2877 Mock::given(method("POST"))
2878 .respond_with(move |request: &wiremock::Request| {
2879 let body: serde_json::Value =
2880 serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
2881 let model = body["model"].as_str().unwrap_or_default().to_string();
2882 observed_calls.lock().push(model.clone());
2883 let sse = if model == "weak" {
2884 weak_sse
2885 } else {
2886 STRONG_SSE
2887 };
2888 ResponseTemplate::new(200)
2889 .insert_header("content-type", "text/event-stream")
2890 .set_body_string(sse)
2891 })
2892 .mount(&server)
2893 .await;
2894
2895 let backend = || {
2896 Backend::OpenAiChat(HttpBackendConfig {
2897 base_url: format!("{}/v1", server.uri()),
2898 api_key: None,
2899 forward_auth: false,
2900 extra_headers: BTreeMap::new(),
2901 extra_body: BTreeMap::new(),
2902 omit_body_fields: BTreeSet::new(),
2903 reasoning_effort: None,
2904 max_retries: 0,
2905 failure_cooldown: std::time::Duration::ZERO,
2906 timeout: None,
2907 })
2908 };
2909 let client = Arc::new(
2910 TranslatingLlmClient::new(&[
2911 ModelConfig::new("weak", backend(), None),
2912 ModelConfig::new("strong", backend(), None),
2913 ])
2914 .expect("building test client"),
2915 );
2916 let algorithm = Arc::new(CandidateAlgorithm {});
2917 let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());
2918 llm_request.stream = true;
2919 let request = Request {
2920 llm_request,
2921 raw_request: None,
2922 metadata: None,
2923 };
2924 let result = run(
2925 algorithm,
2926 ClientRouter::single(client),
2927 request,
2928 to_category_map(&["weak", "strong"]),
2929 None,
2930 )
2931 .await;
2932 (server, calls, result)
2933 }
2934
2935 #[tokio::test]
2936 async fn first_stream_event_overflow_falls_back_to_the_next_candidate() -> Result<()> {
2937 let (_server, calls, result) = run_streaming_candidates(OVERFLOW_SSE).await;
2940 let (_, response) = result?;
2941 assert_eq!(&*calls.lock(), &["weak", "strong"]);
2942 assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2943 let aggregate = response
2944 .llm_response
2945 .into_agg()
2946 .await
2947 .map_err(|source| LibsyError::client_call("strong", source))?;
2948 assert_eq!(completion_text(&aggregate), "ok");
2949
2950 let (_server, calls, result) = run_streaming_candidates(
2953 "data: {\"error\":{\"code\":500,\"message\":\"engine exploded\",\"object\":\"error\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2954 )
2955 .await;
2956 let (_, response) = result?;
2957 assert_eq!(&*calls.lock(), &["weak"]);
2958 let error = response
2959 .llm_response
2960 .into_agg()
2961 .await
2962 .expect_err("the in-band error should surface to the consumer");
2963 assert!(error.to_string().contains("engine exploded"));
2964
2965 let (_server, calls, result) = run_streaming_candidates(
2968 "data: {\"id\":\"turn\",\"model\":\"weak\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"partial\"},\"finish_reason\":null}]}\n\ndata: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens).\",\"object\":\"error\",\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2969 )
2970 .await;
2971 let (_, response) = result?;
2972 assert_eq!(&*calls.lock(), &["weak"]);
2973 let error = response
2974 .llm_response
2975 .into_agg()
2976 .await
2977 .expect_err("the mid-stream overflow should surface to the consumer");
2978 assert!(
2979 error
2980 .to_string()
2981 .contains("exceeds the maximum allowed length")
2982 );
2983
2984 let (_server, calls, result) = run_streaming_candidates("data: [DONE]\n\n").await;
2987 let (_, response) = result?;
2988 assert_eq!(&*calls.lock(), &["weak"]);
2989 let aggregate = response
2990 .llm_response
2991 .into_agg()
2992 .await
2993 .map_err(|source| LibsyError::client_call("weak", source))?;
2994 assert_eq!(completion_text(&aggregate), "");
2995 Ok(())
2996 }
2997}