Skip to main content

switchyard_llm_client/
run.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Drive a libsy algorithm to completion, making the model calls it offloads.
5//!
6//! [`switchyard_libsy::Algorithm::run_stream`] is the whole libsy API: it yields a stream of
7//! steps and expects its consumer to serve every offloaded model call. [`run()`] is that
8//! consumer — it drives the stream with [`switchyard_libsy::drive`], hands routing-time calls to
9//! a [`RoutedLlmClient`], and serves the terminal routing outcome.
10//!
11//! libsy owns the stream mechanics; what this module adds is per-target request preparation,
12//! ordered candidate fallback, and the `libsy.client_call` span around each candidate. Each
13//! completion candidate exhausts its backend retry budget before fallback advances. Routing
14//! calls stop on the first candidate's failure unless the algorithm enables error recovery.
15//! A timeout stops completion calls.
16//!
17//! A Responses continuation can refer to state through `previous_response_id` or `conversation`.
18//! [`ClientRouter`] sends native Responses state back to its provider; for Chat or Anthropic
19//! targets it retains the canonical history needed to materialize the continuation.
20
21use std::collections::{HashMap, HashSet};
22use std::sync::Arc;
23use std::time::Instant;
24
25use futures::{StreamExt, TryStreamExt, stream};
26use http::StatusCode;
27use parking_lot::Mutex;
28use serde_json::{Value, json};
29use switchyard_libsy::{
30    Algorithm, AnswerSummary, Call, CallDecision, LibsyError, OutcomeMetadata, Result,
31    RoutingOutcome, RuntimeModels, drive,
32};
33use switchyard_protocol::{
34    AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,
35    ModelId, Request, Response, ResponseAccumulator, RoutedDecisionClient, RoutedLlmClient,
36    RoutingFallbackReason, Usage, WireFormat,
37};
38use switchyard_translation::prepare_request_for_target;
39
40use crate::observation::{ModelCallObservation, RunObservation, RunObserver};
41use crate::{metrics, observability};
42
43/// Run one request to completion, serving every offloaded model call with `client`.
44///
45/// Returns the model selected by the algorithm and the final [`Response`]. `observer`, when
46/// present, receives metadata for the routing outcome, each completed routing or answer call,
47/// and the routing overhead.
48///
49/// `clients` resolves each offloaded call to the client for the target the algorithm
50/// selected — an algorithm may route among targets served by different providers, so this is
51/// a per-call lookup, not one client for the whole run. Use
52/// [`ClientRouter::single`](ClientRouter::single) when one client serves every target.
53///
54/// Routing calls are buffered; client failures stop the request unless the call enables recovery.
55/// Once routing completes, non-timeout failures may try the outcome's ordered fallback candidates.
56pub async fn run(
57    algorithm: Arc<dyn Algorithm>,
58    clients: ClientRouter,
59    request: Request,
60    models: Arc<RuntimeModels>,
61    observer: Option<RunObserver>,
62) -> Result<(ModelId, Response)> {
63    let algorithm_name = algorithm.name().to_string();
64    let run_started = Instant::now();
65    let routing_clients = clients.clone();
66    // This says if we have an observer, put Some(..) in routing_observations.
67    // No observer means we don't want any routing_observations.
68    let routing_observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new())));
69    let outcome = match clients.stored_state_owner(&request) {
70        Some(owner) => Ok(continue_on(owner, &algorithm_name, request)),
71        None => {
72            drive(Arc::clone(&algorithm), request, models, {
73                let routing_observations = routing_observations.clone();
74                move |call| serve(routing_clients.clone(), call, routing_observations.clone())
75            })
76            .await
77        }
78    };
79    let answered_model = outcome
80        .as_ref()
81        .ok()
82        .and_then(|outcome| outcome.response.as_ref())
83        .and_then(Response::served_model);
84    emit_routing_observations(&observer, &routing_observations, answered_model);
85    let outcome = outcome?;
86    let overhead = run_started.elapsed();
87    metrics::record_routing_overhead(&algorithm_name, overhead);
88
89    let selected_model_id = outcome.selected_model_id()?.clone();
90    if let Some(observer) = &observer
91        && let Some(metadata) = outcome.metadata
92    {
93        observer(RunObservation::Outcome(metadata));
94    }
95    let result = if let Some(response) = outcome.response {
96        let result = clients.remember_state_owner(&outcome.request, response);
97        let served_model = result
98            .as_ref()
99            .ok()
100            .and_then(Response::served_model)
101            .unwrap_or(&selected_model_id);
102        let served_model = clients.provider_model(served_model).clone();
103        metrics::observe_routed_request(&algorithm_name, &served_model, None, result)
104    } else {
105        let observe = |observation| {
106            if let Some(observer) = &observer {
107                observer(RunObservation::AnswerCall(observation));
108            }
109        };
110        call_first_available(
111            &clients,
112            &algorithm_name,
113            &outcome.request,
114            outcome.fallback_request.as_ref(),
115            &outcome.candidate_requests,
116            &outcome.selected_model_ids,
117            &observe,
118        )
119        .await
120        .and_then(|(request, response)| clients.remember_state_owner(request, response))
121    };
122    if let Some(observer) = &observer {
123        observer(RunObservation::RoutingOverhead(overhead));
124    }
125    let mut response = result?;
126    if algorithm.tracks_answers()
127        && let Some(session) = session_id(&outcome.request)
128    {
129        response = report_answer(algorithm, session, response).await;
130    }
131    Ok((selected_model_id, response))
132}
133
134fn session_id(request: &Request) -> Option<String> {
135    let session = request.metadata.as_ref()?.session_id.as_deref()?;
136    (!session.is_empty()).then(|| session.to_string())
137}
138
139/// Reports the served answer to the algorithm; a stream reports it before its final chunk
140/// so the client cannot send its next request first.
141async fn report_answer(
142    algorithm: Arc<dyn Algorithm>,
143    session: String,
144    mut response: Response,
145) -> Response {
146    let stream = match response.llm_response {
147        LlmResponse::Agg(aggregate) => {
148            algorithm
149                .observe_answer(&session, AnswerSummary::of(&aggregate))
150                .await;
151            response.llm_response = LlmResponse::Agg(aggregate);
152            return response;
153        }
154        LlmResponse::Stream(stream) => stream,
155    };
156    algorithm
157        .observe_answer(&session, AnswerSummary { final_reply: false })
158        .await;
159    let report = (algorithm, session, Some(ResponseAccumulator::new()));
160    response.llm_response = LlmResponse::Stream(Box::pin(stream::unfold(
161        (stream, report),
162        |(mut stream, (algorithm, session, mut accumulator))| async move {
163            let item = stream.next().await?;
164            let failed = match &item {
165                Err(_) => true,
166                Ok(event) => event
167                    .normalized()
168                    .iter()
169                    .any(|chunk| matches!(chunk, LlmResponseChunk::StreamError { .. })),
170            };
171            if failed {
172                accumulator = None;
173                algorithm
174                    .observe_answer(&session, AnswerSummary { final_reply: false })
175                    .await;
176            }
177            let mut stopped = false;
178            if let (Ok(event), Some(answer)) = (&item, accumulator.as_mut()) {
179                for chunk in event.normalized() {
180                    stopped |= matches!(chunk, LlmResponseChunk::MessageStop { .. });
181                    answer.push(chunk.clone());
182                }
183            }
184            if stopped && let Some(answer) = accumulator.take() {
185                let answer = AnswerSummary::of(&answer.finish());
186                algorithm.observe_answer(&session, answer).await;
187            }
188            Some((item, (stream, (algorithm, session, accumulator))))
189        },
190    )));
191    response
192}
193
194/// Run an algorithm to a routing decision without serving its terminal completion.
195///
196/// Routing-time calls are served normally. The returned request is prepared only for the selected
197/// target; use [`run`] to execute selected and fallback candidates.
198pub async fn decide(
199    algorithm: Arc<dyn Algorithm>,
200    clients: ClientRouter,
201    request: Request,
202    models: Arc<RuntimeModels>,
203) -> Result<RoutingOutcome> {
204    let routing_clients = clients.clone();
205    let mut outcome = match clients.stored_state_owner(&request) {
206        Some(owner) => continue_on(owner, algorithm.name(), request),
207        None => {
208            drive(algorithm, request, models, move |call| {
209                serve(routing_clients.clone(), call, None)
210            })
211            .await?
212        }
213    };
214    let selected_model_id = outcome.selected_model_id()?.clone();
215    outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
216    outcome.response = outcome
217        .response
218        .map(|response| clients.remember_state_owner(&outcome.request, response))
219        .transpose()?;
220    Ok(outcome)
221}
222
223#[tracing::instrument(
224    name = "libsy.decision_client_call",
225    skip_all,
226    fields(algorithm = %call.algorithm, selected_model = %call.model, outcome = tracing::field::Empty),
227)]
228async fn serve_decision(
229    clients: &ClientRouter,
230    call: CallDecision,
231    observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,
232) -> Result<()> {
233    let client = clients.route_decision(&call.model);
234    let model = call.model.clone();
235    let request = call.request.clone();
236    call.respond(async {
237        let started = Instant::now();
238        let result = async { client?.call(request).await }.await;
239        tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });
240        if let Some(observations) = observations {
241            observations
242                .lock()
243                .push(RunObservation::DecisionCall(ModelCallObservation {
244                    selected_model: model.clone(),
245                    is_success: result.is_ok(),
246                    duration: started.elapsed(),
247                    usage: result.as_ref().ok().map(|response| response.usage.clone()),
248                }));
249        }
250        result.map_err(|error| LibsyError::client_call(model, error))
251    })
252    .await
253}
254
255/// Emits completed routing calls after the outcome reveals whether one response became the answer.
256fn emit_routing_observations(
257    observer: &Option<RunObserver>,
258    observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,
259    answered_model: Option<&ModelId>,
260) {
261    let (Some(observer), Some(observations)) = (observer, observations) else {
262        return;
263    };
264    let mut answer_observed = false;
265    for observation in observations.lock().drain(..) {
266        observer(match observation {
267            RunObservation::LlmCall(call)
268                if !answer_observed && answered_model == Some(&call.selected_model) =>
269            {
270                answer_observed = true;
271                RunObservation::AnswerCall(call)
272            }
273            observation => observation,
274        });
275    }
276}
277
278/// Serve one offloaded call and fulfill its promise.
279///
280/// LLM failures stop the run unless the call enables recovery. Decision results,
281/// including client failures, go back to the algorithm for its routing policy.
282async fn serve(
283    clients: ClientRouter,
284    call: Call,
285    observations: Option<Arc<Mutex<Vec<RunObservation>>>>,
286) -> Result<()> {
287    let call = match call {
288        Call::Model(call) => *call,
289        Call::Decision(call) => return serve_decision(&clients, *call, &observations).await,
290    };
291    let observe = |observation| {
292        if let Some(observations) = &observations {
293            observations
294                .lock()
295                .push(RunObservation::LlmCall(observation));
296        }
297    };
298    let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
299    let request = clients.prepare_routing_request(call.request.clone(), &target);
300    let algorithm = call.algorithm.clone();
301    let candidates = call.models.len();
302    call.respond(call_one(
303        &clients,
304        &target,
305        request,
306        &algorithm,
307        &observe,
308        0,
309        candidates,
310        CallPhase::Routing,
311    ))
312    .await
313}
314
315/// Try candidates in order until one succeeds or a failure stops fallback.
316async fn call_first_available<'a>(
317    clients: &ClientRouter,
318    algorithm: &str,
319    request: &'a Request,
320    fallback_request: Option<&'a Request>,
321    candidate_requests: &'a HashMap<ModelId, Request>,
322    models: &[ModelId],
323    observe: &(dyn Fn(ModelCallObservation) + Send + Sync),
324) -> Result<(&'a Request, Response)> {
325    for (index, target) in models.iter().enumerate() {
326        let request = candidate_requests
327            .get(target)
328            .unwrap_or(match fallback_request {
329                Some(fallback) if index > 0 => fallback,
330                _ => request,
331            });
332        let prepared = clients.prepare_completion_request(request.clone(), target);
333        match call_one(
334            clients,
335            target,
336            prepared,
337            algorithm,
338            observe,
339            index,
340            models.len(),
341            CallPhase::Completion,
342        )
343        .await
344        {
345            Ok(response) => return Ok((request, response)),
346            Err(error) if index + 1 == models.len() => return Err(error),
347            Err(error)
348                if clients.rate_limit_fallback_disabled.contains(target)
349                    && matches!(&error, LibsyError::ClientCall {
350                        source: LlmClientError::UpstreamHttp { status, .. }, ..
351                    } if *status == StatusCode::TOO_MANY_REQUESTS) =>
352            {
353                return Err(error);
354            }
355            Err(error) => match fallback_reason(&error, fallback_request.is_none()) {
356                Some(reason) => tracing::info!(
357                    from = %target,
358                    to = %models[index + 1],
359                    reason = reason.as_str(),
360                    "model call failed; trying next candidate"
361                ),
362                None => return Err(error),
363            },
364        }
365    }
366    Err(LibsyError::NoTargets)
367}
368
369/// The stage of the Switchyard run in which a model call was issued.
370///
371/// A routing-phase call may still supply the final answer. This is deliberately
372/// distinct from the `routing`/`answer` role assigned after the outcome is known.
373#[derive(Clone, Copy)]
374enum CallPhase {
375    Routing,
376    Completion,
377}
378
379impl CallPhase {
380    const fn as_str(self) -> &'static str {
381        match self {
382            Self::Routing => "routing",
383            Self::Completion => "completion",
384        }
385    }
386}
387
388/// Call one candidate model and record its observation and span.
389#[allow(clippy::too_many_arguments)]
390#[tracing::instrument(
391    target = "libsy",
392    name = "libsy.client_call",
393    skip_all,
394    fields(
395        algorithm = algorithm,
396        switchyard.algorithm = algorithm,
397        switchyard.candidate = index + 1,
398        switchyard.candidate_count = count,
399        switchyard.call_phase = phase.as_str(),
400        selected_model = %clients.provider_model(model_id),
401        selected_target = %model_id,
402        otel.kind = "client",
403        otel.name = %format_args!("chat {}", clients.provider_model(model_id)),
404        openinference.span.kind = "LLM",
405        gen_ai.operation.name = "chat",
406        gen_ai.request.model = %clients.provider_model(model_id),
407        gen_ai.request.stream = tracing::field::Empty,
408        gen_ai.request.temperature = tracing::field::Empty,
409        gen_ai.request.top_p = tracing::field::Empty,
410        gen_ai.request.top_k = tracing::field::Empty,
411        gen_ai.request.max_tokens = tracing::field::Empty,
412        gen_ai.request.reasoning.level = tracing::field::Empty,
413        gen_ai.output.type = tracing::field::Empty,
414        gen_ai.conversation.id = tracing::field::Empty,
415        server.address = tracing::field::Empty,
416        server.port = tracing::field::Empty,
417        gen_ai.response.id = tracing::field::Empty,
418        gen_ai.response.model = tracing::field::Empty,
419        gen_ai.usage.input_tokens = tracing::field::Empty,
420        gen_ai.usage.output_tokens = tracing::field::Empty,
421        gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
422        gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
423        gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty,
424        gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
425        outcome = tracing::field::Empty,
426        otel.status_code = tracing::field::Empty,
427        error.type = tracing::field::Empty,
428        error = tracing::field::Empty,
429    )
430)]
431async fn call_one(
432    clients: &ClientRouter,
433    model_id: &ModelId,
434    request: Request,
435    algorithm: &str,
436    observe: &(dyn Fn(ModelCallObservation) + Send + Sync),
437    // index is for span log
438    index: usize,
439    // count is for span log
440    count: usize,
441    phase: CallPhase,
442) -> Result<Response> {
443    let is_routing = matches!(phase, CallPhase::Routing);
444    let span = tracing::Span::current();
445    observability::record_gen_ai_request(&span, &request.llm_request);
446    if let Some(session_id) = request
447        .metadata
448        .as_ref()
449        .and_then(|metadata| metadata.session_id.as_deref())
450    {
451        span.record("gen_ai.conversation.id", session_id);
452    }
453    // Resolved before the clock starts: picking the client is Switchyard's work, not
454    // the provider's, so it belongs in the routing overhead.
455    let client = clients.route(model_id);
456    let started = Instant::now();
457    let mut routing_usage = None;
458    let result = async {
459        let response = async { client?.call(request).await }
460            .await
461            .map_err(|source| LibsyError::client_call(model_id.clone(), source));
462        // Observe dispatch and live I/O once; buffering must not re-observe replay.
463        let mut response = observability::observe_client_call(response)?;
464        if is_routing && let LlmResponse::Stream(chunks) = response.llm_response {
465            let (chunks, usage) = buffer_routing_stream(chunks, &response.upstream_headers)
466                .await
467                .map_err(|source| LibsyError::client_call(model_id.clone(), source))?;
468            routing_usage = usage;
469            response.llm_response = LlmResponse::Stream(chunks);
470        }
471        Ok(response)
472    }
473    .await;
474    let duration = started.elapsed();
475
476    let result = result.map(|mut response| {
477        response.set_served_model(model_id);
478        response
479    });
480    let result = if is_routing {
481        result
482    } else {
483        metrics::observe_routed_request(
484            algorithm,
485            clients.provider_model(model_id),
486            Some(duration),
487            result,
488        )
489    };
490    observe(ModelCallObservation {
491        selected_model: model_id.clone(),
492        is_success: result.is_ok(),
493        duration,
494        usage: result
495            .as_ref()
496            .ok()
497            .and_then(|response| response.llm_response.as_agg())
498            .map(|response| response.usage.clone())
499            .or(routing_usage),
500    });
501    result
502}
503
504// Routing logs need the final usage, but replay must keep signed provider events intact.
505async fn buffer_routing_stream(
506    mut chunks: LlmResponseStream,
507    upstream_headers: &http::HeaderMap,
508) -> std::result::Result<(LlmResponseStream, Option<Usage>), LlmClientError> {
509    let mut events = Vec::new();
510    let mut usage = None;
511    while let Some(event) = chunks.try_next().await? {
512        for chunk in event.normalized() {
513            match chunk {
514                LlmResponseChunk::Usage(latest) => usage = Some(latest.clone()),
515                LlmResponseChunk::DecodeError { message } => {
516                    return Err(LlmClientError::ResponseTranslation(message.clone()));
517                }
518                LlmResponseChunk::StreamError { message } => {
519                    return Err(LlmClientError::UpstreamHttp {
520                        status: StatusCode::BAD_GATEWAY,
521                        body: message.clone(),
522                        headers: Box::new(upstream_headers.clone()),
523                    });
524                }
525                _ => {}
526            }
527        }
528        events.push(event);
529    }
530    Ok((stream::iter(events.into_iter().map(Ok)).boxed(), usage))
531}
532
533/// Whether a failed candidate is worth routing around.
534fn fallback_reason(
535    error: &LibsyError,
536    allow_policy_fallback: bool,
537) -> Option<RoutingFallbackReason> {
538    let LibsyError::ClientCall { source, .. } = error else {
539        return None;
540    };
541    match source {
542        LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),
543        LlmClientError::Transport { .. } | LlmClientError::TemporarilyUnavailable => {
544            Some(RoutingFallbackReason::Unavailable)
545        }
546        LlmClientError::UpstreamHttp { status, body, .. } => {
547            let policy_denial = serde_json::from_str::<Value>(body).is_ok_and(|value| {
548                value["error"]["code"].as_str() == Some("content_policy_violation")
549            });
550            if policy_denial && !allow_policy_fallback {
551                return None;
552            }
553            ((*status == StatusCode::BAD_REQUEST && policy_denial)
554                || matches!(
555                    *status,
556                    StatusCode::FORBIDDEN
557                        | StatusCode::REQUEST_TIMEOUT
558                        | StatusCode::TOO_MANY_REQUESTS
559                )
560                || status.is_server_error())
561            .then_some(RoutingFallbackReason::Unavailable)
562        }
563        _ => None,
564    }
565}
566
567/// Route to `owner` without fallbacks and record why the algorithm did not run.
568fn continue_on(owner: StateOwner, algorithm: &str, mut request: Request) -> RoutingOutcome {
569    tracing::debug!(
570        target = %owner.model,
571        trigger = owner.trigger,
572        "routing Responses continuation to the model that holds its state"
573    );
574    if let Some(history) = owner.history {
575        let mut messages = Vec::with_capacity(history.len + request.llm_request.messages.len());
576        history.extend(&mut messages);
577        messages.append(&mut request.llm_request.messages);
578        request.llm_request.messages = messages;
579    }
580    let mut outcome = RoutingOutcome::route_to(owner.model, Vec::new(), request);
581    outcome.metadata = Some(OutcomeMetadata::new(
582        algorithm.to_string(),
583        Some(json!({"source": "stored_state", "trigger": owner.trigger})),
584    ));
585    outcome
586}
587
588/// The model that served the stored state and the request field that refers to it.
589struct StateOwner {
590    model: ModelId,
591    trigger: &'static str,
592    history: Option<Arc<MessageHistory>>,
593}
594
595struct MessageHistory {
596    parent: Option<Arc<Self>>,
597    segment: Arc<[Message]>,
598    len: usize,
599}
600
601impl MessageHistory {
602    fn new(parent: Option<Arc<Self>>, segment: Arc<[Message]>) -> Self {
603        let len = parent.as_ref().map_or(0, |history| history.len) + segment.len();
604        Self {
605            parent,
606            segment,
607            len,
608        }
609    }
610
611    fn extend(&self, messages: &mut Vec<Message>) {
612        let mut segments = Vec::new();
613        let mut history = Some(self);
614        while let Some(node) = history {
615            segments.push(node.segment.as_ref());
616            history = node.parent.as_deref();
617        }
618        for segment in segments.into_iter().rev() {
619            messages.extend_from_slice(segment);
620        }
621    }
622}
623
624#[derive(Clone)]
625struct StoredState {
626    model: ModelId,
627    /// `None` means the provider owns the state and only routing must be pinned.
628    history: Option<Arc<MessageHistory>>,
629}
630
631#[derive(Clone)]
632struct CanonicalInput {
633    parent: Option<Arc<MessageHistory>>,
634    messages: Arc<[Message]>,
635}
636
637/// Stored Responses IDs and conversation IDs, with their provider or canonical state.
638///
639/// Keep earlier response IDs because a client can continue from any stored response.
640/// Reject new IDs at capacity rather than forgetting where an earlier response is stored.
641#[derive(Default)]
642struct StateOwners {
643    by_id: HashMap<String, StoredState>,
644}
645
646const MAX_STATE_OWNERS: usize = 65_536;
647
648impl StateOwners {
649    fn owner(&self, id: &str) -> Option<&StoredState> {
650        self.by_id.get(id)
651    }
652
653    fn remember(
654        &mut self,
655        response_id: Option<&str>,
656        conversation_id: Option<&str>,
657        model: &ModelId,
658        history: Option<Arc<MessageHistory>>,
659    ) -> std::result::Result<(), LlmClientError> {
660        let materialized = history.is_some();
661        let ids = [
662            response_id,
663            conversation_id.filter(|id| Some(*id) != response_id),
664        ];
665        let mut new_ids = 0;
666        for id in ids.into_iter().flatten() {
667            match self.by_id.get(id) {
668                Some(owner) if owner.model != *model && !materialized => {
669                    return Err(LlmClientError::ResponseStateConflict);
670                }
671                None => new_ids += 1,
672                _ => {}
673            }
674        }
675        if self.by_id.len() + new_ids > MAX_STATE_OWNERS {
676            return Err(LlmClientError::ResponseStateLimitExceeded {
677                limit: MAX_STATE_OWNERS,
678            });
679        }
680        let state = StoredState {
681            model: model.clone(),
682            history,
683        };
684        if let Some(id) = response_id {
685            if materialized {
686                self.by_id.insert(id.to_owned(), state.clone());
687            } else {
688                self.by_id
689                    .entry(id.to_owned())
690                    .or_insert_with(|| state.clone());
691            }
692        }
693        if let Some(id) = conversation_id {
694            self.by_id.entry(id.to_owned()).or_insert(state);
695        }
696        Ok(())
697    }
698}
699
700/// Read the Responses `conversation` ID from a string or an object.
701fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {
702    let conversation = fields.get("conversation")?;
703    conversation
704        .as_str()
705        .or_else(|| conversation.get("id").and_then(Value::as_str))
706}
707
708/// Resolves a routed call's selected model to the client that serves it.
709///
710/// An algorithm routes among named targets; which provider each target lives on is the
711/// host's concern, and two targets in one run may sit on different providers. A router owns
712/// that mapping and resolves the client for each kind of call.
713///
714/// Cloning is cheap — the mapping is shared, so one router can serve every request.
715#[derive(Clone)]
716pub struct ClientRouter {
717    inner: Arc<ClientRouting>,
718    rate_limit_fallback_disabled: Arc<HashSet<ModelId>>,
719}
720
721struct ClientRouting {
722    routing: Routing,
723    decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
724    target_prompts: HashMap<ModelId, String>,
725    routing_answer_target: Option<ModelId>,
726    /// Provider-owned routing pins and materialized cross-format history.
727    state_owners: Mutex<StateOwners>,
728    /// Native Responses state only needs a routing pin when answer targets span clients.
729    track_provider_state: bool,
730}
731
732enum Routing {
733    /// One client serves every model.
734    Single(Arc<dyn RoutedLlmClient>),
735    /// Each model is served by the client configured for it.
736    ByModel(HashMap<ModelId, Arc<dyn RoutedLlmClient>>),
737}
738
739impl ClientRouter {
740    /// Build a router over `model name -> client`, for targets spread across providers.
741    pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
742        Self::new_with_target_prompts(by_model, HashMap::new(), None)
743    }
744
745    /// Build a router with target prompts used by [`run`] and [`decide`].
746    ///
747    /// `routing_answer_target` identifies a routing-time call whose response may become the
748    /// answer. Pass `None` when routing only selects a later completion target. Prompt keys and
749    /// `routing_answer_target` use the resolved model IDs stored in `by_model`.
750    /// This constructor treats every model in `by_model` as a possible completion target.
751    /// Use [`Self::new_with_completion_targets`] to exclude classifier-only models from the
752    /// comparison of completion clients.
753    pub fn new_with_target_prompts(
754        by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
755        target_prompts: HashMap<ModelId, String>,
756        routing_answer_target: Option<ModelId>,
757    ) -> Self {
758        let completion_targets = by_model.keys().cloned().collect::<Vec<_>>();
759        Self::new_with_completion_targets(
760            by_model,
761            target_prompts,
762            routing_answer_target,
763            &completion_targets,
764        )
765    }
766
767    /// Build a router with an explicit list of models that can answer requests.
768    ///
769    /// `by_model` contains callable LLMs, including LLM classifiers. Only the model IDs in
770    /// `completion_targets` determine whether native Responses state needs a routing pin.
771    /// Cross-format state is recorded lazily when a Responses request uses a Chat or Anthropic
772    /// target. `target_prompts` and `routing_answer_target` have the same meaning as in
773    /// [`Self::new_with_target_prompts`].
774    pub fn new_with_completion_targets(
775        by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
776        target_prompts: HashMap<ModelId, String>,
777        routing_answer_target: Option<ModelId>,
778        completion_targets: &[ModelId],
779    ) -> Self {
780        Self::new_with_decision_clients(
781            by_model,
782            HashMap::new(),
783            target_prompts,
784            routing_answer_target,
785            completion_targets,
786        )
787    }
788
789    /// Build a router with separate LLM and decision mappings, shared across clones.
790    /// LLM prompt and completion settings follow [`Self::new_with_completion_targets`].
791    pub fn new_with_decision_clients(
792        by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
793        decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
794        target_prompts: HashMap<ModelId, String>,
795        routing_answer_target: Option<ModelId>,
796        completion_targets: &[ModelId],
797    ) -> Self {
798        let mut clients = completion_targets
799            .iter()
800            .filter_map(|model| by_model.get(model));
801        let spans_providers = clients
802            .next()
803            .is_some_and(|first| clients.any(|client| !Arc::ptr_eq(first, client)));
804        Self {
805            rate_limit_fallback_disabled: Arc::default(),
806            inner: Arc::new(ClientRouting {
807                routing: Routing::ByModel(by_model),
808                decision_clients,
809                target_prompts,
810                routing_answer_target,
811                state_owners: Mutex::default(),
812                track_provider_state: spans_providers,
813            }),
814        }
815    }
816
817    /// A router that serves every model with one client — the single-provider case.
818    ///
819    /// [`TranslatingLlmClient`](crate::TranslatingLlmClient) already maps model names to
820    /// backends internally and rejects ones it does not know, so enumerating them here would
821    /// only duplicate that.
822    pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {
823        Self::single_with_decision_clients(client, HashMap::new())
824    }
825
826    /// Use one client for all LLM targets and explicit clients for decision targets.
827    pub fn single_with_decision_clients(
828        client: Arc<dyn RoutedLlmClient>,
829        decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
830    ) -> Self {
831        Self {
832            rate_limit_fallback_disabled: Arc::default(),
833            inner: Arc::new(ClientRouting {
834                routing: Routing::Single(client),
835                decision_clients,
836                target_prompts: HashMap::new(),
837                routing_answer_target: None,
838                state_owners: Mutex::default(),
839                track_provider_state: false,
840            }),
841        }
842    }
843
844    /// Stop completion fallback on HTTP 429 for the given models.
845    ///
846    /// For translating clients, also use
847    /// [`TranslatingLlmClient::without_rate_limit_cooldown`](crate::TranslatingLlmClient::without_rate_limit_cooldown)
848    /// so later calls are not skipped with a generic cooldown error.
849    pub fn without_rate_limit_fallback(mut self, models: HashSet<ModelId>) -> Self {
850        self.rate_limit_fallback_disabled = Arc::new(models);
851        self
852    }
853
854    /// Resolves a target for model telemetry, including calls that fail before a response.
855    fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
856        self.route(target)
857            .map_or(target, |client| client.provider_model(target))
858    }
859
860    /// The client that serves `model`.
861    ///
862    /// Errors with [`LlmClientError::Configuration`] when the router maps models and has no
863    /// entry for this one, rather than silently sending the call to another provider.
864    pub fn route(
865        &self,
866        model: &ModelId,
867    ) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
868        match &self.inner.routing {
869            Routing::Single(client) => Ok(client),
870            Routing::ByModel(by_model) => {
871                by_model
872                    .get(model)
873                    .ok_or_else(|| LlmClientError::Configuration {
874                        message: format!("no llm client is configured for model {model:?}"),
875                    })
876            }
877        }
878    }
879
880    /// Resolve a decision target without falling back to an LLM client.
881    fn route_decision(
882        &self,
883        model: &ModelId,
884    ) -> std::result::Result<&Arc<dyn RoutedDecisionClient>, LlmClientError> {
885        self.inner
886            .decision_clients
887            .get(model)
888            .ok_or_else(|| LlmClientError::Configuration {
889                message: format!("no decision client is configured for model {model:?}"),
890            })
891    }
892
893    /// Return the recorded model for the requested response or conversation ID.
894    fn stored_state_owner(&self, request: &Request) -> Option<StateOwner> {
895        let fields = &request.llm_request.extensions.fields;
896        let (trigger, id) = match fields.get("previous_response_id").and_then(Value::as_str) {
897            Some(id) => ("previous_response_id", id),
898            None => ("conversation", conversation_id(fields)?),
899        };
900        let state = self.inner.state_owners.lock().owner(id)?.clone();
901        Some(StateOwner {
902            model: state.model,
903            trigger,
904            history: state.history,
905        })
906    }
907
908    fn canonical_input(&self, request: &Request) -> CanonicalInput {
909        let parent = request
910            .llm_request
911            .extensions
912            .fields
913            .get("previous_response_id")
914            .and_then(Value::as_str)
915            .and_then(|id| self.inner.state_owners.lock().owner(id)?.history.clone());
916        let parent_len = parent.as_ref().map_or(0, |history| history.len);
917        let (parent, messages) = match request.llm_request.messages.get(parent_len..) {
918            Some(messages) => (parent, messages.to_vec()),
919            None => (None, request.llm_request.messages.clone()),
920        };
921        CanonicalInput {
922            parent,
923            messages: Arc::from(messages),
924        }
925    }
926
927    /// Record provider-owned Responses state or canonical history for a cross-format response.
928    fn remember_state_owner(&self, request: &Request, mut response: Response) -> Result<Response> {
929        let Some(model) = response.served_model().cloned() else {
930            return Ok(response);
931        };
932        let fields = &request.llm_request.extensions.fields;
933        let store = fields.get("store").and_then(Value::as_bool) != Some(false);
934        let conversation = conversation_id(fields).map(str::to_owned);
935        let responses_format = WireFormat::OpenAiResponses.into();
936        let responses_request = request
937            .llm_request
938            .preservation
939            .requests
940            .contains_key(&responses_format)
941            || fields.contains_key("previous_response_id")
942            || fields.contains_key("conversation");
943        if !responses_request && !self.inner.track_provider_state {
944            return Ok(response);
945        }
946        let canonical_input = responses_request.then(|| self.canonical_input(request));
947        response.llm_response = match response.llm_response {
948            LlmResponse::Agg(agg) => {
949                if let Some(body) = agg.preservation.responses.get(&responses_format) {
950                    if self.inner.track_provider_state {
951                        self.remember_response(body, &model, store, conversation.as_deref())
952                            .map_err(|error| LibsyError::client_call(model.clone(), error))?;
953                    }
954                } else if let Some(input) = &canonical_input {
955                    self.remember_canonical_response(&agg, &model, store, input)
956                        .map_err(|error| LibsyError::client_call(model.clone(), error))?;
957                }
958                LlmResponse::Agg(agg)
959            }
960            LlmResponse::Stream(stream) => {
961                let router = self.clone();
962                LlmResponse::Stream(Box::pin(futures_util::stream::try_unfold(
963                    (
964                        stream,
965                        router,
966                        model,
967                        conversation,
968                        canonical_input,
969                        ResponseAccumulator::new(),
970                        false,
971                        true,
972                    ),
973                    move |(
974                        mut stream,
975                        router,
976                        model,
977                        conversation,
978                        canonical_input,
979                        mut accumulator,
980                        mut native_responses,
981                        mut valid,
982                    )| async move {
983                        let Some(event) = stream.next().await else {
984                            if let Some(input) = &canonical_input
985                                && !native_responses
986                                && valid
987                            {
988                                router.remember_canonical_response(
989                                    &accumulator.finish(),
990                                    &model,
991                                    store,
992                                    input,
993                                )?;
994                            }
995                            return Ok(None);
996                        };
997                        let event = event?;
998                        if let Some(preserved) = event.preservation()
999                            && preserved.source().as_str() == WireFormat::OpenAiResponses.as_str()
1000                        {
1001                            native_responses = true;
1002                            if router.inner.track_provider_state
1003                                && let Some(body) = preserved.raw().get("response")
1004                            {
1005                                router.remember_response(
1006                                    body,
1007                                    &model,
1008                                    store,
1009                                    conversation.as_deref(),
1010                                )?;
1011                            }
1012                        }
1013                        if responses_request {
1014                            for chunk in event.normalized() {
1015                                if matches!(
1016                                    chunk,
1017                                    LlmResponseChunk::DecodeError { .. }
1018                                        | LlmResponseChunk::StreamError { .. }
1019                                ) {
1020                                    valid = false;
1021                                } else {
1022                                    accumulator.push(chunk.clone());
1023                                }
1024                            }
1025                        }
1026                        Ok(Some((
1027                            event,
1028                            (
1029                                stream,
1030                                router,
1031                                model,
1032                                conversation,
1033                                canonical_input,
1034                                accumulator,
1035                                native_responses,
1036                                valid,
1037                            ),
1038                        )))
1039                    },
1040                )))
1041            }
1042        };
1043        Ok(response)
1044    }
1045
1046    /// Check both IDs from a response or stream event before recording either. The response's
1047    /// `store` value overrides the request value; `false` skips only the response ID.
1048    fn remember_response(
1049        &self,
1050        body: &Value,
1051        model: &ModelId,
1052        store: bool,
1053        conversation: Option<&str>,
1054    ) -> std::result::Result<(), LlmClientError> {
1055        let mut owners = self.inner.state_owners.lock();
1056        let response_id = body
1057            .get("id")
1058            .and_then(Value::as_str)
1059            .filter(|_| body.get("store").and_then(Value::as_bool).unwrap_or(store));
1060        let conversation = body.as_object().and_then(conversation_id).or(conversation);
1061        if let Err(error) = owners.remember(response_id, conversation, model, None) {
1062            if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
1063                tracing::warn!(%error, "Responses state owner capacity reached; state was not retained");
1064            } else {
1065                tracing::error!(%error, "could not record Responses state owner");
1066                return Err(error);
1067            }
1068        }
1069        Ok(())
1070    }
1071
1072    fn remember_canonical_response(
1073        &self,
1074        response: &AggLlmResponse,
1075        model: &ModelId,
1076        store: bool,
1077        input: &CanonicalInput,
1078    ) -> std::result::Result<(), LlmClientError> {
1079        let response_id = response.id.as_deref().filter(|_| store);
1080        if response_id.is_none() {
1081            return Ok(());
1082        }
1083        let mut segment = input.messages.to_vec();
1084        segment.extend(
1085            response
1086                .outputs
1087                .iter()
1088                .map(|output| Message::new(output.role, output.content.clone())),
1089        );
1090        let history = Arc::new(MessageHistory::new(
1091            input.parent.clone(),
1092            Arc::from(segment),
1093        ));
1094        let result =
1095            self.inner
1096                .state_owners
1097                .lock()
1098                .remember(response_id, None, model, Some(history));
1099        if let Err(error) = result {
1100            if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
1101                tracing::warn!(%error, "cross-format Responses state capacity reached; history was not retained");
1102                return Ok(());
1103            }
1104            tracing::error!(%error, "could not record cross-format Responses state");
1105            return Err(error);
1106        }
1107        Ok(())
1108    }
1109
1110    /// Prepare a completion candidate with its configured target prompt.
1111    fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request {
1112        let prompt = self.inner.target_prompts.get(target).map(String::as_str);
1113        prepare_request_for_target(&mut request.llm_request, target, prompt);
1114        request
1115    }
1116
1117    /// Prepare a routing call, adding a target prompt only when it generates a candidate answer.
1118    fn prepare_routing_request(&self, mut request: Request, target: &ModelId) -> Request {
1119        let prompt = if self.inner.routing_answer_target.as_ref() == Some(target) {
1120            self.inner.target_prompts.get(target).map(String::as_str)
1121        } else {
1122            None
1123        };
1124        prepare_request_for_target(&mut request.llm_request, target, prompt);
1125        request
1126    }
1127}
1128
1129impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
1130    fn from_iter<I: IntoIterator<Item = (ModelId, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
1131        Self::new(iter.into_iter().collect())
1132    }
1133}
1134
1135#[cfg(test)]
1136mod tests {
1137    use std::collections::{BTreeMap, BTreeSet};
1138
1139    use super::*;
1140    use async_trait::async_trait;
1141    use futures::StreamExt;
1142    use http::StatusCode;
1143    use switchyard_libsy::{Driver, RoutingOutcome};
1144    use switchyard_protocol::{
1145        ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Role,
1146        ToolResult, completion_text, text_request, text_response,
1147    };
1148    use wiremock::matchers::method;
1149    use wiremock::{Mock, MockServer, ResponseTemplate};
1150
1151    use crate::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
1152    use switchyard_protocol::Category;
1153
1154    struct CandidateAlgorithm {}
1155
1156    struct AnsweredAlgorithm {
1157        model: ModelId,
1158    }
1159
1160    #[async_trait]
1161    impl Algorithm for CandidateAlgorithm {
1162        fn name(&self) -> &str {
1163            "candidate_test"
1164        }
1165
1166        async fn route(
1167            self: Arc<Self>,
1168            driver: Driver,
1169            request: Request,
1170        ) -> Result<RoutingOutcome> {
1171            let models = driver.models_for(&Category::Any);
1172            let selected_model = models.first().cloned().ok_or(LibsyError::NoTargets)?;
1173            Ok(RoutingOutcome::route_to(
1174                selected_model,
1175                models.iter().skip(1).cloned().collect(),
1176                request,
1177            ))
1178        }
1179    }
1180
1181    #[async_trait]
1182    impl Algorithm for AnsweredAlgorithm {
1183        fn name(&self) -> &str {
1184            "answered_test"
1185        }
1186
1187        async fn route(
1188            self: Arc<Self>,
1189            driver: Driver,
1190            request: Request,
1191        ) -> Result<RoutingOutcome> {
1192            let response = driver
1193                .call_model(request.clone(), vec![self.model.clone()])
1194                .await?;
1195            Ok(RoutingOutcome::answered(
1196                self.model.clone(),
1197                request,
1198                response,
1199            ))
1200        }
1201    }
1202
1203    /// Rewrites the request for its selected model only.
1204    struct RewritingAlgorithm {}
1205
1206    #[async_trait]
1207    impl Algorithm for RewritingAlgorithm {
1208        fn name(&self) -> &str {
1209            "rewriting_test"
1210        }
1211
1212        async fn route(
1213            self: Arc<Self>,
1214            _driver: Driver,
1215            request: Request,
1216        ) -> Result<RoutingOutcome> {
1217            let mut rewritten = request.clone();
1218            rewritten
1219                .llm_request
1220                .messages
1221                .push(Message::text(Role::User, "for weak only"));
1222            let mut outcome =
1223                RoutingOutcome::route_to("weak".into(), vec!["strong".into()], rewritten);
1224            outcome.fallback_request = Some(request);
1225            Ok(outcome)
1226        }
1227    }
1228
1229    #[derive(Clone, Copy)]
1230    enum FirstOutcome {
1231        ContextWindow,
1232        ContentPolicy(StatusCode),
1233        Unauthorized,
1234        StreamSuccess,
1235        MidStreamError,
1236    }
1237
1238    struct CandidateClient {
1239        calls: Mutex<Vec<ModelId>>,
1240        requests: Mutex<Vec<Request>>,
1241        first: FirstOutcome,
1242    }
1243
1244    #[async_trait]
1245    impl RoutedLlmClient for CandidateClient {
1246        async fn call(&self, request: Request) -> std::result::Result<Response, LlmClientError> {
1247            let model = request.model_id().unwrap_or_default();
1248            self.calls.lock().push(model.clone());
1249            self.requests.lock().push(request);
1250            if model == "weak" {
1251                return match self.first {
1252                    FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded {
1253                        model,
1254                        message: "too long".to_string(),
1255                    }),
1256                    FirstOutcome::ContentPolicy(status) => Err(LlmClientError::UpstreamHttp {
1257                        status,
1258                        body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),
1259                        headers: Box::default(),
1260                    }),
1261                    FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {
1262                        status: StatusCode::UNAUTHORIZED,
1263                        body: "unauthorized".to_string(),
1264                        headers: Box::default(),
1265                    }),
1266                    FirstOutcome::StreamSuccess => Ok(stream_response(vec![
1267                        LlmResponseChunk::TextDelta {
1268                            index: 0,
1269                            text: "streamed".to_string(),
1270                        },
1271                        LlmResponseChunk::MessageStop {
1272                            reason: Some("stop".to_string()),
1273                        },
1274                    ])),
1275                    FirstOutcome::MidStreamError => Ok(stream_response(vec![
1276                        LlmResponseChunk::TextDelta {
1277                            index: 0,
1278                            text: "partial".to_string(),
1279                        },
1280                        LlmResponseChunk::StreamError {
1281                            message: "stream failed".to_string(),
1282                        },
1283                        LlmResponseChunk::MessageStop { reason: Some("stop".into()) },
1284                    ])),
1285                };
1286            }
1287            Ok(Response {
1288                llm_response: LlmResponse::Agg(AggLlmResponse {
1289                    id: Some("candidate_response".into()),
1290                    ..text_response(Some(model.to_string()), model)
1291                }),
1292                metadata: None,
1293                upstream_headers: http::HeaderMap::new(),
1294            })
1295        }
1296    }
1297
1298    fn stream_response(chunks: Vec<LlmResponseChunk>) -> Response {
1299        Response {
1300            llm_response: LlmResponse::Stream(
1301                futures::stream::iter(
1302                    chunks
1303                        .into_iter()
1304                        .map(|chunk| Ok(LlmResponseStreamEvent::from(chunk))),
1305                )
1306                .boxed(),
1307            ),
1308            metadata: None,
1309            upstream_headers: http::HeaderMap::new(),
1310        }
1311    }
1312
1313    fn request() -> Request {
1314        Request {
1315            llm_request: text_request(Some("auto".to_string()), "hello".to_string()),
1316            raw_request: None,
1317            metadata: None,
1318        }
1319    }
1320
1321    #[tokio::test]
1322    async fn system_one_serves_decisions_and_returns_failures_to_the_classifier()
1323    -> std::result::Result<(), Box<dyn std::error::Error>> {
1324        use std::time::Duration;
1325        use switchyard_libsy::{
1326            CapabilityJudgeConfig, DecisionJudgeConfig, LlmClassifierConfig, LlmTaskClassifier,
1327            TaskClassifierConfig,
1328        };
1329        use switchyard_protocol::{
1330            BooleanEstimate, DecisionRequest, DecisionValue, Probability, ScoreValue,
1331        };
1332        use wiremock::matchers::{body_partial_json, header, path};
1333
1334        let server = MockServer::start().await;
1335        let client = Arc::new(crate::SystemOneClient::new(
1336            format!("{}/v1/systemone", server.uri()).parse()?,
1337            "test-key".into(),
1338            Duration::from_secs(2),
1339        )?);
1340        assert!(matches!(
1341            crate::SystemOneClient::new(
1342                format!("{}/v1/systemone", server.uri()).parse()?,
1343                "invalid\nkey".into(),
1344                Duration::from_secs(2),
1345            ),
1346            Err(LlmClientError::Configuration { .. })
1347        ));
1348        let mixed: DecisionRequest = serde_json::from_value(json!({
1349            "model": "judge", "input": [Message::text(Role::User, "A simple task")],
1350            "questions": {
1351                "boolean": {"instructions": "Is it simple?", "kind": {"type": "boolean", "data": {
1352                    "true_description": ["Simple"], "false_description": null
1353                }}},
1354                "route": {"instructions": "Choose", "kind": {"type": "choice", "data": {
1355                    "options": [{"id": "capable", "description": "A wins"}, {"id": "efficient", "description": "B wins"}]
1356                }}},
1357                "score": {"instructions": "Rate difficulty", "kind": {"type": "score", "data": {
1358                    "levels": ["Easy", {"description": "Hard"}]
1359                }}}
1360            }
1361        }))?;
1362        let mut body = json!({
1363            "model": "jev-1.13.0", "usage": {"input_tokens": 42, "output_tokens": 3},
1364            "answers": {
1365                "boolean": {"type": "noul", "noul": 0.9},
1366                "route": {"type": "choice", "choice": "efficient", "confidence": 0.7,
1367                    "probabilities": {"capable": 0.2, "efficient": 0.8}},
1368                "score": {"type": "score", "score": 0.6, "confidence": 0.3,
1369                    "legend": {"0": "Easy", "1": {"description": "Hard"}},
1370                    "probabilities": {"1": 0.6, "0": 0.4}}
1371            }
1372        });
1373        Mock::given(method("POST"))
1374            .and(path("/v1/systemone"))
1375            .and(header("authorization", "Bearer test-key"))
1376            .and(body_partial_json(json!({
1377                "model": "judge", "state": mixed.input,
1378                "questions": {
1379                    "boolean": {"type": "noul", "criteria": {"true": ["Simple"]}},
1380                    "route": {"type": "choice", "instructions": "Choose",
1381                        "criteria": {"capable": "A wins", "efficient": "B wins"}},
1382                    "score": {"type": "score", "criteria": ["Easy", {"description": "Hard"}]}
1383                }
1384            })))
1385            .respond_with(ResponseTemplate::new(200).set_body_json(&body))
1386            .expect(1)
1387            .mount(&server)
1388            .await;
1389        let response = client.call(mixed.clone()).await?;
1390        assert_eq!(response.model.as_deref(), Some("jev-1.13.0"));
1391        assert_eq!(response.id, None);
1392        assert_eq!(response.usage.input_tokens, Some(42));
1393        assert_eq!(response.usage.total_tokens, None);
1394        assert_eq!(
1395            response.answers["boolean"].value,
1396            DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(Probability(0.9)))
1397        );
1398        assert_eq!(
1399            response.answers["score"].value,
1400            DecisionValue::Score {
1401                value: ScoreValue(0.6),
1402                probabilities: Some(vec![Probability(0.4), Probability(0.6)]),
1403            }
1404        );
1405        assert_eq!(
1406            response.answers["route"].provider_confidence.map(|c| c.0),
1407            Some(0.7)
1408        );
1409        server.verify().await;
1410
1411        let mut partial = body.clone();
1412        partial["answers"]
1413            .as_object_mut()
1414            .unwrap()
1415            .remove("boolean");
1416        let mut invalid_responses = vec![json!({"answers": {}}), partial];
1417        for (id, answer) in [
1418            ("unknown-question", json!({"type": "noul", "noul": 0.9})),
1419            ("boolean", json!({"type": "score", "score": 0.0})),
1420            (
1421                "route",
1422                json!({"type": "choice", "choice": "unknown-option",
1423                    "probabilities": {"capable": 0.2, "efficient": 0.8}}),
1424            ),
1425            ("score", json!({"type": "score", "score": -0.1})),
1426            (
1427                "score",
1428                json!({"type": "score", "score": 1.1,
1429                "probabilities": {"0": 0.4, "1": 0.6}}),
1430            ),
1431        ] {
1432            let mut invalid = body.clone();
1433            invalid["answers"][id] = answer;
1434            if id == "unknown-question" {
1435                invalid["answers"]
1436                    .as_object_mut()
1437                    .unwrap()
1438                    .remove("boolean");
1439            }
1440            invalid_responses.push(invalid);
1441        }
1442        for template in invalid_responses
1443            .into_iter()
1444            .map(|body| ResponseTemplate::new(200).set_body_json(body))
1445            .chain([ResponseTemplate::new(200).set_body_string("invalid JSON")])
1446        {
1447            server.reset().await;
1448            Mock::given(method("POST"))
1449                .respond_with(template)
1450                .expect(1)
1451                .mount(&server)
1452                .await;
1453            assert!(matches!(
1454                client.call(mixed.clone()).await,
1455                Err(LlmClientError::ResponseTranslation(_) | LlmClientError::InvalidResponse { .. })
1456            ));
1457            server.verify().await;
1458        }
1459
1460        body["answers"].as_object_mut().unwrap().remove("boolean");
1461        body["answers"].as_object_mut().unwrap().remove("score");
1462        let models = Arc::new(RuntimeModels::new(HashMap::from([
1463            (Category::Judge, vec!["judge".into()]),
1464            (Category::Capable, vec!["capable".into()]),
1465            (Category::Efficient, vec!["efficient".into()]),
1466            (Category::Any, vec!["efficient".into(), "capable".into()]),
1467        ])));
1468        let algorithm = || -> Result<Arc<dyn Algorithm>> {
1469            Ok(Arc::new(LlmTaskClassifier::new(
1470                LlmClassifierConfig::Capability {
1471                    config: TaskClassifierConfig {
1472                        judge: CapabilityJudgeConfig::Decision(DecisionJudgeConfig {
1473                            cutoff: Some(0.4),
1474                            ..DecisionJudgeConfig::default()
1475                        }),
1476                        ..TaskClassifierConfig::default()
1477                    },
1478                },
1479            )?))
1480        };
1481        for (template, expected, is_success) in [
1482            (
1483                ResponseTemplate::new(200).set_body_json(&body),
1484                "efficient",
1485                true,
1486            ),
1487            (
1488                ResponseTemplate::new(503).set_body_string("unavailable"),
1489                "capable",
1490                false,
1491            ),
1492        ] {
1493            server.reset().await;
1494            Mock::given(method("POST"))
1495                .and(path("/v1/systemone"))
1496                .respond_with(template)
1497                .expect(2)
1498                .mount(&server)
1499                .await;
1500            let llm = Arc::new(CandidateClient {
1501                calls: Mutex::default(),
1502                requests: Mutex::default(),
1503                first: FirstOutcome::Unauthorized,
1504            });
1505            let clients = ClientRouter::single_with_decision_clients(
1506                llm.clone(),
1507                HashMap::from([(
1508                    ModelId::from("judge"),
1509                    client.clone() as Arc<dyn RoutedDecisionClient>,
1510                )]),
1511            );
1512            let outcome = decide(algorithm()?, clients.clone(), request(), models.clone()).await?;
1513            assert_eq!(outcome.selected_model_id()?, expected);
1514            assert!(llm.calls.lock().is_empty());
1515            let events = Arc::new(Mutex::new(Vec::new()));
1516            let captured = events.clone();
1517            let (selected, _) = run(
1518                algorithm()?,
1519                clients,
1520                request(),
1521                models.clone(),
1522                Some(Arc::new(move |event| captured.lock().push(event))),
1523            )
1524            .await?;
1525            assert_eq!(selected, expected);
1526            assert_eq!(&*llm.calls.lock(), &[ModelId::from(expected)]);
1527            {
1528                let events = events.lock();
1529                let RunObservation::DecisionCall(call) = &events[0] else {
1530                    panic!("missing decision observation")
1531                };
1532                assert_eq!(call.selected_model, "judge");
1533                assert_eq!(call.is_success, is_success);
1534                assert_eq!(
1535                    call.usage.as_ref().and_then(|u| u.input_tokens),
1536                    is_success.then_some(42)
1537                );
1538                assert!(
1539                    events
1540                        .iter()
1541                        .any(|event| matches!(event, RunObservation::AnswerCall(_)))
1542                );
1543            }
1544            server.verify().await;
1545        }
1546        Ok(())
1547    }
1548
1549    fn instruction_text(request: &Request) -> Vec<&str> {
1550        request
1551            .llm_request
1552            .instructions
1553            .iter()
1554            .flat_map(|instruction| &instruction.content)
1555            .filter_map(|block| match block {
1556                ContentBlock::Text { text } => Some(text.as_str()),
1557                _ => None,
1558            })
1559            .collect()
1560    }
1561
1562    async fn run_candidates(
1563        first: FirstOutcome,
1564    ) -> (Arc<CandidateClient>, Result<(ModelId, Response)>) {
1565        let client = Arc::new(CandidateClient {
1566            calls: Mutex::new(Vec::new()),
1567            requests: Mutex::new(Vec::new()),
1568            first,
1569        });
1570        let algorithm = Arc::new(CandidateAlgorithm {});
1571        let models = to_category_map(&["weak", "strong"]);
1572        let result = run(
1573            algorithm,
1574            ClientRouter::single(client.clone()),
1575            request(),
1576            models,
1577            None,
1578        )
1579        .await;
1580        (client, result)
1581    }
1582
1583    fn to_category_map(names: &[&str]) -> Arc<RuntimeModels> {
1584        Arc::new(RuntimeModels::new(
1585            [(
1586                Category::Any,
1587                names.iter().map(|name| ModelId::from(*name)).collect(),
1588            )]
1589            .into(),
1590        ))
1591    }
1592
1593    #[tokio::test]
1594    async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> {
1595        let client = Arc::new(CandidateClient {
1596            calls: Mutex::new(Vec::new()),
1597            requests: Mutex::new(Vec::new()),
1598            first: FirstOutcome::StreamSuccess,
1599        });
1600        let observations = Arc::new(Mutex::new(Vec::new()));
1601        let observed = Arc::clone(&observations);
1602        let observer: RunObserver = Arc::new(move |event| observed.lock().push(event));
1603
1604        let (selected, response) = run(
1605            Arc::new(AnsweredAlgorithm {
1606                model: "weak".into(),
1607            }),
1608            ClientRouter::single(client.clone()),
1609            request(),
1610            Arc::new(RuntimeModels::default()),
1611            Some(observer),
1612        )
1613        .await?;
1614
1615        assert_eq!(selected, "weak");
1616        assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
1617        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
1618        let observations = observations.lock();
1619        assert!(matches!(observations[0], RunObservation::AnswerCall(_)));
1620        assert!(matches!(
1621            observations[1],
1622            RunObservation::Outcome(ref metadata) if metadata.algorithm == "answered_test"
1623        ));
1624        assert!(matches!(
1625            observations[2],
1626            RunObservation::RoutingOverhead(_)
1627        ));
1628        assert_eq!(observations.len(), 3);
1629        Ok(())
1630    }
1631
1632    #[tokio::test]
1633    async fn served_answer_is_reported_before_the_client_sees_it_end() -> Result<()> {
1634        #[derive(Default)]
1635        struct ReportingAlgorithm {
1636            answers: Mutex<Vec<(String, bool)>>,
1637            tracking: bool,
1638        }
1639
1640        #[async_trait]
1641        impl Algorithm for ReportingAlgorithm {
1642            fn name(&self) -> &str {
1643                "reporting_test"
1644            }
1645
1646            async fn route(
1647                self: Arc<Self>,
1648                _driver: Driver,
1649                request: Request,
1650            ) -> Result<RoutingOutcome> {
1651                Ok(RoutingOutcome::route_to("weak".into(), Vec::new(), request))
1652            }
1653
1654            fn tracks_answers(&self) -> bool {
1655                self.tracking
1656            }
1657
1658            async fn observe_answer(&self, session_id: &str, answer: AnswerSummary) {
1659                self.answers
1660                    .lock()
1661                    .push((session_id.to_string(), answer.final_reply));
1662            }
1663        }
1664
1665        assert!(!CandidateAlgorithm {}.tracks_answers());
1666        for (tracking, first, final_reply, cancel) in [
1667            (false, FirstOutcome::StreamSuccess, true, false),
1668            (true, FirstOutcome::StreamSuccess, true, false),
1669            (true, FirstOutcome::MidStreamError, false, false),
1670            (true, FirstOutcome::StreamSuccess, false, true),
1671        ] {
1672            let algorithm = Arc::new(ReportingAlgorithm {
1673                tracking,
1674                ..Default::default()
1675            });
1676            let client = Arc::new(CandidateClient {
1677                calls: Mutex::new(Vec::new()),
1678                requests: Mutex::new(Vec::new()),
1679                first,
1680            });
1681            let request = Request {
1682                metadata: Some(switchyard_protocol::Metadata {
1683                    session_id: Some("session-1".to_string()),
1684                    ..Default::default()
1685                }),
1686                ..request()
1687            };
1688            let (_, response) = run(
1689                algorithm.clone(),
1690                ClientRouter::single(client),
1691                request,
1692                to_category_map(&["weak"]),
1693                None,
1694            )
1695            .await?;
1696            let LlmResponse::Stream(mut stream) = response.llm_response else {
1697                panic!("expected a streamed answer");
1698            };
1699            if !tracking {
1700                assert!(algorithm.answers.lock().is_empty());
1701                while let Some(item) = stream.next().await {
1702                    assert!(item.is_ok());
1703                }
1704                assert!(algorithm.answers.lock().is_empty());
1705                continue;
1706            }
1707            if cancel {
1708                stream.next().await;
1709                drop(stream);
1710                assert_eq!(
1711                    *algorithm.answers.lock(),
1712                    [("session-1".to_string(), false)]
1713                );
1714                continue;
1715            }
1716            while let Some(item) = stream.next().await {
1717                let stops = item.as_ref().is_ok_and(|event| {
1718                    event
1719                        .normalized()
1720                        .iter()
1721                        .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }))
1722                });
1723                if stops {
1724                    assert_eq!(algorithm.answers.lock().len(), 2);
1725                }
1726            }
1727            assert_eq!(
1728                *algorithm.answers.lock(),
1729                if final_reply {
1730                    vec![
1731                        ("session-1".to_string(), false),
1732                        ("session-1".to_string(), true),
1733                    ]
1734                } else {
1735                    vec![
1736                        ("session-1".to_string(), false),
1737                        ("session-1".to_string(), false),
1738                    ]
1739                }
1740            );
1741        }
1742        Ok(())
1743    }
1744
1745    #[test]
1746    fn answer_observation_keeps_call_order() {
1747        let pending = Some(Arc::new(Mutex::new(
1748            ["answer", "judge"]
1749                .map(|model| ModelCallObservation {
1750                    selected_model: model.into(),
1751                    is_success: true,
1752                    duration: std::time::Duration::ZERO,
1753                    usage: None,
1754                })
1755                .map(RunObservation::LlmCall)
1756                .into(),
1757        )));
1758        let emitted = Arc::new(Mutex::new(Vec::new()));
1759        let captured = Arc::clone(&emitted);
1760        let observer: RunObserver = Arc::new(move |event| captured.lock().push(event));
1761        let answer = ModelId::from("answer");
1762
1763        emit_routing_observations(&Some(observer), &pending, Some(&answer));
1764
1765        assert!(matches!(
1766            &emitted.lock()[..],
1767            [RunObservation::AnswerCall(answer), RunObservation::LlmCall(judge)]
1768                if answer.selected_model == "answer" && judge.selected_model == "judge"
1769        ));
1770    }
1771
1772    #[tokio::test]
1773    async fn responses_state_tracking_uses_upstream_storage_and_format() -> Result<()> {
1774        for (responses, store, conversation) in [
1775            (true, true, None),
1776            (true, false, None),
1777            (true, false, Some(json!("conv_plain"))),
1778            (true, false, Some(json!({"id": "conv_object"}))),
1779            (false, true, None),
1780        ] {
1781            for stream in [false, true] {
1782                let server = MockServer::start().await;
1783                Mock::given(method("POST"))
1784                    .respond_with(move |request: &wiremock::Request| {
1785                        let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1786                        let response = if responses {
1787                            json!({
1788                                "id": "resp_seed", "object": "response", "model": "weak",
1789                                "status": "completed", "output": [], "store": body["store"],
1790                                "conversation": body["conversation"],
1791                            })
1792                        } else if stream {
1793                            json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "delta": {"content": "ok"}, "finish_reason": "stop"}]})
1794                        } else {
1795                            json!({"id": "resp_seed", "model": "weak", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}]})
1796                        };
1797                        if stream {
1798                            let event = if responses { json!({"type": "response.completed", "response": response}) } else { response };
1799                            ResponseTemplate::new(200)
1800                                .insert_header("content-type", "text/event-stream")
1801                                .set_body_string(if responses {
1802                                    format!("event: response.completed\ndata: {event}\n\n")
1803                                } else {
1804                                    format!("data: {event}\n\ndata: [DONE]\n\n")
1805                                })
1806                        } else {
1807                            ResponseTemplate::new(200).set_body_json(response)
1808                        }
1809                    })
1810                    .mount(&server).await;
1811                let mut clients = HashMap::new();
1812                for model in ["weak", "strong"] {
1813                    let config = HttpBackendConfig {
1814                        base_url: server.uri(),
1815                        api_key: None,
1816                        forward_auth: false,
1817                        extra_headers: BTreeMap::new(),
1818                        extra_body: BTreeMap::from([("store".to_string(), json!(store))]),
1819                        omit_body_fields: BTreeSet::new(),
1820                        reasoning_effort: None,
1821                        max_retries: 0,
1822                        failure_cooldown: std::time::Duration::ZERO,
1823                        timeout: None,
1824                    };
1825                    let backend = if responses {
1826                        Backend::OpenAiResponses(config)
1827                    } else {
1828                        Backend::OpenAiChat(config)
1829                    };
1830                    let client: Arc<dyn RoutedLlmClient> = Arc::new(
1831                        TranslatingLlmClient::new(&[ModelConfig::new(model, backend, None)])
1832                            .map_err(|error| LibsyError::external("building test client", error))?,
1833                    );
1834                    clients.insert(ModelId::from(model), client);
1835                }
1836                let clients = ClientRouter::new(clients);
1837                let mut seed = request();
1838                seed.llm_request.stream = stream;
1839                if let Some(conversation) = &conversation {
1840                    seed.llm_request
1841                        .extensions
1842                        .fields
1843                        .insert("conversation".to_string(), conversation.clone());
1844                }
1845                let (_, response) = run(
1846                    Arc::new(switchyard_libsy::Passthrough),
1847                    clients.clone(),
1848                    seed,
1849                    to_category_map(&["weak"]),
1850                    None,
1851                )
1852                .await?;
1853                response
1854                    .llm_response
1855                    .into_agg()
1856                    .await
1857                    .map_err(|error| LibsyError::client_call("weak", error))?;
1858                let mut continuations = vec![(
1859                    "previous_response_id",
1860                    json!("resp_seed"),
1861                    responses && store,
1862                )];
1863                if let Some(conversation) = conversation.clone() {
1864                    continuations.push(("conversation", conversation, true));
1865                }
1866                for (field, id, pinned) in continuations {
1867                    let mut follow = request();
1868                    follow
1869                        .llm_request
1870                        .extensions
1871                        .fields
1872                        .insert(field.to_string(), id);
1873                    let outcome = decide(
1874                        Arc::new(switchyard_libsy::Passthrough),
1875                        clients.clone(),
1876                        follow,
1877                        to_category_map(&["strong"]),
1878                    )
1879                    .await?;
1880                    assert_eq!(
1881                        outcome.selected_model_id()?.as_str(),
1882                        if pinned { "weak" } else { "strong" },
1883                        "responses={responses}, store={store}, stream={stream}, field={field}"
1884                    );
1885                }
1886                let requests = server.received_requests().await.expect("request recording");
1887                assert_eq!(requests.len(), 1);
1888                assert_eq!(
1889                    serde_json::from_slice::<Value>(&requests[0].body).expect("request JSON")["store"],
1890                    store
1891                );
1892            }
1893        }
1894        Ok(())
1895    }
1896
1897    #[tokio::test]
1898    async fn responses_stored_tool_continuation_materializes_for_anthropic() -> Result<()> {
1899        let server = MockServer::start().await;
1900        Mock::given(method("POST"))
1901            .respond_with(|request: &wiremock::Request| {
1902                let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
1903                let blocks = body["messages"]
1904                    .as_array()
1905                    .into_iter()
1906                    .flatten()
1907                    .filter_map(|message| message["content"].as_array())
1908                    .flatten()
1909                    .collect::<Vec<_>>();
1910                let has_result = blocks.iter().any(|block| block["type"] == "tool_result");
1911                let has_call = blocks.iter().any(|block| block["type"] == "tool_use");
1912                let has_thanks = blocks
1913                    .iter()
1914                    .any(|block| block["type"] == "text" && block["text"] == "thanks")
1915                    || body["messages"].as_array().is_some_and(|messages| {
1916                        messages
1917                            .iter()
1918                            .any(|message| message["content"] == "thanks")
1919                    });
1920                if has_result && !has_call {
1921                    return ResponseTemplate::new(400).set_body_json(json!({
1922                        "type": "error",
1923                        "error": {
1924                            "type": "invalid_request_error",
1925                            "message": "tool_result has no corresponding tool_use"
1926                        }
1927                    }));
1928                }
1929                let response = if has_thanks {
1930                    json!({
1931                        "id": "msg_third", "type": "message", "role": "assistant",
1932                        "model": "weak", "content": [{"type": "text", "text": "done"}],
1933                        "stop_reason": "end_turn", "usage": {"input_tokens": 6, "output_tokens": 1}
1934                    })
1935                } else if has_result {
1936                    json!({
1937                        "id": "msg_follow", "type": "message", "role": "assistant",
1938                        "model": "weak", "content": [{"type": "text", "text": "sunny"}],
1939                        "stop_reason": "end_turn", "usage": {"input_tokens": 4, "output_tokens": 1}
1940                    })
1941                } else {
1942                    json!({
1943                        "id": "msg_seed", "type": "message", "role": "assistant",
1944                        "model": "weak",
1945                        "content": [{
1946                            "type": "tool_use", "id": "toolu_weather", "name": "get_weather",
1947                            "input": {"city": "Paris"}
1948                        }],
1949                        "stop_reason": "tool_use", "usage": {"input_tokens": 3, "output_tokens": 1}
1950                    })
1951                };
1952                ResponseTemplate::new(200).set_body_json(response)
1953            })
1954            .mount(&server)
1955            .await;
1956
1957        let client: Arc<dyn RoutedLlmClient> = Arc::new(
1958            TranslatingLlmClient::new(&[ModelConfig::new(
1959                "weak",
1960                Backend::Anthropic(HttpBackendConfig {
1961                    base_url: server.uri(),
1962                    api_key: None,
1963                    forward_auth: false,
1964                    extra_headers: BTreeMap::new(),
1965                    extra_body: BTreeMap::new(),
1966                    omit_body_fields: BTreeSet::new(),
1967                    reasoning_effort: None,
1968                    max_retries: 0,
1969                    failure_cooldown: std::time::Duration::ZERO,
1970                    timeout: None,
1971                }),
1972                None,
1973            )])
1974            .map_err(|error| LibsyError::external("building test client", error))?,
1975        );
1976        let clients = ClientRouter::new(HashMap::from([(ModelId::from("weak"), client)]));
1977        let models = to_category_map(&["weak"]);
1978
1979        let seed = Request {
1980            llm_request: switchyard_translation::decode_request(
1981                WireFormat::OpenAiResponses,
1982                &json!({
1983                    "model": "route",
1984                    "input": "Call get_weather for Paris",
1985                    "tools": [{
1986                        "type": "function", "name": "get_weather",
1987                        "parameters": {"type": "object"}
1988                    }],
1989                    "store": true
1990                }),
1991            )
1992            .map_err(|error| LibsyError::external("decoding seed request", error))?,
1993            raw_request: None,
1994            metadata: None,
1995        };
1996        let (_, response) = run(
1997            Arc::new(switchyard_libsy::Passthrough),
1998            clients.clone(),
1999            seed,
2000            models.clone(),
2001            None,
2002        )
2003        .await?;
2004        assert_eq!(
2005            response
2006                .llm_response
2007                .as_agg()
2008                .and_then(|agg| agg.id.as_deref()),
2009            Some("msg_seed")
2010        );
2011
2012        let follow = Request {
2013            llm_request: switchyard_translation::decode_request(
2014                WireFormat::OpenAiResponses,
2015                &json!({
2016                    "model": "route",
2017                    "previous_response_id": "msg_seed",
2018                    "input": [{
2019                        "type": "function_call_output",
2020                        "call_id": "toolu_weather",
2021                        "output": "Paris is sunny"
2022                    }],
2023                    "tools": [{
2024                        "type": "function", "name": "get_weather",
2025                        "parameters": {"type": "object"}
2026                    }],
2027                    "store": true
2028                }),
2029            )
2030            .map_err(|error| LibsyError::external("decoding continuation request", error))?,
2031            raw_request: None,
2032            metadata: None,
2033        };
2034        let (_, response) = run(
2035            Arc::new(switchyard_libsy::Passthrough),
2036            clients.clone(),
2037            follow,
2038            models.clone(),
2039            None,
2040        )
2041        .await?;
2042        assert_eq!(
2043            completion_text(
2044                response
2045                    .llm_response
2046                    .as_agg()
2047                    .expect("buffered Anthropic response"),
2048            ),
2049            "sunny"
2050        );
2051
2052        let third = Request {
2053            llm_request: switchyard_translation::decode_request(
2054                WireFormat::OpenAiResponses,
2055                &json!({
2056                    "model": "route",
2057                    "previous_response_id": "msg_follow",
2058                    "input": "thanks",
2059                    "store": true
2060                }),
2061            )
2062            .map_err(|error| LibsyError::external("decoding chained request", error))?,
2063            raw_request: None,
2064            metadata: None,
2065        };
2066        let history = clients
2067            .stored_state_owner(&third)
2068            .and_then(|owner| owner.history)
2069            .expect("chained canonical history");
2070        assert_eq!(history.len, 4);
2071        assert_eq!(history.segment.len(), 2);
2072        assert!(history.parent.is_some());
2073        let (_, response) = run(
2074            Arc::new(switchyard_libsy::Passthrough),
2075            clients,
2076            third,
2077            models,
2078            None,
2079        )
2080        .await?;
2081        assert_eq!(
2082            completion_text(
2083                response
2084                    .llm_response
2085                    .as_agg()
2086                    .expect("buffered Anthropic response"),
2087            ),
2088            "done"
2089        );
2090
2091        let requests = server.received_requests().await.expect("request recording");
2092        assert_eq!(requests.len(), 3);
2093        let follow: Value = serde_json::from_slice(&requests[1].body)
2094            .map_err(|error| LibsyError::external("decoding captured request", error))?;
2095        assert_eq!(follow["messages"][1]["content"][0]["type"], "tool_use");
2096        assert_eq!(follow["messages"][2]["content"][0]["type"], "tool_result");
2097        let third: Value = serde_json::from_slice(&requests[2].body)
2098            .map_err(|error| LibsyError::external("decoding captured request", error))?;
2099        assert_eq!(third["messages"][1]["content"][0]["type"], "tool_use");
2100        assert_eq!(third["messages"][2]["content"][0]["type"], "tool_result");
2101        assert_eq!(third["messages"][3]["content"], "sunny");
2102        assert_eq!(third["messages"][4]["content"][0]["text"], "thanks");
2103        Ok(())
2104    }
2105
2106    #[tokio::test]
2107    async fn streamed_cross_format_state_is_recorded_only_after_completion() -> Result<()> {
2108        let client = Arc::new(CandidateClient {
2109            calls: Mutex::new(Vec::new()),
2110            requests: Mutex::new(Vec::new()),
2111            first: FirstOutcome::StreamSuccess,
2112        });
2113        let clients = ClientRouter::new(HashMap::from([(
2114            ModelId::from("weak"),
2115            client as Arc<dyn RoutedLlmClient>,
2116        )]));
2117        let mut seed = request();
2118        seed.llm_request.messages = vec![Message::text(Role::User, "inspect")];
2119        seed.llm_request
2120            .preservation
2121            .requests
2122            .insert(WireFormat::OpenAiResponses.into(), json!({}));
2123        let mut response = stream_response(vec![
2124            LlmResponseChunk::MessageStart {
2125                id: Some("msg_stream".to_string()),
2126                model: Some("weak".to_string()),
2127            },
2128            LlmResponseChunk::ToolCallDelta {
2129                index: 0,
2130                id: Some("toolu_stream".to_string()),
2131                name: Some("lookup".to_string()),
2132                arguments_delta: Some("{}".to_string()),
2133            },
2134            LlmResponseChunk::MessageStop {
2135                reason: Some("tool_use".to_string()),
2136            },
2137        ]);
2138        response.set_served_model(&ModelId::from("weak"));
2139        let response = clients.remember_state_owner(&seed, response)?;
2140
2141        let mut follow = request();
2142        follow
2143            .llm_request
2144            .extensions
2145            .fields
2146            .insert("previous_response_id".to_string(), json!("msg_stream"));
2147        follow.llm_request.messages = vec![Message::new(
2148            Role::Tool,
2149            vec![ContentBlock::ToolResult(ToolResult {
2150                tool_call_id: "toolu_stream".to_string(),
2151                content: vec![ContentBlock::Text {
2152                    text: "ready".to_string(),
2153                }],
2154                is_error: None,
2155            })],
2156        )];
2157        assert!(clients.stored_state_owner(&follow).is_none());
2158        response
2159            .llm_response
2160            .into_agg()
2161            .await
2162            .map_err(|error| LibsyError::client_call("weak", error))?;
2163
2164        let outcome = continue_on(
2165            clients
2166                .stored_state_owner(&follow)
2167                .expect("completed stream state"),
2168            "passthrough",
2169            follow,
2170        );
2171        assert_eq!(outcome.request.llm_request.messages.len(), 3);
2172        assert!(matches!(
2173            outcome.request.llm_request.messages[1].content.as_slice(),
2174            [ContentBlock::ToolCall(call)] if call.id == "toolu_stream"
2175        ));
2176        assert!(matches!(
2177            outcome.request.llm_request.messages[2].content.as_slice(),
2178            [ContentBlock::ToolResult(result)] if result.tool_call_id == "toolu_stream"
2179        ));
2180        Ok(())
2181    }
2182
2183    #[test]
2184    fn cross_format_response_with_store_false_is_not_retained() -> Result<()> {
2185        let client = Arc::new(CandidateClient {
2186            calls: Mutex::new(Vec::new()),
2187            requests: Mutex::new(Vec::new()),
2188            first: FirstOutcome::StreamSuccess,
2189        });
2190        let clients = ClientRouter::new(HashMap::from([(
2191            ModelId::from("weak"),
2192            client as Arc<dyn RoutedLlmClient>,
2193        )]));
2194        let mut seed = request();
2195        seed.llm_request
2196            .extensions
2197            .fields
2198            .insert("store".to_string(), json!(false));
2199        seed.llm_request
2200            .preservation
2201            .requests
2202            .insert(WireFormat::OpenAiResponses.into(), json!({}));
2203        let mut agg = text_response(Some("weak".to_string()), "done".to_string());
2204        agg.id = Some("msg_not_stored".to_string());
2205        let mut response = Response {
2206            llm_response: LlmResponse::Agg(agg),
2207            metadata: None,
2208            upstream_headers: http::HeaderMap::new(),
2209        };
2210        response.set_served_model(&ModelId::from("weak"));
2211        clients.remember_state_owner(&seed, response)?;
2212
2213        let mut follow = request();
2214        follow
2215            .llm_request
2216            .extensions
2217            .fields
2218            .insert("previous_response_id".to_string(), json!("msg_not_stored"));
2219        assert!(clients.stored_state_owner(&follow).is_none());
2220        Ok(())
2221    }
2222
2223    #[tokio::test]
2224    async fn state_capacity_does_not_fail_completed_responses() -> Result<()> {
2225        let client = Arc::new(CandidateClient {
2226            calls: Mutex::new(Vec::new()),
2227            requests: Mutex::new(Vec::new()),
2228            first: FirstOutcome::StreamSuccess,
2229        });
2230        let clients = ClientRouter::new(HashMap::from([(
2231            ModelId::from("weak"),
2232            client as Arc<dyn RoutedLlmClient>,
2233        )]));
2234        let model = ModelId::from("weak");
2235        clients.inner.state_owners.lock().by_id = (0..MAX_STATE_OWNERS)
2236            .map(|id| {
2237                (
2238                    format!("existing_{id}"),
2239                    StoredState {
2240                        model: model.clone(),
2241                        history: None,
2242                    },
2243                )
2244            })
2245            .collect();
2246
2247        let mut request = request();
2248        request
2249            .llm_request
2250            .preservation
2251            .requests
2252            .insert(WireFormat::OpenAiResponses.into(), json!({}));
2253        let mut agg = text_response(Some("weak".to_string()), "done".to_string());
2254        agg.id = Some("overflow_buffered".to_string());
2255        let mut buffered = Response {
2256            llm_response: LlmResponse::Agg(agg),
2257            metadata: None,
2258            upstream_headers: http::HeaderMap::new(),
2259        };
2260        buffered.set_served_model(&model);
2261        clients.remember_state_owner(&request, buffered)?;
2262
2263        let mut streamed = stream_response(vec![
2264            LlmResponseChunk::MessageStart {
2265                id: Some("overflow_streamed".to_string()),
2266                model: Some("weak".to_string()),
2267            },
2268            LlmResponseChunk::TextDelta {
2269                index: 0,
2270                text: "done".to_string(),
2271            },
2272            LlmResponseChunk::MessageStop {
2273                reason: Some("stop".to_string()),
2274            },
2275        ]);
2276        streamed.set_served_model(&model);
2277        clients
2278            .remember_state_owner(&request, streamed)?
2279            .llm_response
2280            .into_agg()
2281            .await
2282            .map_err(|error| LibsyError::client_call("weak", error))?;
2283
2284        assert_eq!(
2285            clients.inner.state_owners.lock().by_id.len(),
2286            MAX_STATE_OWNERS
2287        );
2288        Ok(())
2289    }
2290
2291    #[test]
2292    fn state_owners_preserve_existing_records_at_capacity()
2293    -> std::result::Result<(), LlmClientError> {
2294        let mut owners = StateOwners::default();
2295        let model = ModelId::from("model/a");
2296        for id in 0..MAX_STATE_OWNERS - 1 {
2297            owners.remember(Some(&format!("resp_{id}")), None, &model, None)?;
2298        }
2299        assert!(matches!(
2300            owners.remember(Some("new_response"), Some("new_conversation"), &model, None,),
2301            Err(LlmClientError::ResponseStateLimitExceeded {
2302                limit: MAX_STATE_OWNERS
2303            })
2304        ));
2305        assert!(owners.owner("new_response").is_none());
2306        assert!(owners.owner("new_conversation").is_none());
2307        owners.remember(Some("last_id"), Some("last_id"), &model, None)?;
2308        owners.remember(Some("last_id"), Some("resp_0"), &model, None)?;
2309        assert!(matches!(
2310            owners.remember(Some("over_limit"), None, &model, None),
2311            Err(LlmClientError::ResponseStateLimitExceeded { .. })
2312        ));
2313        assert!(matches!(
2314            owners.remember(Some("resp_0"), None, &ModelId::from("model/b"), None,),
2315            Err(LlmClientError::ResponseStateConflict)
2316        ));
2317        assert_eq!(
2318            owners.owner("resp_0").map(|state| &state.model),
2319            Some(&model)
2320        );
2321        assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
2322        Ok(())
2323    }
2324
2325    #[test]
2326    fn concurrent_state_owners_do_not_exceed_capacity() {
2327        let model = ModelId::from("model/a");
2328        let owners = Arc::new(Mutex::new(StateOwners {
2329            by_id: (0..MAX_STATE_OWNERS - 16)
2330                .map(|id| {
2331                    (
2332                        format!("existing_{id}"),
2333                        StoredState {
2334                            model: model.clone(),
2335                            history: None,
2336                        },
2337                    )
2338                })
2339                .collect(),
2340        }));
2341        let barrier = Arc::new(std::sync::Barrier::new(32));
2342        let accepted = std::thread::scope(|scope| {
2343            let handles = (0..32)
2344                .map(|id| {
2345                    let owners = owners.clone();
2346                    let barrier = barrier.clone();
2347                    let model = model.clone();
2348                    scope.spawn(move || {
2349                        barrier.wait();
2350                        let result = owners.lock().remember(
2351                            Some(&format!("resp_{id}")),
2352                            Some(&format!("conv_{id}")),
2353                            &model,
2354                            None,
2355                        );
2356                        match result {
2357                            Ok(()) => Some(id),
2358                            Err(LlmClientError::ResponseStateLimitExceeded { .. }) => None,
2359                            other => panic!("unexpected state tracking result: {other:?}"),
2360                        }
2361                    })
2362                })
2363                .collect::<Vec<_>>();
2364            handles
2365                .into_iter()
2366                .filter_map(|handle| handle.join().expect("request thread panicked"))
2367                .collect::<Vec<_>>()
2368        });
2369        let owners = owners.lock();
2370        assert_eq!(accepted.len(), 8);
2371        assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
2372        assert_eq!(
2373            owners.owner("existing_0").map(|state| &state.model),
2374            Some(&model)
2375        );
2376        for id in accepted {
2377            assert_eq!(
2378                owners
2379                    .owner(&format!("resp_{id}"))
2380                    .map(|state| &state.model),
2381                Some(&model)
2382            );
2383            assert_eq!(
2384                owners
2385                    .owner(&format!("conv_{id}"))
2386                    .map(|state| &state.model),
2387                Some(&model)
2388            );
2389        }
2390    }
2391
2392    #[tokio::test]
2393    async fn each_fallback_candidate_receives_only_its_own_prompt() -> Result<()> {
2394        let client = Arc::new(CandidateClient {
2395            calls: Mutex::new(Vec::new()),
2396            requests: Mutex::new(Vec::new()),
2397            first: FirstOutcome::ContextWindow,
2398        });
2399        let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
2400        let clients = ClientRouter::new_with_target_prompts(
2401            HashMap::from([
2402                (ModelId::from("weak"), Arc::clone(&routed_client)),
2403                (ModelId::from("strong"), routed_client),
2404            ]),
2405            HashMap::from([
2406                ("weak".into(), "weak prompt".to_string()),
2407                ("strong".into(), "strong prompt".to_string()),
2408            ]),
2409            None,
2410        );
2411
2412        run(
2413            Arc::new(CandidateAlgorithm {}),
2414            clients,
2415            request(),
2416            to_category_map(&["weak", "strong"]),
2417            None,
2418        )
2419        .await?;
2420
2421        let calls = client.requests.lock();
2422        assert_eq!(calls.len(), 2);
2423        assert_eq!(instruction_text(&calls[0]), ["weak prompt"]);
2424        assert_eq!(instruction_text(&calls[1]), ["strong prompt"]);
2425        Ok(())
2426    }
2427
2428    #[tokio::test]
2429    async fn decision_prompts_a_routing_response_target() -> Result<()> {
2430        let client = Arc::new(CandidateClient {
2431            calls: Mutex::new(Vec::new()),
2432            requests: Mutex::new(Vec::new()),
2433            first: FirstOutcome::StreamSuccess,
2434        });
2435        let routed_client: Arc<dyn RoutedLlmClient> = client.clone();
2436        let clients = ClientRouter::new_with_target_prompts(
2437            HashMap::from([(ModelId::from("answer"), routed_client)]),
2438            HashMap::from([("answer".into(), "answer prompt".to_string())]),
2439            Some("answer".into()),
2440        );
2441
2442        decide(
2443            Arc::new(AnsweredAlgorithm {
2444                model: "answer".into(),
2445            }),
2446            clients,
2447            request(),
2448            Arc::new(RuntimeModels::default()),
2449        )
2450        .await?;
2451
2452        let calls = client.requests.lock();
2453        assert_eq!(calls.len(), 1);
2454        assert_eq!(instruction_text(&calls[0]), ["answer prompt"]);
2455        Ok(())
2456    }
2457
2458    #[tokio::test]
2459    async fn decision_prepares_only_the_selected_request() -> Result<()> {
2460        let client = Arc::new(CandidateClient {
2461            calls: Mutex::new(Vec::new()),
2462            requests: Mutex::new(Vec::new()),
2463            first: FirstOutcome::StreamSuccess,
2464        });
2465        let routed_client: Arc<dyn RoutedLlmClient> = client;
2466        let clients = ClientRouter::new_with_target_prompts(
2467            HashMap::from([
2468                (ModelId::from("weak"), Arc::clone(&routed_client)),
2469                (ModelId::from("strong"), routed_client),
2470            ]),
2471            HashMap::from([
2472                (ModelId::from("weak"), "weak prompt".to_string()),
2473                (ModelId::from("strong"), "strong prompt".to_string()),
2474            ]),
2475            None,
2476        );
2477
2478        let outcome = decide(
2479            Arc::new(CandidateAlgorithm {}),
2480            clients,
2481            request(),
2482            to_category_map(&["weak", "strong"]),
2483        )
2484        .await?;
2485
2486        assert_eq!(
2487            outcome.selected_model_ids,
2488            [ModelId::from("weak"), ModelId::from("strong")]
2489        );
2490        assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);
2491        Ok(())
2492    }
2493
2494    #[test]
2495    fn fallback_only_accepts_context_policy_and_unavailable_failures() {
2496        let error = |source| LibsyError::client_call("target", source);
2497        assert_eq!(
2498            fallback_reason(
2499                &error(LlmClientError::ContextWindowExceeded {
2500                    model: "target".into(),
2501                    message: "too long".to_string(),
2502                }),
2503                true
2504            ),
2505            Some(RoutingFallbackReason::ContextWindow)
2506        );
2507        for status in [
2508            StatusCode::FORBIDDEN,
2509            StatusCode::REQUEST_TIMEOUT,
2510            StatusCode::TOO_MANY_REQUESTS,
2511            StatusCode::INTERNAL_SERVER_ERROR,
2512            StatusCode::from_u16(599).expect("599 is a valid status"),
2513        ] {
2514            assert_eq!(
2515                fallback_reason(
2516                    &error(LlmClientError::UpstreamHttp {
2517                        status,
2518                        body: "failed".to_string(),
2519                        headers: Box::default(),
2520                    }),
2521                    true
2522                ),
2523                Some(RoutingFallbackReason::Unavailable)
2524            );
2525        }
2526        for (body, expected) in [
2527            (
2528                r#"{"error":{"code":"content_policy_violation"}}"#,
2529                Some(RoutingFallbackReason::Unavailable),
2530            ),
2531            (
2532                r#"{"error":{"code":"invalid_request_error"},"metadata":"content_policy_violation"}"#,
2533                None,
2534            ),
2535            (r#"{"error":{"message":"content_policy_violation"}}"#, None),
2536            ("content_policy_violation", None),
2537        ] {
2538            assert_eq!(
2539                fallback_reason(
2540                    &error(LlmClientError::UpstreamHttp {
2541                        status: StatusCode::BAD_REQUEST,
2542                        body: body.to_string(),
2543                        headers: Box::default(),
2544                    }),
2545                    true
2546                ),
2547                expected,
2548                "{body}"
2549            );
2550        }
2551        for status in [
2552            StatusCode::BAD_REQUEST,
2553            StatusCode::UNAUTHORIZED,
2554            StatusCode::NOT_FOUND,
2555            StatusCode::CONFLICT,
2556            StatusCode::from_u16(499).expect("499 is a valid status"),
2557            StatusCode::from_u16(600).expect("600 is a valid status"),
2558        ] {
2559            assert_eq!(
2560                fallback_reason(
2561                    &error(LlmClientError::UpstreamHttp {
2562                        status,
2563                        body: "failed".to_string(),
2564                        headers: Box::default(),
2565                    }),
2566                    true
2567                ),
2568                None
2569            );
2570        }
2571    }
2572
2573    #[tokio::test]
2574    async fn candidate_failures_follow_the_fallback_policy() -> Result<()> {
2575        // Context overflow is retryable across candidates.
2576        let (client, result) = run_candidates(FirstOutcome::ContextWindow).await;
2577        let (_, response) = result?;
2578        assert_eq!(
2579            &*client.calls.lock(),
2580            &[ModelId::from("weak"), "strong".into()]
2581        );
2582        assert_eq!(
2583            response
2584                .llm_response
2585                .as_agg()
2586                .map(|response| response.model.as_deref()),
2587            Some(Some("strong"))
2588        );
2589        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2590
2591        // A provider-specific policy denial also permits another candidate.
2592        let (client, result) =
2593            run_candidates(FirstOutcome::ContentPolicy(StatusCode::BAD_REQUEST)).await;
2594        let (_, response) = result?;
2595        assert_eq!(
2596            &*client.calls.lock(),
2597            &[ModelId::from("weak"), "strong".into()]
2598        );
2599        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2600
2601        // Authentication failure is not retryable, so the second candidate is untouched.
2602        let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;
2603        assert!(matches!(
2604            result,
2605            Err(LibsyError::ClientCall {
2606                source: LlmClientError::UpstreamHttp {
2607                    status: StatusCode::UNAUTHORIZED,
2608                    ..
2609                },
2610                ..
2611            })
2612        ));
2613        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2614        Ok(())
2615    }
2616
2617    #[tokio::test]
2618    async fn fallback_candidates_receive_the_fallback_request() -> Result<()> {
2619        let client = Arc::new(CandidateClient {
2620            calls: Mutex::new(Vec::new()),
2621            requests: Mutex::new(Vec::new()),
2622            first: FirstOutcome::ContextWindow,
2623        });
2624        let clients = ClientRouter::single(client.clone());
2625        let mut seed = request();
2626        seed.llm_request
2627            .preservation
2628            .requests
2629            .insert(WireFormat::OpenAiResponses.into(), json!({}));
2630        run(
2631            Arc::new(RewritingAlgorithm {}),
2632            clients.clone(),
2633            seed,
2634            to_category_map(&["weak", "strong"]),
2635            None,
2636        )
2637        .await?;
2638        let sent: Vec<_> = client
2639            .requests
2640            .lock()
2641            .iter()
2642            .map(|request| request.llm_request.messages.clone())
2643            .collect();
2644        assert_eq!(sent.len(), 2);
2645        assert_eq!(sent[0].len(), request().llm_request.messages.len() + 1);
2646        assert_eq!(sent[1], request().llm_request.messages);
2647        let mut follow = request();
2648        follow
2649            .llm_request
2650            .extensions
2651            .fields
2652            .insert("previous_response_id".into(), json!("candidate_response"));
2653        follow.llm_request.messages = vec![Message::text(Role::User, "continue")];
2654        let owner = clients
2655            .stored_state_owner(&follow)
2656            .expect("fallback Responses state");
2657        let continued = continue_on(owner, "test", follow);
2658        let mut expected = sent[1].clone();
2659        expected.push(Message::text(Role::Assistant, "strong"));
2660        expected.push(Message::text(Role::User, "continue"));
2661        assert_eq!(continued.request.llm_request.messages, expected);
2662        Ok(())
2663    }
2664
2665    #[tokio::test]
2666    async fn fallback_request_does_not_bypass_content_policy() -> Result<()> {
2667        for status in [StatusCode::BAD_REQUEST, StatusCode::FORBIDDEN] {
2668            let client = Arc::new(CandidateClient {
2669                calls: Mutex::new(Vec::new()),
2670                requests: Mutex::new(Vec::new()),
2671                first: FirstOutcome::ContentPolicy(status),
2672            });
2673            let result = run(
2674                Arc::new(RewritingAlgorithm {}),
2675                ClientRouter::single(client.clone()),
2676                request(),
2677                to_category_map(&["weak", "strong"]),
2678                None,
2679            )
2680            .await;
2681            assert!(
2682                matches!(result, Err(LibsyError::ClientCall { source: LlmClientError::UpstreamHttp { status: actual, .. }, .. }) if actual == status)
2683            );
2684            assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2685        }
2686        Ok(())
2687    }
2688
2689    #[tokio::test]
2690    async fn retry_exhaustion_falls_back_and_optional_cooldown_skips_failed_backend() -> Result<()>
2691    {
2692        let server = MockServer::start().await;
2693        let calls = Arc::new(Mutex::new(Vec::new()));
2694        let observed_calls = Arc::clone(&calls);
2695        Mock::given(method("POST"))
2696            .respond_with(move |request: &wiremock::Request| {
2697                let body: serde_json::Value =
2698                    serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
2699                let model = body["model"].as_str().unwrap_or_default().to_string();
2700                observed_calls.lock().push(model.clone());
2701                if model == "weak" {
2702                    ResponseTemplate::new(503)
2703                        .insert_header("retry-after", "0")
2704                        .set_body_string("unavailable")
2705                } else {
2706                    ResponseTemplate::new(200).set_body_json(serde_json::json!({
2707                        "id": "answer",
2708                        "model": "strong",
2709                        "choices": [{
2710                            "index": 0,
2711                            "message": {"role": "assistant", "content": "ok"},
2712                            "finish_reason": "stop"
2713                        }],
2714                        "usage": {}
2715                    }))
2716                }
2717            })
2718            .mount(&server)
2719            .await;
2720
2721        let client_with_cooldown = |cooldown| {
2722            let backend = || {
2723                Backend::OpenAiChat(HttpBackendConfig {
2724                    base_url: format!("{}/v1", server.uri()),
2725                    api_key: None,
2726                    forward_auth: false,
2727                    extra_headers: BTreeMap::new(),
2728                    extra_body: BTreeMap::new(),
2729                    omit_body_fields: BTreeSet::new(),
2730                    reasoning_effort: None,
2731                    max_retries: 2,
2732                    failure_cooldown: cooldown,
2733                    timeout: None,
2734                })
2735            };
2736            TranslatingLlmClient::new(&[
2737                ModelConfig::new("weak", backend(), None),
2738                ModelConfig::new("strong", backend(), None),
2739            ])
2740            .map(Arc::new)
2741            .map_err(|error| LibsyError::external("building test client", error))
2742        };
2743        let call = async |client: Arc<TranslatingLlmClient>| {
2744            run(
2745                Arc::new(CandidateAlgorithm {}),
2746                ClientRouter::single(client),
2747                request(),
2748                to_category_map(&["weak", "strong"]),
2749                None,
2750            )
2751            .await?;
2752            Ok::<_, LibsyError>(std::mem::take(&mut *calls.lock()))
2753        };
2754
2755        let client = client_with_cooldown(std::time::Duration::ZERO)?;
2756        assert_eq!(
2757            call(client.clone()).await?,
2758            ["weak", "weak", "weak", "strong"]
2759        );
2760        assert_eq!(call(client).await?, ["weak", "weak", "weak", "strong"]);
2761
2762        let cooldown = std::time::Duration::from_secs(60);
2763        let client = client_with_cooldown(cooldown)?;
2764        assert_eq!(
2765            call(client.clone()).await?,
2766            ["weak", "weak", "weak", "strong"]
2767        );
2768        assert_eq!(call(client.clone()).await?, ["strong"]);
2769
2770        tokio::time::pause();
2771        tokio::time::advance(cooldown).await;
2772        tokio::time::resume();
2773
2774        assert_eq!(call(client).await?, ["weak", "weak", "weak", "strong"]);
2775        Ok(())
2776    }
2777
2778    #[tokio::test]
2779    async fn refused_connection_falls_back_to_the_next_candidate() -> Result<()> {
2780        let server = MockServer::start().await;
2781        Mock::given(method("POST"))
2782            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
2783                "id": "answer",
2784                "model": "strong",
2785                "choices": [{
2786                    "index": 0,
2787                    "message": {"role": "assistant", "content": "ok"},
2788                    "finish_reason": "stop"
2789                }],
2790                "usage": {}
2791            })))
2792            .mount(&server)
2793            .await;
2794
2795        let backend = |base_url: String| {
2796            Backend::OpenAiChat(HttpBackendConfig {
2797                base_url,
2798                api_key: None,
2799                forward_auth: false,
2800                extra_headers: BTreeMap::new(),
2801                extra_body: BTreeMap::new(),
2802                omit_body_fields: BTreeSet::new(),
2803                reasoning_effort: None,
2804                max_retries: 0,
2805                failure_cooldown: std::time::Duration::ZERO,
2806                timeout: None,
2807            })
2808        };
2809        let client = Arc::new(
2810            TranslatingLlmClient::new(&[
2811                // Nothing listens on port 1, so the connection is refused.
2812                ModelConfig::new("weak", backend("http://127.0.0.1:1/v1".to_string()), None),
2813                ModelConfig::new("strong", backend(format!("{}/v1", server.uri())), None),
2814            ])
2815            .map_err(|error| LibsyError::external("building test client", error))?,
2816        );
2817        let (_, response) = run(
2818            Arc::new(CandidateAlgorithm {}),
2819            ClientRouter::single(client),
2820            request(),
2821            to_category_map(&["weak", "strong"]),
2822            None,
2823        )
2824        .await?;
2825
2826        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2827        Ok(())
2828    }
2829
2830    #[tokio::test]
2831    async fn streams_are_outside_the_candidate_fallback_boundary() -> Result<()> {
2832        // Receiving a stream handle is a successful call and ends candidate selection.
2833        let (client, result) = run_candidates(FirstOutcome::StreamSuccess).await;
2834        let (_, response) = result?;
2835        assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));
2836        let aggregate = response
2837            .llm_response
2838            .into_agg()
2839            .await
2840            .map_err(|source| LibsyError::client_call("weak", source))?;
2841        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2842        assert_eq!(completion_text(&aggregate), "streamed");
2843
2844        // A later in-stream failure surfaces during aggregation without trying another model.
2845        let (client, result) = run_candidates(FirstOutcome::MidStreamError).await;
2846        let (_, response) = result?;
2847        let error = response
2848            .llm_response
2849            .into_agg()
2850            .await
2851            .expect_err("the stream should fail during aggregation");
2852        assert!(error.to_string().contains("stream failed"));
2853        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
2854        Ok(())
2855    }
2856
2857    // Verbatim shape of an SGLang admission rejection on a streaming session:
2858    // HTTP 200, and the overflow arrives as the first SSE event.
2859    const OVERFLOW_SSE: &str = "data: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.\",\"object\":\"error\",\"param\":null,\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n";
2860
2861    // A normal one-turn streamed answer, as the second candidate serves it.
2862    const STRONG_SSE: &str = "data: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
2863
2864    /// Runs a streaming request through the weak→strong candidate loop, serving
2865    /// `weak_sse` from the first candidate and [`STRONG_SSE`] from the second.
2866    /// The server is returned so response streams stay readable after the call.
2867    async fn run_streaming_candidates(
2868        weak_sse: &'static str,
2869    ) -> (
2870        MockServer,
2871        Arc<Mutex<Vec<String>>>,
2872        Result<(ModelId, Response)>,
2873    ) {
2874        let server = MockServer::start().await;
2875        let calls = Arc::new(Mutex::new(Vec::new()));
2876        let observed_calls = Arc::clone(&calls);
2877        Mock::given(method("POST"))
2878            .respond_with(move |request: &wiremock::Request| {
2879                let body: serde_json::Value =
2880                    serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
2881                let model = body["model"].as_str().unwrap_or_default().to_string();
2882                observed_calls.lock().push(model.clone());
2883                let sse = if model == "weak" {
2884                    weak_sse
2885                } else {
2886                    STRONG_SSE
2887                };
2888                ResponseTemplate::new(200)
2889                    .insert_header("content-type", "text/event-stream")
2890                    .set_body_string(sse)
2891            })
2892            .mount(&server)
2893            .await;
2894
2895        let backend = || {
2896            Backend::OpenAiChat(HttpBackendConfig {
2897                base_url: format!("{}/v1", server.uri()),
2898                api_key: None,
2899                forward_auth: false,
2900                extra_headers: BTreeMap::new(),
2901                extra_body: BTreeMap::new(),
2902                omit_body_fields: BTreeSet::new(),
2903                reasoning_effort: None,
2904                max_retries: 0,
2905                failure_cooldown: std::time::Duration::ZERO,
2906                timeout: None,
2907            })
2908        };
2909        let client = Arc::new(
2910            TranslatingLlmClient::new(&[
2911                ModelConfig::new("weak", backend(), None),
2912                ModelConfig::new("strong", backend(), None),
2913            ])
2914            .expect("building test client"),
2915        );
2916        let algorithm = Arc::new(CandidateAlgorithm {});
2917        let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());
2918        llm_request.stream = true;
2919        let request = Request {
2920            llm_request,
2921            raw_request: None,
2922            metadata: None,
2923        };
2924        let result = run(
2925            algorithm,
2926            ClientRouter::single(client),
2927            request,
2928            to_category_map(&["weak", "strong"]),
2929            None,
2930        )
2931        .await;
2932        (server, calls, result)
2933    }
2934
2935    #[tokio::test]
2936    async fn first_stream_event_overflow_falls_back_to_the_next_candidate() -> Result<()> {
2937        // Nothing has reached the caller when the first event arrives, so an overflow
2938        // there falls through to the next candidate exactly as its buffered twin does.
2939        let (_server, calls, result) = run_streaming_candidates(OVERFLOW_SSE).await;
2940        let (_, response) = result?;
2941        assert_eq!(&*calls.lock(), &["weak", "strong"]);
2942        assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
2943        let aggregate = response
2944            .llm_response
2945            .into_agg()
2946            .await
2947            .map_err(|source| LibsyError::client_call("strong", source))?;
2948        assert_eq!(completion_text(&aggregate), "ok");
2949
2950        // Only a classified overflow re-routes; any other in-band error event still
2951        // streams through to the consumer without trying another candidate.
2952        let (_server, calls, result) = run_streaming_candidates(
2953            "data: {\"error\":{\"code\":500,\"message\":\"engine exploded\",\"object\":\"error\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2954        )
2955        .await;
2956        let (_, response) = result?;
2957        assert_eq!(&*calls.lock(), &["weak"]);
2958        let error = response
2959            .llm_response
2960            .into_agg()
2961            .await
2962            .expect_err("the in-band error should surface to the consumer");
2963        assert!(error.to_string().contains("engine exploded"));
2964
2965        // Once content has streamed, a retry would repeat delivered output, so a later
2966        // overflow event keeps today's semantics and surfaces during aggregation.
2967        let (_server, calls, result) = run_streaming_candidates(
2968            "data: {\"id\":\"turn\",\"model\":\"weak\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"partial\"},\"finish_reason\":null}]}\n\ndata: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens).\",\"object\":\"error\",\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
2969        )
2970        .await;
2971        let (_, response) = result?;
2972        assert_eq!(&*calls.lock(), &["weak"]);
2973        let error = response
2974            .llm_response
2975            .into_agg()
2976            .await
2977            .expect_err("the mid-stream overflow should surface to the consumer");
2978        assert!(
2979            error
2980                .to_string()
2981                .contains("exceeds the maximum allowed length")
2982        );
2983
2984        // A stream that ends before any event is not a classified failure; it still
2985        // aggregates to an empty response.
2986        let (_server, calls, result) = run_streaming_candidates("data: [DONE]\n\n").await;
2987        let (_, response) = result?;
2988        assert_eq!(&*calls.lock(), &["weak"]);
2989        let aggregate = response
2990            .llm_response
2991            .into_agg()
2992            .await
2993            .map_err(|source| LibsyError::client_call("weak", source))?;
2994        assert_eq!(completion_text(&aggregate), "");
2995        Ok(())
2996    }
2997}