Skip to main content

switchyard_llm_client/
client.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! [`TranslatingLlmClient`]: encode a neutral
5//! request, call the configured backend over HTTP, decode the neutral response.
6
7use std::borrow::Cow;
8use std::collections::{BTreeMap, BTreeSet, HashMap};
9use std::future::ready;
10use std::sync::atomic::{AtomicU64, Ordering};
11use std::time::{Duration, SystemTime};
12
13use async_trait::async_trait;
14use futures_util::{StreamExt, stream};
15use http::StatusCode;
16use reqwest::RequestBuilder;
17use reqwest::header::{HeaderMap, HeaderName, HeaderValue, RETRY_AFTER};
18use serde_json::{Map, Value, json};
19use switchyard_protocol::{
20    LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata,
21    ModelId, Request, Response, RoutedLlmClient,
22};
23use switchyard_translation::{TranslationError, WireFormat, decode_stream, remap_reasoning_effort};
24use tokio::time::Instant;
25use tracing::Instrument;
26
27use crate::backend::{Backend, openai_url};
28use crate::error::{LlmClientError, Result};
29use crate::metrics;
30use crate::observed_translation;
31
32// Caller headers safe to send when caller auth forwarding is disabled.
33const ALLOWED_METADATA_HEADERS: &[&str] = &["x-request-id"];
34
35// Headers tied to the inbound connection, destination, or body. The HTTP client
36// must rebuild these for the upstream request, even when forwarding auth.
37const NON_FORWARDABLE_HEADERS: &[&str] = &[
38    "host",
39    "content-length",
40    "connection",
41    "proxy-connection",
42    "keep-alive",
43    "proxy-authenticate",
44    "proxy-authorization",
45    "te",
46    "trailer",
47    "transfer-encoding",
48    "upgrade",
49    "content-type",
50    "content-encoding",
51    "accept-encoding",
52    "expect",
53];
54
55const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250);
56const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2);
57const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
58
59/// A named target's provider model and backends.
60#[derive(Clone, Debug)]
61pub struct ModelConfig {
62    target_name: ModelId,
63    provider_model: Option<ModelId>,
64    default_backend: Backend,
65    other_backends: Option<Vec<Backend>>,
66    responses_reasoning: crate::ResponsesReasoningPolicy,
67    reasoning_effort_map: BTreeMap<String, String>,
68}
69
70impl ModelConfig {
71    /// Configures a target. Its name is also the provider model ID unless overridden.
72    pub fn new(
73        target_name: impl Into<ModelId>,
74        default_backend: Backend,
75        other_backends: Option<Vec<Backend>>,
76    ) -> Self {
77        Self {
78            target_name: target_name.into(),
79            provider_model: None,
80            default_backend,
81            other_backends,
82            responses_reasoning: crate::ResponsesReasoningPolicy::default(),
83            reasoning_effort_map: BTreeMap::new(),
84        }
85    }
86
87    /// Sets how Responses reasoning items are replayed to this target.
88    #[must_use]
89    pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self {
90        self.responses_reasoning = policy;
91        self
92    }
93
94    /// Remaps matching caller efforts for this target. A backend's fixed effort takes precedence.
95    #[must_use]
96    pub fn with_reasoning_effort_map(mut self, effort_map: BTreeMap<String, String>) -> Self {
97        self.reasoning_effort_map = effort_map;
98        self
99    }
100
101    /// Sends `provider_model` upstream in place of `target_name`. The client is still
102    /// called with `target_name`, so several entries can serve one provider model with
103    /// different backend settings.
104    pub fn with_provider_model(mut self, provider_model: impl Into<ModelId>) -> Self {
105        self.provider_model = Some(provider_model.into());
106        self
107    }
108}
109
110/// A model-bearing provider operation outside the normal completion endpoint.
111#[derive(Clone, Copy, Debug, Eq, PartialEq)]
112pub enum AuxiliaryOperation {
113    /// Anthropic Messages input-token counting.
114    AnthropicCountTokens,
115    /// OpenAI Responses input-token counting.
116    ResponsesInputTokens,
117    /// OpenAI Responses compaction.
118    ResponsesCompact,
119}
120
121impl AuxiliaryOperation {
122    const fn wire_format(self) -> WireFormat {
123        match self {
124            Self::AnthropicCountTokens => WireFormat::AnthropicMessages,
125            Self::ResponsesInputTokens | Self::ResponsesCompact => WireFormat::OpenAiResponses,
126        }
127    }
128
129    fn url(self, backend: &Backend) -> String {
130        let suffix = match self {
131            Self::AnthropicCountTokens => return backend.count_tokens_url(),
132            Self::ResponsesInputTokens => "/responses/input_tokens",
133            Self::ResponsesCompact => "/responses/compact",
134        };
135        let base_url = backend.url();
136        openai_url(&base_url, suffix).unwrap_or_else(|error| {
137            tracing::error!(%error, "Unable to build OpenAI auxiliary endpoint URL");
138            base_url
139        })
140    }
141}
142
143/// A client that dispatches neutral-IR requests to named targets.
144///
145/// Each [`ModelConfig`] defines a target's provider model and backends.
146/// Each call resolves the target and wire format, encodes the request,
147/// applies auth and forwarded headers, sends the HTTP request with a shared
148/// [`reqwest::Client`], and decodes the response back to the neutral IR (buffered
149/// or streamed).
150pub struct TranslatingLlmClient {
151    configs_by_target: HashMap<ModelId, ModelConfig>,
152    client: reqwest::Client,
153    forward_auth_client: reqwest::Client,
154    is_rate_limit_cooldown_enabled: bool,
155    cooldown_epoch: Instant,
156    // Milliseconds since cooldown_epoch. Each atomic is independent, so relaxed ordering suffices.
157    unavailable_until: HashMap<ModelId, AtomicU64>,
158}
159
160impl TranslatingLlmClient {
161    /// Builds a client over the given [`ModelConfig`]s, with a fresh shared HTTP
162    /// client and the built-in translation codecs.
163    pub fn new(model_configs: &[ModelConfig]) -> Result<Self> {
164        // Pre-populate so concurrent calls update only atomics and keep map access lock-free.
165        let mut unavailable_until = HashMap::new();
166        for config in model_configs {
167            for backend in std::iter::once(&config.default_backend)
168                .chain(config.other_backends.iter().flatten())
169            {
170                backend.validate_configured_headers(&config.target_name)?;
171            }
172            unavailable_until.insert(config.target_name.clone(), AtomicU64::new(0));
173        }
174        let build_client = |builder: reqwest::ClientBuilder| {
175            builder.build().map_err(|error| LlmClientError::Transport {
176                source: Box::new(error),
177            })
178        };
179        let client = build_client(reqwest::Client::builder())?;
180        // A redirect could move provider-specific headers to another origin.
181        // Forwarded credentials are sent only to the configured URL.
182        let forward_auth_client =
183            build_client(reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()))?;
184        let configs_by_target = model_configs
185            .iter()
186            .map(|config| (config.target_name.clone(), config.clone()))
187            .collect();
188
189        Ok(Self {
190            configs_by_target,
191            client,
192            forward_auth_client,
193            is_rate_limit_cooldown_enabled: true,
194            cooldown_epoch: Instant::now(),
195            unavailable_until,
196        })
197    }
198
199    /// Disable HTTP 429 cooldown so subsequent calls can receive the upstream error.
200    pub fn without_rate_limit_cooldown(mut self) -> Self {
201        self.is_rate_limit_cooldown_enabled = false;
202        self
203    }
204
205    /// The target's backend for `format`, or `None` if no matching backend exists.
206    pub fn backend_for(&self, target_name: &ModelId, format: WireFormat) -> Option<&Backend> {
207        self.configs_by_target.get(target_name).and_then(|config| {
208            if config.default_backend.wire_format() == format {
209                Some(&config.default_backend)
210            } else {
211                config
212                    .other_backends
213                    .as_ref()
214                    .and_then(|backends| backends.iter().find(|b| b.wire_format() == format))
215            }
216        })
217    }
218
219    /// The target's provider model ID, falling back to its name when none is configured.
220    pub fn provider_model<'a>(&'a self, target_name: &'a ModelId) -> &'a ModelId {
221        self.configs_by_target
222            .get(target_name)
223            .and_then(|config| config.provider_model.as_ref())
224            .unwrap_or(target_name)
225    }
226
227    /// Whether the target has a backend for `operation`.
228    pub fn supports_auxiliary(&self, target_name: &ModelId, operation: AuxiliaryOperation) -> bool {
229        self.backend_for(target_name, operation.wire_format())
230            .is_some()
231    }
232
233    /// Reads llama.cpp model properties through the backend's configured auth and headers.
234    ///
235    /// Redirects are disabled so credentials cannot move to another origin.
236    pub async fn get_model_properties(
237        &self,
238        target_name: &ModelId,
239        format: WireFormat,
240        timeout: Duration,
241    ) -> Result<Value> {
242        let backend =
243            self.backend_for(target_name, format)
244                .ok_or_else(|| LlmClientError::Configuration {
245                    message: format!("model {target_name} has no backend for {format:?}"),
246                })?;
247        let mut url = reqwest::Url::parse(backend.base_url()).map_err(|error| {
248            LlmClientError::Configuration {
249                message: format!("model {target_name} has an invalid backend URL: {error}"),
250            }
251        })?;
252        let root = url.path().trim_end_matches('/');
253        let root = root
254            .strip_suffix("/chat/completions")
255            .or_else(|| root.strip_suffix("/responses"))
256            .unwrap_or(root);
257        let path = format!(
258            "{}/props",
259            root.strip_suffix("/v1")
260                .unwrap_or(root)
261                .trim_end_matches('/')
262        );
263        url.set_path(&path);
264        url.set_query(None);
265
266        let builder = self.forward_auth_client.get(url).timeout(timeout);
267        let builder = apply_extra_headers(builder, backend);
268        let response = backend
269            .apply_auth(builder)
270            .send()
271            .await
272            .map_err(convert_reqwest_error)?
273            .error_for_status()
274            .map_err(convert_reqwest_error)?;
275        response
276            .json()
277            .await
278            .map_err(|source| LlmClientError::InvalidResponse {
279                source: Box::new(source),
280            })
281    }
282
283    /// Calls a model-bearing auxiliary provider operation.
284    ///
285    /// Returns an error when the model has no compatible backend or the upstream
286    /// request fails or returns invalid JSON.
287    pub async fn call_auxiliary(
288        &self,
289        target_name: &ModelId,
290        request: Request,
291        operation: AuxiliaryOperation,
292    ) -> Result<Value> {
293        let wire_format = operation.wire_format();
294        let backend = self.backend_for(target_name, wire_format).ok_or_else(|| {
295            LlmClientError::Configuration {
296                message: format!("model {target_name} has no backend for {operation:?}"),
297            }
298        })?;
299        let Request {
300            mut llm_request,
301            metadata,
302            ..
303        } = request;
304        llm_request.model = Some(self.provider_model(target_name).to_string());
305        let http_response = self
306            .send_encoded(
307                backend,
308                wire_format,
309                llm_request,
310                metadata.as_ref(),
311                target_name,
312                UpstreamEndpoint::Auxiliary(operation),
313            )
314            .await?;
315        let EncodedResponse::Buffered { body, .. } = http_response else {
316            return Err(LlmClientError::InvalidRequest {
317                message: "auxiliary endpoints do not support streaming".to_string(),
318            });
319        };
320        serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {
321            source: Box::new(error),
322        })
323    }
324
325    /// Encode `llm_request` for `wire_format`, POST it to `url` with the request's
326    /// forwarded headers plus the backend's static headers and auth, and return the
327    /// successful upstream response. A
328    /// buffered response is fully collected within the retry boundary; a streamed
329    /// response has its first decoded event checked within that boundary. The backend's
330    /// deadline, when set, spans every attempt, retry delay, and subsequent stream read. A
331    /// non-success status maps to a typed error — a 400 is classified as a context-window
332    /// overflow via the backend's provider rules. Shared by
333    /// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the
334    /// backend's completion URL and decodes a response) and
335    /// the model-bearing auxiliary operations, which return raw JSON.
336    async fn send_encoded(
337        &self,
338        backend: &Backend,
339        wire_format: WireFormat,
340        llm_request: LlmRequest,
341        metadata: Option<&Metadata>,
342        target_name: &ModelId,
343        endpoint: UpstreamEndpoint,
344    ) -> Result<EncodedResponse> {
345        let cooldown = backend.failure_cooldown();
346        let unavailable_until =
347            if matches!(endpoint, UpstreamEndpoint::Completion) && !cooldown.is_zero() {
348                self.unavailable_until.get(target_name)
349            } else {
350                None
351            };
352        if let Some(until) = unavailable_until
353            && until.load(Ordering::Relaxed) > duration_millis(self.cooldown_epoch.elapsed())
354        {
355            tracing::debug!(model = %target_name, wire_format = %wire_format, "skipping backend during failure cooldown");
356            return Err(LlmClientError::TemporarilyUnavailable);
357        }
358
359        let mut body = observed_translation::encode_request(&llm_request, wire_format)
360            .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
361        // `encode_request` round-trips a preserved same-format body verbatim,
362        // which keeps the caller's original `model`. Set the provider model explicitly.
363        let provider_model_id = self.provider_model(target_name);
364        set_json_model(&mut body, provider_model_id);
365        if matches!(backend, Backend::OpenAiResponses(_)) {
366            sanitize_openai_responses_provider_body(&mut body);
367        }
368        // Strip before `merge_extra_body` so a target can reinstate either field
369        // deliberately via `extra_body`.
370        if matches!(backend, Backend::Anthropic(_)) {
371            strip_anthropic_incompatible_fields(&mut body);
372            strip_unsigned_thinking_blocks(&mut body);
373        }
374        omit_configured_body_fields(&mut body, backend.omit_body_fields());
375        let target_config = self.configs_by_target.get(target_name);
376        if matches!(endpoint, UpstreamEndpoint::Completion)
377            && backend.reasoning_effort().is_none()
378            && let Some(config) = target_config
379        {
380            remap_reasoning_effort(
381                &mut body,
382                wire_format.as_str(),
383                &config.reasoning_effort_map,
384            );
385        }
386        merge_extra_body(&mut body, backend.extra_body());
387        if matches!(backend, Backend::OpenAiResponses(_)) {
388            let outbound_model_id = body
389                .get("model")
390                .and_then(Value::as_str)
391                .unwrap_or(provider_model_id)
392                .to_owned();
393            target_config
394                .map(|config| config.responses_reasoning)
395                .unwrap_or_default()
396                .normalize(&mut body, backend.base_url(), &outbound_model_id);
397        }
398        // After the merge on purpose: the effort override must win over both the caller's
399        // value and any `reasoning` default a target set through `extra_body`. Token counting
400        // does not depend on effort, and Anthropic's count endpoint may reject the field.
401        if matches!(endpoint, UpstreamEndpoint::Completion) {
402            apply_reasoning_effort(&mut body, backend);
403        }
404        if matches!(backend, Backend::OpenAiResponses(_)) {
405            normalize_codex_string_input(&backend.url(), &mut body);
406        }
407        if matches!(backend, Backend::Anthropic(_)) {
408            if nvidia_inference_api_backend(backend) {
409                strip_anthropic_tool_strict(&mut body);
410            }
411            enable_anthropic_prompt_caching(&mut body, llm_request.temporary_note_count);
412        }
413        if matches!(
414            backend,
415            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_)
416        ) {
417            enable_openai_guidance_caching(
418                &mut body,
419                wire_format,
420                provider_model_id,
421                llm_request.temporary_note_count,
422            );
423        }
424        if matches!(backend, Backend::OpenAiChat(_)) {
425            ensure_openai_stream_usage(&mut body);
426        }
427        let streaming = endpoint.allows_streaming()
428            && body.get("stream").and_then(Value::as_bool).unwrap_or(false);
429        let url = endpoint.url(backend);
430        record_gen_ai_request(&url, provider_model_id, streaming);
431
432        let result = self
433            .send_with_retries(&url, backend, &body, metadata, target_name, streaming)
434            .await;
435        // Keep quota failures request-local when forwarding auth or returning 429 to callers.
436        if let Some(until) = unavailable_until
437            && let Err(error) = &result
438            && is_transient_failure(error)
439            && !((backend.is_forwarding_auth() || !self.is_rate_limit_cooldown_enabled)
440                && matches!(error, LlmClientError::UpstreamHttp { status, .. } if *status == StatusCode::TOO_MANY_REQUESTS))
441        {
442            let deadline = duration_millis(self.cooldown_epoch.elapsed().saturating_add(cooldown));
443            until.fetch_max(deadline, Ordering::Relaxed);
444        }
445        // Let cooldown expire naturally; a successful in-flight call may overlap a newer failure.
446        result
447    }
448
449    // Sends the encoded body, retrying retryable failures within the backend's retry budget.
450    async fn send_with_retries(
451        &self,
452        url: &str,
453        backend: &Backend,
454        body: &Value,
455        metadata: Option<&Metadata>,
456        target_name: &ModelId,
457        streaming: bool,
458    ) -> Result<EncodedResponse> {
459        let max_retries = u64::from(backend.max_retries());
460        let max_attempts = max_retries
461            + 1
462            + u64::from(
463                matches!(backend, Backend::OpenAiChat(_))
464                    && body.get("chat_template_kwargs").is_some(),
465            );
466        let mut body = Cow::Borrowed(body);
467        let mut template_retry = false;
468        let expires_at = backend
469            .timeout()
470            .map(|timeout| (tokio::time::sleep(timeout).deadline(), timeout));
471        let deadline = async {
472            match expires_at {
473                Some((at, timeout)) => {
474                    tokio::time::sleep_until(at).await;
475                    timeout
476                }
477                None => std::future::pending().await,
478            }
479        };
480        tokio::pin!(deadline);
481        let mut attempt = 0_u64;
482        loop {
483            if let Some((at, timeout)) = expires_at
484                && tokio::time::Instant::now() >= at
485            {
486                return Err(deadline_error(timeout));
487            }
488            let span = tracing::debug_span!(
489                target: "libsy",
490                "libsy.upstream_attempt",
491                model = %target_name,
492                wire_format = %backend.wire_format(),
493                attempt = attempt + 1 + u64::from(template_retry),
494                max_attempts,
495                retry = attempt > 0 || template_retry,
496                openinference.span.kind = "CHAIN",
497                outcome = tracing::field::Empty,
498                status_code = tracing::field::Empty,
499                will_retry = tracing::field::Empty,
500                retry_delay_ms = tracing::field::Empty,
501            );
502            let mut attempt_started = false;
503            let result = tokio::select! {
504                biased;
505                timeout = &mut deadline => {
506                    if attempt_started {
507                        metrics::record_upstream_attempt(None);
508                    }
509                    span.record("outcome", "error");
510                    span.record("will_retry", false);
511                    return Err(deadline_error(timeout));
512                }
513                result = async {
514                    attempt_started = true;
515                    self.send_once(url, backend, body.as_ref(), metadata, target_name, streaming).await
516                }.instrument(span.clone()) => result,
517            };
518            // The retained handle updates this same attempt span with its outcome.
519            match result {
520                Ok(response) => {
521                    span.record("outcome", "ok");
522                    span.record("status_code", response.status());
523                    span.record("will_retry", false);
524                    if attempt > 0 || template_retry {
525                        metrics::record_retry_recovered();
526                    }
527                    return Ok(match response {
528                        EncodedResponse::Streaming {
529                            status,
530                            chunks,
531                            upstream_headers,
532                        } => EncodedResponse::Streaming {
533                            status,
534                            chunks: stream_with_deadline(chunks, expires_at),
535                            upstream_headers,
536                        },
537                        buffered => buffered,
538                    });
539                }
540                Err(failure) => {
541                    let retry_without_template = matches!(backend, Backend::OpenAiChat(_))
542                        && body.get("chat_template_kwargs")
543                            == Some(&json!({"enable_thinking": false}))
544                        && rejects_template_options(&failure.error);
545                    let will_retry =
546                        retry_without_template || (attempt < max_retries && failure.is_retryable());
547                    span.record("outcome", "error");
548                    if let Some(status) = failure.status {
549                        span.record("status_code", status.as_u16());
550                    }
551                    span.record("will_retry", will_retry);
552                    if !will_retry {
553                        return Err(failure.error);
554                    }
555
556                    if retry_without_template {
557                        if let Some(object) = body.to_mut().as_object_mut() {
558                            object.remove("chat_template_kwargs");
559                        }
560                        template_retry = true;
561                        span.record("retry_delay_ms", 0_u64);
562                        continue;
563                    }
564                    let delay = retry_delay(attempt, failure.retry_after);
565                    span.record("retry_delay_ms", duration_millis(delay));
566                    // Close the attempt span before sleeping so backoff is not attempt latency.
567                    drop(span);
568                    tokio::select! {
569                        biased;
570                        timeout = &mut deadline => return Err(deadline_error(timeout)),
571                        _ = tokio::time::sleep(delay) => {}
572                    }
573                    attempt += 1;
574                }
575            }
576        }
577    }
578
579    // Performs one HTTP attempt and retains the retry metadata alongside any error.
580    async fn send_once(
581        &self,
582        url: &str,
583        backend: &Backend,
584        body: &Value,
585        metadata: Option<&Metadata>,
586        target_name: &ModelId,
587        streaming: bool,
588    ) -> std::result::Result<EncodedResponse, AttemptFailure> {
589        let client = if backend.is_forwarding_auth() {
590            &self.forward_auth_client
591        } else {
592            &self.client
593        };
594        let builder = client.post(url).json(body);
595        let builder = forward_metadata_headers(builder, metadata, backend);
596        let builder = backend.apply_forwarded_auth(builder, metadata);
597        let builder = apply_extra_headers(builder, backend);
598        let builder = backend.apply_auth(builder);
599
600        let response = match async {
601            let request = builder.build()?;
602            if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) {
603                crate::payload_capture::record("request", || String::from_utf8_lossy(body));
604            }
605            client.execute(request).await
606        }
607        .await
608        {
609            Ok(response) => response,
610            Err(error) => {
611                metrics::record_upstream_attempt(None);
612                return Err(AttemptFailure {
613                    error: convert_reqwest_error(error),
614                    status: None,
615                    retry_after: None,
616                });
617            }
618        };
619        let status = response.status();
620        if status.is_success() {
621            if streaming {
622                let upstream_headers = response.headers().clone();
623                let chunks = match prepare_response_stream(response, backend, target_name).await {
624                    Ok(chunks) => chunks,
625                    Err(error) => {
626                        metrics::record_upstream_attempt(None);
627                        return Err(AttemptFailure {
628                            error,
629                            status: Some(status),
630                            retry_after: None,
631                        });
632                    }
633                };
634                metrics::record_upstream_attempt(Some(status.as_u16()));
635                return Ok(EncodedResponse::Streaming {
636                    status: status.as_u16(),
637                    chunks,
638                    upstream_headers,
639                });
640            }
641            let upstream_headers = response.headers().clone();
642            let body = match response.bytes().await {
643                Ok(body) => body,
644                Err(error) => {
645                    metrics::record_upstream_attempt(None);
646                    return Err(AttemptFailure {
647                        error: convert_reqwest_error(error),
648                        status: Some(status),
649                        retry_after: None,
650                    });
651                }
652            };
653            metrics::record_upstream_attempt(Some(status.as_u16()));
654            crate::payload_capture::record("response", || String::from_utf8_lossy(&body));
655            return Ok(EncodedResponse::Buffered {
656                status: status.as_u16(),
657                body: body.to_vec(),
658                upstream_headers,
659            });
660        }
661
662        let retry_after = retry_after_delay(response.headers());
663        let headers = Box::new(response.headers().clone());
664        let body = match response.text().await {
665            Ok(body) => body,
666            Err(error) => {
667                metrics::record_upstream_attempt(None);
668                return Err(AttemptFailure {
669                    error: convert_reqwest_error(error),
670                    status: Some(status),
671                    retry_after,
672                });
673            }
674        };
675        let body = redact_forwarded_headers(body, metadata, backend.is_forwarding_auth());
676        metrics::record_upstream_attempt(Some(status.as_u16()));
677        let error =
678            if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
679                LlmClientError::ContextWindowExceeded {
680                    model: target_name.clone(),
681                    message: body,
682                }
683            } else {
684                LlmClientError::UpstreamHttp {
685                    status,
686                    body,
687                    headers,
688                }
689            };
690        Err(AttemptFailure {
691            error,
692            status: Some(status),
693            retry_after,
694        })
695    }
696
697    /// Calls the selected target using its configured provider model and backend.
698    /// `target_name` takes precedence over the request's model field.
699    /// Request metadata selects the wire format, falling back to the target's default.
700    pub async fn call_rewrite_model(
701        &self,
702        request: Request,
703        target_name: Option<&ModelId>,
704    ) -> Result<Response> {
705        let Request {
706            mut llm_request,
707            metadata,
708            ..
709        } = request;
710
711        let selected_target = target_name
712            .cloned()
713            .or_else(|| llm_request.model.map(ModelId::from))
714            .ok_or_else(|| LlmClientError::InvalidRequest {
715                message: "no model given".to_string(),
716            })?;
717        llm_request.model = Some(self.provider_model(&selected_target).to_string());
718
719        let orig_format = metadata.as_ref().and_then(|m| m.wire_format);
720        let wire_format = orig_format.unwrap_or(
721            self.configs_by_target
722                .get(&selected_target)
723                .map(|config| config.default_backend.wire_format())
724                .ok_or_else(|| LlmClientError::Configuration {
725                    message: format!("no backend configured for model {selected_target:?}"),
726                })?,
727        );
728        let backend = self
729            .backend_for(&selected_target, wire_format)
730            .ok_or_else(|| LlmClientError::Configuration {
731                message: format!(
732                    "model {selected_target:?} has no backend for format {wire_format}"
733                ),
734            })?;
735
736        let http_response = self
737            .send_encoded(
738                backend,
739                wire_format,
740                llm_request,
741                metadata.as_ref(),
742                &selected_target,
743                UpstreamEndpoint::Completion,
744            )
745            .await?;
746
747        let (llm_response, upstream_headers) = match http_response {
748            EncodedResponse::Streaming {
749                chunks,
750                upstream_headers,
751                ..
752            } => (LlmResponse::Stream(chunks), upstream_headers),
753            EncodedResponse::Buffered {
754                body,
755                upstream_headers,
756                ..
757            } => {
758                let body = serde_json::from_slice::<Value>(&body).map_err(|error| {
759                    LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))
760                })?;
761                // Map a provider's failed generation to 502, even under HTTP 200.
762                // Redact forwarded credentials before returning the provider error.
763                let mut agg =
764                    observed_translation::decode_response(&body, wire_format).map_err(|error| {
765                        match error {
766                            TranslationError::UpstreamFailure { error } => {
767                                LlmClientError::UpstreamHttp {
768                                    status: StatusCode::BAD_GATEWAY,
769                                    body: redact_forwarded_headers(
770                                        json!({ "error": error }).to_string(),
771                                        metadata.as_ref(),
772                                        backend.is_forwarding_auth(),
773                                    ),
774                                    headers: Box::new(upstream_headers.clone()),
775                                }
776                            }
777                            error => LlmClientError::ResponseTranslation(error.to_string()),
778                        }
779                    })?;
780                // The gateway may report a buffered charge only in its response header.
781                if agg.usage.provider_cost.is_none() {
782                    agg.usage.provider_cost = upstream_headers
783                        .get("x-litellm-response-cost")
784                        .and_then(|value| value.to_str().ok())
785                        .and_then(|value| value.trim().parse::<f64>().ok())
786                        .filter(|usd| usd.is_finite() && *usd >= 0.0);
787                }
788                (LlmResponse::Agg(agg), upstream_headers)
789            }
790        };
791
792        Ok(Response {
793            llm_response,
794            metadata,
795            upstream_headers,
796        })
797    }
798}
799
800#[async_trait]
801impl RoutedLlmClient for TranslatingLlmClient {
802    fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
803        TranslatingLlmClient::provider_model(self, target)
804    }
805
806    async fn call(&self, request: Request) -> Result<Response> {
807        self.call_rewrite_model(request, None).await
808    }
809}
810
811#[derive(Clone, Copy)]
812enum UpstreamEndpoint {
813    Completion,
814    Auxiliary(AuxiliaryOperation),
815}
816
817impl UpstreamEndpoint {
818    fn url(self, backend: &Backend) -> String {
819        match self {
820            UpstreamEndpoint::Completion => backend.url(),
821            UpstreamEndpoint::Auxiliary(operation) => operation.url(backend),
822        }
823    }
824
825    fn allows_streaming(self) -> bool {
826        matches!(self, UpstreamEndpoint::Completion)
827    }
828}
829
830enum EncodedResponse {
831    Buffered {
832        status: u16,
833        body: Vec<u8>,
834        upstream_headers: HeaderMap,
835    },
836    Streaming {
837        status: u16,
838        chunks: LlmResponseStream,
839        upstream_headers: HeaderMap,
840    },
841}
842
843impl EncodedResponse {
844    fn status(&self) -> u16 {
845        match self {
846            EncodedResponse::Buffered { status, .. }
847            | EncodedResponse::Streaming { status, .. } => *status,
848        }
849    }
850}
851
852// The typed error decides retry eligibility; status and Retry-After feed
853// attempt telemetry and delay selection.
854struct AttemptFailure {
855    error: LlmClientError,
856    status: Option<StatusCode>,
857    retry_after: Option<Duration>,
858}
859
860fn deadline_error(timeout: Duration) -> LlmClientError {
861    LlmClientError::Timeout {
862        source: Box::new(std::io::Error::new(
863            std::io::ErrorKind::TimedOut,
864            format!(
865                "response did not finish within {} ms, retries included",
866                duration_millis(timeout)
867            ),
868        )),
869    }
870}
871
872impl AttemptFailure {
873    fn is_retryable(&self) -> bool {
874        is_transient_failure(&self.error)
875    }
876}
877
878fn is_transient_failure(error: &LlmClientError) -> bool {
879    match error {
880        LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
881        LlmClientError::UpstreamHttp { status, .. } => {
882            metrics::is_retryable_http_status(status.as_u16())
883        }
884        _ => false,
885    }
886}
887
888fn stream_with_deadline(
889    chunks: LlmResponseStream,
890    deadline: Option<(tokio::time::Instant, Duration)>,
891) -> LlmResponseStream {
892    let Some((at, timeout)) = deadline else {
893        return chunks;
894    };
895    stream::unfold(Some(chunks), move |chunks| async move {
896        let mut chunks = chunks?;
897        let chunk = tokio::select! {
898            biased;
899            _ = tokio::time::sleep_until(at) => return Some((Err(deadline_error(timeout)), None)),
900            chunk = chunks.next() => chunk?,
901        };
902        let next = chunk.is_ok().then_some(chunks);
903        Some((chunk, next))
904    })
905    .boxed()
906}
907
908async fn prepare_response_stream(
909    response: reqwest::Response,
910    backend: &Backend,
911    target_name: &ModelId,
912) -> Result<LlmResponseStream> {
913    let bytes = response.bytes_stream().map(|chunk| {
914        chunk
915            .map(|bytes| bytes.to_vec())
916            .map_err(convert_reqwest_error)
917    });
918    let mut chunks = decode_stream(bytes, backend.wire_format())?;
919    match chunks.next().await {
920        None => Ok(stream::empty().boxed()),
921        // Nothing has reached the caller, so transport failures can still be retried.
922        Some(Err(error @ (LlmClientError::Transport { .. } | LlmClientError::Timeout { .. }))) => {
923            Err(error)
924        }
925        Some(first) => {
926            // An in-band context overflow skips retries and advances to another candidate.
927            if let Some(message) = first_event_overflow(&first, backend) {
928                return Err(LlmClientError::ContextWindowExceeded {
929                    model: target_name.clone(),
930                    message,
931                });
932            }
933            Ok(stream::once(ready(first)).chain(chunks).boxed())
934        }
935    }
936}
937
938// The overflow message when a stream's first event is an in-band provider rejection
939// of the whole request, rather than the start of a response.
940fn first_event_overflow(
941    first: &Result<LlmResponseStreamEvent>,
942    backend: &Backend,
943) -> Option<String> {
944    first
945        .as_ref()
946        .ok()?
947        .normalized()
948        .iter()
949        .find_map(|chunk| match chunk {
950            LlmResponseChunk::StreamError { message } if backend.is_context_overflow(message) => {
951                Some(message.clone())
952            }
953            _ => None,
954        })
955}
956
957// Uses Retry-After when supplied, capped so an upstream cannot stall a request indefinitely.
958fn retry_after_delay(headers: &HeaderMap) -> Option<Duration> {
959    let value = headers.get(RETRY_AFTER)?.to_str().ok()?;
960    let delay = if let Ok(seconds) = value.parse::<u64>() {
961        Duration::from_secs(seconds)
962    } else {
963        let retry_at = httpdate::parse_http_date(value).ok()?;
964        retry_at
965            .duration_since(SystemTime::now())
966            .unwrap_or(Duration::ZERO)
967    };
968    Some(delay.min(MAX_RETRY_AFTER))
969}
970
971fn retry_delay(retry_number: u64, retry_after: Option<Duration>) -> Duration {
972    // Retry-After wins; otherwise double 250 ms up to the two-second cap.
973    retry_after.unwrap_or_else(|| {
974        let multiplier = 1_u32 << retry_number.min(3);
975        INITIAL_RETRY_DELAY
976            .saturating_mul(multiplier)
977            .min(MAX_RETRY_BACKOFF)
978    })
979}
980
981fn duration_millis(duration: Duration) -> u64 {
982    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
983}
984
985fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {
986    let span = tracing::Span::current();
987    span.record("gen_ai.request.model", model);
988    if streaming {
989        span.record("gen_ai.request.stream", true);
990    }
991    if let Ok(url) = reqwest::Url::parse(url) {
992        if let Some(host) = url.host_str() {
993            span.record("server.address", host);
994        }
995        if let Some(port) = url.port_or_known_default() {
996            span.record("server.port", i64::from(port));
997        }
998    }
999}
1000
1001pub(crate) fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
1002    // Reqwest labels truncated or otherwise unreadable response bodies as decode
1003    // errors, so distinguish them from serde JSON failures at the call site.
1004    let error = error.without_url();
1005    // reqwest reports a connect-phase timeout as both `is_connect()` and
1006    // `is_timeout()`. Check connect first: no request was sent on the failed
1007    // connection, so it is a transport failure like a refused connection.
1008    if error.is_connect() {
1009        LlmClientError::Transport {
1010            source: Box::new(error),
1011        }
1012    } else if error.is_timeout() {
1013        LlmClientError::Timeout {
1014            source: Box::new(error),
1015        }
1016    } else if error.is_builder() {
1017        LlmClientError::Configuration {
1018            message: format!("failed to build upstream request: {error}"),
1019        }
1020    } else {
1021        LlmClientError::Transport {
1022            source: Box::new(error),
1023        }
1024    }
1025}
1026
1027// Forwards safe metadata headers, or all application headers when forwarding auth.
1028fn forward_metadata_headers(
1029    builder: RequestBuilder,
1030    metadata: Option<&Metadata>,
1031    backend: &Backend,
1032) -> RequestBuilder {
1033    let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
1034        return builder;
1035    };
1036    let mut forwarded = HeaderMap::with_capacity(headers.len());
1037    for (name, value) in headers {
1038        let is_allowed = if backend.is_forwarding_auth() {
1039            !is_non_forwardable_header(name.as_str(), headers)
1040                && !backend.is_provider_owned_header(name.as_str())
1041        } else {
1042            is_allowed_metadata_header(name.as_str())
1043        };
1044        if !is_allowed {
1045            continue;
1046        }
1047        forwarded.append(name, value.clone());
1048    }
1049    builder.headers(forwarded)
1050}
1051
1052// Configured headers replace existing values for the same case-insensitive name.
1053fn apply_extra_headers(builder: RequestBuilder, backend: &Backend) -> RequestBuilder {
1054    let mut headers = HeaderMap::with_capacity(backend.extra_headers().len());
1055    // Names and values are validated by TranslatingLlmClient::new.
1056    for (name, value) in backend.extra_headers() {
1057        let name = HeaderName::from_bytes(name.as_bytes()).expect("validated header name");
1058        let value = HeaderValue::from_bytes(value.as_bytes()).expect("validated header value");
1059        headers.insert(name, value);
1060    }
1061    builder.headers(headers)
1062}
1063
1064// Overwrites the outbound body's `model` field with the resolved model id.
1065fn set_json_model(body: &mut Value, model: &str) {
1066    if let Value::Object(object) = body {
1067        object.insert("model".to_string(), Value::String(model.to_string()));
1068    }
1069}
1070
1071// The Codex Responses endpoint requires item-list input even on the preserved-body path.
1072fn normalize_codex_string_input(url: &str, body: &mut Value) {
1073    let codex = reqwest::Url::parse(url).ok().is_some_and(|url| {
1074        url.path()
1075            .trim_end_matches('/')
1076            .ends_with("/backend-api/codex/responses")
1077    });
1078    if codex && let Some(input @ Value::String(_)) = body.get_mut("input") {
1079        let text = input.take();
1080        *input = json!([{"type": "message", "role": "user", "content": text}]);
1081    }
1082}
1083
1084fn rejects_template_options(error: &LlmClientError) -> bool {
1085    let LlmClientError::UpstreamHttp { status, body, .. } = error else {
1086        return false;
1087    };
1088    if !matches!(
1089        *status,
1090        StatusCode::BAD_REQUEST | StatusCode::UNPROCESSABLE_ENTITY
1091    ) {
1092        return false;
1093    }
1094    let parsed = serde_json::from_str::<Value>(body).ok();
1095    let error = parsed.as_ref().and_then(|value| value.get("error"));
1096    let message = match &parsed {
1097        Some(_) => error
1098            .and_then(|error| error.get("message"))
1099            .and_then(Value::as_str)
1100            .unwrap_or_default(),
1101        None => body,
1102    }
1103    .to_ascii_lowercase();
1104    let field = error
1105        .and_then(|error| error.get("param"))
1106        .and_then(Value::as_str);
1107    let code = error
1108        .and_then(|error| error.get("code"))
1109        .and_then(Value::as_str);
1110    (field == Some("chat_template_kwargs") || message.contains("chat_template_kwargs"))
1111        && (matches!(code, Some("unknown_parameter" | "unsupported_parameter"))
1112            || [
1113                "unrecognized request argument",
1114                "unknown parameter",
1115                "unsupported parameter",
1116                "unexpected keyword argument",
1117                "unknown field",
1118                "unsupported field",
1119                "extra inputs are not permitted",
1120            ]
1121            .iter()
1122            .any(|phrase| message.contains(phrase)))
1123}
1124
1125const CODEX_NAMESPACE_SEPARATOR: &str = "__";
1126
1127/// Rewrites a Responses body into the shape OpenAI-compatible providers accept.
1128///
1129/// Codex extends Responses with namespace containers and namespaced function calls, and
1130/// OpenAI-compatible providers expect a flat Responses tool namespace.
1131fn sanitize_openai_responses_provider_body(body: &mut Value) {
1132    let Value::Object(object) = body else {
1133        return;
1134    };
1135    sanitize_openai_responses_input_for_provider(object.get_mut("input"));
1136    sanitize_openai_responses_tools_for_provider(object.get_mut("tools"));
1137    sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice"));
1138}
1139
1140fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) {
1141    let Some(Value::Array(items)) = input else {
1142        return;
1143    };
1144    for item in items {
1145        let Some(object) = item.as_object_mut() else {
1146            continue;
1147        };
1148        if object.get("type").and_then(Value::as_str) == Some("function_call") {
1149            qualify_responses_function_name(object);
1150        }
1151    }
1152}
1153
1154fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) {
1155    let Some(Value::Array(tools)) = tools else {
1156        return;
1157    };
1158    let mut flat_tools = Vec::with_capacity(tools.len());
1159    for tool in std::mem::take(tools) {
1160        push_sanitized_openai_responses_tool(&mut flat_tools, tool);
1161    }
1162    *tools = flat_tools;
1163}
1164
1165fn push_sanitized_openai_responses_tool(out: &mut Vec<Value>, tool: Value) {
1166    let Value::Object(mut object) = tool else {
1167        out.push(tool);
1168        return;
1169    };
1170    if object.get("type").and_then(Value::as_str) == Some("namespace") {
1171        let namespace = object
1172            .get("name")
1173            .and_then(Value::as_str)
1174            .unwrap_or_default()
1175            .to_string();
1176        let Some(Value::Array(children)) = object.remove("tools") else {
1177            out.push(Value::Object(object));
1178            return;
1179        };
1180        for child in children {
1181            push_sanitized_namespaced_tool(out, &namespace, child);
1182        }
1183        return;
1184    }
1185    ensure_responses_function_tool_description(&mut object);
1186    out.push(Value::Object(object));
1187}
1188
1189fn push_sanitized_namespaced_tool(out: &mut Vec<Value>, namespace: &str, tool: Value) {
1190    let Value::Object(mut object) = tool else {
1191        out.push(tool);
1192        return;
1193    };
1194    if object.get("type").and_then(Value::as_str) == Some("function") {
1195        qualify_responses_function_name_with_namespace(&mut object, namespace);
1196        ensure_responses_function_tool_description(&mut object);
1197    }
1198    out.push(Value::Object(object));
1199}
1200
1201fn sanitize_openai_responses_tool_choice_for_provider(tool_choice: Option<&mut Value>) {
1202    let Some(Value::Object(object)) = tool_choice else {
1203        return;
1204    };
1205    if object.get("type").and_then(Value::as_str) == Some("function") {
1206        qualify_responses_function_name(object);
1207    }
1208}
1209
1210fn qualify_responses_function_name(object: &mut Map<String, Value>) {
1211    let namespace = object
1212        .remove("namespace")
1213        .and_then(|value| value.as_str().map(ToOwned::to_owned));
1214    let Some(namespace) = namespace.as_deref() else {
1215        return;
1216    };
1217    qualify_responses_function_name_with_namespace(object, namespace);
1218}
1219
1220fn qualify_responses_function_name_with_namespace(
1221    object: &mut Map<String, Value>,
1222    namespace: &str,
1223) {
1224    if namespace.is_empty() {
1225        return;
1226    }
1227    let Some(name) = object.get("name").and_then(Value::as_str) else {
1228        return;
1229    };
1230    let prefix = format!("{namespace}{CODEX_NAMESPACE_SEPARATOR}");
1231    if name.starts_with(&prefix) {
1232        return;
1233    }
1234    object.insert("name".to_string(), Value::String(format!("{prefix}{name}")));
1235}
1236
1237fn ensure_responses_function_tool_description(object: &mut Map<String, Value>) {
1238    if object.get("type").and_then(Value::as_str) == Some("function")
1239        && !matches!(object.get("description"), Some(Value::String(_)))
1240    {
1241        object.insert("description".to_string(), Value::String(String::new()));
1242    }
1243}
1244
1245// Drops fields accepted by OpenAI-like APIs but rejected by Anthropic Messages.
1246//
1247// A router can serve earlier turns of a session from an OpenAI-format target and
1248// later turns from an Anthropic one. Clients such as Claude Code send
1249// `context_management` on every turn, so the Anthropic leg must strip it or the
1250// upstream rejects the request (for example `clear_thinking_20251015` strategy
1251// requires `thinking` to be enabled or adaptive).
1252fn strip_anthropic_incompatible_fields(body: &mut Value) {
1253    if let Value::Object(object) = body {
1254        object.remove("reasoning_effort");
1255        object.remove("context_management");
1256        // `strict: false` is the default, and Bedrock rejects the field outright.
1257        if let Some(Value::Array(tools)) = object.get_mut("tools") {
1258            for tool in tools {
1259                if let Value::Object(tool) = tool
1260                    && tool.get("strict") == Some(&Value::Bool(false))
1261                {
1262                    tool.remove("strict");
1263                }
1264            }
1265        }
1266    }
1267}
1268
1269// NVIDIA Inference API's Anthropic-compatible schema rejects the strict tool-use
1270// extension that native Anthropic accepts.
1271fn nvidia_inference_api_backend(backend: &Backend) -> bool {
1272    reqwest::Url::parse(backend.base_url())
1273        .ok()
1274        .is_some_and(|url| {
1275            url.host_str()
1276                .is_some_and(|host| host.eq_ignore_ascii_case("inference-api.nvidia.com"))
1277        })
1278}
1279
1280fn strip_anthropic_tool_strict(body: &mut Value) {
1281    let Some(tools) = body.get_mut("tools").and_then(Value::as_array_mut) else {
1282        return;
1283    };
1284    for tool in tools.iter_mut().filter_map(Value::as_object_mut) {
1285        tool.remove("strict");
1286    }
1287}
1288
1289// Removes replayed `thinking` blocks that carry no signature.
1290//
1291// Anthropic requires signed thinking blocks on replay. A router can serve earlier
1292// turns of a session from an OpenAI-format target whose thinking blocks are
1293// unsigned, so the Anthropic leg must drop them or the upstream rejects the
1294// request. Bedrock enforces this (surfacing as a SigV4 signature mismatch) where
1295// Azure-hosted Anthropic currently does not.
1296fn strip_unsigned_thinking_blocks(body: &mut Value) {
1297    let Value::Object(object) = body else {
1298        return;
1299    };
1300    let Some(Value::Array(messages)) = object.get_mut("messages") else {
1301        return;
1302    };
1303    for message in messages {
1304        strip_unsigned_thinking_from_message(message);
1305    }
1306}
1307
1308// Drops unsigned thinking blocks from one message, collapsing content that ends
1309// up empty to an empty string so the message stays valid.
1310fn strip_unsigned_thinking_from_message(message: &mut Value) {
1311    let Value::Object(message) = message else {
1312        return;
1313    };
1314    let Some(Value::Array(blocks)) = message.get("content") else {
1315        return;
1316    };
1317    if !blocks.iter().any(is_unsigned_thinking_block) {
1318        return;
1319    }
1320    let Some(Value::Array(blocks)) = message.get_mut("content") else {
1321        return;
1322    };
1323    blocks.retain(|block| !is_unsigned_thinking_block(block));
1324    if blocks.is_empty() {
1325        message.insert("content".to_string(), Value::String(String::new()));
1326    }
1327}
1328
1329// A thinking block is unsigned when `signature` is absent or empty.
1330fn is_unsigned_thinking_block(block: &Value) -> bool {
1331    if block.get("type").and_then(Value::as_str) != Some("thinking") {
1332        return false;
1333    }
1334    !matches!(
1335        block.get("signature").and_then(Value::as_str),
1336        Some(signature) if !signature.is_empty()
1337    )
1338}
1339
1340// Forces the target's configured reasoning effort onto the outbound body, replacing the
1341// caller's value. Unlike `extra_body`, this is an override: a route that sends one model at a
1342// higher effort than the client asked for is the point of the setting.
1343fn apply_reasoning_effort(body: &mut Value, backend: &Backend) {
1344    let Some(effort) = backend.reasoning_effort() else {
1345        return;
1346    };
1347    let Value::Object(object) = body else {
1348        return;
1349    };
1350    match backend {
1351        // Responses nests effort under `reasoning`, Anthropic under `output_config`.
1352        Backend::OpenAiResponses(_) => set_nested_effort(object, "reasoning", effort),
1353        Backend::Anthropic(_) => set_nested_effort(object, "output_config", effort),
1354        Backend::OpenAiChat(_) => {
1355            // Chat Completions takes effort as a top-level field.
1356            object.insert(
1357                "reasoning_effort".to_string(),
1358                Value::String(effort.to_string()),
1359            );
1360        }
1361    }
1362}
1363
1364// Replaces only `effort` inside `key`, keeping fields the caller set next to it (`summary` or
1365// `format`, for example). A non-object value there is malformed and is replaced whole.
1366fn set_nested_effort(object: &mut serde_json::Map<String, Value>, key: &str, effort: &str) {
1367    let container = object
1368        .entry(key.to_string())
1369        .or_insert_with(|| Value::Object(serde_json::Map::new()));
1370    if !container.is_object() {
1371        *container = Value::Object(serde_json::Map::new());
1372    }
1373    if let Value::Object(container) = container {
1374        container.insert("effort".to_string(), Value::String(effort.to_string()));
1375    }
1376}
1377
1378// Applies target defaults without overriding fields supplied by the caller.
1379fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {
1380    let Value::Object(object) = body else {
1381        return;
1382    };
1383    for (key, value) in extra_body {
1384        object.entry(key.clone()).or_insert_with(|| value.clone());
1385    }
1386}
1387
1388fn omit_configured_body_fields(body: &mut Value, omit_body_fields: &BTreeSet<String>) {
1389    let Value::Object(object) = body else {
1390        return;
1391    };
1392    for key in omit_body_fields {
1393        object.remove(key);
1394    }
1395}
1396
1397// Anthropic and Bedrock both cap a request at four blocks carrying
1398// `cache_control`, counting tools, system blocks and message blocks together.
1399const MAX_CACHE_CONTROL_BLOCKS: usize = 4;
1400
1401// Counts the blocks upstream will see carrying a `cache_control` marker. The
1402// marker's own value holds no such key, so it is never counted twice.
1403fn count_cache_control_blocks(value: &Value) -> usize {
1404    match value {
1405        Value::Object(map) => {
1406            usize::from(map.contains_key("cache_control"))
1407                + map.values().map(count_cache_control_blocks).sum::<usize>()
1408        }
1409        Value::Array(items) => items.iter().map(count_cache_control_blocks).sum(),
1410        _ => 0,
1411    }
1412}
1413
1414// This helper marks the final caller content block before any temporary notes, if policy permits.
1415fn enable_anthropic_prompt_caching(body: &mut Value, temporary_note_count: usize) {
1416    // Anthropic rejects a fifth cache marker with HTTP 400.
1417    // This check preserves the caller's markers when the request has reached that limit.
1418    if count_cache_control_blocks(body) >= MAX_CACHE_CONTROL_BLOCKS {
1419        return;
1420    }
1421    // Requests with only a top-level `1h` policy can return HTTP 400.
1422    // New block markers match the caller's TTL on turns with or without notes.
1423    let marker = body
1424        .get("cache_control")
1425        .cloned()
1426        .unwrap_or_else(|| serde_json::json!({"type":"ephemeral"}));
1427    let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
1428        return;
1429    };
1430    let mut remaining = temporary_note_count;
1431    // Temporary guidance is absent next turn, so the breakpoint must precede it.
1432    for message in messages.iter_mut().rev() {
1433        match message.get_mut("content") {
1434            Some(content @ Value::String(_)) if remaining == 0 => {
1435                let text = std::mem::take(content);
1436                *content = serde_json::json!([{
1437                    "type": "text", "text": text, "cache_control": marker
1438                }]);
1439                return;
1440            }
1441            Some(Value::String(_)) => remaining = remaining.saturating_sub(1),
1442            Some(Value::Array(blocks)) => {
1443                if remaining >= blocks.len() {
1444                    remaining -= blocks.len();
1445                    continue;
1446                }
1447                let index = blocks.len() - remaining - 1;
1448                if let Some(block) = blocks[index].as_object_mut() {
1449                    block.entry("cache_control".to_string()).or_insert(marker);
1450                }
1451                return;
1452            }
1453            _ => {}
1454        }
1455    }
1456}
1457
1458// GPT-5.6 and GPT-6 cache writes end at selected message or content boundaries.
1459// A temporary suffix needs a separate cache entry before it so the next request can reuse the prefix.
1460fn enable_openai_guidance_caching(
1461    body: &mut Value,
1462    format: WireFormat,
1463    model: &ModelId,
1464    suffix: usize,
1465) {
1466    let name = model.as_str().rsplit('/').next().unwrap_or_default();
1467    let supported = name == "gpt-5.6"
1468        || name.starts_with("gpt-5.6-")
1469        || name == "gpt-6"
1470        || name.starts_with("gpt-6-")
1471        || name.starts_with("gpt-6.");
1472    // Explicit-only callers choose their write locations.
1473    // This helper adds markers only for names recognized by its model-name check.
1474    if suffix == 0
1475        || !supported
1476        || body
1477            .pointer("/prompt_cache_options/mode")
1478            .and_then(Value::as_str)
1479            == Some("explicit")
1480    {
1481        return;
1482    }
1483    let key = if format == WireFormat::OpenAiResponses {
1484        "input"
1485    } else {
1486        "messages"
1487    };
1488    let Some(items) = body.get_mut(key).and_then(Value::as_array_mut) else {
1489        return;
1490    };
1491    let marker_count = items
1492        .iter()
1493        .map(|item| {
1494            ["content", "output"]
1495                .iter()
1496                .filter_map(|key| item.get(key).and_then(Value::as_array))
1497                .flatten()
1498                .filter(|block| block.get("prompt_cache_breakpoint").is_some())
1499                .count()
1500        })
1501        .sum::<usize>();
1502    // Implicit mode reserves one of the four write slots for the provider's final breakpoint.
1503    if marker_count >= 3 {
1504        return;
1505    }
1506    let Some(prefix_len) = items.len().checked_sub(suffix) else {
1507        return;
1508    };
1509    for item in items[..prefix_len].iter_mut().rev() {
1510        let field = if item.get("type").and_then(Value::as_str) == Some("function_call_output") {
1511            "output"
1512        } else if matches!(
1513            item.get("role").and_then(Value::as_str),
1514            Some("user" | "tool" | "developer" | "system")
1515        ) {
1516            "content"
1517        } else {
1518            continue;
1519        };
1520        let Some(content) = item.get_mut(field) else {
1521            continue;
1522        };
1523        if let Value::String(text) = content {
1524            let text = std::mem::take(text);
1525            let kind = if format == WireFormat::OpenAiResponses {
1526                "input_text"
1527            } else {
1528                "text"
1529            };
1530            *content = serde_json::json!([{"type":kind,"text":text}]);
1531        }
1532        // Empty content arrays have no block to mark, so the search continues through earlier items.
1533        if let Some(block) = content
1534            .as_array_mut()
1535            .and_then(|blocks| blocks.last_mut())
1536            .and_then(Value::as_object_mut)
1537        {
1538            block
1539                .entry("prompt_cache_breakpoint".to_string())
1540                .or_insert_with(|| serde_json::json!({"mode":"explicit"}));
1541            return;
1542        }
1543    }
1544}
1545
1546// Requests streamed Chat usage by default while preserving an explicit caller choice.
1547fn ensure_openai_stream_usage(body: &mut Value) {
1548    let Value::Object(object) = body else {
1549        return;
1550    };
1551    if object.get("stream").and_then(Value::as_bool) != Some(true) {
1552        return;
1553    }
1554
1555    match object.get_mut("stream_options") {
1556        Some(Value::Object(options)) => {
1557            options
1558                .entry("include_usage".to_string())
1559                .or_insert(Value::Bool(true));
1560        }
1561        _ => {
1562            let mut options = Map::new();
1563            options.insert("include_usage".to_string(), Value::Bool(true));
1564            object.insert("stream_options".to_string(), Value::Object(options));
1565        }
1566    }
1567}
1568
1569fn is_allowed_metadata_header(name: &str) -> bool {
1570    ALLOWED_METADATA_HEADERS
1571        .iter()
1572        .any(|allowed| name.eq_ignore_ascii_case(allowed))
1573}
1574
1575fn is_non_forwardable_header(name: &str, headers: &HeaderMap) -> bool {
1576    NON_FORWARDABLE_HEADERS
1577        .iter()
1578        .any(|blocked| name.eq_ignore_ascii_case(blocked))
1579        || headers.get_all("connection").iter().any(|value| {
1580            value
1581                .as_bytes()
1582                .split(|byte| *byte == b',')
1583                .any(|option| option.trim_ascii().eq_ignore_ascii_case(name.as_bytes()))
1584        })
1585}
1586
1587// Unknown application headers can carry credentials when auth forwarding is enabled.
1588fn redact_forwarded_headers(
1589    mut body: String,
1590    metadata: Option<&Metadata>,
1591    is_forwarding_auth: bool,
1592) -> String {
1593    if !is_forwarding_auth {
1594        return body;
1595    }
1596    let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
1597        return body;
1598    };
1599    for (name, value) in headers {
1600        if is_non_forwardable_header(name.as_str(), headers) {
1601            continue;
1602        }
1603        let Ok(value) = value.to_str() else {
1604            continue;
1605        };
1606        if !value.is_empty() {
1607            body = body.replace(value, "[REDACTED]");
1608        }
1609    }
1610    body
1611}
1612
1613#[cfg(test)]
1614mod tests {
1615    use std::collections::BTreeMap;
1616    use std::error::Error;
1617    use std::io::{Read, Write};
1618    use std::sync::Arc;
1619    use std::sync::atomic::{AtomicUsize, Ordering};
1620    use std::thread::JoinHandle;
1621
1622    use serde_json::json;
1623    use switchyard_protocol::{completion_text, text_request};
1624    use wiremock::matchers::{method, path};
1625    use wiremock::{Mock, MockServer, ResponseTemplate};
1626
1627    use super::*;
1628    use crate::backend::HttpBackendConfig;
1629
1630    /// Decodes a wire body into a request, as a server endpoint does before calling the client.
1631    fn wire_request(
1632        body: Value,
1633        http_headers: Option<http::HeaderMap>,
1634        wire_format: WireFormat,
1635    ) -> switchyard_translation::Result<Request> {
1636        Ok(Request {
1637            llm_request: switchyard_translation::decode_request(wire_format, &body)?,
1638            raw_request: None,
1639            metadata: Some(Metadata {
1640                http_headers,
1641                ..Default::default()
1642            }),
1643        })
1644    }
1645
1646    fn assert_tool_call_name(agg: &switchyard_protocol::AggLlmResponse, name: &str) {
1647        let Some(switchyard_protocol::ContentBlock::ToolCall(call)) =
1648            agg.outputs[0].content.first()
1649        else {
1650            panic!("expected a tool call, got {:?}", agg.outputs[0].content);
1651        };
1652        assert_eq!(call.name, name);
1653    }
1654
1655    fn config(base_url: &str) -> HttpBackendConfig {
1656        HttpBackendConfig {
1657            base_url: base_url.to_string(),
1658            api_key: Some("secret".to_string()),
1659            forward_auth: false,
1660            extra_headers: BTreeMap::new(),
1661            extra_body: BTreeMap::new(),
1662            omit_body_fields: BTreeSet::new(),
1663            reasoning_effort: None,
1664            max_retries: 0,
1665            failure_cooldown: Duration::ZERO,
1666            timeout: None,
1667        }
1668    }
1669
1670    #[test]
1671    fn provider_endpoint_urls_preserve_query_and_fragment() {
1672        let invalid_url = "not a URL/";
1673        let backend = Backend::OpenAiResponses(config(invalid_url));
1674        assert!(openai_url(invalid_url, "/responses").is_err());
1675        assert_eq!(backend.url(), invalid_url);
1676        assert_eq!(
1677            AuxiliaryOperation::ResponsesCompact.url(&backend),
1678            invalid_url
1679        );
1680
1681        let anthropic = Backend::Anthropic(config(invalid_url));
1682        assert_eq!(anthropic.url(), invalid_url);
1683        assert_eq!(anthropic.count_tokens_url(), invalid_url);
1684
1685        for path in ["", "/v1/", "/v1/messages"] {
1686            for tail in [
1687                "",
1688                "?api-version=2026-09-01&value=%2F/",
1689                "#fragment?value=/",
1690            ] {
1691                let base_url = format!("https://example.com{path}{tail}");
1692                let backend = Backend::Anthropic(config(&base_url));
1693                assert_eq!(
1694                    backend.url(),
1695                    format!("https://example.com/v1/messages{tail}")
1696                );
1697                assert_eq!(
1698                    AuxiliaryOperation::AnthropicCountTokens.url(&backend),
1699                    format!("https://example.com/v1/messages/count_tokens{tail}")
1700                );
1701            }
1702        }
1703
1704        for path in ["/v1", "/v1/responses/", "/v1/chat/completions"] {
1705            for tail in [
1706                "",
1707                "?api-version=2026-09-01&value=%2F/",
1708                "#fragment?value=/",
1709            ] {
1710                let base_url = format!("https://example.com{path}{tail}");
1711                let responses = Backend::OpenAiResponses(config(&base_url));
1712                let chat = Backend::OpenAiChat(config(&base_url));
1713                for (actual, endpoint) in [
1714                    (responses.url(), "/responses"),
1715                    (chat.url(), "/chat/completions"),
1716                    (
1717                        AuxiliaryOperation::ResponsesCompact.url(&responses),
1718                        "/responses/compact",
1719                    ),
1720                    (
1721                        AuxiliaryOperation::ResponsesInputTokens.url(&responses),
1722                        "/responses/input_tokens",
1723                    ),
1724                ] {
1725                    assert_eq!(actual, format!("https://example.com/v1{endpoint}{tail}"));
1726                }
1727            }
1728        }
1729    }
1730
1731    fn config_with_retries(base_url: &str, max_retries: u32) -> HttpBackendConfig {
1732        HttpBackendConfig {
1733            max_retries,
1734            ..config(base_url)
1735        }
1736    }
1737
1738    fn forwarding_config(base_url: &str) -> HttpBackendConfig {
1739        HttpBackendConfig {
1740            api_key: None,
1741            forward_auth: true,
1742            ..config(base_url)
1743        }
1744    }
1745
1746    // A one-model config list: "gpt" served over OpenAI Chat at base_url.
1747    fn chat_map(base_url: &str) -> Vec<ModelConfig> {
1748        vec![ModelConfig::new(
1749            "gpt",
1750            Backend::OpenAiChat(config(base_url)),
1751            None,
1752        )]
1753    }
1754
1755    fn responses_map(base_url: &str) -> Vec<ModelConfig> {
1756        vec![ModelConfig::new(
1757            "gpt",
1758            Backend::OpenAiResponses(config(base_url)),
1759            None,
1760        )]
1761    }
1762
1763    fn chat_map_with_extra_body(
1764        base_url: &str,
1765        extra_body: BTreeMap<String, Value>,
1766    ) -> Vec<ModelConfig> {
1767        let mut backend = config(base_url);
1768        backend.extra_body = extra_body;
1769        vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
1770    }
1771
1772    fn chat_map_with_omit_body_fields(
1773        base_url: &str,
1774        omit_body_fields: BTreeSet<String>,
1775    ) -> Vec<ModelConfig> {
1776        let mut backend = config(base_url);
1777        backend.omit_body_fields = omit_body_fields;
1778        vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
1779    }
1780
1781    fn chat_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
1782        let mut backend = config(base_url);
1783        backend.reasoning_effort = Some(effort.to_string());
1784        vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
1785    }
1786
1787    fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
1788        let mut backend = config(base_url);
1789        backend.reasoning_effort = Some(effort.to_string());
1790        vec![ModelConfig::new(
1791            "gpt",
1792            Backend::OpenAiResponses(backend),
1793            None,
1794        )]
1795    }
1796
1797    fn anthropic_map(base_url: &str) -> Vec<ModelConfig> {
1798        vec![ModelConfig::new(
1799            "claude",
1800            Backend::Anthropic(config(base_url)),
1801            None,
1802        )]
1803    }
1804
1805    #[tokio::test]
1806    async fn model_properties_uses_configured_auth_and_backend_path()
1807    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
1808        for (base_path, props_path) in [
1809            ("/v1", "/props"),
1810            ("/v1/", "/props"),
1811            ("/v1/chat/completions", "/props"),
1812            ("/v1/responses/", "/props"),
1813            ("/prefix/v1", "/prefix/props"),
1814            ("/prefix/v1/chat/completions", "/prefix/props"),
1815            ("/prefix/v1/responses", "/prefix/props"),
1816            ("/prefix/", "/prefix/props"),
1817        ] {
1818            let server = MockServer::start().await;
1819            Mock::given(method("GET"))
1820                .and(path(props_path))
1821                .and(wiremock::matchers::header("authorization", "Bearer secret"))
1822                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1823                    "default_generation_settings": {"n_ctx": 65_536}
1824                })))
1825                .expect(1)
1826                .mount(&server)
1827                .await;
1828            let client =
1829                TranslatingLlmClient::new(&chat_map(&format!("{}{base_path}", server.uri())))?;
1830            let properties = client
1831                .get_model_properties(
1832                    &ModelId::from("gpt"),
1833                    WireFormat::OpenAiChat,
1834                    Duration::from_secs(1),
1835                )
1836                .await?;
1837            assert_eq!(properties["default_generation_settings"]["n_ctx"], 65_536);
1838        }
1839        Ok(())
1840    }
1841
1842    fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec<ModelConfig> {
1843        vec![ModelConfig::new(
1844            "gpt",
1845            Backend::OpenAiChat(config_with_retries(base_url, max_retries)),
1846            None,
1847        )]
1848    }
1849
1850    fn chat_success_response() -> ResponseTemplate {
1851        ResponseTemplate::new(200).set_body_json(json!({
1852            "id": "chatcmpl-1",
1853            "model": "gpt",
1854            "choices": [{
1855                "index": 0,
1856                "message": {"role": "assistant", "content": "recovered"},
1857                "finish_reason": "stop"
1858            }],
1859            "usage": {}
1860        }))
1861    }
1862
1863    fn truncated_response_server(
1864        content_type: &str,
1865        body: &str,
1866    ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
1867        response_sequence_server(vec![format!(
1868            "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\n\
1869             Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
1870            body.len() + 100
1871        )])
1872    }
1873
1874    fn response_sequence_server(
1875        responses: Vec<String>,
1876    ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
1877        let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
1878        let address = listener.local_addr()?;
1879        let handle = std::thread::spawn(move || {
1880            for response in responses {
1881                let (mut stream, _) = listener.accept()?;
1882                let mut request = [0_u8; 1024];
1883                if stream.read(&mut request)? == 0 {
1884                    return Err(std::io::Error::new(
1885                        std::io::ErrorKind::UnexpectedEof,
1886                        "client closed before sending a request",
1887                    ));
1888                }
1889                stream.write_all(response.as_bytes())?;
1890            }
1891            Ok(())
1892        });
1893        Ok((format!("http://{address}/v1"), handle))
1894    }
1895
1896    fn raw_chat_success_response() -> String {
1897        let body = r#"{"id":"chatcmpl-1","model":"gpt","choices":[{"index":0,"message":{"role":"assistant","content":"recovered"},"finish_reason":"stop"}],"usage":{}}"#;
1898        format!(
1899            "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
1900             Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
1901            body.len()
1902        )
1903    }
1904
1905    fn request_for(model: Option<&str>, stream: bool) -> Request {
1906        let mut llm_request = text_request(model.map(str::to_string), "hi");
1907        llm_request.stream = stream;
1908        Request {
1909            llm_request,
1910            raw_request: None,
1911            metadata: None,
1912        }
1913    }
1914
1915    fn request_with_headers(model: &str, headers: HeaderMap) -> Request {
1916        let mut request = request_for(Some(model), false);
1917        request.metadata = Some(Metadata {
1918            http_headers: Some(headers),
1919            ..Default::default()
1920        });
1921        request
1922    }
1923
1924    #[tokio::test]
1925    async fn transport_errors_drop_the_upstream_url() {
1926        let error = reqwest::Client::new()
1927            .post("http://127.0.0.1:1/v1?key=CANARY")
1928            .send()
1929            .await
1930            .expect_err("closed port");
1931
1932        assert!(!convert_reqwest_error(error).to_string().contains("CANARY"));
1933    }
1934
1935    #[tokio::test]
1936    async fn captured_request_build_failure_remains_a_configuration_error() {
1937        let client = TranslatingLlmClient::new(&chat_map("://invalid")).unwrap();
1938        let result =
1939            crate::with_payload_capture(client.call(request_for(Some("gpt"), false))).await;
1940        assert!(matches!(result, Err(LlmClientError::Configuration { .. })));
1941    }
1942
1943    #[test]
1944    fn nvidia_anthropic_tools_drop_unsupported_strict_field() {
1945        let backend = Backend::Anthropic(config("https://inference-api.nvidia.com"));
1946        assert!(nvidia_inference_api_backend(&backend));
1947        assert!(!nvidia_inference_api_backend(&Backend::Anthropic(config(
1948            "https://inference-api.nvidia.com.example.test"
1949        ))));
1950        let mut body = json!({"tools": [{"name": "a", "input_schema": {}, "strict": true}]});
1951        strip_anthropic_tool_strict(&mut body);
1952        assert!(body["tools"][0].get("strict").is_none());
1953    }
1954
1955    #[test]
1956    fn anthropic_prompt_caching_marks_final_message() {
1957        let caller_managed = json!({
1958            "cache_control": {"type": "ephemeral", "ttl": "1h"},
1959            "messages": [{"role": "user", "content": "hello"}]
1960        });
1961        let mut body = caller_managed.clone();
1962        enable_anthropic_prompt_caching(&mut body, 0);
1963        assert_eq!(body["cache_control"], caller_managed["cache_control"]);
1964        assert_eq!(
1965            body["messages"][0]["content"],
1966            json!([{
1967                "type": "text", "text": "hello",
1968                "cache_control": {"type": "ephemeral", "ttl": "1h"}
1969            }])
1970        );
1971
1972        let mut body = json!({
1973            "messages": [{"role": "user", "content": "hello"}]
1974        });
1975
1976        enable_anthropic_prompt_caching(&mut body, 0);
1977
1978        assert_eq!(
1979            body["messages"][0]["content"][0]["cache_control"],
1980            json!({"type": "ephemeral"})
1981        );
1982    }
1983
1984    // Counts the blocks upstream will see carrying a `cache_control` marker.
1985    fn cache_control_blocks(value: &Value) -> usize {
1986        count_cache_control_blocks(value)
1987    }
1988
1989    // A body whose four breakpoints are all spent elsewhere: the caller manages
1990    // its own caching and left the final block unmarked on purpose.
1991    fn body_at_the_cache_control_limit() -> Value {
1992        json!({
1993            "system": [
1994                {"type": "text", "text": "a", "cache_control": {"type": "ephemeral"}},
1995                {"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}
1996            ],
1997            "tools": [
1998                {"name": "t", "input_schema": {"type": "object"},
1999                 "cache_control": {"type": "ephemeral"}}
2000            ],
2001            "messages": [
2002                {"role": "user", "content": [
2003                    {"type": "text", "text": "old", "cache_control": {"type": "ephemeral"}}
2004                ]},
2005                {"role": "assistant", "content": [{"type": "text", "text": "ok"}]},
2006                {"role": "user", "content": [{"type": "text", "text": "new"}]}
2007            ]
2008        })
2009    }
2010
2011    #[test]
2012    fn anthropic_prompt_caching_abstains_at_the_cache_control_limit() {
2013        let mut body = body_at_the_cache_control_limit();
2014        assert_eq!(cache_control_blocks(&body), 4);
2015
2016        enable_anthropic_prompt_caching(&mut body, 0);
2017
2018        // Anthropic and Bedrock both reject a fifth marker with HTTP 400, which
2019        // would fail a request that was valid before it reached us.
2020        assert_eq!(cache_control_blocks(&body), 4);
2021        assert!(
2022            body["messages"][2]["content"][0]
2023                .get("cache_control")
2024                .is_none()
2025        );
2026    }
2027
2028    #[test]
2029    fn anthropic_prompt_caching_still_marks_one_below_the_limit() {
2030        let mut body = body_at_the_cache_control_limit();
2031        // Free one breakpoint, so there is room for ours.
2032        body["system"].as_array_mut().unwrap().pop();
2033        assert_eq!(cache_control_blocks(&body), 3);
2034
2035        enable_anthropic_prompt_caching(&mut body, 0);
2036
2037        assert_eq!(cache_control_blocks(&body), 4);
2038        assert_eq!(
2039            body["messages"][2]["content"][0]["cache_control"],
2040            json!({"type": "ephemeral"})
2041        );
2042    }
2043
2044    // A request that pins `format` in its metadata, so the client resolves that
2045    // wire format instead of the model's default backend.
2046    fn request_with_wire_format(model: &str, format: WireFormat) -> Request {
2047        let mut request = request_for(Some(model), false);
2048        request.metadata = Some(Metadata {
2049            session_id: None,
2050            agent_id: None,
2051            task_id: None,
2052            correlation_id: None,
2053            extra_metadata: None,
2054            http_headers: None,
2055            wire_format: Some(format),
2056            ..Default::default()
2057        });
2058        request
2059    }
2060
2061    #[tokio::test]
2062    async fn missing_model_errors()
2063    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2064        let client = TranslatingLlmClient::new(&[])?;
2065        let Err(error) = client
2066            .call_rewrite_model(request_for(None, false), None)
2067            .await
2068        else {
2069            panic!("expected an error");
2070        };
2071        assert!(matches!(
2072            error,
2073            LlmClientError::InvalidRequest { message } if message == "no model given"
2074        ));
2075        Ok(())
2076    }
2077
2078    #[tokio::test]
2079    async fn unknown_model_errors()
2080    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2081        let client = TranslatingLlmClient::new(&[])?;
2082        let Err(error) = client
2083            .call_rewrite_model(request_for(Some("gpt"), false), None)
2084            .await
2085        else {
2086            panic!("expected an error");
2087        };
2088        assert!(matches!(
2089            error,
2090            LlmClientError::Configuration { message }
2091                if message.contains("gpt")
2092        ));
2093        Ok(())
2094    }
2095
2096    #[tokio::test]
2097    async fn unknown_model_format_errors()
2098    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2099        // "gpt" exists but only over OpenAI Chat; the request pins Anthropic.
2100        let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
2101        let Err(error) = client
2102            .call_rewrite_model(
2103                request_with_wire_format("gpt", WireFormat::AnthropicMessages),
2104                None,
2105            )
2106            .await
2107        else {
2108            panic!("expected an error");
2109        };
2110        assert!(matches!(
2111            error,
2112            LlmClientError::Configuration { message }
2113                if message.contains("gpt")
2114                    && message.contains(&WireFormat::AnthropicMessages.to_string())
2115        ));
2116        Ok(())
2117    }
2118
2119    #[test]
2120    fn backend_for_resolves_configured_format()
2121    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2122        let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
2123        // "gpt" is served over OpenAI Chat only; other formats and models miss.
2124        assert!(
2125            client
2126                .backend_for(&ModelId::from("gpt"), WireFormat::OpenAiChat)
2127                .is_some()
2128        );
2129        assert!(
2130            client
2131                .backend_for(&ModelId::from("gpt"), WireFormat::AnthropicMessages)
2132                .is_none()
2133        );
2134        assert!(
2135            client
2136                .backend_for(&ModelId::from("missing"), WireFormat::OpenAiChat)
2137                .is_none()
2138        );
2139        Ok(())
2140    }
2141
2142    #[tokio::test]
2143    async fn target_name_arg_wins_over_request_model()
2144    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2145        let client = TranslatingLlmClient::new(&[])?;
2146        // Arg "b" is looked up (and reported), not the request's "a".
2147        let Err(error) = client
2148            .call_rewrite_model(request_for(Some("a"), false), Some(&ModelId::from("b")))
2149            .await
2150        else {
2151            panic!("expected an error");
2152        };
2153        assert!(matches!(
2154            error,
2155            LlmClientError::Configuration { message }
2156                if message.contains("\"b\"")
2157        ));
2158        Ok(())
2159    }
2160
2161    #[tokio::test]
2162    async fn buffered_openai_chat_round_trips()
2163    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2164        let server = MockServer::start().await;
2165        Mock::given(method("POST"))
2166            .and(path("/v1/chat/completions"))
2167            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2168                "id": "chatcmpl-1",
2169                "model": "gpt",
2170                "choices": [{
2171                    "index": 0,
2172                    "message": {"role": "assistant", "content": "Hi there"},
2173                    "finish_reason": "stop"
2174                }],
2175                "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
2176            })))
2177            .mount(&server)
2178            .await;
2179
2180        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
2181
2182        let response = client
2183            .call_rewrite_model(request_for(Some("gpt"), false), None)
2184            .await?;
2185        let agg = response.llm_response.into_agg().await?;
2186        assert_eq!(completion_text(&agg), "Hi there");
2187
2188        Ok(())
2189    }
2190
2191    #[tokio::test]
2192    async fn invalid_json_is_a_response_translation_error()
2193    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2194        let server = MockServer::start().await;
2195        let calls = Arc::new(AtomicUsize::new(0));
2196        let observed_calls = Arc::clone(&calls);
2197        Mock::given(method("POST"))
2198            .respond_with(move |_: &wiremock::Request| {
2199                observed_calls.fetch_add(1, Ordering::SeqCst);
2200                ResponseTemplate::new(200).set_body_raw("not json", "application/json")
2201            })
2202            .mount(&server)
2203            .await;
2204
2205        let client =
2206            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
2207        let Err(error) = client
2208            .call_rewrite_model(request_for(Some("gpt"), false), None)
2209            .await
2210        else {
2211            panic!("expected invalid JSON to fail");
2212        };
2213
2214        assert!(matches!(
2215            error,
2216            LlmClientError::ResponseTranslation(message)
2217                if message.contains("invalid upstream JSON")
2218        ));
2219        assert_eq!(calls.load(Ordering::SeqCst), 1);
2220        Ok(())
2221    }
2222
2223    #[tokio::test]
2224    async fn response_body_io_failure_is_a_transport_error()
2225    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2226        let (base_url, server) = truncated_response_server("application/json", "{}")?;
2227        let client = TranslatingLlmClient::new(&chat_map(&base_url))?;
2228        let result = client
2229            .call_rewrite_model(request_for(Some("gpt"), false), None)
2230            .await;
2231        server
2232            .join()
2233            .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2234
2235        let Err(error) = result else {
2236            panic!("expected the truncated response body to fail");
2237        };
2238        let LlmClientError::Transport { source } = error else {
2239            panic!("expected a transport error");
2240        };
2241        let Some(source) = source.downcast_ref::<reqwest::Error>() else {
2242            panic!("expected the reqwest transport source");
2243        };
2244        assert!(source.is_decode());
2245        assert!(
2246            !std::error::Error::source(&source)
2247                .is_some_and(|source| source.is::<serde_json::Error>())
2248        );
2249        Ok(())
2250    }
2251
2252    #[tokio::test]
2253    async fn response_body_transport_failures_are_retried()
2254    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2255        let truncated_responses = [
2256            "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
2257             Content-Length: 102\r\nConnection: close\r\n\r\n{}",
2258            "HTTP/1.1 401 Unauthorized\r\nContent-Type: text/plain\r\n\
2259             Content-Length: 100\r\nConnection: close\r\n\r\nbad",
2260        ];
2261
2262        for truncated in truncated_responses {
2263            let (base_url, server) =
2264                response_sequence_server(vec![truncated.to_string(), raw_chat_success_response()])?;
2265            let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
2266            let response = client
2267                .call_rewrite_model(request_for(Some("gpt"), false), None)
2268                .await?;
2269            server
2270                .join()
2271                .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2272
2273            assert_eq!(
2274                completion_text(&response.llm_response.into_agg().await?),
2275                "recovered"
2276            );
2277        }
2278        Ok(())
2279    }
2280
2281    #[tokio::test]
2282    async fn pre_first_event_transport_failure_uses_retry_budget()
2283    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2284        let truncated = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\
2285                         Content-Length: 100\r\nConnection: close\r\n\r\n";
2286        let body = "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"recovered\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
2287        let success = format!(
2288            "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\
2289             Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
2290            body.len()
2291        );
2292
2293        for second in [success, truncated.to_string()] {
2294            let is_exhausted = second == truncated;
2295            let (base_url, server) = response_sequence_server(vec![truncated.to_string(), second])?;
2296            let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
2297            let result = client
2298                .call_rewrite_model(request_for(Some("gpt"), true), None)
2299                .await;
2300            if is_exhausted {
2301                assert!(matches!(result, Err(LlmClientError::Transport { .. })));
2302            } else {
2303                assert_eq!(
2304                    completion_text(&result?.llm_response.into_agg().await?),
2305                    "recovered"
2306                );
2307            }
2308            server
2309                .join()
2310                .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2311        }
2312        Ok(())
2313    }
2314
2315    #[tokio::test]
2316    async fn streaming_body_io_failure_preserves_transport_error()
2317    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2318        let body = "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n";
2319        let (base_url, server) = truncated_response_server("text/event-stream", body)?;
2320        let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 2))?;
2321        let response = client
2322            .call_rewrite_model(request_for(Some("gpt"), true), None)
2323            .await?;
2324        let result = response.llm_response.into_agg().await;
2325        server
2326            .join()
2327            .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2328
2329        let Err(error) = result else {
2330            panic!("expected the truncated stream body to fail");
2331        };
2332
2333        assert!(matches!(error, LlmClientError::Transport { .. }));
2334        Ok(())
2335    }
2336
2337    #[tokio::test]
2338    async fn rewrites_model_to_resolved_upstream_id()
2339    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2340        // Inbound body says "switchyard"; the upstream must receive "gpt".
2341        let server = MockServer::start().await;
2342        Mock::given(method("POST"))
2343            .and(path("/v1/chat/completions"))
2344            .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"})))
2345            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2346                "id": "1", "model": "gpt",
2347                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
2348                "usage": {}
2349            })))
2350            .mount(&server)
2351            .await;
2352
2353        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
2354        // Inbound model differs from the map key / resolved model.
2355        client
2356            .call_rewrite_model(
2357                request_for(Some("switchyard"), false),
2358                Some(&ModelId::from("gpt")),
2359            )
2360            .await?;
2361        // The body_partial_json matcher asserts the upstream saw model "gpt".
2362        Ok(())
2363    }
2364
2365    /// A configured reasoning effort replaces the caller's value on both OpenAI wire formats,
2366    /// which `extra_body` (defaults only) cannot do.
2367    #[tokio::test]
2368    async fn reasoning_effort_override_replaces_the_callers_effort()
2369    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2370        let server = MockServer::start().await;
2371        Mock::given(method("POST"))
2372            .and(path("/v1/chat/completions"))
2373            .and(wiremock::matchers::body_partial_json(json!({
2374                "model": "gpt",
2375                "reasoning_effort": "max"
2376            })))
2377            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2378                "id": "1",
2379                "model": "gpt",
2380                "choices": [{
2381                    "index": 0,
2382                    "message": {"role": "assistant", "content": "ok"},
2383                    "finish_reason": "stop"
2384                }],
2385                "usage": {}
2386            })))
2387            .mount(&server)
2388            .await;
2389        let client = TranslatingLlmClient::new(&chat_map_with_effort(
2390            &format!("{}/v1", server.uri()),
2391            "max",
2392        ))?;
2393        client
2394            .call_rewrite_model(
2395                wire_request(
2396                    json!({
2397                        "model": "client-facing",
2398                        "messages": [{"role": "user", "content": "hi"}],
2399                        "reasoning_effort": "high"
2400                    }),
2401                    None,
2402                    WireFormat::OpenAiChat,
2403                )?,
2404                Some(&ModelId::from("gpt")),
2405            )
2406            .await?;
2407
2408        let server = MockServer::start().await;
2409        Mock::given(method("POST"))
2410            .and(path("/v1/responses"))
2411            .and(wiremock::matchers::body_partial_json(json!({
2412                "model": "gpt",
2413                "reasoning": {"effort": "max", "summary": "auto"}
2414            })))
2415            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2416                "id": "resp_1",
2417                "object": "response",
2418                "model": "gpt",
2419                "status": "completed",
2420                "output": [{
2421                    "type": "message",
2422                    "id": "msg_1",
2423                    "role": "assistant",
2424                    "status": "completed",
2425                    "content": [{"type": "output_text", "text": "ok"}]
2426                }],
2427                "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
2428            })))
2429            .mount(&server)
2430            .await;
2431        let client = TranslatingLlmClient::new(&responses_map_with_effort(
2432            &format!("{}/v1", server.uri()),
2433            "max",
2434        ))?;
2435        client
2436            .call_rewrite_model(
2437                wire_request(
2438                    json!({
2439                        "model": "client-facing",
2440                        "input": [{"role": "user", "content": "hi"}],
2441                        "reasoning": {"effort": "high", "summary": "auto"}
2442                    }),
2443                    None,
2444                    WireFormat::OpenAiResponses,
2445                )?,
2446                Some(&ModelId::from("gpt")),
2447            )
2448            .await?;
2449        Ok(())
2450    }
2451
2452    #[tokio::test]
2453    async fn reasoning_effort_map_only_rewrites_matching_effort_for_the_selected_target()
2454    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2455        let server = MockServer::start().await;
2456        Mock::given(method("POST"))
2457            .and(path("/v1/chat/completions"))
2458            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2459                "id": "1", "model": "gpt",
2460                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
2461                "usage": {}
2462            })))
2463            .expect(4)
2464            .mount(&server)
2465            .await;
2466        let backend = Backend::OpenAiChat(config(&format!("{}/v1", server.uri())));
2467        let models = [
2468            ModelConfig::new("mapped", backend.clone(), None)
2469                .with_provider_model("gpt")
2470                .with_reasoning_effort_map(BTreeMap::from([("minimal".into(), "low".into())])),
2471            ModelConfig::new("unmapped", backend, None).with_provider_model("gpt"),
2472        ];
2473        let client = TranslatingLlmClient::new(&models)?;
2474        let cases = [
2475            ("mapped", Some("minimal"), Some("low")),
2476            ("mapped", Some("high"), Some("high")),
2477            ("mapped", None, None),
2478            ("unmapped", Some("minimal"), Some("minimal")),
2479        ];
2480        for (target, effort, _) in cases {
2481            let mut body = json!({
2482                "model": "client-facing",
2483                "messages": [{"role": "user", "content": "hi"}]
2484            });
2485            if let Some(effort) = effort {
2486                body["reasoning_effort"] = json!(effort);
2487            }
2488            client
2489                .call_rewrite_model(
2490                    wire_request(body, None, WireFormat::OpenAiChat)?,
2491                    Some(&ModelId::from(target)),
2492                )
2493                .await?;
2494        }
2495        let calls = server.received_requests().await.ok_or("missing requests")?;
2496        for (call, (_, _, expected)) in calls.iter().zip(cases) {
2497            let body: Value = serde_json::from_slice(&call.body)?;
2498            assert_eq!(body["model"], "gpt");
2499            assert_eq!(
2500                body.get("reasoning_effort"),
2501                expected.map(Value::from).as_ref()
2502            );
2503        }
2504        Ok(())
2505    }
2506
2507    // Covers both a translated request and an Anthropic caller's body sent as is.
2508    #[tokio::test]
2509    async fn anthropic_requests_drop_tool_strict_false()
2510    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2511        let server = MockServer::start().await;
2512        Mock::given(method("POST"))
2513            .and(path("/v1/messages"))
2514            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2515                "id": "msg_1",
2516                "type": "message",
2517                "role": "assistant",
2518                "model": "claude",
2519                "content": [{"type": "text", "text": "ok"}],
2520                "stop_reason": "end_turn",
2521                "usage": {"input_tokens": 1, "output_tokens": 1}
2522            })))
2523            .expect(2)
2524            .mount(&server)
2525            .await;
2526        let client = TranslatingLlmClient::new(&[ModelConfig::new(
2527            "claude",
2528            Backend::Anthropic(config(&server.uri())),
2529            None,
2530        )])?;
2531        let schema = json!({"type": "object"});
2532        let requests = [
2533            (
2534                WireFormat::OpenAiChat,
2535                json!({
2536                    "model": "client-facing",
2537                    "messages": [{"role": "user", "content": "hi"}],
2538                    "tools": [
2539                        {"type": "function", "function": {"name": "loose", "parameters": schema, "strict": false}},
2540                        {"type": "function", "function": {"name": "exact", "parameters": schema, "strict": true}}
2541                    ]
2542                }),
2543            ),
2544            (
2545                WireFormat::AnthropicMessages,
2546                json!({
2547                    "model": "client-facing",
2548                    "max_tokens": 8,
2549                    "messages": [{"role": "user", "content": "hi"}],
2550                    "tools": [
2551                        {"name": "loose", "input_schema": schema, "strict": false},
2552                        {"name": "exact", "input_schema": schema, "strict": true}
2553                    ]
2554                }),
2555            ),
2556        ];
2557        for (format, request) in requests {
2558            client
2559                .call_rewrite_model(
2560                    wire_request(request, None, format)?,
2561                    Some(&ModelId::from("claude")),
2562                )
2563                .await?;
2564        }
2565        for received in server.received_requests().await.unwrap_or_default() {
2566            let body: Value = serde_json::from_slice(&received.body)?;
2567            assert!(body["tools"][0].get("strict").is_none(), "{body}");
2568            assert_eq!(body["tools"][1]["strict"], true, "{body}");
2569        }
2570        Ok(())
2571    }
2572
2573    #[tokio::test]
2574    async fn template_rejection_retries_once_without_changing_other_fields()
2575    -> std::result::Result<(), Box<dyn Error + Sync + Send>> {
2576        for (message, reject_retry, omit) in [
2577            (
2578                "Unrecognized request argument supplied: chat_template_kwargs",
2579                false,
2580                false,
2581            ),
2582            (
2583                "Unrecognized request argument supplied: chat_template_kwargs",
2584                true,
2585                false,
2586            ),
2587            (
2588                "Invalid temperature for chat_template_kwargs request",
2589                false,
2590                false,
2591            ),
2592            (
2593                "Unrecognized request argument supplied: chat_template_kwargs",
2594                false,
2595                true,
2596            ),
2597        ] {
2598            let server = MockServer::start().await;
2599            Mock::given(method("POST"))
2600                .respond_with(move |request: &wiremock::Request| {
2601                    let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
2602                    if body.get("chat_template_kwargs").is_some() || reject_retry {
2603                        ResponseTemplate::new(400).set_body_json(json!({"error": {"message": message}}))
2604                    } else {
2605                        ResponseTemplate::new(200).set_body_json(json!({
2606                            "id": "1", "model": "gpt", "choices": [{"index": 0,
2607                                "message": {"role": "assistant", "content": "yes"}, "finish_reason": "stop"}],
2608                            "usage": {}
2609                        }))
2610                    }
2611                }).mount(&server).await;
2612            let fields = if omit {
2613                BTreeSet::from(["chat_template_kwargs".into()])
2614            } else {
2615                BTreeSet::new()
2616            };
2617            let client = TranslatingLlmClient::new(&chat_map_with_omit_body_fields(
2618                &format!("{}/v1", server.uri()),
2619                fields,
2620            ))?;
2621            let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "judge"}],
2622                "reasoning_effort": "none", "logprobs": true, "top_logprobs": 8,
2623                "chat_template_kwargs": {"enable_thinking": false}});
2624            let result = client
2625                .call_rewrite_model(
2626                    wire_request(raw.clone(), None, WireFormat::OpenAiChat)?,
2627                    Some(&ModelId::from("gpt")),
2628                )
2629                .await;
2630            let rejects_field = message.starts_with("Unrecognized");
2631            assert_eq!(result.is_ok(), omit || (rejects_field && !reject_retry));
2632            let calls = server.received_requests().await.ok_or("missing requests")?;
2633            assert_eq!(calls.len(), if !omit && rejects_field { 2 } else { 1 });
2634            for (index, call) in calls.iter().enumerate() {
2635                let mut expected = raw.clone();
2636                if omit || index > 0 {
2637                    expected
2638                        .as_object_mut()
2639                        .unwrap()
2640                        .remove("chat_template_kwargs");
2641                }
2642                assert_eq!(serde_json::from_slice::<Value>(&call.body)?, expected);
2643            }
2644        }
2645        Ok(())
2646    }
2647
2648    #[tokio::test]
2649    async fn template_compatibility_preserves_the_transient_retry_budget()
2650    -> std::result::Result<(), Box<dyn Error + Sync + Send>> {
2651        let server = MockServer::start().await;
2652        let count = Arc::new(AtomicUsize::new(0));
2653        let observed = Arc::clone(&count);
2654        Mock::given(method("POST"))
2655            .respond_with(move |request: &wiremock::Request| {
2656                let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
2657                match observed.fetch_add(1, Ordering::SeqCst) {
2658                    0 => {
2659                        assert!(body.get("chat_template_kwargs").is_some());
2660                        ResponseTemplate::new(422).set_body_json(json!({"error": {
2661                            "code": "unsupported_parameter", "param": "chat_template_kwargs"
2662                        }}))
2663                    }
2664                    1 => {
2665                        assert!(body.get("chat_template_kwargs").is_none());
2666                        ResponseTemplate::new(503).insert_header("retry-after", "0")
2667                    }
2668                    _ => ResponseTemplate::new(200).set_body_json(json!({
2669                        "id": "1", "model": "gpt", "choices": [{"index": 0,
2670                            "message": {"role": "assistant", "content": "yes"}, "finish_reason": "stop"}],
2671                        "usage": {}
2672                    })),
2673                }
2674            }).mount(&server).await;
2675        let client =
2676            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
2677        client
2678            .call_rewrite_model(
2679                wire_request(
2680                    json!({"model": "gpt", "messages": [{"role": "user", "content": "judge"}],
2681                "chat_template_kwargs": {"enable_thinking": false}}),
2682                    None,
2683                    WireFormat::OpenAiChat,
2684                )?,
2685                Some(&ModelId::from("gpt")),
2686            )
2687            .await?;
2688        assert_eq!(count.load(Ordering::SeqCst), 3);
2689        Ok(())
2690    }
2691
2692    #[tokio::test]
2693    async fn preserved_codex_string_input_is_normalized_at_the_client_boundary()
2694    -> std::result::Result<(), Box<dyn Error + Sync + Send>> {
2695        for root in ["/backend-api/codex", "/v1"] {
2696            for input in [
2697                json!("hello"),
2698                json!([{"role": "user", "content": "hello"}]),
2699            ] {
2700                let server = MockServer::start().await;
2701                Mock::given(method("POST"))
2702                    .and(path(format!("{root}/responses")))
2703                    .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2704                        "id": "resp_1", "object": "response", "model": "gpt", "status": "completed", "output": [], "usage": {}
2705                    }))).expect(1).mount(&server).await;
2706                let client =
2707                    TranslatingLlmClient::new(&responses_map(&format!("{}{root}", server.uri())))?;
2708                let raw = json!({"model": "client-name", "input": input,
2709                    "instructions": "Be brief.", "store": false, "metadata": {"task": "judge"},
2710                    "custom_field": {"keep": true}});
2711                client
2712                    .call_rewrite_model(
2713                        wire_request(raw.clone(), None, WireFormat::OpenAiResponses)?,
2714                        Some(&ModelId::from("gpt")),
2715                    )
2716                    .await?;
2717                let mut expected = raw;
2718                expected["model"] = json!("gpt");
2719                if root == "/backend-api/codex" && input.is_string() {
2720                    expected["input"] =
2721                        json!([{"type": "message", "role": "user", "content": input}]);
2722                }
2723                let calls = server.received_requests().await.ok_or("missing requests")?;
2724                assert_eq!(serde_json::from_slice::<Value>(&calls[0].body)?, expected);
2725            }
2726        }
2727        Ok(())
2728    }
2729
2730    #[tokio::test]
2731    async fn token_counting_does_not_send_the_effort_override()
2732    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2733        let server = MockServer::start().await;
2734        Mock::given(method("POST"))
2735            .and(path("/v1/messages/count_tokens"))
2736            .respond_with(ResponseTemplate::new(200).set_body_json(json!({"input_tokens": 3})))
2737            .expect(1)
2738            .mount(&server)
2739            .await;
2740        let mut backend = config(&server.uri());
2741        backend.reasoning_effort = Some("max".to_string());
2742        let client = TranslatingLlmClient::new(&[ModelConfig::new(
2743            "claude",
2744            Backend::Anthropic(backend),
2745            None,
2746        )])?;
2747        let llm_request = switchyard_translation::decode_request(
2748            WireFormat::AnthropicMessages,
2749            &json!({"model": "claude", "messages": [{"role": "user", "content": "hi"}]}),
2750        )?;
2751        let request = Request {
2752            llm_request,
2753            raw_request: None,
2754            metadata: None,
2755        };
2756        client
2757            .call_auxiliary(
2758                &ModelId::from("claude"),
2759                request,
2760                AuxiliaryOperation::AnthropicCountTokens,
2761            )
2762            .await?;
2763        let received = server.received_requests().await.unwrap_or_default();
2764        let body: Value = serde_json::from_slice(&received[0].body)?;
2765        assert!(body.get("output_config").is_none(), "{body}");
2766        Ok(())
2767    }
2768
2769    #[tokio::test]
2770    async fn reasoning_effort_override_sets_anthropic_output_config_effort()
2771    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2772        let server = MockServer::start().await;
2773        let format = json!({"type": "json_schema", "schema": {"type": "object"}});
2774        Mock::given(method("POST"))
2775            .and(path("/v1/messages"))
2776            .and(wiremock::matchers::body_partial_json(json!({
2777                "model": "claude",
2778                "output_config": {"effort": "max", "format": format}
2779            })))
2780            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2781                "id": "msg_1",
2782                "type": "message",
2783                "role": "assistant",
2784                "model": "claude",
2785                "content": [{"type": "text", "text": "ok"}],
2786                "stop_reason": "end_turn",
2787                "usage": {"input_tokens": 1, "output_tokens": 1}
2788            })))
2789            .expect(1)
2790            .mount(&server)
2791            .await;
2792        let mut backend = config(&server.uri());
2793        backend.reasoning_effort = Some("max".to_string());
2794        let client = TranslatingLlmClient::new(&[ModelConfig::new(
2795            "claude",
2796            Backend::Anthropic(backend),
2797            None,
2798        )])?;
2799        client
2800            .call_rewrite_model(
2801                wire_request(
2802                    json!({
2803                        "model": "client-facing",
2804                        "max_tokens": 7,
2805                        "messages": [{"role": "user", "content": "hi"}],
2806                        "output_config": {"effort": "high", "format": format}
2807                    }),
2808                    None,
2809                    WireFormat::AnthropicMessages,
2810                )?,
2811                Some(&ModelId::from("claude")),
2812            )
2813            .await?;
2814        Ok(())
2815    }
2816
2817    #[tokio::test]
2818    async fn extra_body_adds_defaults_without_overriding_the_request()
2819    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2820        let server = MockServer::start().await;
2821        Mock::given(method("POST"))
2822            .and(path("/v1/chat/completions"))
2823            .and(wiremock::matchers::body_partial_json(json!({
2824                "model": "gpt",
2825                "max_tokens": 7,
2826                "service_tier": "priority"
2827            })))
2828            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2829                "id": "1",
2830                "model": "gpt",
2831                "choices": [{
2832                    "index": 0,
2833                    "message": {"role": "assistant", "content": "ok"},
2834                    "finish_reason": "stop"
2835                }],
2836                "usage": {}
2837            })))
2838            .mount(&server)
2839            .await;
2840
2841        let extra_body = BTreeMap::from([
2842            ("max_tokens".to_string(), json!(999)),
2843            ("service_tier".to_string(), json!("priority")),
2844        ]);
2845        let client = TranslatingLlmClient::new(&chat_map_with_extra_body(
2846            &format!("{}/v1", server.uri()),
2847            extra_body,
2848        ))?;
2849        let raw = json!({
2850            "model": "client-facing",
2851            "messages": [{"role": "user", "content": "hi"}],
2852            "max_tokens": 7
2853        });
2854
2855        client
2856            .call_rewrite_model(
2857                wire_request(raw, None, WireFormat::OpenAiChat)?,
2858                Some(&ModelId::from("gpt")),
2859            )
2860            .await?;
2861        Ok(())
2862    }
2863
2864    #[tokio::test]
2865    async fn omit_body_fields_strips_a_field_extra_body_cannot_override()
2866    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2867        let server = MockServer::start().await;
2868        Mock::given(method("POST"))
2869            .and(path("/v1/chat/completions"))
2870            .and(|request: &wiremock::Request| {
2871                let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
2872                body.get("max_tokens").is_none()
2873            })
2874            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2875                "id": "1",
2876                "model": "gpt",
2877                "choices": [{
2878                    "index": 0,
2879                    "message": {"role": "assistant", "content": "ok"},
2880                    "finish_reason": "stop"
2881                }],
2882                "usage": {}
2883            })))
2884            .mount(&server)
2885            .await;
2886
2887        let client = TranslatingLlmClient::new(&chat_map_with_omit_body_fields(
2888            &format!("{}/v1", server.uri()),
2889            BTreeSet::from(["max_tokens".to_string()]),
2890        ))?;
2891        let raw = json!({
2892            "model": "client-facing",
2893            "messages": [{"role": "user", "content": "hi"}],
2894            "max_tokens": 7
2895        });
2896
2897        client
2898            .call_rewrite_model(
2899                wire_request(raw, None, WireFormat::OpenAiChat)?,
2900                Some(&ModelId::from("gpt")),
2901            )
2902            .await?;
2903        Ok(())
2904    }
2905
2906    // A weak OpenAI-format tier emits thinking blocks with no signature. Replaying
2907    // them to Anthropic is rejected (Bedrock reports it as a SigV4 mismatch), so
2908    // the Anthropic leg must drop them while keeping signed ones.
2909    #[tokio::test]
2910    async fn anthropic_requests_drop_unsigned_thinking_blocks()
2911    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2912        let server = MockServer::start().await;
2913        Mock::given(method("POST"))
2914            .and(path("/v1/messages"))
2915            .and(|request: &wiremock::Request| {
2916                let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
2917                let messages = body.get("messages").and_then(Value::as_array).cloned();
2918                let Some(messages) = messages else {
2919                    return false;
2920                };
2921                // The unsigned block is gone, the signed one survives, and the
2922                // message whose only block was unsigned is not left with an empty
2923                // content array.
2924                let blocks: Vec<&Value> = messages
2925                    .iter()
2926                    .filter_map(|message| message.get("content"))
2927                    .filter_map(Value::as_array)
2928                    .flatten()
2929                    .collect();
2930                let thinking: Vec<&&Value> = blocks
2931                    .iter()
2932                    .filter(|block| block.get("type").and_then(Value::as_str) == Some("thinking"))
2933                    .collect();
2934                thinking.len() == 1
2935                    && thinking[0].get("signature").and_then(Value::as_str) == Some("sig-abc")
2936                    && messages
2937                        .iter()
2938                        .all(|message| message.get("content") != Some(&json!([])))
2939            })
2940            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2941                "id": "msg_1",
2942                "type": "message",
2943                "role": "assistant",
2944                "model": "claude",
2945                "content": [{"type": "text", "text": "ok"}],
2946                "stop_reason": "end_turn",
2947                "usage": {"input_tokens": 1, "output_tokens": 1}
2948            })))
2949            .mount(&server)
2950            .await;
2951
2952        let client = TranslatingLlmClient::new(&anthropic_map(&server.uri()))?;
2953        let raw = json!({
2954            "model": "client-facing",
2955            "max_tokens": 7,
2956            "messages": [
2957                {"role": "user", "content": "fix the build"},
2958                {"role": "assistant", "content": [
2959                    {"type": "thinking", "thinking": "weak tier reasoning", "signature": ""}
2960                ]},
2961                {"role": "assistant", "content": [
2962                    {"type": "thinking", "thinking": "signed reasoning", "signature": "sig-abc"},
2963                    {"type": "text", "text": "here goes"}
2964                ]},
2965                {"role": "user", "content": "continue"}
2966            ]
2967        });
2968
2969        client
2970            .call_rewrite_model(
2971                wire_request(raw, None, WireFormat::AnthropicMessages)?,
2972                Some(&ModelId::from("claude")),
2973            )
2974            .await?;
2975        Ok(())
2976    }
2977
2978    #[tokio::test]
2979    async fn responses_reasoning_policy_preserves_replay_history()
2980    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2981        use crate::ResponsesReasoningPolicy::{Auto, Drop, PreserveEncrypted, PreservePlaintext};
2982        let summary = json!([{"type": "summary_text", "text": "Check the parser."}]);
2983        let history = json!([
2984            {"type": "message", "role": "user", "content": "continue"},
2985            {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]},
2986            {"type": "reasoning", "id": "rs_stored", "status": "completed", "summary": []},
2987            // GLM's plaintext forms, as Codex replays them.
2988            {"type": "reasoning", "id": "rs_glm", "summary": summary, "encrypted_content": null},
2989            {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "Run the tests."}], "encrypted_content": ""},
2990            {"type": "reasoning"},
2991            {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"},
2992            {"type": "function_call_output", "call_id": "call_1", "output": "ok"}
2993        ]);
2994        // An explicit policy wins over the target. `Auto` follows the target.
2995        let cases = [
2996            (Auto, "openai/gpt-5", PreserveEncrypted),
2997            (Auto, "nvidia/zai-org/glm-5.3-flash", PreservePlaintext),
2998            (
2999                PreserveEncrypted,
3000                "nvidia/zai-org/glm-5.3-flash",
3001                PreserveEncrypted,
3002            ),
3003            (Drop, "openai/gpt-5", Drop),
3004        ];
3005        for store in [Some(true), Some(false), None] {
3006            for (policy, model, behavior) in cases {
3007                for has_input_override in [false, true] {
3008                    let server = MockServer::start().await;
3009                    Mock::given(method("POST"))
3010                        .and(path("/v1/responses"))
3011                        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3012                            "id": "resp_1", "object": "response", "model": "gpt",
3013                            "status": "completed", "output": [],
3014                            "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
3015                        })))
3016                        .expect(1)
3017                        .mount(&server)
3018                        .await;
3019                    let mut backend = config(&format!("{}/v1", server.uri()));
3020                    if has_input_override {
3021                        backend.omit_body_fields.insert("input".to_string());
3022                        backend
3023                            .extra_body
3024                            .insert("input".to_string(), history.clone());
3025                    }
3026                    let client = TranslatingLlmClient::new(&[ModelConfig::new(
3027                        model,
3028                        Backend::OpenAiResponses(backend),
3029                        None,
3030                    )
3031                    .with_responses_reasoning(policy)])?;
3032                    let mut request = json!({"model": "switchyard", "input": history});
3033                    if let Some(store) = store {
3034                        request["store"] = json!(store);
3035                    }
3036                    client
3037                        .call_rewrite_model(
3038                            wire_request(request, None, WireFormat::OpenAiResponses)?,
3039                            Some(&ModelId::from(model)),
3040                        )
3041                        .await?;
3042                    let requests = server.received_requests().await.ok_or("missing requests")?;
3043                    let body: Value = serde_json::from_slice(&requests[0].body)?;
3044                    let stored = store != Some(false);
3045                    let mut expected = vec![history[0].clone()];
3046                    match behavior {
3047                        // OpenAI reads its own payload and stored IDs, but rejects plaintext.
3048                        PreserveEncrypted => {
3049                            let indexes: &[usize] = if stored { &[1, 2, 3] } else { &[1] };
3050                            for &index in indexes {
3051                                let mut item = history[index].clone();
3052                                item["content"] = json!([]);
3053                                expected.push(item);
3054                            }
3055                        }
3056                        // Other targets keep plaintext and lose payloads of unknown origin.
3057                        PreservePlaintext => {
3058                            let mut item = history[1].clone();
3059                            item.as_object_mut()
3060                                .ok_or("object")?
3061                                .remove("encrypted_content");
3062                            expected.push(item);
3063                            if stored {
3064                                expected.push(history[2].clone());
3065                            }
3066                            expected.extend([history[3].clone(), history[4].clone()]);
3067                        }
3068                        _ => {}
3069                    }
3070                    expected.extend([history[6].clone(), history[7].clone()]);
3071                    assert_eq!(
3072                        body["input"],
3073                        json!(expected),
3074                        "store={store:?}, policy={policy:?}, model={model}, replace={has_input_override}"
3075                    );
3076                }
3077            }
3078        }
3079
3080        Ok(())
3081    }
3082
3083    #[tokio::test]
3084    async fn nvidia_anthropic_requests_strip_strict_from_extra_body()
3085    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3086        let server = MockServer::start().await;
3087        Mock::given(method("POST"))
3088            .and(path("/v1/messages"))
3089            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3090                "id": "msg_1",
3091                "type": "message",
3092                "role": "assistant",
3093                "model": "claude",
3094                "content": [{"type": "text", "text": "ok"}],
3095                "stop_reason": "end_turn",
3096                "usage": {"input_tokens": 1, "output_tokens": 1}
3097            })))
3098            .expect(1)
3099            .mount(&server)
3100            .await;
3101
3102        let mut backend = config(&format!(
3103            "http://inference-api.nvidia.com:{}",
3104            server.address().port()
3105        ));
3106        backend.extra_body.insert(
3107            "tools".into(),
3108            json!([{
3109                "name": "lookup",
3110                "input_schema": {"type": "object", "properties": {}},
3111                "strict": true
3112            }]),
3113        );
3114        let mut client = TranslatingLlmClient::new(&[ModelConfig::new(
3115            "claude",
3116            Backend::Anthropic(backend),
3117            None,
3118        )])?;
3119        client.client = reqwest::Client::builder()
3120            .no_proxy()
3121            .resolve("inference-api.nvidia.com", *server.address())
3122            .build()?;
3123        client
3124            .call_rewrite_model(
3125                wire_request(
3126                    json!({
3127                        "model": "claude",
3128                        "max_tokens": 7,
3129                        "messages": [{"role": "user", "content": "hi"}]
3130                    }),
3131                    None,
3132                    WireFormat::AnthropicMessages,
3133                )?,
3134                Some(&ModelId::from("claude")),
3135            )
3136            .await?;
3137
3138        let requests = server.received_requests().await.unwrap();
3139        let body: Value = serde_json::from_slice(&requests[0].body)?;
3140        assert_eq!(body["tools"][0]["name"], "lookup");
3141        assert!(body["tools"][0].get("strict").is_none());
3142        Ok(())
3143    }
3144
3145    // A router can serve earlier turns from an OpenAI target and later turns from
3146    // an Anthropic one, so the Anthropic leg must drop OpenAI-only fields the
3147    // caller keeps sending or the upstream rejects the whole request.
3148    #[tokio::test]
3149    async fn anthropic_requests_drop_openai_only_fields()
3150    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3151        let server = MockServer::start().await;
3152        Mock::given(method("POST"))
3153            .and(path("/v1/messages"))
3154            .and(|request: &wiremock::Request| {
3155                let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
3156                body.get("context_management").is_none() && body.get("reasoning_effort").is_none()
3157            })
3158            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3159                "id": "msg_1",
3160                "type": "message",
3161                "role": "assistant",
3162                "model": "claude",
3163                "content": [{"type": "text", "text": "ok"}],
3164                "stop_reason": "end_turn",
3165                "usage": {"input_tokens": 1, "output_tokens": 1}
3166            })))
3167            .mount(&server)
3168            .await;
3169
3170        let client = TranslatingLlmClient::new(&anthropic_map(&server.uri()))?;
3171        let raw = json!({
3172            "model": "client-facing",
3173            "messages": [{"role": "user", "content": "hi"}],
3174            "max_tokens": 7,
3175            "reasoning_effort": "high",
3176            "context_management": {
3177                "edits": [{"type": "clear_thinking_20251015"}]
3178            }
3179        });
3180
3181        client
3182            .call_rewrite_model(
3183                wire_request(raw, None, WireFormat::AnthropicMessages)?,
3184                Some(&ModelId::from("claude")),
3185            )
3186            .await?;
3187        Ok(())
3188    }
3189
3190    #[tokio::test]
3191    async fn streaming_openai_chat_aggregates()
3192    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3193        let server = MockServer::start().await;
3194        let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
3195             data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
3196             data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n\
3197             data: [DONE]\n\n";
3198        Mock::given(method("POST"))
3199            .and(path("/v1/chat/completions"))
3200            .and(wiremock::matchers::body_partial_json(json!({
3201                "stream": true,
3202                "stream_options": {"include_usage": true}
3203            })))
3204            .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
3205            .mount(&server)
3206            .await;
3207
3208        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3209
3210        let response = client
3211            .call_rewrite_model(request_for(Some("gpt"), true), None)
3212            .await?;
3213        assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
3214        let agg = response.llm_response.into_agg().await?;
3215        assert_eq!(completion_text(&agg), "Hello world");
3216        assert_eq!(agg.usage.input_tokens, Some(1));
3217        assert_eq!(agg.usage.output_tokens, Some(2));
3218        assert_eq!(agg.usage.total_tokens, Some(3));
3219        Ok(())
3220    }
3221
3222    #[tokio::test]
3223    async fn streaming_openai_chat_preserves_usage_opt_out()
3224    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3225        let server = MockServer::start().await;
3226        Mock::given(method("POST"))
3227            .and(path("/v1/chat/completions"))
3228            .and(wiremock::matchers::body_partial_json(json!({
3229                "stream": true,
3230                "stream_options": {"include_usage": false}
3231            })))
3232            .respond_with(
3233                ResponseTemplate::new(200).set_body_raw("data: [DONE]\n\n", "text/event-stream"),
3234            )
3235            .mount(&server)
3236            .await;
3237
3238        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3239        let raw = json!({
3240            "model": "client-facing",
3241            "messages": [{"role": "user", "content": "hi"}],
3242            "stream": true,
3243            "stream_options": {"include_usage": false}
3244        });
3245
3246        let response = client
3247            .call_rewrite_model(
3248                wire_request(raw, None, WireFormat::OpenAiChat)?,
3249                Some(&ModelId::from("gpt")),
3250            )
3251            .await?;
3252        assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
3253        Ok(())
3254    }
3255
3256    #[tokio::test]
3257    async fn upstream_500_is_upstream_http()
3258    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3259        let server = MockServer::start().await;
3260        Mock::given(method("POST"))
3261            .respond_with(ResponseTemplate::new(500).set_body_string("boom"))
3262            .mount(&server)
3263            .await;
3264
3265        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3266
3267        let Err(error) = client
3268            .call_rewrite_model(request_for(Some("gpt"), false), None)
3269            .await
3270        else {
3271            panic!("expected an error");
3272        };
3273        assert!(matches!(
3274            error,
3275            LlmClientError::UpstreamHttp {
3276                status: StatusCode::INTERNAL_SERVER_ERROR,
3277                ..
3278            }
3279        ));
3280        Ok(())
3281    }
3282
3283    #[tokio::test]
3284    async fn retryable_http_failure_recovers_within_budget()
3285    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3286        let server = MockServer::start().await;
3287        let calls = Arc::new(AtomicUsize::new(0));
3288        let observed_calls = Arc::clone(&calls);
3289        Mock::given(method("POST"))
3290            .respond_with(move |_: &wiremock::Request| {
3291                if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {
3292                    ResponseTemplate::new(503)
3293                        .insert_header("retry-after", "0")
3294                        .set_body_string("temporarily unavailable")
3295                } else {
3296                    chat_success_response()
3297                }
3298            })
3299            .mount(&server)
3300            .await;
3301
3302        let client =
3303            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
3304        let response = client
3305            .call_rewrite_model(request_for(Some("gpt"), false), None)
3306            .await?;
3307        let agg = response.llm_response.into_agg().await?;
3308
3309        assert_eq!(completion_text(&agg), "recovered");
3310        assert_eq!(calls.load(Ordering::SeqCst), 2);
3311        Ok(())
3312    }
3313
3314    #[tokio::test]
3315    async fn deterministic_http_failure_is_not_retried()
3316    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3317        let server = MockServer::start().await;
3318        let calls = Arc::new(AtomicUsize::new(0));
3319        let observed_calls = Arc::clone(&calls);
3320        Mock::given(method("POST"))
3321            .respond_with(move |_: &wiremock::Request| {
3322                observed_calls.fetch_add(1, Ordering::SeqCst);
3323                ResponseTemplate::new(401).set_body_string("invalid key")
3324            })
3325            .mount(&server)
3326            .await;
3327
3328        let client =
3329            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
3330        let Err(error) = client
3331            .call_rewrite_model(request_for(Some("gpt"), false), None)
3332            .await
3333        else {
3334            panic!("expected an upstream error");
3335        };
3336
3337        assert!(matches!(
3338            error,
3339            LlmClientError::UpstreamHttp {
3340                status: StatusCode::UNAUTHORIZED,
3341                body,
3342                ..
3343            } if body == "invalid key"
3344        ));
3345        assert_eq!(calls.load(Ordering::SeqCst), 1);
3346        Ok(())
3347    }
3348
3349    #[tokio::test]
3350    async fn rate_limit_cooldown_depends_on_shared_credentials()
3351    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3352        for forward_auth in [true, false] {
3353            let server = MockServer::start().await;
3354            Mock::given(method("POST"))
3355                .respond_with(ResponseTemplate::new(429).insert_header("retry-after", "0"))
3356                .expect(if forward_auth { 6 } else { 3 })
3357                .mount(&server)
3358                .await;
3359            let backend = HttpBackendConfig {
3360                forward_auth,
3361                failure_cooldown: Duration::from_secs(60),
3362                ..config_with_retries(&format!("{}/v1", server.uri()), 2)
3363            };
3364            let client = TranslatingLlmClient::new(&[ModelConfig::new(
3365                "gpt",
3366                Backend::OpenAiChat(backend),
3367                None,
3368            )])?;
3369            for credential in ["Bearer first-user", "Bearer second-user"] {
3370                let mut headers = http::HeaderMap::new();
3371                headers.insert("authorization", http::HeaderValue::from_static(credential));
3372                let result = client
3373                    .call_rewrite_model(request_with_headers("gpt", headers), None)
3374                    .await;
3375                if !forward_auth && credential == "Bearer second-user" {
3376                    assert!(matches!(
3377                        result,
3378                        Err(LlmClientError::TemporarilyUnavailable)
3379                    ));
3380                } else {
3381                    assert!(matches!(
3382                        result,
3383                        Err(LlmClientError::UpstreamHttp {
3384                            status: StatusCode::TOO_MANY_REQUESTS,
3385                            ..
3386                        })
3387                    ));
3388                }
3389            }
3390        }
3391        Ok(())
3392    }
3393
3394    #[tokio::test]
3395    async fn retry_exhaustion_returns_the_final_upstream_error()
3396    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3397        let server = MockServer::start().await;
3398        let calls = Arc::new(AtomicUsize::new(0));
3399        let observed_calls = Arc::clone(&calls);
3400        Mock::given(method("POST"))
3401            .respond_with(move |_: &wiremock::Request| {
3402                let attempt = observed_calls.fetch_add(1, Ordering::SeqCst) + 1;
3403                ResponseTemplate::new(500)
3404                    .insert_header("retry-after", "0")
3405                    .set_body_string(format!("attempt {attempt}"))
3406            })
3407            .mount(&server)
3408            .await;
3409
3410        let client =
3411            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
3412        let Err(error) = client
3413            .call_rewrite_model(request_for(Some("gpt"), false), None)
3414            .await
3415        else {
3416            panic!("expected retry exhaustion");
3417        };
3418
3419        assert!(matches!(
3420            error,
3421            LlmClientError::UpstreamHttp {
3422                status: StatusCode::INTERNAL_SERVER_ERROR,
3423                body,
3424                ..
3425            } if body == "attempt 3"
3426        ));
3427        assert_eq!(calls.load(Ordering::SeqCst), 3);
3428        Ok(())
3429    }
3430
3431    // The error keeps every header of the last attempt; consumers choose what to forward.
3432    #[tokio::test]
3433    async fn upstream_error_carries_headers_from_the_last_attempt()
3434    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3435        let server = MockServer::start().await;
3436        // A zero `retry-after` keeps the retry from sleeping.
3437        let reply = |id: &str| {
3438            ResponseTemplate::new(429)
3439                .insert_header("retry-after", "0")
3440                .insert_header("x-request-id", id)
3441                .insert_header("set-cookie", "a=b")
3442        };
3443        Mock::given(method("POST"))
3444            .respond_with(reply("first"))
3445            .up_to_n_times(1)
3446            .mount(&server)
3447            .await;
3448        Mock::given(method("POST"))
3449            .respond_with(reply("second"))
3450            .mount(&server)
3451            .await;
3452        let client =
3453            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
3454        let Err(LlmClientError::UpstreamHttp { headers, .. }) = client
3455            .call_rewrite_model(request_for(Some("gpt"), false), None)
3456            .await
3457        else {
3458            panic!("expected an upstream HTTP error");
3459        };
3460        assert_eq!(headers["x-request-id"], "second");
3461        assert_eq!(headers["retry-after"], "0");
3462        assert_eq!(headers["set-cookie"], "a=b");
3463        Ok(())
3464    }
3465
3466    #[tokio::test]
3467    async fn deadline_expires_before_send_or_stream_poll()
3468    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3469        let server = MockServer::start().await;
3470        Mock::given(method("POST"))
3471            .respond_with(ResponseTemplate::new(200).set_body_raw(
3472                "data: {\"id\":\"test\",\"model\":\"gpt\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
3473                "text/event-stream",
3474            ))
3475            .mount(&server)
3476            .await;
3477        for timeout in [Duration::ZERO, Duration::from_millis(200)] {
3478            let mut backend = config(&server.uri());
3479            backend.timeout = Some(timeout);
3480            let client = TranslatingLlmClient::new(&[ModelConfig::new(
3481                "gpt",
3482                Backend::OpenAiChat(backend),
3483                None,
3484            )])?;
3485            let result = client
3486                .call_rewrite_model(request_for(Some("gpt"), true), None)
3487                .await;
3488            if timeout.is_zero() {
3489                assert!(matches!(result, Err(LlmClientError::Timeout { .. })));
3490                assert!(
3491                    server
3492                        .received_requests()
3493                        .await
3494                        .expect("recorded requests")
3495                        .is_empty()
3496                );
3497            } else {
3498                let LlmResponse::Stream(mut chunks) = result?.llm_response else {
3499                    return Err("expected a streaming response".into());
3500                };
3501                tokio::time::sleep(Duration::from_millis(300)).await;
3502                assert!(matches!(
3503                    chunks.next().await,
3504                    Some(Err(LlmClientError::Timeout { .. }))
3505                ));
3506                assert!(chunks.next().await.is_none());
3507            }
3508        }
3509        Ok(())
3510    }
3511
3512    #[tokio::test]
3513    async fn timeout_is_retried_before_a_response_is_returned()
3514    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3515        let server = MockServer::start().await;
3516        let calls = Arc::new(AtomicUsize::new(0));
3517        let observed_calls = Arc::clone(&calls);
3518        Mock::given(method("POST"))
3519            .respond_with(move |_: &wiremock::Request| {
3520                if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {
3521                    ResponseTemplate::new(200).set_delay(Duration::from_millis(500))
3522                } else {
3523                    chat_success_response()
3524                }
3525            })
3526            .mount(&server)
3527            .await;
3528
3529        let mut client =
3530            TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
3531        client.client = reqwest::Client::builder()
3532            .timeout(Duration::from_millis(100))
3533            .build()?;
3534        let response = client
3535            .call_rewrite_model(request_for(Some("gpt"), false), None)
3536            .await?;
3537
3538        assert_eq!(
3539            completion_text(&response.llm_response.into_agg().await?),
3540            "recovered"
3541        );
3542        assert_eq!(calls.load(Ordering::SeqCst), 2);
3543        Ok(())
3544    }
3545
3546    #[test]
3547    fn retryable_error_classes_are_explicit() {
3548        let transport = AttemptFailure {
3549            error: LlmClientError::Transport {
3550                source: std::io::Error::other("disconnected").into(),
3551            },
3552            status: None,
3553            retry_after: None,
3554        };
3555        assert!(transport.is_retryable());
3556
3557        for status in [
3558            StatusCode::REQUEST_TIMEOUT,
3559            StatusCode::TOO_MANY_REQUESTS,
3560            StatusCode::INTERNAL_SERVER_ERROR,
3561            StatusCode::SERVICE_UNAVAILABLE,
3562            StatusCode::from_u16(599)
3563                .expect("599 is a syntactically valid HTTP code in the http crate"),
3564        ] {
3565            let failure = AttemptFailure {
3566                error: LlmClientError::UpstreamHttp {
3567                    status,
3568                    body: String::new(),
3569                    headers: Box::default(),
3570                },
3571                status: Some(status),
3572                retry_after: None,
3573            };
3574            assert!(failure.is_retryable(), "HTTP {status} should retry");
3575        }
3576        for status in [
3577            StatusCode::BAD_REQUEST,
3578            StatusCode::UNAUTHORIZED,
3579            StatusCode::CONFLICT,
3580            StatusCode::from_u16(600)
3581                .expect("600 is a syntactically valid HTTP code in the http crate"),
3582        ] {
3583            let failure = AttemptFailure {
3584                error: LlmClientError::UpstreamHttp {
3585                    status,
3586                    body: String::new(),
3587                    headers: Box::default(),
3588                },
3589                status: Some(status),
3590                retry_after: None,
3591            };
3592            assert!(!failure.is_retryable(), "HTTP {status} should fail fast");
3593        }
3594
3595        let configuration = AttemptFailure {
3596            error: LlmClientError::Configuration {
3597                message: "invalid header".to_string(),
3598            },
3599            status: None,
3600            retry_after: None,
3601        };
3602        assert!(!configuration.is_retryable());
3603
3604        let context_window = AttemptFailure {
3605            error: LlmClientError::ContextWindowExceeded {
3606                model: ModelId::from("gpt"),
3607                message: "too long".to_string(),
3608            },
3609            status: Some(StatusCode::BAD_REQUEST),
3610            retry_after: None,
3611        };
3612        assert!(!context_window.is_retryable());
3613    }
3614
3615    #[test]
3616    fn retry_after_supports_seconds_and_http_dates() {
3617        let mut headers = HeaderMap::new();
3618        headers.insert(RETRY_AFTER, reqwest::header::HeaderValue::from_static("3"));
3619        assert_eq!(retry_after_delay(&headers), Some(Duration::from_secs(3)));
3620
3621        let retry_at = SystemTime::now() + Duration::from_secs(2);
3622        let value = httpdate::fmt_http_date(retry_at);
3623        let Ok(value) = reqwest::header::HeaderValue::from_str(&value) else {
3624            panic!("formatted HTTP date should be a valid header");
3625        };
3626        headers.insert(RETRY_AFTER, value);
3627        let Some(delay) = retry_after_delay(&headers) else {
3628            panic!("HTTP date should produce a retry delay");
3629        };
3630        assert!(delay <= Duration::from_secs(2));
3631    }
3632
3633    #[tokio::test]
3634    async fn routed_llm_client_exposes_timeout_variant()
3635    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3636        let server = MockServer::start().await;
3637        Mock::given(method("POST"))
3638            .respond_with(
3639                ResponseTemplate::new(200).set_delay(std::time::Duration::from_millis(100)),
3640            )
3641            .mount(&server)
3642            .await;
3643
3644        let mut client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3645        client.client = reqwest::Client::builder()
3646            .timeout(std::time::Duration::from_millis(10))
3647            .build()?;
3648
3649        let Err(error) = client.call(request_for(Some("gpt"), false)).await else {
3650            panic!("expected a timeout");
3651        };
3652        let LlmClientError::Timeout { source } = error else {
3653            panic!("expected the protocol timeout variant");
3654        };
3655        let Some(source) = source.downcast_ref::<reqwest::Error>() else {
3656            panic!("expected the reqwest timeout source");
3657        };
3658        assert!(source.is_timeout());
3659        Ok(())
3660    }
3661
3662    // A connect that never completes sends nothing, like a refused connection.
3663    #[cfg(target_os = "linux")]
3664    #[tokio::test]
3665    async fn connect_timeout_is_a_transport_error()
3666    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3667        // Linux drops SYNs once a listener's accept queue is full, so later
3668        // connects hang like an upstream behind a dropping firewall.
3669        let socket = tokio::net::TcpSocket::new_v4()?;
3670        socket.bind("127.0.0.1:0".parse()?)?;
3671        let listener = socket.listen(0)?;
3672        let addr = listener.local_addr()?;
3673        let _queued: Vec<_> = (0..2)
3674            .map(|_| {
3675                std::net::TcpStream::connect_timeout(&addr, std::time::Duration::from_millis(100))
3676            })
3677            .collect();
3678
3679        let mut client = TranslatingLlmClient::new(&chat_map(&format!("http://{addr}/v1")))?;
3680        // Stands in for the OS connect timeout (30 s on Linux via reqwest's default
3681        // TCP_USER_TIMEOUT); both surface as `is_connect() && is_timeout()`.
3682        client.client = reqwest::Client::builder()
3683            .connect_timeout(std::time::Duration::from_millis(100))
3684            .build()?;
3685
3686        let Err(error) = client.call(request_for(Some("gpt"), false)).await else {
3687            panic!("expected the connect to fail");
3688        };
3689        let LlmClientError::Transport { source } = error else {
3690            panic!("expected Transport, got {error:?}");
3691        };
3692        let Some(source) = source.downcast_ref::<reqwest::Error>() else {
3693            panic!("expected the reqwest connect-timeout source");
3694        };
3695        assert!(source.is_connect());
3696        assert!(source.is_timeout());
3697        Ok(())
3698    }
3699
3700    #[tokio::test]
3701    async fn llama_cpp_context_overflow_400_is_mapped()
3702    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3703        let server = MockServer::start().await;
3704        Mock::given(method("POST"))
3705            .respond_with(ResponseTemplate::new(400).set_body_json(json!({
3706                "error": {
3707                    "message": "request (6016 tokens) exceeds the available context size (4096 tokens), try increasing it"
3708                }
3709            })))
3710            .mount(&server)
3711            .await;
3712
3713        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3714
3715        let Err(error) = client
3716            .call_rewrite_model(request_for(Some("gpt"), false), None)
3717            .await
3718        else {
3719            panic!("expected an error");
3720        };
3721        assert!(matches!(
3722            error,
3723            LlmClientError::ContextWindowExceeded { model, .. } if model == "gpt"
3724        ));
3725        Ok(())
3726    }
3727
3728    #[tokio::test]
3729    async fn content_policy_400_preserves_upstream_error()
3730    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3731        let server = MockServer::start().await;
3732        let body = json!({
3733            "error": {
3734                "code": "content_policy_violation",
3735                "message": "request blocked by content policy",
3736                "type": "invalid_request_error"
3737            },
3738            "metadata": {"documentation_section": "context window"}
3739        });
3740        Mock::given(method("POST"))
3741            .respond_with(ResponseTemplate::new(400).set_body_json(&body))
3742            .expect(1)
3743            .mount(&server)
3744            .await;
3745        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3746        let Err(error) = client
3747            .call_rewrite_model(request_for(Some("gpt"), false), None)
3748            .await
3749        else {
3750            panic!("expected a policy denial");
3751        };
3752        let LlmClientError::UpstreamHttp {
3753            status,
3754            body: actual,
3755            ..
3756        } = error
3757        else {
3758            panic!("expected the upstream HTTP error, got {error:?}");
3759        };
3760        assert_eq!(status, StatusCode::BAD_REQUEST);
3761        assert_eq!(serde_json::from_str::<serde_json::Value>(&actual)?, body);
3762        Ok(())
3763    }
3764
3765    #[tokio::test]
3766    async fn forwards_only_allowlisted_metadata_headers()
3767    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3768        let server = MockServer::start().await;
3769        Mock::given(method("POST"))
3770            .and(wiremock::matchers::header("x-request-id", "abc"))
3771            // A forwarded Authorization must NOT override the backend's bearer key.
3772            .and(wiremock::matchers::header("authorization", "Bearer secret"))
3773            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3774                "id": "1", "model": "gpt",
3775                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
3776                "usage": {}
3777            })))
3778            .mount(&server)
3779            .await;
3780
3781        let mut headers = http::HeaderMap::new();
3782        headers.insert("x-request-id", http::HeaderValue::from_static("abc"));
3783        headers.insert(
3784            "authorization",
3785            http::HeaderValue::from_static("Bearer client-key"),
3786        );
3787        headers.insert(
3788            "x-goog-api-key",
3789            http::HeaderValue::from_static("client-google-key"),
3790        );
3791        headers.insert(
3792            "x-custom-internal-secret",
3793            http::HeaderValue::from_static("client-custom-key"),
3794        );
3795        let request = request_with_headers("gpt", headers);
3796
3797        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3798
3799        // The allowed request id survives and caller authorization cannot replace
3800        // the backend's credential.
3801        client.call_rewrite_model(request, None).await?;
3802        let received = server
3803            .received_requests()
3804            .await
3805            .ok_or("request recording should be enabled")?;
3806        let received = received.first().ok_or("expected one upstream request")?;
3807        assert!(!received.headers.contains_key("x-goog-api-key"));
3808        assert!(!received.headers.contains_key("x-custom-internal-secret"));
3809        Ok(())
3810    }
3811
3812    #[tokio::test]
3813    async fn forwards_application_headers_with_forward_auth()
3814    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3815        let server = MockServer::start().await;
3816        Mock::given(method("POST"))
3817            .and(wiremock::matchers::header(
3818                "authorization",
3819                "Bearer client-key",
3820            ))
3821            .and(wiremock::matchers::header(
3822                "x-goog-api-key",
3823                "client-google-key",
3824            ))
3825            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3826                "id": "1", "model": "gpt",
3827                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
3828                "usage": {}
3829            })))
3830            .mount(&server)
3831            .await;
3832
3833        let mut headers = http::HeaderMap::new();
3834        headers.insert(
3835            "authorization",
3836            http::HeaderValue::from_static("Bearer client-key"),
3837        );
3838        headers.insert(
3839            "x-goog-api-key",
3840            http::HeaderValue::from_static("client-google-key"),
3841        );
3842        headers.append("x-custom", http::HeaderValue::from_static("caller-first"));
3843        headers.append("x-custom", http::HeaderValue::from_static("caller-second"));
3844        headers.insert("host", http::HeaderValue::from_static("client.example"));
3845        headers.insert("content-type", http::HeaderValue::from_static("text/plain"));
3846        headers.insert("connection", http::HeaderValue::from_static("x-hop-by-hop"));
3847        headers.insert(
3848            "x-hop-by-hop",
3849            http::HeaderValue::from_static("client-only"),
3850        );
3851        let request = request_with_headers("gpt", headers);
3852        let mut backend_config = forwarding_config(&format!("{}/v1", server.uri()));
3853        backend_config.extra_headers = BTreeMap::from([
3854            ("X-Custom".into(), "configured".into()),
3855            ("X-Configured-Only".into(), "static".into()),
3856        ]);
3857        let client = TranslatingLlmClient::new(&[ModelConfig::new(
3858            "gpt",
3859            Backend::OpenAiChat(backend_config),
3860            None,
3861        )])?;
3862
3863        client.call_rewrite_model(request, None).await?;
3864        let received = server
3865            .received_requests()
3866            .await
3867            .ok_or("request recording should be enabled")?;
3868        let received = received.first().ok_or("expected one upstream request")?;
3869        assert_ne!(received.headers["host"], "client.example");
3870        assert_eq!(received.headers["content-type"], "application/json");
3871        assert!(!received.headers.contains_key("connection"));
3872        assert!(!received.headers.contains_key("x-hop-by-hop"));
3873        assert_eq!(received.headers.get_all("x-custom").iter().count(), 1);
3874        assert_eq!(received.headers["x-custom"], "configured");
3875        assert_eq!(received.headers["x-configured-only"], "static");
3876        Ok(())
3877    }
3878
3879    #[tokio::test]
3880    async fn forward_auth_preserves_anthropic_headers_and_redacts_errors()
3881    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3882        let server = MockServer::start().await;
3883        Mock::given(method("POST"))
3884            .and(path("/v1/messages"))
3885            .and(|request: &wiremock::Request| {
3886                request
3887                    .headers
3888                    .get("anthropic-beta")
3889                    .and_then(|value| value.to_str().ok())
3890                    == Some("oauth-2025-04-20")
3891                    && request.headers.get_all("anthropic-version").iter().count() == 1
3892                    && request
3893                        .headers
3894                        .get("anthropic-version")
3895                        .and_then(|value| value.to_str().ok())
3896                        == Some("2023-06-01")
3897            })
3898            .respond_with(ResponseTemplate::new(401).set_body_json(json!({
3899                "error": {"message": "rejected client-google-key"}
3900            })))
3901            .mount(&server)
3902            .await;
3903
3904        let mut headers = http::HeaderMap::new();
3905        headers.insert(
3906            "authorization",
3907            http::HeaderValue::from_static("Bearer client-key"),
3908        );
3909        headers.insert(
3910            "x-goog-api-key",
3911            http::HeaderValue::from_static("client-google-key"),
3912        );
3913        headers.insert(
3914            "anthropic-beta",
3915            http::HeaderValue::from_static("oauth-2025-04-20,prompt-caching-2024-07-31"),
3916        );
3917        headers.insert(
3918            "anthropic-version",
3919            http::HeaderValue::from_static("caller-version"),
3920        );
3921        let backend_config = forwarding_config(&server.uri());
3922        let client = TranslatingLlmClient::new(&[ModelConfig::new(
3923            "claude",
3924            Backend::Anthropic(backend_config),
3925            None,
3926        )])?;
3927        let raw = json!({
3928            "model": "claude",
3929            "max_tokens": 8,
3930            "messages": [{"role": "user", "content": "hello"}]
3931        });
3932
3933        let Err(LlmClientError::UpstreamHttp { body, .. }) = client
3934            .call_rewrite_model(
3935                wire_request(raw, Some(headers), WireFormat::AnthropicMessages)?,
3936                Some(&ModelId::from("claude")),
3937            )
3938            .await
3939        else {
3940            panic!("expected an upstream HTTP error");
3941        };
3942        assert_eq!(body, r#"{"error":{"message":"rejected [REDACTED]"}}"#);
3943        Ok(())
3944    }
3945
3946    // Exercises the `RoutedLlmClient` impl: `call` uses the model already materialized in the
3947    // request and round-trips a buffered response.
3948    #[tokio::test]
3949    async fn routed_llm_client_serves_the_request_model()
3950    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3951        let server = MockServer::start().await;
3952        Mock::given(method("POST"))
3953            .and(path("/v1/chat/completions"))
3954            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3955                "id": "chatcmpl-1",
3956                "model": "gpt",
3957                "choices": [{
3958                    "index": 0,
3959                    "message": {"role": "assistant", "content": "routed hi"},
3960                    "finish_reason": "stop"
3961                }],
3962                "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
3963            })))
3964            .mount(&server)
3965            .await;
3966
3967        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3968        let response = client.call(request_for(Some("gpt"), false)).await?;
3969        let agg = response.llm_response.into_agg().await?;
3970        assert_eq!(completion_text(&agg), "routed hi");
3971        Ok(())
3972    }
3973
3974    // A Codex namespace is folded into the upstream tool name.
3975    #[tokio::test]
3976    async fn chat_backend_folds_codex_mcp_namespace_into_the_tool_name()
3977    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3978        let server = MockServer::start().await;
3979        Mock::given(method("POST"))
3980            .and(path("/v1/chat/completions"))
3981            .and(wiremock::matchers::body_partial_json(json!({
3982                "tools": [{
3983                    "type": "function",
3984                    "function": {"name": "mcp__open_websearch__search"}
3985                }]
3986            })))
3987            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3988                "id": "chatcmpl-1",
3989                "model": "gpt",
3990                "choices": [{
3991                    "index": 0,
3992                    "message": {
3993                        "role": "assistant",
3994                        "content": null,
3995                        "tool_calls": [{
3996                            "id": "call_1",
3997                            "type": "function",
3998                            "function": {
3999                                "name": "mcp__open_websearch__search",
4000                                "arguments": "{\"q\":\"rust\"}"
4001                            }
4002                        }]
4003                    },
4004                    "finish_reason": "tool_calls"
4005                }],
4006                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
4007            })))
4008            .mount(&server)
4009            .await;
4010
4011        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
4012        let raw = json!({
4013            "model": "client-facing",
4014            "input": "Search for Rust.",
4015            "tools": [{
4016                "type": "namespace",
4017                "name": "mcp__open_websearch",
4018                "tools": [{
4019                    "type": "function",
4020                    "name": "search",
4021                    "description": "Search the web",
4022                    "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}
4023                }]
4024            }]
4025        });
4026
4027        let response = client
4028            .call_rewrite_model(
4029                wire_request(raw, None, WireFormat::OpenAiResponses)?,
4030                Some(&ModelId::from("gpt")),
4031            )
4032            .await?;
4033        let agg = response.llm_response.into_agg().await?;
4034
4035        assert_tool_call_name(&agg, "mcp__open_websearch__search");
4036        Ok(())
4037    }
4038
4039    #[tokio::test]
4040    async fn openai_responses_backend_flattens_codex_namespaces_before_upstream()
4041    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
4042        let server = MockServer::start().await;
4043        Mock::given(method("POST"))
4044            .and(path("/v1/responses"))
4045            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
4046                "id": "resp_1",
4047                "model": "gpt",
4048                "object": "response",
4049                "created_at": 0,
4050                "status": "completed",
4051                "output": [{
4052                    "type": "function_call",
4053                    "call_id": "call_1",
4054                    "name": "mcp__open_websearch__search",
4055                    "arguments": "{\"q\":\"rust\"}"
4056                }],
4057                "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
4058            })))
4059            .mount(&server)
4060            .await;
4061
4062        let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?;
4063        let parameters = json!({"type": "object", "properties": {"q": {"type": "string"}}});
4064        let raw = json!({
4065            "model": "client-facing",
4066            "input": [{
4067                "type": "function_call",
4068                "call_id": "call_0",
4069                "name": "search",
4070                "namespace": "mcp__open_websearch",
4071                "arguments": "{}"
4072            }],
4073            "tool_choice": {
4074                "type": "function",
4075                "name": "search",
4076                "namespace": "mcp__open_websearch"
4077            },
4078            "tools": [{
4079                "type": "namespace",
4080                "name": "mcp__open_websearch",
4081                "tools": [{
4082                    "type": "function",
4083                    "name": "search",
4084                    "parameters": parameters.clone()
4085                }]
4086            }]
4087        });
4088
4089        let response = client
4090            .call_rewrite_model(
4091                wire_request(raw, None, WireFormat::OpenAiResponses)?,
4092                Some(&ModelId::from("gpt")),
4093            )
4094            .await?;
4095        let agg = response.llm_response.into_agg().await?;
4096
4097        let received = server
4098            .received_requests()
4099            .await
4100            .ok_or("request recording should be enabled")?;
4101        let request_body: Value = serde_json::from_slice(&received[0].body)?;
4102        assert_eq!(request_body["model"], "gpt");
4103        assert_eq!(
4104            request_body["tools"],
4105            json!([{
4106                "type": "function",
4107                "name": "mcp__open_websearch__search",
4108                "description": "",
4109                "parameters": parameters
4110            }])
4111        );
4112        assert_eq!(
4113            request_body["tool_choice"],
4114            json!({"type": "function", "name": "mcp__open_websearch__search"})
4115        );
4116        assert_eq!(
4117            request_body["input"][0]["name"],
4118            "mcp__open_websearch__search"
4119        );
4120        assert!(request_body["input"][0].get("namespace").is_none());
4121        assert_tool_call_name(&agg, "mcp__open_websearch__search");
4122        Ok(())
4123    }
4124
4125    // A stage route can answer early steps of a Codex turn from Claude and a later step from
4126    // GPT, and Codex replays Claude's reasoning items on every step. OpenAI returns 400 for two
4127    // kinds: items with an ID but no `encrypted_content` when the request sets `store: false`,
4128    // and items whose `encrypted_content` holds Anthropic thinking that Switchyard packed for
4129    // the client. The Responses leg must drop both and keep GPT's own encrypted reasoning.
4130    // A non-OpenAI target instead keeps the plaintext and drops both payloads.
4131    #[tokio::test]
4132    async fn openai_responses_backend_drops_reasoning_the_provider_cannot_read()
4133    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
4134        let server = MockServer::start().await;
4135        Mock::given(method("POST"))
4136            .and(path("/v1/responses"))
4137            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
4138                "id": "resp_1",
4139                "model": "gpt",
4140                "object": "response",
4141                "created_at": 0,
4142                "status": "completed",
4143                "output": [{
4144                    "type": "message",
4145                    "role": "assistant",
4146                    "content": [{"type": "output_text", "text": "ok"}]
4147                }],
4148                "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
4149            })))
4150            .mount(&server)
4151            .await;
4152
4153        let mut claude_reasoning = json!({
4154            "type": "reasoning",
4155            "id": "rs_b99feb9a-763d-4f7f-b6b2-2c0199b47ded",
4156            "summary": [{"type": "summary_text", "text": "Read the files first."}],
4157            "encrypted_content": null
4158        });
4159        // Build the item the way Switchyard does when it returns a Claude reply to a Responses
4160        // client, so the test follows any change to that format.
4161        let claude_reply = switchyard_translation::TranslationEngine::default()
4162            .translate_response(
4163                WireFormat::AnthropicMessages,
4164                WireFormat::OpenAiResponses,
4165                &json!({
4166                    "id": "msg_1",
4167                    "type": "message",
4168                    "role": "assistant",
4169                    "model": "claude",
4170                    "content": [
4171                        {"type": "thinking", "thinking": "Plan the edit.", "signature": "sig-abc"},
4172                        {"type": "text", "text": "ok"}
4173                    ],
4174                    "stop_reason": "end_turn",
4175                    "usage": {"input_tokens": 1, "output_tokens": 1}
4176                }),
4177                &switchyard_translation::TranslationPolicy::default(),
4178            )?
4179            .body;
4180        let switchyard_reasoning = claude_reply["output"]
4181            .as_array()
4182            .and_then(|output| output.iter().find(|item| item["type"] == "reasoning"))
4183            .cloned()
4184            .ok_or("translated Claude reply should contain a reasoning item")?;
4185        assert!(switchyard_translation::is_anthropic_thinking_envelope(
4186            switchyard_reasoning["encrypted_content"]
4187                .as_str()
4188                .unwrap_or_default()
4189        ));
4190        let mut gpt_reasoning = json!({
4191            "type": "reasoning",
4192            "id": "rs_gpt",
4193            "summary": [],
4194            "encrypted_content": "opaque-encrypted-reasoning"
4195        });
4196        let mut empty_reasoning =
4197            json!({"type": "reasoning", "id": "rs_empty", "summary": [], "encrypted_content": ""});
4198        let user = json!({"type": "message", "role": "user", "content": "fix the build"});
4199        let call = json!({"type": "function_call", "call_id": "call_1", "name": "exec", "arguments": "{}"});
4200        let output = json!({"type": "function_call_output", "call_id": "call_1", "output": "ok"});
4201        let input = json!([
4202            user,
4203            claude_reasoning,
4204            switchyard_reasoning,
4205            call,
4206            output,
4207            gpt_reasoning,
4208            empty_reasoning
4209        ]);
4210
4211        let models = ["openai/gpt-5", "nvidia/zai-org/glm-5.3-flash"].map(|model| {
4212            ModelConfig::new(
4213                model,
4214                Backend::OpenAiResponses(config(&format!("{}/v1", server.uri()))),
4215                None,
4216            )
4217        });
4218        let client = TranslatingLlmClient::new(&models)?;
4219        for model in ["openai/gpt-5", "nvidia/zai-org/glm-5.3-flash"] {
4220            for store in [json!(false), json!(true)] {
4221                let raw = json!({"model": "client-facing", "store": store, "input": input.clone()});
4222                client
4223                    .call_rewrite_model(
4224                        wire_request(raw, None, WireFormat::OpenAiResponses)?,
4225                        Some(&ModelId::from(model)),
4226                    )
4227                    .await?;
4228            }
4229        }
4230
4231        let received = server
4232            .received_requests()
4233            .await
4234            .ok_or("request recording should be enabled")?;
4235        let sent: Vec<Value> = received
4236            .iter()
4237            .map(|request| serde_json::from_slice::<Value>(&request.body))
4238            .collect::<std::result::Result<_, _>>()?;
4239        assert_eq!(sent.len(), 4);
4240        for request in &sent {
4241            assert!(
4242                request["input"]
4243                    .as_array()
4244                    .ok_or("expected input array")?
4245                    .iter()
4246                    .all(|item| !item["encrypted_content"]
4247                        .as_str()
4248                        .is_some_and(switchyard_translation::is_anthropic_thinking_envelope)),
4249                "provider cannot read translated Anthropic reasoning"
4250            );
4251        }
4252
4253        // GLM keeps the plaintext beside the Anthropic envelope and drops GPT's payload.
4254        let mut switchyard_plaintext = switchyard_reasoning.clone();
4255        switchyard_plaintext
4256            .as_object_mut()
4257            .ok_or("object")?
4258            .remove("encrypted_content");
4259        assert_eq!(
4260            sent[2]["input"],
4261            json!([user, claude_reasoning, switchyard_plaintext, call, output])
4262        );
4263        assert_eq!(
4264            sent[3]["input"],
4265            json!([
4266                user,
4267                claude_reasoning,
4268                switchyard_plaintext,
4269                call,
4270                output,
4271                empty_reasoning
4272            ])
4273        );
4274
4275        for item in [
4276            &mut claude_reasoning,
4277            &mut gpt_reasoning,
4278            &mut empty_reasoning,
4279        ] {
4280            item["content"] = json!([]);
4281        }
4282        // With `store: false`, only GPT's own encrypted reasoning survives.
4283        assert_eq!(sent[0]["input"], json!([user, call, output, gpt_reasoning]));
4284        // With `store: true`, IDs can refer to stored reasoning, but OpenAI still cannot
4285        // decrypt the Switchyard item.
4286        assert_eq!(
4287            sent[1]["input"],
4288            json!([
4289                user,
4290                claude_reasoning,
4291                call,
4292                output,
4293                gpt_reasoning,
4294                empty_reasoning
4295            ])
4296        );
4297        Ok(())
4298    }
4299
4300    // Allowed caller headers reach the upstream.
4301    #[tokio::test]
4302    async fn call_rewrite_model_forwards_caller_headers()
4303    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
4304        let server = MockServer::start().await;
4305        Mock::given(method("POST"))
4306            .and(wiremock::matchers::header("x-request-id", "abc"))
4307            // A forwarded authorization must NOT override the backend's bearer key.
4308            .and(wiremock::matchers::header("authorization", "Bearer secret"))
4309            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
4310                "id": "1", "model": "gpt",
4311                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
4312                "usage": {}
4313            })))
4314            .mount(&server)
4315            .await;
4316
4317        let mut headers = http::HeaderMap::new();
4318        headers.insert("x-request-id", http::HeaderValue::from_static("abc"));
4319        headers.insert(
4320            "authorization",
4321            http::HeaderValue::from_static("Bearer client-key"),
4322        );
4323
4324        let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
4325        let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]});
4326        // The allowed request id survives and caller authorization cannot replace
4327        // the backend's credential.
4328        client
4329            .call_rewrite_model(
4330                wire_request(raw, Some(headers), WireFormat::OpenAiChat)?,
4331                Some(&ModelId::from("gpt")),
4332            )
4333            .await?;
4334        Ok(())
4335    }
4336}