switchyard_llm_client/
backend.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Per-provider backend configuration: wire format, upstream URL, and auth.
5
6use std::time::Duration;
7use std::{
8    collections::{BTreeMap, BTreeSet},
9    fmt,
10};
11
12use reqwest::RequestBuilder;
13use reqwest::header::{HeaderName, HeaderValue};
14use serde_json::Value;
15use switchyard_protocol::{Metadata, WireFormat};
16
17use crate::error::{LlmClientError, Result, is_overflow_body};
18
19const ANTHROPIC_VERSION: &str = "2023-06-01";
20
21/// Default number of retries for server-configured upstream calls.
22pub const DEFAULT_MAX_RETRIES: u32 = 2;
23
24// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
25// provider-wrap is a one-line entry here, not a fork of the parsing logic.
26const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
27    "maximum context length",
28    "context length exceeded",
29    "context window",
30    "context length is only",
31    "please reduce the length of the input",
32    "exceeds the maximum allowed input length",
33    "exceeds the maximum allowed length",
34    "exceeds the available context size",
35    "is longer than the model's context length",
36    // llama-server, both of its ERROR_TYPE_EXCEED_CONTEXT_SIZE messages
37    // (tools/server/server-context.cpp). The structured check below catches
38    // them from `error.type`; these cover a proxy that rewrites the envelope
39    // and keeps the message.
40    "exceeds the available context size",
41    "larger than the max context size",
42];
43
44// Anthropic has no structured `error.code`, so detection is phrase-based only.
45const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
46    "prompt is too long",
47    "maximum number of tokens",
48    "context window",
49    "context length",
50];
51
52/// Shared HTTP configuration for one upstream backend.
53#[derive(Clone)]
54pub struct HttpBackendConfig {
55    /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
56    pub base_url: String,
57    /// API key for the provider, loaded by the caller. `None` sends no configured auth.
58    /// Client construction rejects active values that cannot form the provider's auth header.
59    pub api_key: Option<String>,
60    /// Whether this backend forwards the caller's provider credential and application headers.
61    ///
62    /// The forwarding backends in one route must use one credential family (OpenAI or
63    /// Anthropic) unless they all use the same scheme, host, and port, such as one LLM
64    /// gateway. Backends that send a configured key are not restricted.
65    pub forward_auth: bool,
66    /// Custom headers added to every outbound call to this backend.
67    ///
68    /// Provider-owned headers are rejected so a static value cannot replace
69    /// configured or forwarded auth. Names and values must be valid HTTP header bytes;
70    /// header names are case-insensitive.
71    /// Configured values replace caller values for the same header name.
72    pub extra_headers: BTreeMap<String, String>,
73    /// Default top-level request fields, applied only when the request omits the key.
74    pub extra_body: BTreeMap<String, Value>,
75    /// Top-level request fields dropped from the outbound body before `extra_body` is merged,
76    /// so a target can reinstate one deliberately. For providers that reject an otherwise
77    /// standard field (e.g. an internal API that 400s on `max_output_tokens`).
78    pub omit_body_fields: BTreeSet<String>,
79    /// Reasoning effort forced on every request to this backend, replacing whatever the caller
80    /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`,
81    /// and Anthropic Messages as `output_config.effort`.
82    pub reasoning_effort: Option<String>,
83    /// Additional attempts for transient upstream failures.
84    /// One template-compatibility retry may also run within the same timeout.
85    pub max_retries: u32,
86    /// Cooldown after an exhausted transient completion failure. Zero disables it.
87    pub failure_cooldown: Duration,
88    /// Deadline for one complete response, including retries, retry delays, and stream reads.
89    /// `None` leaves the wait unbounded.
90    pub timeout: Option<Duration>,
91}
92
93impl fmt::Debug for HttpBackendConfig {
94    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
95        f.debug_struct("HttpBackendConfig")
96            .field("base_url", &self.base_url)
97            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
98            .field("forward_auth", &self.forward_auth)
99            .field("extra_header_names", &self.extra_headers.keys())
100            .field("extra_body_keys", &self.extra_body.keys())
101            .field("omit_body_fields", &self.omit_body_fields)
102            .field("reasoning_effort", &self.reasoning_effort)
103            .field("max_retries", &self.max_retries)
104            .field("failure_cooldown", &self.failure_cooldown)
105            .field("timeout", &self.timeout)
106            .finish()
107    }
108}
109
110/// A configured upstream backend, one variant per built-in wire format.
111///
112/// The variant fixes the wire format, URL path, and auth scheme together so no
113/// invalid combination can be constructed.
114#[derive(Clone, Debug)]
115pub enum Backend {
116    /// OpenAI-compatible Chat Completions API.
117    OpenAiChat(HttpBackendConfig),
118    /// OpenAI Responses API.
119    OpenAiResponses(HttpBackendConfig),
120    /// Anthropic Messages API.
121    Anthropic(HttpBackendConfig),
122}
123
124impl Backend {
125    // Matches reqwest's header conversions before the client can send a request.
126    pub(crate) fn validate_configured_headers(&self, model_name: &str) -> Result<()> {
127        for (name, value) in &self.config().extra_headers {
128            if HeaderName::from_bytes(name.as_bytes()).is_err() {
129                return Err(LlmClientError::Configuration {
130                    message: format!(
131                        "model {model_name:?} extra_headers contains invalid HTTP header name {name:?}"
132                    ),
133                });
134            }
135            if HeaderValue::from_bytes(value.as_bytes()).is_err() {
136                return Err(LlmClientError::Configuration {
137                    message: format!(
138                        "model {model_name:?} has invalid HTTP header value for extra_headers entry {name:?}"
139                    ),
140                });
141            }
142        }
143
144        let invalid_name = self.config().extra_headers.keys().find(|name| match self {
145            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
146                name.eq_ignore_ascii_case("authorization")
147                    || (self.is_forwarding_auth()
148                        && (name.eq_ignore_ascii_case("chatgpt-account-id")
149                            || name.eq_ignore_ascii_case("x-openai-fedramp")))
150            }
151            Backend::Anthropic(_) => {
152                name.eq_ignore_ascii_case("x-api-key")
153                    || name.eq_ignore_ascii_case("anthropic-version")
154                    || (self.is_forwarding_auth()
155                        && (name.eq_ignore_ascii_case("authorization")
156                            || name.eq_ignore_ascii_case("anthropic-beta")))
157            }
158        });
159        if let Some(name) = invalid_name {
160            return Err(LlmClientError::Configuration {
161                message: format!(
162                    "model {model_name:?} extra_headers cannot set {name:?}; extra_headers is only for additional headers"
163                ),
164            });
165        }
166
167        let Some(api_key) = self.configured_api_key() else {
168            return Ok(());
169        };
170        let valid_api_key = match self {
171            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
172                HeaderValue::try_from(format!("Bearer {api_key}")).is_ok()
173            }
174            Backend::Anthropic(_) => HeaderValue::from_str(api_key).is_ok(),
175        };
176        if !valid_api_key {
177            return Err(LlmClientError::Configuration {
178                message: format!(
179                    "model {model_name:?} api_key cannot be encoded as an HTTP header"
180                ),
181            });
182        }
183        Ok(())
184    }
185
186    /// The wire format the request IR is encoded to for this backend.
187    pub fn wire_format(&self) -> WireFormat {
188        match self {
189            Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
190            Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
191            Backend::Anthropic(_) => WireFormat::AnthropicMessages,
192        }
193    }
194
195    // Shared HTTP config, regardless of variant.
196    fn config(&self) -> &HttpBackendConfig {
197        match self {
198            Backend::OpenAiChat(config)
199            | Backend::OpenAiResponses(config)
200            | Backend::Anthropic(config) => config,
201        }
202    }
203
204    // Static credentials are unused when the caller's authorization is forwarded.
205    fn configured_api_key(&self) -> Option<&str> {
206        if self.is_forwarding_auth() {
207            None
208        } else {
209            self.config().api_key.as_deref()
210        }
211    }
212
213    pub(crate) fn base_url(&self) -> &str {
214        &self.config().base_url
215    }
216
217    /// The fully resolved upstream URL for this backend's endpoint.
218    ///
219    /// Tolerates base URLs that already include the provider path (or a bare
220    /// `/v1`), matching the join rules of the existing native backends.
221    pub fn url(&self) -> String {
222        let base_url = &self.config().base_url;
223        let result = match self {
224            Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
225            Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
226            Backend::Anthropic(_) => anthropic_url(base_url, ""),
227        };
228        result.unwrap_or_else(|error| {
229            tracing::error!(%error, "Unable to build provider endpoint URL");
230            base_url.clone()
231        })
232    }
233
234    /// Applies this backend's configured auth and version headers to a request builder.
235    ///
236    /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
237    /// `x-api-key: <key>` plus the required `anthropic-version` header. A backend
238    /// with `forward_auth` uses the caller's provider credential instead.
239    pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
240        let api_key = self.configured_api_key();
241        match self {
242            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
243                if let Some(api_key) = api_key {
244                    builder = builder.bearer_auth(api_key);
245                }
246            }
247            Backend::Anthropic(_) => {
248                builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
249                if let Some(api_key) = api_key {
250                    builder = builder.header("x-api-key", api_key);
251                }
252            }
253        }
254        builder
255    }
256
257    pub(crate) fn is_forwarding_auth(&self) -> bool {
258        self.config().forward_auth
259    }
260
261    pub(crate) fn is_provider_owned_header(&self, name: &str) -> bool {
262        match self {
263            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
264                ["authorization", "chatgpt-account-id", "x-openai-fedramp"]
265                    .iter()
266                    .any(|owned| name.eq_ignore_ascii_case(owned))
267            }
268            Backend::Anthropic(_) => [
269                "authorization",
270                "x-api-key",
271                "anthropic-beta",
272                "anthropic-version",
273            ]
274            .iter()
275            .any(|owned| name.eq_ignore_ascii_case(owned)),
276        }
277    }
278
279    /// Applies only the caller credential accepted by this provider.
280    pub(crate) fn apply_forwarded_auth(
281        &self,
282        mut builder: RequestBuilder,
283        metadata: Option<&Metadata>,
284    ) -> RequestBuilder {
285        if !self.is_forwarding_auth() {
286            return builder;
287        }
288        let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
289            return builder;
290        };
291        match self {
292            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
293                for name in ["authorization", "chatgpt-account-id", "x-openai-fedramp"] {
294                    if let Some(value) = headers.get(name) {
295                        builder = builder.header(name, sensitive_header(value));
296                    }
297                }
298            }
299            Backend::Anthropic(_) => {
300                for name in ["authorization", "x-api-key"] {
301                    if let Some(value) = headers.get(name) {
302                        builder = builder.header(name, sensitive_header(value));
303                    }
304                }
305                if let Some(value) = headers.get("anthropic-beta")
306                    && let Some(value) = oauth_beta_header(value)
307                {
308                    builder = builder.header("anthropic-beta", value);
309                }
310            }
311        }
312        builder
313    }
314
315    /// Custom per-backend headers to forward on every call.
316    pub fn extra_headers(&self) -> &BTreeMap<String, String> {
317        &self.config().extra_headers
318    }
319
320    /// Default top-level fields to merge into outbound request bodies.
321    pub fn extra_body(&self) -> &BTreeMap<String, Value> {
322        &self.config().extra_body
323    }
324
325    /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.
326    pub fn omit_body_fields(&self) -> &BTreeSet<String> {
327        &self.config().omit_body_fields
328    }
329
330    /// Reasoning effort forced on outbound requests, if the target configures one.
331    pub fn reasoning_effort(&self) -> Option<&str> {
332        self.config().reasoning_effort.as_deref()
333    }
334
335    /// Additional attempts allowed after the initial request.
336    pub fn max_retries(&self) -> u32 {
337        self.config().max_retries
338    }
339
340    /// Cooldown after an exhausted transient completion failure.
341    pub fn failure_cooldown(&self) -> Duration {
342        self.config().failure_cooldown
343    }
344
345    /// Deadline for all attempts and the complete response; `None` leaves the wait unbounded.
346    pub fn timeout(&self) -> Option<Duration> {
347        self.config().timeout
348    }
349
350    /// Whether this backend speaks the Anthropic Messages wire format — the only
351    /// one with a `count_tokens` endpoint.
352    pub fn is_anthropic(&self) -> bool {
353        matches!(self, Backend::Anthropic(_))
354    }
355
356    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
357    /// URL join as [`url`](Self::url).
358    pub fn count_tokens_url(&self) -> String {
359        let base_url = &self.config().base_url;
360        anthropic_url(base_url, "/count_tokens").unwrap_or_else(|error| {
361            tracing::error!(%error, "Unable to build Anthropic token-counting URL");
362            base_url.clone()
363        })
364    }
365
366    /// Whether an upstream 400 `body` looks like a context-window overflow for
367    /// this backend's provider.
368    pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
369        match self {
370            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
371                body,
372                |value| {
373                    let error = value.get("error");
374                    let field = |name| {
375                        error
376                            .and_then(|err| err.get(name))
377                            .and_then(serde_json::Value::as_str)
378                    };
379                    // llama-server puts the HTTP status in `code` as a number and
380                    // names the reason in `type`, so the canonical code never
381                    // matches. Its two overflow messages are worded unlike any
382                    // other provider's, which leaves the type as the signal that
383                    // holds across releases.
384                    field("code") == Some("context_length_exceeded")
385                        || field("type") == Some("exceed_context_size_error")
386                },
387                OPENAI_OVERFLOW_PHRASES,
388            ),
389            Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
390        }
391    }
392}
393
394// Retains OAuth markers while keeping provider feature betas backend-owned.
395fn sensitive_header(value: &HeaderValue) -> HeaderValue {
396    let mut value = value.clone();
397    value.set_sensitive(true);
398    value
399}
400
401fn oauth_beta_header(value: &HeaderValue) -> Option<HeaderValue> {
402    let oauth_betas = value
403        .to_str()
404        .ok()?
405        .split(',')
406        .map(str::trim)
407        .filter(|beta| {
408            beta.get(..6)
409                .is_some_and(|prefix| prefix.eq_ignore_ascii_case("oauth-"))
410        });
411    let value = oauth_betas.collect::<Vec<_>>().join(",");
412    if value.is_empty() {
413        return None;
414    }
415    let mut value = HeaderValue::from_str(&value).ok()?;
416    value.set_sensitive(true);
417    Some(value)
418}
419
420// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
421pub(crate) fn openai_url(base_url: &str, suffix: &str) -> Result<String> {
422    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
423        message: format!("Invalid OpenAI base URL: {error}"),
424    })?;
425    let base_path = url.path().trim_end_matches('/');
426    let base_root = base_path
427        .strip_suffix("/chat/completions")
428        .or_else(|| base_path.strip_suffix("/responses"))
429        .unwrap_or(base_path);
430    url.set_path(&format!("{base_root}{suffix}"));
431    Ok(url.into())
432}
433
434// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
435fn anthropic_url(base_url: &str, suffix: &str) -> Result<String> {
436    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
437        message: format!("Invalid Anthropic base URL: {error}"),
438    })?;
439    let base_path = url.path().trim_end_matches('/');
440    let messages_path = if base_path.ends_with("/v1/messages") {
441        base_path.to_string()
442    } else if base_path.ends_with("/v1") {
443        format!("{base_path}/messages")
444    } else {
445        format!("{base_path}/v1/messages")
446    };
447    url.set_path(&format!("{messages_path}{suffix}"));
448    Ok(url.into())
449}
450
451#[cfg(test)]
452mod tests {
453    use super::*;
454
455    fn config(base_url: &str) -> HttpBackendConfig {
456        HttpBackendConfig {
457            base_url: base_url.to_string(),
458            api_key: Some("secret".to_string()),
459            forward_auth: false,
460            extra_headers: BTreeMap::new(),
461            extra_body: BTreeMap::new(),
462            omit_body_fields: BTreeSet::new(),
463            reasoning_effort: None,
464            max_retries: 0,
465            failure_cooldown: Duration::ZERO,
466            timeout: None,
467        }
468    }
469
470    #[test]
471    fn openai_chat_url_joins_bare_v1() {
472        let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
473        assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
474    }
475
476    #[test]
477    fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
478        assert_eq!(
479            Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
480            "https://api.openai.com/v1/chat/completions"
481        );
482        assert_eq!(
483            Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
484            "https://api.openai.com/v1/chat/completions"
485        );
486    }
487
488    #[test]
489    fn openai_responses_url_uses_responses_path() {
490        assert_eq!(
491            Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
492            "https://api.openai.com/v1/responses"
493        );
494    }
495
496    #[test]
497    fn anthropic_url_join_cases() {
498        assert_eq!(
499            Backend::Anthropic(config("https://api.anthropic.com")).url(),
500            "https://api.anthropic.com/v1/messages"
501        );
502        assert_eq!(
503            Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
504            "https://api.anthropic.com/v1/messages"
505        );
506        assert_eq!(
507            Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
508            "https://api.anthropic.com/v1/messages"
509        );
510    }
511
512    #[test]
513    fn wire_format_matches_variant() {
514        assert_eq!(
515            Backend::OpenAiChat(config("x")).wire_format(),
516            WireFormat::OpenAiChat
517        );
518        assert_eq!(
519            Backend::OpenAiResponses(config("x")).wire_format(),
520            WireFormat::OpenAiResponses
521        );
522        assert_eq!(
523            Backend::Anthropic(config("x")).wire_format(),
524            WireFormat::AnthropicMessages
525        );
526    }
527
528    // Header validation follows reqwest for both accepted and rejected bytes.
529    #[test]
530    fn validates_additional_header_bytes() {
531        let cases = [
532            ("x-display-name", "café", None),
533            (
534                "bad header",
535                "value",
536                Some("invalid HTTP header name \"bad header\""),
537            ),
538            (
539                "x-test-header",
540                "bad\nvalue",
541                Some("invalid HTTP header value for extra_headers entry \"x-test-header\""),
542            ),
543        ];
544
545        for (name, value, expected) in cases {
546            let mut config = config("x");
547            config
548                .extra_headers
549                .insert(name.to_string(), value.to_string());
550            let result = Backend::OpenAiChat(config).validate_configured_headers("model");
551            match expected {
552                Some(expected) => assert!(
553                    result.is_err_and(|error| error.to_string().contains(expected)),
554                    "expected {expected:?}"
555                ),
556                None => result.expect("encodable header must pass validation"),
557            }
558        }
559    }
560
561    // Only static credentials that apply_auth would send are validated.
562    #[test]
563    fn configured_api_key_validation_matches_auth_application() {
564        const INVALID_KEY: &str = "canary\nsecret";
565        let mut config = config("x");
566        config.api_key = Some(INVALID_KEY.to_string());
567        let builders: [fn(HttpBackendConfig) -> Backend; 2] =
568            [Backend::OpenAiChat, Backend::Anthropic];
569        let client = reqwest::Client::new();
570
571        for build_backend in builders {
572            let error = build_backend(config.clone())
573                .validate_configured_headers("model")
574                .expect_err("invalid API key must fail")
575                .to_string();
576            assert!(
577                error.contains("api_key cannot be encoded as an HTTP header"),
578                "{error}"
579            );
580            assert!(!error.contains(INVALID_KEY), "API key leaked in: {error}");
581
582            let mut forwarded = config.clone();
583            forwarded.forward_auth = true;
584            let backend = build_backend(forwarded);
585            backend
586                .validate_configured_headers("model")
587                .expect("unused API key must not fail validation");
588            let request = backend
589                .apply_auth(client.get("https://example.test"))
590                .build()
591                .expect("request");
592            assert!(!request.headers().contains_key("authorization"));
593            assert!(!request.headers().contains_key("x-api-key"));
594        }
595    }
596
597    #[test]
598    fn openai_detects_canonical_and_wrapped_overflow() {
599        let backend = Backend::OpenAiChat(config("x"));
600        assert!(
601            backend.is_context_overflow(
602                r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
603            )
604        );
605        // NVIDIA/LiteLLM message wrap with no structured code.
606        assert!(backend.is_context_overflow(
607            r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
608        ));
609        assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
610        // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
611        assert!(backend.is_context_overflow(
612            r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
613        ));
614        // Native SGLang: top-level error envelope (no `error` key).
615        // KV-pool rejection (managers/utils.py) and declared-context rejection
616        // (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.
617        assert!(backend.is_context_overflow(
618            r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#
619        ));
620        assert!(backend.is_context_overflow(
621            r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#
622        ));
623    }
624
625    #[test]
626    fn openai_detects_llama_server_overflow() {
627        let backend = Backend::OpenAiChat(config("x"));
628        // llama-server, numeric `code` and the reason in `type`. Both of the
629        // messages ERROR_TYPE_EXCEED_CONTEXT_SIZE can carry are here.
630        assert!(backend.is_context_overflow(
631            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#
632        ));
633        assert!(backend.is_context_overflow(
634            r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#
635        ));
636        // The type alone is enough, so a reworded message still falls back.
637        assert!(backend.is_context_overflow(
638            r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#
639        ));
640        // And the message alone is enough, for a proxy that drops the type.
641        assert!(backend.is_context_overflow(
642            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#
643        ));
644        // llama-server's other 400s must still surface to the caller.
645        assert!(!backend.is_context_overflow(
646            r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#
647        ));
648    }
649
650    #[test]
651    fn anthropic_detects_prompt_too_long() {
652        let backend = Backend::Anthropic(config("x"));
653        assert!(
654            backend.is_context_overflow(
655                r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
656            )
657        );
658        assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
659    }
660}
switchyard_llm_client/backend.rs

switchyard_llm_client/
backend.rs
