Skip to main content

switchyard_llm_client/
backend.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Per-provider backend configuration: wire format, upstream URL, and auth.
5
6use std::time::Duration;
7use std::{
8    collections::{BTreeMap, BTreeSet},
9    fmt,
10};
11
12use reqwest::RequestBuilder;
13use reqwest::header::{HeaderName, HeaderValue};
14use serde_json::Value;
15use switchyard_protocol::{Metadata, WireFormat};
16
17use crate::error::{LlmClientError, Result, is_overflow_body};
18
19const ANTHROPIC_VERSION: &str = "2023-06-01";
20
21/// Default number of retries for server-configured upstream calls.
22pub const DEFAULT_MAX_RETRIES: u32 = 2;
23
24// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
25// provider-wrap is a one-line entry here, not a fork of the parsing logic.
26const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
27    "maximum context length",
28    "context length exceeded",
29    "context window",
30    "context length is only",
31    "please reduce the length of the input",
32    "exceeds the maximum allowed input length",
33    "exceeds the maximum allowed length",
34    "exceeds the available context size",
35    "is longer than the model's context length",
36    // llama-server, both of its ERROR_TYPE_EXCEED_CONTEXT_SIZE messages
37    // (tools/server/server-context.cpp). The structured check below catches
38    // them from `error.type`; these cover a proxy that rewrites the envelope
39    // and keeps the message.
40    "exceeds the available context size",
41    "larger than the max context size",
42];
43
44// Anthropic has no structured `error.code`, so detection is phrase-based only.
45const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
46    "prompt is too long",
47    "maximum number of tokens",
48    "context window",
49    "context length",
50];
51
52/// Shared HTTP configuration for one upstream backend.
53#[derive(Clone)]
54pub struct HttpBackendConfig {
55    /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
56    pub base_url: String,
57    /// API key for the provider, loaded by the caller. `None` sends no configured auth.
58    /// Client construction rejects active values that cannot form the provider's auth header.
59    pub api_key: Option<String>,
60    /// Whether this backend forwards the caller's provider credential and application headers.
61    ///
62    /// The forwarding backends in one route must use one credential family (OpenAI or
63    /// Anthropic) unless they all use the same scheme, host, and port, such as one LLM
64    /// gateway. Backends that send a configured key are not restricted.
65    pub forward_auth: bool,
66    /// Custom headers added to every outbound call to this backend.
67    ///
68    /// Provider-owned headers are rejected so a static value cannot replace
69    /// configured or forwarded auth. Names and values must be valid HTTP header bytes;
70    /// header names are case-insensitive.
71    /// Configured values replace caller values for the same header name.
72    pub extra_headers: BTreeMap<String, String>,
73    /// Default top-level request fields, applied only when the request omits the key.
74    pub extra_body: BTreeMap<String, Value>,
75    /// Top-level request fields dropped from the outbound body before `extra_body` is merged,
76    /// so a target can reinstate one deliberately. For providers that reject an otherwise
77    /// standard field (e.g. an internal API that 400s on `max_output_tokens`).
78    pub omit_body_fields: BTreeSet<String>,
79    /// Reasoning effort forced on every request to this backend, replacing whatever the caller
80    /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`,
81    /// and Anthropic Messages as `output_config.effort`.
82    pub reasoning_effort: Option<String>,
83    /// Additional attempts for transient upstream failures.
84    /// One template-compatibility retry may also run within the same timeout.
85    pub max_retries: u32,
86    /// Cooldown after an exhausted transient completion failure. Zero disables it.
87    pub failure_cooldown: Duration,
88    /// Deadline for one complete response, including retries, retry delays, and stream reads.
89    /// `None` leaves the wait unbounded.
90    pub timeout: Option<Duration>,
91}
92
93impl fmt::Debug for HttpBackendConfig {
94    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
95        f.debug_struct("HttpBackendConfig")
96            .field("base_url", &self.base_url)
97            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
98            .field("forward_auth", &self.forward_auth)
99            .field("extra_header_names", &self.extra_headers.keys())
100            .field("extra_body_keys", &self.extra_body.keys())
101            .field("omit_body_fields", &self.omit_body_fields)
102            .field("reasoning_effort", &self.reasoning_effort)
103            .field("max_retries", &self.max_retries)
104            .field("failure_cooldown", &self.failure_cooldown)
105            .field("timeout", &self.timeout)
106            .finish()
107    }
108}
109
110/// A configured upstream backend, one variant per built-in wire format.
111///
112/// The variant fixes the wire format, URL path, and auth scheme together so no
113/// invalid combination can be constructed.
114#[derive(Clone, Debug)]
115pub enum Backend {
116    /// OpenAI-compatible Chat Completions API.
117    OpenAiChat(HttpBackendConfig),
118    /// OpenAI Responses API.
119    OpenAiResponses(HttpBackendConfig),
120    /// Anthropic Messages API.
121    Anthropic(HttpBackendConfig),
122}
123
124impl Backend {
125    // Matches reqwest's header conversions before the client can send a request.
126    pub(crate) fn validate_configured_headers(&self, model_name: &str) -> Result<()> {
127        for (name, value) in &self.config().extra_headers {
128            if HeaderName::from_bytes(name.as_bytes()).is_err() {
129                return Err(LlmClientError::Configuration {
130                    message: format!(
131                        "model {model_name:?} extra_headers contains invalid HTTP header name {name:?}"
132                    ),
133                });
134            }
135            if HeaderValue::from_bytes(value.as_bytes()).is_err() {
136                return Err(LlmClientError::Configuration {
137                    message: format!(
138                        "model {model_name:?} has invalid HTTP header value for extra_headers entry {name:?}"
139                    ),
140                });
141            }
142        }
143
144        let invalid_name = self.config().extra_headers.keys().find(|name| match self {
145            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
146                name.eq_ignore_ascii_case("authorization")
147                    || (self.is_forwarding_auth()
148                        && (name.eq_ignore_ascii_case("chatgpt-account-id")
149                            || name.eq_ignore_ascii_case("x-openai-fedramp")))
150            }
151            Backend::Anthropic(_) => {
152                name.eq_ignore_ascii_case("x-api-key")
153                    || name.eq_ignore_ascii_case("anthropic-version")
154                    || (self.is_forwarding_auth()
155                        && (name.eq_ignore_ascii_case("authorization")
156                            || name.eq_ignore_ascii_case("anthropic-beta")))
157            }
158        });
159        if let Some(name) = invalid_name {
160            return Err(LlmClientError::Configuration {
161                message: format!(
162                    "model {model_name:?} extra_headers cannot set {name:?}; extra_headers is only for additional headers"
163                ),
164            });
165        }
166
167        let Some(api_key) = self.configured_api_key() else {
168            return Ok(());
169        };
170        let valid_api_key = match self {
171            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
172                HeaderValue::try_from(format!("Bearer {api_key}")).is_ok()
173            }
174            Backend::Anthropic(_) => HeaderValue::from_str(api_key).is_ok(),
175        };
176        if !valid_api_key {
177            return Err(LlmClientError::Configuration {
178                message: format!(
179                    "model {model_name:?} api_key cannot be encoded as an HTTP header"
180                ),
181            });
182        }
183        Ok(())
184    }
185
186    /// The wire format the request IR is encoded to for this backend.
187    pub fn wire_format(&self) -> WireFormat {
188        match self {
189            Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
190            Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
191            Backend::Anthropic(_) => WireFormat::AnthropicMessages,
192        }
193    }
194
195    // Shared HTTP config, regardless of variant.
196    fn config(&self) -> &HttpBackendConfig {
197        match self {
198            Backend::OpenAiChat(config)
199            | Backend::OpenAiResponses(config)
200            | Backend::Anthropic(config) => config,
201        }
202    }
203
204    // Static credentials are unused when the caller's authorization is forwarded.
205    fn configured_api_key(&self) -> Option<&str> {
206        if self.is_forwarding_auth() {
207            None
208        } else {
209            self.config().api_key.as_deref()
210        }
211    }
212
213    pub(crate) fn base_url(&self) -> &str {
214        &self.config().base_url
215    }
216
217    /// The fully resolved upstream URL for this backend's endpoint.
218    ///
219    /// Tolerates base URLs that already include the provider path (or a bare
220    /// `/v1`), matching the join rules of the existing native backends.
221    pub fn url(&self) -> String {
222        let base_url = &self.config().base_url;
223        let result = match self {
224            Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
225            Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
226            Backend::Anthropic(_) => anthropic_url(base_url, ""),
227        };
228        result.unwrap_or_else(|error| {
229            tracing::error!(%error, "Unable to build provider endpoint URL");
230            base_url.clone()
231        })
232    }
233
234    /// Applies this backend's configured auth and version headers to a request builder.
235    ///
236    /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
237    /// `x-api-key: <key>` plus the required `anthropic-version` header. A backend
238    /// with `forward_auth` uses the caller's provider credential instead.
239    pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
240        let api_key = self.configured_api_key();
241        match self {
242            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
243                if let Some(api_key) = api_key {
244                    builder = builder.bearer_auth(api_key);
245                }
246            }
247            Backend::Anthropic(_) => {
248                builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
249                if let Some(api_key) = api_key {
250                    builder = builder.header("x-api-key", api_key);
251                }
252            }
253        }
254        builder
255    }
256
257    pub(crate) fn is_forwarding_auth(&self) -> bool {
258        self.config().forward_auth
259    }
260
261    pub(crate) fn is_provider_owned_header(&self, name: &str) -> bool {
262        match self {
263            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
264                ["authorization", "chatgpt-account-id", "x-openai-fedramp"]
265                    .iter()
266                    .any(|owned| name.eq_ignore_ascii_case(owned))
267            }
268            Backend::Anthropic(_) => [
269                "authorization",
270                "x-api-key",
271                "anthropic-beta",
272                "anthropic-version",
273            ]
274            .iter()
275            .any(|owned| name.eq_ignore_ascii_case(owned)),
276        }
277    }
278
279    /// Applies only the caller credential accepted by this provider.
280    pub(crate) fn apply_forwarded_auth(
281        &self,
282        mut builder: RequestBuilder,
283        metadata: Option<&Metadata>,
284    ) -> RequestBuilder {
285        if !self.is_forwarding_auth() {
286            return builder;
287        }
288        let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
289            return builder;
290        };
291        match self {
292            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
293                for name in ["authorization", "chatgpt-account-id", "x-openai-fedramp"] {
294                    if let Some(value) = headers.get(name) {
295                        builder = builder.header(name, sensitive_header(value));
296                    }
297                }
298            }
299            Backend::Anthropic(_) => {
300                for name in ["authorization", "x-api-key"] {
301                    if let Some(value) = headers.get(name) {
302                        builder = builder.header(name, sensitive_header(value));
303                    }
304                }
305                if let Some(value) = headers.get("anthropic-beta")
306                    && let Some(value) = oauth_beta_header(value)
307                {
308                    builder = builder.header("anthropic-beta", value);
309                }
310            }
311        }
312        builder
313    }
314
315    /// Custom per-backend headers to forward on every call.
316    pub fn extra_headers(&self) -> &BTreeMap<String, String> {
317        &self.config().extra_headers
318    }
319
320    /// Default top-level fields to merge into outbound request bodies.
321    pub fn extra_body(&self) -> &BTreeMap<String, Value> {
322        &self.config().extra_body
323    }
324
325    /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.
326    pub fn omit_body_fields(&self) -> &BTreeSet<String> {
327        &self.config().omit_body_fields
328    }
329
330    /// Reasoning effort forced on outbound requests, if the target configures one.
331    pub fn reasoning_effort(&self) -> Option<&str> {
332        self.config().reasoning_effort.as_deref()
333    }
334
335    /// Additional attempts allowed after the initial request.
336    pub fn max_retries(&self) -> u32 {
337        self.config().max_retries
338    }
339
340    /// Cooldown after an exhausted transient completion failure.
341    pub fn failure_cooldown(&self) -> Duration {
342        self.config().failure_cooldown
343    }
344
345    /// Deadline for all attempts and the complete response; `None` leaves the wait unbounded.
346    pub fn timeout(&self) -> Option<Duration> {
347        self.config().timeout
348    }
349
350    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
351    /// URL join as [`url`](Self::url).
352    pub fn count_tokens_url(&self) -> String {
353        let base_url = &self.config().base_url;
354        anthropic_url(base_url, "/count_tokens").unwrap_or_else(|error| {
355            tracing::error!(%error, "Unable to build Anthropic token-counting URL");
356            base_url.clone()
357        })
358    }
359
360    /// Whether an upstream 400 `body` looks like a context-window overflow for
361    /// this backend's provider.
362    pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
363        match self {
364            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
365                body,
366                |value| {
367                    let error = value.get("error");
368                    let field = |name| {
369                        error
370                            .and_then(|err| err.get(name))
371                            .and_then(serde_json::Value::as_str)
372                    };
373                    // llama-server puts the HTTP status in `code` as a number and
374                    // names the reason in `type`, so the canonical code never
375                    // matches. Its two overflow messages are worded unlike any
376                    // other provider's, which leaves the type as the signal that
377                    // holds across releases.
378                    field("code") == Some("context_length_exceeded")
379                        || field("type") == Some("exceed_context_size_error")
380                },
381                OPENAI_OVERFLOW_PHRASES,
382            ),
383            Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
384        }
385    }
386}
387
388// Retains OAuth markers while keeping provider feature betas backend-owned.
389fn sensitive_header(value: &HeaderValue) -> HeaderValue {
390    let mut value = value.clone();
391    value.set_sensitive(true);
392    value
393}
394
395fn oauth_beta_header(value: &HeaderValue) -> Option<HeaderValue> {
396    let oauth_betas = value
397        .to_str()
398        .ok()?
399        .split(',')
400        .map(str::trim)
401        .filter(|beta| {
402            beta.get(..6)
403                .is_some_and(|prefix| prefix.eq_ignore_ascii_case("oauth-"))
404        });
405    let value = oauth_betas.collect::<Vec<_>>().join(",");
406    if value.is_empty() {
407        return None;
408    }
409    let mut value = HeaderValue::from_str(&value).ok()?;
410    value.set_sensitive(true);
411    Some(value)
412}
413
414// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
415pub(crate) fn openai_url(base_url: &str, suffix: &str) -> Result<String> {
416    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
417        message: format!("Invalid OpenAI base URL: {error}"),
418    })?;
419    let base_path = url.path().trim_end_matches('/');
420    let base_root = base_path
421        .strip_suffix("/chat/completions")
422        .or_else(|| base_path.strip_suffix("/responses"))
423        .unwrap_or(base_path);
424    url.set_path(&format!("{base_root}{suffix}"));
425    Ok(url.into())
426}
427
428// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
429fn anthropic_url(base_url: &str, suffix: &str) -> Result<String> {
430    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
431        message: format!("Invalid Anthropic base URL: {error}"),
432    })?;
433    let base_path = url.path().trim_end_matches('/');
434    let messages_path = if base_path.ends_with("/v1/messages") {
435        base_path.to_string()
436    } else if base_path.ends_with("/v1") {
437        format!("{base_path}/messages")
438    } else {
439        format!("{base_path}/v1/messages")
440    };
441    url.set_path(&format!("{messages_path}{suffix}"));
442    Ok(url.into())
443}
444
445#[cfg(test)]
446mod tests {
447    use super::*;
448
449    fn config(base_url: &str) -> HttpBackendConfig {
450        HttpBackendConfig {
451            base_url: base_url.to_string(),
452            api_key: Some("secret".to_string()),
453            forward_auth: false,
454            extra_headers: BTreeMap::new(),
455            extra_body: BTreeMap::new(),
456            omit_body_fields: BTreeSet::new(),
457            reasoning_effort: None,
458            max_retries: 0,
459            failure_cooldown: Duration::ZERO,
460            timeout: None,
461        }
462    }
463
464    #[test]
465    fn openai_chat_url_joins_bare_v1() {
466        let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
467        assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
468    }
469
470    #[test]
471    fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
472        assert_eq!(
473            Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
474            "https://api.openai.com/v1/chat/completions"
475        );
476        assert_eq!(
477            Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
478            "https://api.openai.com/v1/chat/completions"
479        );
480    }
481
482    #[test]
483    fn openai_responses_url_uses_responses_path() {
484        assert_eq!(
485            Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
486            "https://api.openai.com/v1/responses"
487        );
488    }
489
490    #[test]
491    fn anthropic_url_join_cases() {
492        assert_eq!(
493            Backend::Anthropic(config("https://api.anthropic.com")).url(),
494            "https://api.anthropic.com/v1/messages"
495        );
496        assert_eq!(
497            Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
498            "https://api.anthropic.com/v1/messages"
499        );
500        assert_eq!(
501            Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
502            "https://api.anthropic.com/v1/messages"
503        );
504    }
505
506    #[test]
507    fn wire_format_matches_variant() {
508        assert_eq!(
509            Backend::OpenAiChat(config("x")).wire_format(),
510            WireFormat::OpenAiChat
511        );
512        assert_eq!(
513            Backend::OpenAiResponses(config("x")).wire_format(),
514            WireFormat::OpenAiResponses
515        );
516        assert_eq!(
517            Backend::Anthropic(config("x")).wire_format(),
518            WireFormat::AnthropicMessages
519        );
520    }
521
522    // Header validation follows reqwest for both accepted and rejected bytes.
523    #[test]
524    fn validates_additional_header_bytes() {
525        let cases = [
526            ("x-display-name", "café", None),
527            (
528                "bad header",
529                "value",
530                Some("invalid HTTP header name \"bad header\""),
531            ),
532            (
533                "x-test-header",
534                "bad\nvalue",
535                Some("invalid HTTP header value for extra_headers entry \"x-test-header\""),
536            ),
537        ];
538
539        for (name, value, expected) in cases {
540            let mut config = config("x");
541            config
542                .extra_headers
543                .insert(name.to_string(), value.to_string());
544            let result = Backend::OpenAiChat(config).validate_configured_headers("model");
545            match expected {
546                Some(expected) => assert!(
547                    result.is_err_and(|error| error.to_string().contains(expected)),
548                    "expected {expected:?}"
549                ),
550                None => result.expect("encodable header must pass validation"),
551            }
552        }
553    }
554
555    // Only static credentials that apply_auth would send are validated.
556    #[test]
557    fn configured_api_key_validation_matches_auth_application() {
558        const INVALID_KEY: &str = "canary\nsecret";
559        let mut config = config("x");
560        config.api_key = Some(INVALID_KEY.to_string());
561        let builders: [fn(HttpBackendConfig) -> Backend; 2] =
562            [Backend::OpenAiChat, Backend::Anthropic];
563        let client = reqwest::Client::new();
564
565        for build_backend in builders {
566            let error = build_backend(config.clone())
567                .validate_configured_headers("model")
568                .expect_err("invalid API key must fail")
569                .to_string();
570            assert!(
571                error.contains("api_key cannot be encoded as an HTTP header"),
572                "{error}"
573            );
574            assert!(!error.contains(INVALID_KEY), "API key leaked in: {error}");
575
576            let mut forwarded = config.clone();
577            forwarded.forward_auth = true;
578            let backend = build_backend(forwarded);
579            backend
580                .validate_configured_headers("model")
581                .expect("unused API key must not fail validation");
582            let request = backend
583                .apply_auth(client.get("https://example.test"))
584                .build()
585                .expect("request");
586            assert!(!request.headers().contains_key("authorization"));
587            assert!(!request.headers().contains_key("x-api-key"));
588        }
589    }
590
591    #[test]
592    fn openai_detects_canonical_and_wrapped_overflow() {
593        let backend = Backend::OpenAiChat(config("x"));
594        assert!(
595            backend.is_context_overflow(
596                r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
597            )
598        );
599        // NVIDIA/LiteLLM message wrap with no structured code.
600        assert!(backend.is_context_overflow(
601            r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
602        ));
603        assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
604        // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
605        assert!(backend.is_context_overflow(
606            r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
607        ));
608        // Native SGLang: top-level error envelope (no `error` key).
609        // KV-pool rejection (managers/utils.py) and declared-context rejection
610        // (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.
611        assert!(backend.is_context_overflow(
612            r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#
613        ));
614        assert!(backend.is_context_overflow(
615            r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#
616        ));
617    }
618
619    #[test]
620    fn openai_detects_llama_server_overflow() {
621        let backend = Backend::OpenAiChat(config("x"));
622        // llama-server, numeric `code` and the reason in `type`. Both of the
623        // messages ERROR_TYPE_EXCEED_CONTEXT_SIZE can carry are here.
624        assert!(backend.is_context_overflow(
625            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#
626        ));
627        assert!(backend.is_context_overflow(
628            r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#
629        ));
630        // The type alone is enough, so a reworded message still falls back.
631        assert!(backend.is_context_overflow(
632            r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#
633        ));
634        // And the message alone is enough, for a proxy that drops the type.
635        assert!(backend.is_context_overflow(
636            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#
637        ));
638        // llama-server's other 400s must still surface to the caller.
639        assert!(!backend.is_context_overflow(
640            r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#
641        ));
642    }
643
644    #[test]
645    fn anthropic_detects_prompt_too_long() {
646        let backend = Backend::Anthropic(config("x"));
647        assert!(
648            backend.is_context_overflow(
649                r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
650            )
651        );
652        assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
653    }
654}