switchyard_llm_client/
backend.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Per-provider backend configuration: wire format, upstream URL, and auth.
5
6use std::time::Duration;
7use std::{
8    collections::{BTreeMap, BTreeSet},
9    fmt,
10};
11
12use reqwest::RequestBuilder;
13use reqwest::header::{HeaderName, HeaderValue};
14use serde_json::Value;
15use switchyard_protocol::{Metadata, WireFormat};
16
17use crate::error::{LlmClientError, Result, is_overflow_body};
18
19const ANTHROPIC_VERSION: &str = "2023-06-01";
20
21/// Default number of retries for server-configured upstream calls.
22pub const DEFAULT_MAX_RETRIES: u32 = 2;
23
24// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
25// provider-wrap is a one-line entry here, not a fork of the parsing logic.
26const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
27    "maximum context length",
28    "context length exceeded",
29    "context window",
30    "context length is only",
31    "please reduce the length of the input",
32    "exceeds the maximum allowed input length",
33    "exceeds the maximum allowed length",
34    "exceeds the available context size",
35    "is longer than the model's context length",
36    // llama-server, both of its ERROR_TYPE_EXCEED_CONTEXT_SIZE messages
37    // (tools/server/server-context.cpp). The structured check below catches
38    // them from `error.type`; these cover a proxy that rewrites the envelope
39    // and keeps the message.
40    "exceeds the available context size",
41    "larger than the max context size",
42];
43
44// Anthropic has no structured `error.code`, so detection is phrase-based only.
45const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
46    "prompt is too long",
47    "maximum number of tokens",
48    "context window",
49    "context length",
50];
51
52/// Shared HTTP configuration for one upstream backend.
53#[derive(Clone)]
54pub struct HttpBackendConfig {
55    /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
56    pub base_url: String,
57    /// API key for the provider, loaded by the caller. `None` sends no configured auth.
58    /// Client construction rejects active values that cannot form the provider's auth header.
59    pub api_key: Option<String>,
60    /// Whether this backend forwards the caller's provider credential and application headers.
61    ///
62    /// The forwarding backends in one route must use one credential family (OpenAI or
63    /// Anthropic) unless they all use the same scheme, host, and port, such as one LLM
64    /// gateway. Backends that send a configured key are not restricted.
65    pub forward_auth: bool,
66    /// Custom headers added to every outbound call to this backend.
67    ///
68    /// Provider-owned headers are rejected so a static value cannot replace
69    /// configured or forwarded auth. Names and values must be valid HTTP header bytes;
70    /// header names are case-insensitive.
71    pub extra_headers: BTreeMap<String, String>,
72    /// Default top-level request fields, applied only when the request omits the key.
73    pub extra_body: BTreeMap<String, Value>,
74    /// Top-level request fields dropped from the outbound body before `extra_body` is merged,
75    /// so a target can reinstate one deliberately. For providers that reject an otherwise
76    /// standard field (e.g. an internal API that 400s on `max_output_tokens`).
77    pub omit_body_fields: BTreeSet<String>,
78    /// Reasoning effort forced on every request to this backend, replacing whatever the caller
79    /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;
80    /// Anthropic has no equivalent and rejects the setting at configuration time.
81    pub reasoning_effort: Option<String>,
82    /// Additional attempts for transient upstream failures.
83    /// One template-compatibility retry may also run within the same timeout.
84    pub max_retries: u32,
85    /// Cooldown after an exhausted transient completion failure. Zero disables it.
86    pub failure_cooldown: Duration,
87    /// Deadline for one complete response, including retries, retry delays, and stream reads.
88    /// `None` leaves the wait unbounded.
89    pub timeout: Option<Duration>,
90}
91
92impl fmt::Debug for HttpBackendConfig {
93    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
94        f.debug_struct("HttpBackendConfig")
95            .field("base_url", &self.base_url)
96            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
97            .field("forward_auth", &self.forward_auth)
98            .field("extra_header_names", &self.extra_headers.keys())
99            .field("extra_body_keys", &self.extra_body.keys())
100            .field("omit_body_fields", &self.omit_body_fields)
101            .field("reasoning_effort", &self.reasoning_effort)
102            .field("max_retries", &self.max_retries)
103            .field("failure_cooldown", &self.failure_cooldown)
104            .field("timeout", &self.timeout)
105            .finish()
106    }
107}
108
109/// A configured upstream backend, one variant per built-in wire format.
110///
111/// The variant fixes the wire format, URL path, and auth scheme together so no
112/// invalid combination can be constructed.
113#[derive(Clone, Debug)]
114pub enum Backend {
115    /// OpenAI-compatible Chat Completions API.
116    OpenAiChat(HttpBackendConfig),
117    /// OpenAI Responses API.
118    OpenAiResponses(HttpBackendConfig),
119    /// Anthropic Messages API.
120    Anthropic(HttpBackendConfig),
121}
122
123impl Backend {
124    // Matches reqwest's header conversions before the client can send a request.
125    pub(crate) fn validate_configured_headers(&self, model_name: &str) -> Result<()> {
126        for (name, value) in &self.config().extra_headers {
127            if HeaderName::from_bytes(name.as_bytes()).is_err() {
128                return Err(LlmClientError::Configuration {
129                    message: format!(
130                        "model {model_name:?} extra_headers contains invalid HTTP header name {name:?}"
131                    ),
132                });
133            }
134            if HeaderValue::from_bytes(value.as_bytes()).is_err() {
135                return Err(LlmClientError::Configuration {
136                    message: format!(
137                        "model {model_name:?} has invalid HTTP header value for extra_headers entry {name:?}"
138                    ),
139                });
140            }
141        }
142
143        let invalid_name = self.config().extra_headers.keys().find(|name| match self {
144            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
145                name.eq_ignore_ascii_case("authorization")
146                    || (self.is_forwarding_auth()
147                        && (name.eq_ignore_ascii_case("chatgpt-account-id")
148                            || name.eq_ignore_ascii_case("x-openai-fedramp")))
149            }
150            Backend::Anthropic(_) => {
151                name.eq_ignore_ascii_case("x-api-key")
152                    || name.eq_ignore_ascii_case("anthropic-version")
153                    || (self.is_forwarding_auth()
154                        && (name.eq_ignore_ascii_case("authorization")
155                            || name.eq_ignore_ascii_case("anthropic-beta")))
156            }
157        });
158        if let Some(name) = invalid_name {
159            return Err(LlmClientError::Configuration {
160                message: format!(
161                    "model {model_name:?} extra_headers cannot set {name:?}; extra_headers is only for additional headers"
162                ),
163            });
164        }
165
166        let Some(api_key) = self.configured_api_key() else {
167            return Ok(());
168        };
169        let valid_api_key = match self {
170            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
171                HeaderValue::try_from(format!("Bearer {api_key}")).is_ok()
172            }
173            Backend::Anthropic(_) => HeaderValue::from_str(api_key).is_ok(),
174        };
175        if !valid_api_key {
176            return Err(LlmClientError::Configuration {
177                message: format!(
178                    "model {model_name:?} api_key cannot be encoded as an HTTP header"
179                ),
180            });
181        }
182        Ok(())
183    }
184
185    /// The wire format the request IR is encoded to for this backend.
186    pub fn wire_format(&self) -> WireFormat {
187        match self {
188            Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
189            Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
190            Backend::Anthropic(_) => WireFormat::AnthropicMessages,
191        }
192    }
193
194    // Shared HTTP config, regardless of variant.
195    fn config(&self) -> &HttpBackendConfig {
196        match self {
197            Backend::OpenAiChat(config)
198            | Backend::OpenAiResponses(config)
199            | Backend::Anthropic(config) => config,
200        }
201    }
202
203    // Static credentials are unused when the caller's authorization is forwarded.
204    fn configured_api_key(&self) -> Option<&str> {
205        if self.is_forwarding_auth() {
206            None
207        } else {
208            self.config().api_key.as_deref()
209        }
210    }
211
212    pub(crate) fn base_url(&self) -> &str {
213        &self.config().base_url
214    }
215
216    /// The fully resolved upstream URL for this backend's endpoint.
217    ///
218    /// Tolerates base URLs that already include the provider path (or a bare
219    /// `/v1`), matching the join rules of the existing native backends.
220    pub fn url(&self) -> String {
221        let base_url = &self.config().base_url;
222        let result = match self {
223            Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
224            Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
225            Backend::Anthropic(_) => anthropic_url(base_url, ""),
226        };
227        result.unwrap_or_else(|error| {
228            tracing::error!(%error, "Unable to build provider endpoint URL");
229            base_url.clone()
230        })
231    }
232
233    /// Applies this backend's configured auth and version headers to a request builder.
234    ///
235    /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
236    /// `x-api-key: <key>` plus the required `anthropic-version` header. A backend
237    /// with `forward_auth` uses the caller's provider credential instead.
238    pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
239        let api_key = self.configured_api_key();
240        match self {
241            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
242                if let Some(api_key) = api_key {
243                    builder = builder.bearer_auth(api_key);
244                }
245            }
246            Backend::Anthropic(_) => {
247                builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
248                if let Some(api_key) = api_key {
249                    builder = builder.header("x-api-key", api_key);
250                }
251            }
252        }
253        builder
254    }
255
256    pub(crate) fn is_forwarding_auth(&self) -> bool {
257        self.config().forward_auth
258    }
259
260    pub(crate) fn is_provider_owned_header(&self, name: &str) -> bool {
261        match self {
262            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
263                ["authorization", "chatgpt-account-id", "x-openai-fedramp"]
264                    .iter()
265                    .any(|owned| name.eq_ignore_ascii_case(owned))
266            }
267            Backend::Anthropic(_) => [
268                "authorization",
269                "x-api-key",
270                "anthropic-beta",
271                "anthropic-version",
272            ]
273            .iter()
274            .any(|owned| name.eq_ignore_ascii_case(owned)),
275        }
276    }
277
278    /// Applies only the caller credential accepted by this provider.
279    pub(crate) fn apply_forwarded_auth(
280        &self,
281        mut builder: RequestBuilder,
282        metadata: Option<&Metadata>,
283    ) -> RequestBuilder {
284        if !self.is_forwarding_auth() {
285            return builder;
286        }
287        let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
288            return builder;
289        };
290        match self {
291            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
292                for name in ["authorization", "chatgpt-account-id", "x-openai-fedramp"] {
293                    if let Some(value) = headers.get(name) {
294                        builder = builder.header(name, sensitive_header(value));
295                    }
296                }
297            }
298            Backend::Anthropic(_) => {
299                for name in ["authorization", "x-api-key"] {
300                    if let Some(value) = headers.get(name) {
301                        builder = builder.header(name, sensitive_header(value));
302                    }
303                }
304                if let Some(value) = headers.get("anthropic-beta")
305                    && let Some(value) = oauth_beta_header(value)
306                {
307                    builder = builder.header("anthropic-beta", value);
308                }
309            }
310        }
311        builder
312    }
313
314    /// Custom per-backend headers to forward on every call.
315    pub fn extra_headers(&self) -> &BTreeMap<String, String> {
316        &self.config().extra_headers
317    }
318
319    /// Default top-level fields to merge into outbound request bodies.
320    pub fn extra_body(&self) -> &BTreeMap<String, Value> {
321        &self.config().extra_body
322    }
323
324    /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.
325    pub fn omit_body_fields(&self) -> &BTreeSet<String> {
326        &self.config().omit_body_fields
327    }
328
329    /// Reasoning effort forced on outbound requests, if the target configures one.
330    pub fn reasoning_effort(&self) -> Option<&str> {
331        self.config().reasoning_effort.as_deref()
332    }
333
334    /// Additional attempts allowed after the initial request.
335    pub fn max_retries(&self) -> u32 {
336        self.config().max_retries
337    }
338
339    /// Cooldown after an exhausted transient completion failure.
340    pub fn failure_cooldown(&self) -> Duration {
341        self.config().failure_cooldown
342    }
343
344    /// Deadline for all attempts and the complete response; `None` leaves the wait unbounded.
345    pub fn timeout(&self) -> Option<Duration> {
346        self.config().timeout
347    }
348
349    /// Whether this backend speaks the Anthropic Messages wire format — the only
350    /// one with a `count_tokens` endpoint.
351    pub fn is_anthropic(&self) -> bool {
352        matches!(self, Backend::Anthropic(_))
353    }
354
355    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
356    /// URL join as [`url`](Self::url).
357    pub fn count_tokens_url(&self) -> String {
358        let base_url = &self.config().base_url;
359        anthropic_url(base_url, "/count_tokens").unwrap_or_else(|error| {
360            tracing::error!(%error, "Unable to build Anthropic token-counting URL");
361            base_url.clone()
362        })
363    }
364
365    /// Whether an upstream 400 `body` looks like a context-window overflow for
366    /// this backend's provider.
367    pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
368        match self {
369            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
370                body,
371                |value| {
372                    let error = value.get("error");
373                    let field = |name| {
374                        error
375                            .and_then(|err| err.get(name))
376                            .and_then(serde_json::Value::as_str)
377                    };
378                    // llama-server puts the HTTP status in `code` as a number and
379                    // names the reason in `type`, so the canonical code never
380                    // matches. Its two overflow messages are worded unlike any
381                    // other provider's, which leaves the type as the signal that
382                    // holds across releases.
383                    field("code") == Some("context_length_exceeded")
384                        || field("type") == Some("exceed_context_size_error")
385                },
386                OPENAI_OVERFLOW_PHRASES,
387            ),
388            Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
389        }
390    }
391}
392
393// Retains OAuth markers while keeping provider feature betas backend-owned.
394fn sensitive_header(value: &HeaderValue) -> HeaderValue {
395    let mut value = value.clone();
396    value.set_sensitive(true);
397    value
398}
399
400fn oauth_beta_header(value: &HeaderValue) -> Option<HeaderValue> {
401    let oauth_betas = value
402        .to_str()
403        .ok()?
404        .split(',')
405        .map(str::trim)
406        .filter(|beta| {
407            beta.get(..6)
408                .is_some_and(|prefix| prefix.eq_ignore_ascii_case("oauth-"))
409        });
410    let value = oauth_betas.collect::<Vec<_>>().join(",");
411    if value.is_empty() {
412        return None;
413    }
414    let mut value = HeaderValue::from_str(&value).ok()?;
415    value.set_sensitive(true);
416    Some(value)
417}
418
419// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
420pub(crate) fn openai_url(base_url: &str, suffix: &str) -> Result<String> {
421    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
422        message: format!("Invalid OpenAI base URL: {error}"),
423    })?;
424    let base_path = url.path().trim_end_matches('/');
425    let base_root = base_path
426        .strip_suffix("/chat/completions")
427        .or_else(|| base_path.strip_suffix("/responses"))
428        .unwrap_or(base_path);
429    url.set_path(&format!("{base_root}{suffix}"));
430    Ok(url.into())
431}
432
433// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
434fn anthropic_url(base_url: &str, suffix: &str) -> Result<String> {
435    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
436        message: format!("Invalid Anthropic base URL: {error}"),
437    })?;
438    let base_path = url.path().trim_end_matches('/');
439    let messages_path = if base_path.ends_with("/v1/messages") {
440        base_path.to_string()
441    } else if base_path.ends_with("/v1") {
442        format!("{base_path}/messages")
443    } else {
444        format!("{base_path}/v1/messages")
445    };
446    url.set_path(&format!("{messages_path}{suffix}"));
447    Ok(url.into())
448}
449
450#[cfg(test)]
451mod tests {
452    use super::*;
453
454    fn config(base_url: &str) -> HttpBackendConfig {
455        HttpBackendConfig {
456            base_url: base_url.to_string(),
457            api_key: Some("secret".to_string()),
458            forward_auth: false,
459            extra_headers: BTreeMap::new(),
460            extra_body: BTreeMap::new(),
461            omit_body_fields: BTreeSet::new(),
462            reasoning_effort: None,
463            max_retries: 0,
464            failure_cooldown: Duration::ZERO,
465            timeout: None,
466        }
467    }
468
469    #[test]
470    fn openai_chat_url_joins_bare_v1() {
471        let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
472        assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
473    }
474
475    #[test]
476    fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
477        assert_eq!(
478            Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
479            "https://api.openai.com/v1/chat/completions"
480        );
481        assert_eq!(
482            Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
483            "https://api.openai.com/v1/chat/completions"
484        );
485    }
486
487    #[test]
488    fn openai_responses_url_uses_responses_path() {
489        assert_eq!(
490            Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
491            "https://api.openai.com/v1/responses"
492        );
493    }
494
495    #[test]
496    fn anthropic_url_join_cases() {
497        assert_eq!(
498            Backend::Anthropic(config("https://api.anthropic.com")).url(),
499            "https://api.anthropic.com/v1/messages"
500        );
501        assert_eq!(
502            Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
503            "https://api.anthropic.com/v1/messages"
504        );
505        assert_eq!(
506            Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
507            "https://api.anthropic.com/v1/messages"
508        );
509    }
510
511    #[test]
512    fn wire_format_matches_variant() {
513        assert_eq!(
514            Backend::OpenAiChat(config("x")).wire_format(),
515            WireFormat::OpenAiChat
516        );
517        assert_eq!(
518            Backend::OpenAiResponses(config("x")).wire_format(),
519            WireFormat::OpenAiResponses
520        );
521        assert_eq!(
522            Backend::Anthropic(config("x")).wire_format(),
523            WireFormat::AnthropicMessages
524        );
525    }
526
527    // Header validation follows reqwest for both accepted and rejected bytes.
528    #[test]
529    fn validates_additional_header_bytes() {
530        let cases = [
531            ("x-display-name", "café", None),
532            (
533                "bad header",
534                "value",
535                Some("invalid HTTP header name \"bad header\""),
536            ),
537            (
538                "x-test-header",
539                "bad\nvalue",
540                Some("invalid HTTP header value for extra_headers entry \"x-test-header\""),
541            ),
542        ];
543
544        for (name, value, expected) in cases {
545            let mut config = config("x");
546            config
547                .extra_headers
548                .insert(name.to_string(), value.to_string());
549            let result = Backend::OpenAiChat(config).validate_configured_headers("model");
550            match expected {
551                Some(expected) => assert!(
552                    result.is_err_and(|error| error.to_string().contains(expected)),
553                    "expected {expected:?}"
554                ),
555                None => result.expect("encodable header must pass validation"),
556            }
557        }
558    }
559
560    // Only static credentials that apply_auth would send are validated.
561    #[test]
562    fn configured_api_key_validation_matches_auth_application() {
563        const INVALID_KEY: &str = "canary\nsecret";
564        let mut config = config("x");
565        config.api_key = Some(INVALID_KEY.to_string());
566        let builders: [fn(HttpBackendConfig) -> Backend; 2] =
567            [Backend::OpenAiChat, Backend::Anthropic];
568        let client = reqwest::Client::new();
569
570        for build_backend in builders {
571            let error = build_backend(config.clone())
572                .validate_configured_headers("model")
573                .expect_err("invalid API key must fail")
574                .to_string();
575            assert!(
576                error.contains("api_key cannot be encoded as an HTTP header"),
577                "{error}"
578            );
579            assert!(!error.contains(INVALID_KEY), "API key leaked in: {error}");
580
581            let mut forwarded = config.clone();
582            forwarded.forward_auth = true;
583            let backend = build_backend(forwarded);
584            backend
585                .validate_configured_headers("model")
586                .expect("unused API key must not fail validation");
587            let request = backend
588                .apply_auth(client.get("https://example.test"))
589                .build()
590                .expect("request");
591            assert!(!request.headers().contains_key("authorization"));
592            assert!(!request.headers().contains_key("x-api-key"));
593        }
594    }
595
596    #[test]
597    fn openai_detects_canonical_and_wrapped_overflow() {
598        let backend = Backend::OpenAiChat(config("x"));
599        assert!(
600            backend.is_context_overflow(
601                r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
602            )
603        );
604        // NVIDIA/LiteLLM message wrap with no structured code.
605        assert!(backend.is_context_overflow(
606            r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
607        ));
608        assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
609        // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
610        assert!(backend.is_context_overflow(
611            r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
612        ));
613        // Native SGLang: top-level error envelope (no `error` key).
614        // KV-pool rejection (managers/utils.py) and declared-context rejection
615        // (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.
616        assert!(backend.is_context_overflow(
617            r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#
618        ));
619        assert!(backend.is_context_overflow(
620            r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#
621        ));
622    }
623
624    #[test]
625    fn openai_detects_llama_server_overflow() {
626        let backend = Backend::OpenAiChat(config("x"));
627        // llama-server, numeric `code` and the reason in `type`. Both of the
628        // messages ERROR_TYPE_EXCEED_CONTEXT_SIZE can carry are here.
629        assert!(backend.is_context_overflow(
630            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#
631        ));
632        assert!(backend.is_context_overflow(
633            r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#
634        ));
635        // The type alone is enough, so a reworded message still falls back.
636        assert!(backend.is_context_overflow(
637            r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#
638        ));
639        // And the message alone is enough, for a proxy that drops the type.
640        assert!(backend.is_context_overflow(
641            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#
642        ));
643        // llama-server's other 400s must still surface to the caller.
644        assert!(!backend.is_context_overflow(
645            r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#
646        ));
647    }
648
649    #[test]
650    fn anthropic_detects_prompt_too_long() {
651        let backend = Backend::Anthropic(config("x"));
652        assert!(
653            backend.is_context_overflow(
654                r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
655            )
656        );
657        assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
658    }
659}
switchyard_llm_client/backend.rs

switchyard_llm_client/
backend.rs
