switchyard_llm_client/
backend.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Per-provider backend configuration: wire format, upstream URL, and auth.
5
6use std::time::Duration;
7use std::{
8    collections::{BTreeMap, BTreeSet},
9    fmt,
10};
11
12use reqwest::RequestBuilder;
13use reqwest::header::{HeaderName, HeaderValue};
14use serde_json::Value;
15use switchyard_protocol::{Metadata, WireFormat};
16
17use crate::error::{LlmClientError, Result, is_overflow_body};
18
19const ANTHROPIC_VERSION: &str = "2023-06-01";
20
21/// Default number of retries for server-configured upstream calls.
22pub const DEFAULT_MAX_RETRIES: u32 = 2;
23
24// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
25// provider-wrap is a one-line entry here, not a fork of the parsing logic.
26const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
27    "maximum context length",
28    "context length exceeded",
29    "context window",
30    "context length is only",
31    "please reduce the length of the input",
32    "exceeds the maximum allowed input length",
33    "exceeds the maximum allowed length",
34    "exceeds the available context size",
35    "is longer than the model's context length",
36    // llama-server, both of its ERROR_TYPE_EXCEED_CONTEXT_SIZE messages
37    // (tools/server/server-context.cpp). The structured check below catches
38    // them from `error.type`; these cover a proxy that rewrites the envelope
39    // and keeps the message.
40    "exceeds the available context size",
41    "larger than the max context size",
42];
43
44// Anthropic has no structured `error.code`, so detection is phrase-based only.
45const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
46    "prompt is too long",
47    "maximum number of tokens",
48    "context window",
49    "context length",
50];
51
52/// Shared HTTP configuration for one upstream backend.
53#[derive(Clone)]
54pub struct HttpBackendConfig {
55    /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
56    pub base_url: String,
57    /// API key for the provider, loaded by the caller. `None` sends no configured auth.
58    /// Client construction rejects active values that cannot form the provider's auth header.
59    pub api_key: Option<String>,
60    /// Whether this backend forwards the caller's provider credential and application headers.
61    ///
62    /// The forwarding backends in one route must use one credential family (OpenAI or
63    /// Anthropic) unless they all use the same scheme, host, and port, such as one LLM
64    /// gateway. Backends that send a configured key are not restricted.
65    pub forward_auth: bool,
66    /// Custom headers added to every outbound call to this backend.
67    ///
68    /// Provider-owned headers are rejected so a static value cannot replace
69    /// configured or forwarded auth. Names and values must be valid HTTP header bytes;
70    /// header names are case-insensitive.
71    pub extra_headers: BTreeMap<String, String>,
72    /// Default top-level request fields, applied only when the request omits the key.
73    pub extra_body: BTreeMap<String, Value>,
74    /// Top-level request fields dropped from the outbound body before `extra_body` is merged,
75    /// so a target can reinstate one deliberately. For providers that reject an otherwise
76    /// standard field (e.g. an internal API that 400s on `max_output_tokens`).
77    pub omit_body_fields: BTreeSet<String>,
78    /// Reasoning effort forced on every request to this backend, replacing whatever the caller
79    /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;
80    /// Anthropic has no equivalent and rejects the setting at configuration time.
81    pub reasoning_effort: Option<String>,
82    /// Additional attempts after the initial upstream request.
83    pub max_retries: u32,
84    /// Cooldown after an exhausted transient completion failure. Zero disables it.
85    pub failure_cooldown: Duration,
86    /// Deadline for one complete response, including retries, retry delays, and stream reads.
87    /// `None` leaves the wait unbounded.
88    pub timeout: Option<Duration>,
89}
90
91impl fmt::Debug for HttpBackendConfig {
92    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
93        f.debug_struct("HttpBackendConfig")
94            .field("base_url", &self.base_url)
95            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
96            .field("forward_auth", &self.forward_auth)
97            .field("extra_header_names", &self.extra_headers.keys())
98            .field("extra_body_keys", &self.extra_body.keys())
99            .field("omit_body_fields", &self.omit_body_fields)
100            .field("reasoning_effort", &self.reasoning_effort)
101            .field("max_retries", &self.max_retries)
102            .field("failure_cooldown", &self.failure_cooldown)
103            .field("timeout", &self.timeout)
104            .finish()
105    }
106}
107
108/// A configured upstream backend, one variant per built-in wire format.
109///
110/// The variant fixes the wire format, URL path, and auth scheme together so no
111/// invalid combination can be constructed.
112#[derive(Clone, Debug)]
113pub enum Backend {
114    /// OpenAI-compatible Chat Completions API.
115    OpenAiChat(HttpBackendConfig),
116    /// OpenAI Responses API.
117    OpenAiResponses(HttpBackendConfig),
118    /// Anthropic Messages API.
119    Anthropic(HttpBackendConfig),
120}
121
122impl Backend {
123    // Matches reqwest's header conversions before the client can send a request.
124    pub(crate) fn validate_configured_headers(&self, model_name: &str) -> Result<()> {
125        for (name, value) in &self.config().extra_headers {
126            if HeaderName::from_bytes(name.as_bytes()).is_err() {
127                return Err(LlmClientError::Configuration {
128                    message: format!(
129                        "model {model_name:?} extra_headers contains invalid HTTP header name {name:?}"
130                    ),
131                });
132            }
133            if HeaderValue::from_bytes(value.as_bytes()).is_err() {
134                return Err(LlmClientError::Configuration {
135                    message: format!(
136                        "model {model_name:?} has invalid HTTP header value for extra_headers entry {name:?}"
137                    ),
138                });
139            }
140        }
141
142        let invalid_name = self.config().extra_headers.keys().find(|name| match self {
143            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
144                name.eq_ignore_ascii_case("authorization")
145                    || (self.is_forwarding_auth()
146                        && (name.eq_ignore_ascii_case("chatgpt-account-id")
147                            || name.eq_ignore_ascii_case("x-openai-fedramp")))
148            }
149            Backend::Anthropic(_) => {
150                name.eq_ignore_ascii_case("x-api-key")
151                    || name.eq_ignore_ascii_case("anthropic-version")
152                    || (self.is_forwarding_auth()
153                        && (name.eq_ignore_ascii_case("authorization")
154                            || name.eq_ignore_ascii_case("anthropic-beta")))
155            }
156        });
157        if let Some(name) = invalid_name {
158            return Err(LlmClientError::Configuration {
159                message: format!(
160                    "model {model_name:?} extra_headers cannot set {name:?}; extra_headers is only for additional headers"
161                ),
162            });
163        }
164
165        let Some(api_key) = self.configured_api_key() else {
166            return Ok(());
167        };
168        let valid_api_key = match self {
169            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
170                HeaderValue::try_from(format!("Bearer {api_key}")).is_ok()
171            }
172            Backend::Anthropic(_) => HeaderValue::from_str(api_key).is_ok(),
173        };
174        if !valid_api_key {
175            return Err(LlmClientError::Configuration {
176                message: format!(
177                    "model {model_name:?} api_key cannot be encoded as an HTTP header"
178                ),
179            });
180        }
181        Ok(())
182    }
183
184    /// The wire format the request IR is encoded to for this backend.
185    pub fn wire_format(&self) -> WireFormat {
186        match self {
187            Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
188            Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
189            Backend::Anthropic(_) => WireFormat::AnthropicMessages,
190        }
191    }
192
193    // Shared HTTP config, regardless of variant.
194    fn config(&self) -> &HttpBackendConfig {
195        match self {
196            Backend::OpenAiChat(config)
197            | Backend::OpenAiResponses(config)
198            | Backend::Anthropic(config) => config,
199        }
200    }
201
202    // Static credentials are unused when the caller's authorization is forwarded.
203    fn configured_api_key(&self) -> Option<&str> {
204        if self.is_forwarding_auth() {
205            None
206        } else {
207            self.config().api_key.as_deref()
208        }
209    }
210
211    pub(crate) fn base_url(&self) -> &str {
212        &self.config().base_url
213    }
214
215    /// The fully resolved upstream URL for this backend's endpoint.
216    ///
217    /// Tolerates base URLs that already include the provider path (or a bare
218    /// `/v1`), matching the join rules of the existing native backends.
219    pub fn url(&self) -> String {
220        let base_url = &self.config().base_url;
221        let result = match self {
222            Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
223            Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
224            Backend::Anthropic(_) => anthropic_url(base_url, ""),
225        };
226        result.unwrap_or_else(|error| {
227            tracing::error!(%error, "Unable to build provider endpoint URL");
228            base_url.clone()
229        })
230    }
231
232    /// Applies this backend's configured auth and version headers to a request builder.
233    ///
234    /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
235    /// `x-api-key: <key>` plus the required `anthropic-version` header. A backend
236    /// with `forward_auth` uses the caller's provider credential instead.
237    pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
238        let api_key = self.configured_api_key();
239        match self {
240            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
241                if let Some(api_key) = api_key {
242                    builder = builder.bearer_auth(api_key);
243                }
244            }
245            Backend::Anthropic(_) => {
246                builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
247                if let Some(api_key) = api_key {
248                    builder = builder.header("x-api-key", api_key);
249                }
250            }
251        }
252        builder
253    }
254
255    pub(crate) fn is_forwarding_auth(&self) -> bool {
256        self.config().forward_auth
257    }
258
259    pub(crate) fn is_provider_owned_header(&self, name: &str) -> bool {
260        match self {
261            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
262                ["authorization", "chatgpt-account-id", "x-openai-fedramp"]
263                    .iter()
264                    .any(|owned| name.eq_ignore_ascii_case(owned))
265            }
266            Backend::Anthropic(_) => [
267                "authorization",
268                "x-api-key",
269                "anthropic-beta",
270                "anthropic-version",
271            ]
272            .iter()
273            .any(|owned| name.eq_ignore_ascii_case(owned)),
274        }
275    }
276
277    /// Applies only the caller credential accepted by this provider.
278    pub(crate) fn apply_forwarded_auth(
279        &self,
280        mut builder: RequestBuilder,
281        metadata: Option<&Metadata>,
282    ) -> RequestBuilder {
283        if !self.is_forwarding_auth() {
284            return builder;
285        }
286        let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
287            return builder;
288        };
289        match self {
290            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
291                for name in ["authorization", "chatgpt-account-id", "x-openai-fedramp"] {
292                    if let Some(value) = headers.get(name) {
293                        builder = builder.header(name, sensitive_header(value));
294                    }
295                }
296            }
297            Backend::Anthropic(_) => {
298                for name in ["authorization", "x-api-key"] {
299                    if let Some(value) = headers.get(name) {
300                        builder = builder.header(name, sensitive_header(value));
301                    }
302                }
303                if let Some(value) = headers.get("anthropic-beta")
304                    && let Some(value) = oauth_beta_header(value)
305                {
306                    builder = builder.header("anthropic-beta", value);
307                }
308            }
309        }
310        builder
311    }
312
313    /// Custom per-backend headers to forward on every call.
314    pub fn extra_headers(&self) -> &BTreeMap<String, String> {
315        &self.config().extra_headers
316    }
317
318    /// Default top-level fields to merge into outbound request bodies.
319    pub fn extra_body(&self) -> &BTreeMap<String, Value> {
320        &self.config().extra_body
321    }
322
323    /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.
324    pub fn omit_body_fields(&self) -> &BTreeSet<String> {
325        &self.config().omit_body_fields
326    }
327
328    /// Reasoning effort forced on outbound requests, if the target configures one.
329    pub fn reasoning_effort(&self) -> Option<&str> {
330        self.config().reasoning_effort.as_deref()
331    }
332
333    /// Additional attempts allowed after the initial request.
334    pub fn max_retries(&self) -> u32 {
335        self.config().max_retries
336    }
337
338    /// Cooldown after an exhausted transient completion failure.
339    pub fn failure_cooldown(&self) -> Duration {
340        self.config().failure_cooldown
341    }
342
343    /// Deadline for all attempts and the complete response; `None` leaves the wait unbounded.
344    pub fn timeout(&self) -> Option<Duration> {
345        self.config().timeout
346    }
347
348    /// Whether this backend speaks the Anthropic Messages wire format — the only
349    /// one with a `count_tokens` endpoint.
350    pub fn is_anthropic(&self) -> bool {
351        matches!(self, Backend::Anthropic(_))
352    }
353
354    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
355    /// URL join as [`url`](Self::url).
356    pub fn count_tokens_url(&self) -> String {
357        let base_url = &self.config().base_url;
358        anthropic_url(base_url, "/count_tokens").unwrap_or_else(|error| {
359            tracing::error!(%error, "Unable to build Anthropic token-counting URL");
360            base_url.clone()
361        })
362    }
363
364    /// Whether an upstream 400 `body` looks like a context-window overflow for
365    /// this backend's provider.
366    pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
367        match self {
368            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
369                body,
370                |value| {
371                    let error = value.get("error");
372                    let field = |name| {
373                        error
374                            .and_then(|err| err.get(name))
375                            .and_then(serde_json::Value::as_str)
376                    };
377                    // llama-server puts the HTTP status in `code` as a number and
378                    // names the reason in `type`, so the canonical code never
379                    // matches. Its two overflow messages are worded unlike any
380                    // other provider's, which leaves the type as the signal that
381                    // holds across releases.
382                    field("code") == Some("context_length_exceeded")
383                        || field("type") == Some("exceed_context_size_error")
384                },
385                OPENAI_OVERFLOW_PHRASES,
386            ),
387            Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
388        }
389    }
390}
391
392// Retains OAuth markers while keeping provider feature betas backend-owned.
393fn sensitive_header(value: &HeaderValue) -> HeaderValue {
394    let mut value = value.clone();
395    value.set_sensitive(true);
396    value
397}
398
399fn oauth_beta_header(value: &HeaderValue) -> Option<HeaderValue> {
400    let oauth_betas = value
401        .to_str()
402        .ok()?
403        .split(',')
404        .map(str::trim)
405        .filter(|beta| {
406            beta.get(..6)
407                .is_some_and(|prefix| prefix.eq_ignore_ascii_case("oauth-"))
408        });
409    let value = oauth_betas.collect::<Vec<_>>().join(",");
410    if value.is_empty() {
411        return None;
412    }
413    let mut value = HeaderValue::from_str(&value).ok()?;
414    value.set_sensitive(true);
415    Some(value)
416}
417
418// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
419pub(crate) fn openai_url(base_url: &str, suffix: &str) -> Result<String> {
420    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
421        message: format!("Invalid OpenAI base URL: {error}"),
422    })?;
423    let base_path = url.path().trim_end_matches('/');
424    let base_root = base_path
425        .strip_suffix("/chat/completions")
426        .or_else(|| base_path.strip_suffix("/responses"))
427        .unwrap_or(base_path);
428    url.set_path(&format!("{base_root}{suffix}"));
429    Ok(url.into())
430}
431
432// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
433fn anthropic_url(base_url: &str, suffix: &str) -> Result<String> {
434    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
435        message: format!("Invalid Anthropic base URL: {error}"),
436    })?;
437    let base_path = url.path().trim_end_matches('/');
438    let messages_path = if base_path.ends_with("/v1/messages") {
439        base_path.to_string()
440    } else if base_path.ends_with("/v1") {
441        format!("{base_path}/messages")
442    } else {
443        format!("{base_path}/v1/messages")
444    };
445    url.set_path(&format!("{messages_path}{suffix}"));
446    Ok(url.into())
447}
448
449#[cfg(test)]
450mod tests {
451    use super::*;
452
453    fn config(base_url: &str) -> HttpBackendConfig {
454        HttpBackendConfig {
455            base_url: base_url.to_string(),
456            api_key: Some("secret".to_string()),
457            forward_auth: false,
458            extra_headers: BTreeMap::new(),
459            extra_body: BTreeMap::new(),
460            omit_body_fields: BTreeSet::new(),
461            reasoning_effort: None,
462            max_retries: 0,
463            failure_cooldown: Duration::ZERO,
464            timeout: None,
465        }
466    }
467
468    #[test]
469    fn openai_chat_url_joins_bare_v1() {
470        let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
471        assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
472    }
473
474    #[test]
475    fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
476        assert_eq!(
477            Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
478            "https://api.openai.com/v1/chat/completions"
479        );
480        assert_eq!(
481            Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
482            "https://api.openai.com/v1/chat/completions"
483        );
484    }
485
486    #[test]
487    fn openai_responses_url_uses_responses_path() {
488        assert_eq!(
489            Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
490            "https://api.openai.com/v1/responses"
491        );
492    }
493
494    #[test]
495    fn anthropic_url_join_cases() {
496        assert_eq!(
497            Backend::Anthropic(config("https://api.anthropic.com")).url(),
498            "https://api.anthropic.com/v1/messages"
499        );
500        assert_eq!(
501            Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
502            "https://api.anthropic.com/v1/messages"
503        );
504        assert_eq!(
505            Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
506            "https://api.anthropic.com/v1/messages"
507        );
508    }
509
510    #[test]
511    fn wire_format_matches_variant() {
512        assert_eq!(
513            Backend::OpenAiChat(config("x")).wire_format(),
514            WireFormat::OpenAiChat
515        );
516        assert_eq!(
517            Backend::OpenAiResponses(config("x")).wire_format(),
518            WireFormat::OpenAiResponses
519        );
520        assert_eq!(
521            Backend::Anthropic(config("x")).wire_format(),
522            WireFormat::AnthropicMessages
523        );
524    }
525
526    // Header validation follows reqwest for both accepted and rejected bytes.
527    #[test]
528    fn validates_additional_header_bytes() {
529        let cases = [
530            ("x-display-name", "café", None),
531            (
532                "bad header",
533                "value",
534                Some("invalid HTTP header name \"bad header\""),
535            ),
536            (
537                "x-test-header",
538                "bad\nvalue",
539                Some("invalid HTTP header value for extra_headers entry \"x-test-header\""),
540            ),
541        ];
542
543        for (name, value, expected) in cases {
544            let mut config = config("x");
545            config
546                .extra_headers
547                .insert(name.to_string(), value.to_string());
548            let result = Backend::OpenAiChat(config).validate_configured_headers("model");
549            match expected {
550                Some(expected) => assert!(
551                    result.is_err_and(|error| error.to_string().contains(expected)),
552                    "expected {expected:?}"
553                ),
554                None => result.expect("encodable header must pass validation"),
555            }
556        }
557    }
558
559    // Only static credentials that apply_auth would send are validated.
560    #[test]
561    fn configured_api_key_validation_matches_auth_application() {
562        const INVALID_KEY: &str = "canary\nsecret";
563        let mut config = config("x");
564        config.api_key = Some(INVALID_KEY.to_string());
565        let builders: [fn(HttpBackendConfig) -> Backend; 2] =
566            [Backend::OpenAiChat, Backend::Anthropic];
567        let client = reqwest::Client::new();
568
569        for build_backend in builders {
570            let error = build_backend(config.clone())
571                .validate_configured_headers("model")
572                .expect_err("invalid API key must fail")
573                .to_string();
574            assert!(
575                error.contains("api_key cannot be encoded as an HTTP header"),
576                "{error}"
577            );
578            assert!(!error.contains(INVALID_KEY), "API key leaked in: {error}");
579
580            let mut forwarded = config.clone();
581            forwarded.forward_auth = true;
582            let backend = build_backend(forwarded);
583            backend
584                .validate_configured_headers("model")
585                .expect("unused API key must not fail validation");
586            let request = backend
587                .apply_auth(client.get("https://example.test"))
588                .build()
589                .expect("request");
590            assert!(!request.headers().contains_key("authorization"));
591            assert!(!request.headers().contains_key("x-api-key"));
592        }
593    }
594
595    #[test]
596    fn openai_detects_canonical_and_wrapped_overflow() {
597        let backend = Backend::OpenAiChat(config("x"));
598        assert!(
599            backend.is_context_overflow(
600                r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
601            )
602        );
603        // NVIDIA/LiteLLM message wrap with no structured code.
604        assert!(backend.is_context_overflow(
605            r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
606        ));
607        assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
608        // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
609        assert!(backend.is_context_overflow(
610            r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
611        ));
612        // Native SGLang: top-level error envelope (no `error` key).
613        // KV-pool rejection (managers/utils.py) and declared-context rejection
614        // (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.
615        assert!(backend.is_context_overflow(
616            r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#
617        ));
618        assert!(backend.is_context_overflow(
619            r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#
620        ));
621    }
622
623    #[test]
624    fn openai_detects_llama_server_overflow() {
625        let backend = Backend::OpenAiChat(config("x"));
626        // llama-server, numeric `code` and the reason in `type`. Both of the
627        // messages ERROR_TYPE_EXCEED_CONTEXT_SIZE can carry are here.
628        assert!(backend.is_context_overflow(
629            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#
630        ));
631        assert!(backend.is_context_overflow(
632            r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#
633        ));
634        // The type alone is enough, so a reworded message still falls back.
635        assert!(backend.is_context_overflow(
636            r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#
637        ));
638        // And the message alone is enough, for a proxy that drops the type.
639        assert!(backend.is_context_overflow(
640            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#
641        ));
642        // llama-server's other 400s must still surface to the caller.
643        assert!(!backend.is_context_overflow(
644            r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#
645        ));
646    }
647
648    #[test]
649    fn anthropic_detects_prompt_too_long() {
650        let backend = Backend::Anthropic(config("x"));
651        assert!(
652            backend.is_context_overflow(
653                r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
654            )
655        );
656        assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
657    }
658}
switchyard_llm_client/backend.rs

switchyard_llm_client/
backend.rs
