1use std::time::Duration;
7use std::{
8 collections::{BTreeMap, BTreeSet},
9 fmt,
10};
11
12use reqwest::RequestBuilder;
13use reqwest::header::{HeaderName, HeaderValue};
14use serde_json::Value;
15use switchyard_protocol::{Metadata, WireFormat};
16
17use crate::error::{LlmClientError, Result, is_overflow_body};
18
19const ANTHROPIC_VERSION: &str = "2023-06-01";
20
21pub const DEFAULT_MAX_RETRIES: u32 = 2;
23
24const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
27 "maximum context length",
28 "context length exceeded",
29 "context window",
30 "context length is only",
31 "please reduce the length of the input",
32 "exceeds the maximum allowed input length",
33 "exceeds the maximum allowed length",
34 "exceeds the available context size",
35 "is longer than the model's context length",
36 "exceeds the available context size",
41 "larger than the max context size",
42];
43
44const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
46 "prompt is too long",
47 "maximum number of tokens",
48 "context window",
49 "context length",
50];
51
52#[derive(Clone)]
54pub struct HttpBackendConfig {
55 pub base_url: String,
57 pub api_key: Option<String>,
60 pub forward_auth: bool,
66 pub extra_headers: BTreeMap<String, String>,
73 pub extra_body: BTreeMap<String, Value>,
75 pub omit_body_fields: BTreeSet<String>,
79 pub reasoning_effort: Option<String>,
83 pub max_retries: u32,
86 pub failure_cooldown: Duration,
88 pub timeout: Option<Duration>,
91}
92
93impl fmt::Debug for HttpBackendConfig {
94 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
95 f.debug_struct("HttpBackendConfig")
96 .field("base_url", &self.base_url)
97 .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
98 .field("forward_auth", &self.forward_auth)
99 .field("extra_header_names", &self.extra_headers.keys())
100 .field("extra_body_keys", &self.extra_body.keys())
101 .field("omit_body_fields", &self.omit_body_fields)
102 .field("reasoning_effort", &self.reasoning_effort)
103 .field("max_retries", &self.max_retries)
104 .field("failure_cooldown", &self.failure_cooldown)
105 .field("timeout", &self.timeout)
106 .finish()
107 }
108}
109
110#[derive(Clone, Debug)]
115pub enum Backend {
116 OpenAiChat(HttpBackendConfig),
118 OpenAiResponses(HttpBackendConfig),
120 Anthropic(HttpBackendConfig),
122}
123
124impl Backend {
125 pub(crate) fn validate_configured_headers(&self, model_name: &str) -> Result<()> {
127 for (name, value) in &self.config().extra_headers {
128 if HeaderName::from_bytes(name.as_bytes()).is_err() {
129 return Err(LlmClientError::Configuration {
130 message: format!(
131 "model {model_name:?} extra_headers contains invalid HTTP header name {name:?}"
132 ),
133 });
134 }
135 if HeaderValue::from_bytes(value.as_bytes()).is_err() {
136 return Err(LlmClientError::Configuration {
137 message: format!(
138 "model {model_name:?} has invalid HTTP header value for extra_headers entry {name:?}"
139 ),
140 });
141 }
142 }
143
144 let invalid_name = self.config().extra_headers.keys().find(|name| match self {
145 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
146 name.eq_ignore_ascii_case("authorization")
147 || (self.is_forwarding_auth()
148 && (name.eq_ignore_ascii_case("chatgpt-account-id")
149 || name.eq_ignore_ascii_case("x-openai-fedramp")))
150 }
151 Backend::Anthropic(_) => {
152 name.eq_ignore_ascii_case("x-api-key")
153 || name.eq_ignore_ascii_case("anthropic-version")
154 || (self.is_forwarding_auth()
155 && (name.eq_ignore_ascii_case("authorization")
156 || name.eq_ignore_ascii_case("anthropic-beta")))
157 }
158 });
159 if let Some(name) = invalid_name {
160 return Err(LlmClientError::Configuration {
161 message: format!(
162 "model {model_name:?} extra_headers cannot set {name:?}; extra_headers is only for additional headers"
163 ),
164 });
165 }
166
167 let Some(api_key) = self.configured_api_key() else {
168 return Ok(());
169 };
170 let valid_api_key = match self {
171 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
172 HeaderValue::try_from(format!("Bearer {api_key}")).is_ok()
173 }
174 Backend::Anthropic(_) => HeaderValue::from_str(api_key).is_ok(),
175 };
176 if !valid_api_key {
177 return Err(LlmClientError::Configuration {
178 message: format!(
179 "model {model_name:?} api_key cannot be encoded as an HTTP header"
180 ),
181 });
182 }
183 Ok(())
184 }
185
186 pub fn wire_format(&self) -> WireFormat {
188 match self {
189 Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
190 Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
191 Backend::Anthropic(_) => WireFormat::AnthropicMessages,
192 }
193 }
194
195 fn config(&self) -> &HttpBackendConfig {
197 match self {
198 Backend::OpenAiChat(config)
199 | Backend::OpenAiResponses(config)
200 | Backend::Anthropic(config) => config,
201 }
202 }
203
204 fn configured_api_key(&self) -> Option<&str> {
206 if self.is_forwarding_auth() {
207 None
208 } else {
209 self.config().api_key.as_deref()
210 }
211 }
212
213 pub(crate) fn base_url(&self) -> &str {
214 &self.config().base_url
215 }
216
217 pub fn url(&self) -> String {
222 let base_url = &self.config().base_url;
223 let result = match self {
224 Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
225 Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
226 Backend::Anthropic(_) => anthropic_url(base_url, ""),
227 };
228 result.unwrap_or_else(|error| {
229 tracing::error!(%error, "Unable to build provider endpoint URL");
230 base_url.clone()
231 })
232 }
233
234 pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
240 let api_key = self.configured_api_key();
241 match self {
242 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
243 if let Some(api_key) = api_key {
244 builder = builder.bearer_auth(api_key);
245 }
246 }
247 Backend::Anthropic(_) => {
248 builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
249 if let Some(api_key) = api_key {
250 builder = builder.header("x-api-key", api_key);
251 }
252 }
253 }
254 builder
255 }
256
257 pub(crate) fn is_forwarding_auth(&self) -> bool {
258 self.config().forward_auth
259 }
260
261 pub(crate) fn is_provider_owned_header(&self, name: &str) -> bool {
262 match self {
263 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
264 ["authorization", "chatgpt-account-id", "x-openai-fedramp"]
265 .iter()
266 .any(|owned| name.eq_ignore_ascii_case(owned))
267 }
268 Backend::Anthropic(_) => [
269 "authorization",
270 "x-api-key",
271 "anthropic-beta",
272 "anthropic-version",
273 ]
274 .iter()
275 .any(|owned| name.eq_ignore_ascii_case(owned)),
276 }
277 }
278
279 pub(crate) fn apply_forwarded_auth(
281 &self,
282 mut builder: RequestBuilder,
283 metadata: Option<&Metadata>,
284 ) -> RequestBuilder {
285 if !self.is_forwarding_auth() {
286 return builder;
287 }
288 let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
289 return builder;
290 };
291 match self {
292 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
293 for name in ["authorization", "chatgpt-account-id", "x-openai-fedramp"] {
294 if let Some(value) = headers.get(name) {
295 builder = builder.header(name, sensitive_header(value));
296 }
297 }
298 }
299 Backend::Anthropic(_) => {
300 for name in ["authorization", "x-api-key"] {
301 if let Some(value) = headers.get(name) {
302 builder = builder.header(name, sensitive_header(value));
303 }
304 }
305 if let Some(value) = headers.get("anthropic-beta")
306 && let Some(value) = oauth_beta_header(value)
307 {
308 builder = builder.header("anthropic-beta", value);
309 }
310 }
311 }
312 builder
313 }
314
315 pub fn extra_headers(&self) -> &BTreeMap<String, String> {
317 &self.config().extra_headers
318 }
319
320 pub fn extra_body(&self) -> &BTreeMap<String, Value> {
322 &self.config().extra_body
323 }
324
325 pub fn omit_body_fields(&self) -> &BTreeSet<String> {
327 &self.config().omit_body_fields
328 }
329
330 pub fn reasoning_effort(&self) -> Option<&str> {
332 self.config().reasoning_effort.as_deref()
333 }
334
335 pub fn max_retries(&self) -> u32 {
337 self.config().max_retries
338 }
339
340 pub fn failure_cooldown(&self) -> Duration {
342 self.config().failure_cooldown
343 }
344
345 pub fn timeout(&self) -> Option<Duration> {
347 self.config().timeout
348 }
349
350 pub fn count_tokens_url(&self) -> String {
353 let base_url = &self.config().base_url;
354 anthropic_url(base_url, "/count_tokens").unwrap_or_else(|error| {
355 tracing::error!(%error, "Unable to build Anthropic token-counting URL");
356 base_url.clone()
357 })
358 }
359
360 pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
363 match self {
364 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
365 body,
366 |value| {
367 let error = value.get("error");
368 let field = |name| {
369 error
370 .and_then(|err| err.get(name))
371 .and_then(serde_json::Value::as_str)
372 };
373 field("code") == Some("context_length_exceeded")
379 || field("type") == Some("exceed_context_size_error")
380 },
381 OPENAI_OVERFLOW_PHRASES,
382 ),
383 Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
384 }
385 }
386}
387
388fn sensitive_header(value: &HeaderValue) -> HeaderValue {
390 let mut value = value.clone();
391 value.set_sensitive(true);
392 value
393}
394
395fn oauth_beta_header(value: &HeaderValue) -> Option<HeaderValue> {
396 let oauth_betas = value
397 .to_str()
398 .ok()?
399 .split(',')
400 .map(str::trim)
401 .filter(|beta| {
402 beta.get(..6)
403 .is_some_and(|prefix| prefix.eq_ignore_ascii_case("oauth-"))
404 });
405 let value = oauth_betas.collect::<Vec<_>>().join(",");
406 if value.is_empty() {
407 return None;
408 }
409 let mut value = HeaderValue::from_str(&value).ok()?;
410 value.set_sensitive(true);
411 Some(value)
412}
413
414pub(crate) fn openai_url(base_url: &str, suffix: &str) -> Result<String> {
416 let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
417 message: format!("Invalid OpenAI base URL: {error}"),
418 })?;
419 let base_path = url.path().trim_end_matches('/');
420 let base_root = base_path
421 .strip_suffix("/chat/completions")
422 .or_else(|| base_path.strip_suffix("/responses"))
423 .unwrap_or(base_path);
424 url.set_path(&format!("{base_root}{suffix}"));
425 Ok(url.into())
426}
427
428fn anthropic_url(base_url: &str, suffix: &str) -> Result<String> {
430 let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
431 message: format!("Invalid Anthropic base URL: {error}"),
432 })?;
433 let base_path = url.path().trim_end_matches('/');
434 let messages_path = if base_path.ends_with("/v1/messages") {
435 base_path.to_string()
436 } else if base_path.ends_with("/v1") {
437 format!("{base_path}/messages")
438 } else {
439 format!("{base_path}/v1/messages")
440 };
441 url.set_path(&format!("{messages_path}{suffix}"));
442 Ok(url.into())
443}
444
445#[cfg(test)]
446mod tests {
447 use super::*;
448
449 fn config(base_url: &str) -> HttpBackendConfig {
450 HttpBackendConfig {
451 base_url: base_url.to_string(),
452 api_key: Some("secret".to_string()),
453 forward_auth: false,
454 extra_headers: BTreeMap::new(),
455 extra_body: BTreeMap::new(),
456 omit_body_fields: BTreeSet::new(),
457 reasoning_effort: None,
458 max_retries: 0,
459 failure_cooldown: Duration::ZERO,
460 timeout: None,
461 }
462 }
463
464 #[test]
465 fn openai_chat_url_joins_bare_v1() {
466 let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
467 assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
468 }
469
470 #[test]
471 fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
472 assert_eq!(
473 Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
474 "https://api.openai.com/v1/chat/completions"
475 );
476 assert_eq!(
477 Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
478 "https://api.openai.com/v1/chat/completions"
479 );
480 }
481
482 #[test]
483 fn openai_responses_url_uses_responses_path() {
484 assert_eq!(
485 Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
486 "https://api.openai.com/v1/responses"
487 );
488 }
489
490 #[test]
491 fn anthropic_url_join_cases() {
492 assert_eq!(
493 Backend::Anthropic(config("https://api.anthropic.com")).url(),
494 "https://api.anthropic.com/v1/messages"
495 );
496 assert_eq!(
497 Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
498 "https://api.anthropic.com/v1/messages"
499 );
500 assert_eq!(
501 Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
502 "https://api.anthropic.com/v1/messages"
503 );
504 }
505
506 #[test]
507 fn wire_format_matches_variant() {
508 assert_eq!(
509 Backend::OpenAiChat(config("x")).wire_format(),
510 WireFormat::OpenAiChat
511 );
512 assert_eq!(
513 Backend::OpenAiResponses(config("x")).wire_format(),
514 WireFormat::OpenAiResponses
515 );
516 assert_eq!(
517 Backend::Anthropic(config("x")).wire_format(),
518 WireFormat::AnthropicMessages
519 );
520 }
521
522 #[test]
524 fn validates_additional_header_bytes() {
525 let cases = [
526 ("x-display-name", "café", None),
527 (
528 "bad header",
529 "value",
530 Some("invalid HTTP header name \"bad header\""),
531 ),
532 (
533 "x-test-header",
534 "bad\nvalue",
535 Some("invalid HTTP header value for extra_headers entry \"x-test-header\""),
536 ),
537 ];
538
539 for (name, value, expected) in cases {
540 let mut config = config("x");
541 config
542 .extra_headers
543 .insert(name.to_string(), value.to_string());
544 let result = Backend::OpenAiChat(config).validate_configured_headers("model");
545 match expected {
546 Some(expected) => assert!(
547 result.is_err_and(|error| error.to_string().contains(expected)),
548 "expected {expected:?}"
549 ),
550 None => result.expect("encodable header must pass validation"),
551 }
552 }
553 }
554
555 #[test]
557 fn configured_api_key_validation_matches_auth_application() {
558 const INVALID_KEY: &str = "canary\nsecret";
559 let mut config = config("x");
560 config.api_key = Some(INVALID_KEY.to_string());
561 let builders: [fn(HttpBackendConfig) -> Backend; 2] =
562 [Backend::OpenAiChat, Backend::Anthropic];
563 let client = reqwest::Client::new();
564
565 for build_backend in builders {
566 let error = build_backend(config.clone())
567 .validate_configured_headers("model")
568 .expect_err("invalid API key must fail")
569 .to_string();
570 assert!(
571 error.contains("api_key cannot be encoded as an HTTP header"),
572 "{error}"
573 );
574 assert!(!error.contains(INVALID_KEY), "API key leaked in: {error}");
575
576 let mut forwarded = config.clone();
577 forwarded.forward_auth = true;
578 let backend = build_backend(forwarded);
579 backend
580 .validate_configured_headers("model")
581 .expect("unused API key must not fail validation");
582 let request = backend
583 .apply_auth(client.get("https://example.test"))
584 .build()
585 .expect("request");
586 assert!(!request.headers().contains_key("authorization"));
587 assert!(!request.headers().contains_key("x-api-key"));
588 }
589 }
590
591 #[test]
592 fn openai_detects_canonical_and_wrapped_overflow() {
593 let backend = Backend::OpenAiChat(config("x"));
594 assert!(
595 backend.is_context_overflow(
596 r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
597 )
598 );
599 assert!(backend.is_context_overflow(
601 r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
602 ));
603 assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
604 assert!(backend.is_context_overflow(
606 r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
607 ));
608 assert!(backend.is_context_overflow(
612 r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#
613 ));
614 assert!(backend.is_context_overflow(
615 r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#
616 ));
617 }
618
619 #[test]
620 fn openai_detects_llama_server_overflow() {
621 let backend = Backend::OpenAiChat(config("x"));
622 assert!(backend.is_context_overflow(
625 r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#
626 ));
627 assert!(backend.is_context_overflow(
628 r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#
629 ));
630 assert!(backend.is_context_overflow(
632 r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#
633 ));
634 assert!(backend.is_context_overflow(
636 r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#
637 ));
638 assert!(!backend.is_context_overflow(
640 r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#
641 ));
642 }
643
644 #[test]
645 fn anthropic_detects_prompt_too_long() {
646 let backend = Backend::Anthropic(config("x"));
647 assert!(
648 backend.is_context_overflow(
649 r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
650 )
651 );
652 assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
653 }
654}