1use std::borrow::Cow;
8use std::collections::{BTreeMap, BTreeSet, HashMap};
9use std::future::ready;
10use std::sync::atomic::{AtomicU64, Ordering};
11use std::time::{Duration, SystemTime};
12
13use async_trait::async_trait;
14use futures_util::{StreamExt, stream};
15use http::StatusCode;
16use reqwest::RequestBuilder;
17use reqwest::header::{HeaderMap, HeaderName, HeaderValue, RETRY_AFTER};
18use serde_json::{Map, Value, json};
19use switchyard_protocol::{
20 LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata,
21 ModelId, Request, Response, RoutedLlmClient,
22};
23use switchyard_translation::{TranslationError, WireFormat, decode_stream, remap_reasoning_effort};
24use tokio::time::Instant;
25use tracing::Instrument;
26
27use crate::backend::{Backend, openai_url};
28use crate::error::{LlmClientError, Result};
29use crate::metrics;
30use crate::observed_translation;
31
32const ALLOWED_METADATA_HEADERS: &[&str] = &["x-request-id"];
34
35const NON_FORWARDABLE_HEADERS: &[&str] = &[
38 "host",
39 "content-length",
40 "connection",
41 "proxy-connection",
42 "keep-alive",
43 "proxy-authenticate",
44 "proxy-authorization",
45 "te",
46 "trailer",
47 "transfer-encoding",
48 "upgrade",
49 "content-type",
50 "content-encoding",
51 "accept-encoding",
52 "expect",
53];
54
55const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250);
56const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2);
57const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
58
59#[derive(Clone, Debug)]
61pub struct ModelConfig {
62 target_name: ModelId,
63 provider_model: Option<ModelId>,
64 default_backend: Backend,
65 other_backends: Option<Vec<Backend>>,
66 responses_reasoning: crate::ResponsesReasoningPolicy,
67 reasoning_effort_map: BTreeMap<String, String>,
68}
69
70impl ModelConfig {
71 pub fn new(
73 target_name: impl Into<ModelId>,
74 default_backend: Backend,
75 other_backends: Option<Vec<Backend>>,
76 ) -> Self {
77 Self {
78 target_name: target_name.into(),
79 provider_model: None,
80 default_backend,
81 other_backends,
82 responses_reasoning: crate::ResponsesReasoningPolicy::default(),
83 reasoning_effort_map: BTreeMap::new(),
84 }
85 }
86
87 #[must_use]
89 pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self {
90 self.responses_reasoning = policy;
91 self
92 }
93
94 #[must_use]
96 pub fn with_reasoning_effort_map(mut self, effort_map: BTreeMap<String, String>) -> Self {
97 self.reasoning_effort_map = effort_map;
98 self
99 }
100
101 pub fn with_provider_model(mut self, provider_model: impl Into<ModelId>) -> Self {
105 self.provider_model = Some(provider_model.into());
106 self
107 }
108}
109
110#[derive(Clone, Copy, Debug, Eq, PartialEq)]
112pub enum AuxiliaryOperation {
113 AnthropicCountTokens,
115 ResponsesInputTokens,
117 ResponsesCompact,
119}
120
121impl AuxiliaryOperation {
122 const fn wire_format(self) -> WireFormat {
123 match self {
124 Self::AnthropicCountTokens => WireFormat::AnthropicMessages,
125 Self::ResponsesInputTokens | Self::ResponsesCompact => WireFormat::OpenAiResponses,
126 }
127 }
128
129 fn url(self, backend: &Backend) -> String {
130 let suffix = match self {
131 Self::AnthropicCountTokens => return backend.count_tokens_url(),
132 Self::ResponsesInputTokens => "/responses/input_tokens",
133 Self::ResponsesCompact => "/responses/compact",
134 };
135 let base_url = backend.url();
136 openai_url(&base_url, suffix).unwrap_or_else(|error| {
137 tracing::error!(%error, "Unable to build OpenAI auxiliary endpoint URL");
138 base_url
139 })
140 }
141}
142
143pub struct TranslatingLlmClient {
151 configs_by_target: HashMap<ModelId, ModelConfig>,
152 client: reqwest::Client,
153 forward_auth_client: reqwest::Client,
154 is_rate_limit_cooldown_enabled: bool,
155 cooldown_epoch: Instant,
156 unavailable_until: HashMap<ModelId, AtomicU64>,
158}
159
160impl TranslatingLlmClient {
161 pub fn new(model_configs: &[ModelConfig]) -> Result<Self> {
164 let mut unavailable_until = HashMap::new();
166 for config in model_configs {
167 for backend in std::iter::once(&config.default_backend)
168 .chain(config.other_backends.iter().flatten())
169 {
170 backend.validate_configured_headers(&config.target_name)?;
171 }
172 unavailable_until.insert(config.target_name.clone(), AtomicU64::new(0));
173 }
174 let build_client = |builder: reqwest::ClientBuilder| {
175 builder.build().map_err(|error| LlmClientError::Transport {
176 source: Box::new(error),
177 })
178 };
179 let client = build_client(reqwest::Client::builder())?;
180 let forward_auth_client =
183 build_client(reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()))?;
184 let configs_by_target = model_configs
185 .iter()
186 .map(|config| (config.target_name.clone(), config.clone()))
187 .collect();
188
189 Ok(Self {
190 configs_by_target,
191 client,
192 forward_auth_client,
193 is_rate_limit_cooldown_enabled: true,
194 cooldown_epoch: Instant::now(),
195 unavailable_until,
196 })
197 }
198
199 pub fn without_rate_limit_cooldown(mut self) -> Self {
201 self.is_rate_limit_cooldown_enabled = false;
202 self
203 }
204
205 pub fn backend_for(&self, target_name: &ModelId, format: WireFormat) -> Option<&Backend> {
207 self.configs_by_target.get(target_name).and_then(|config| {
208 if config.default_backend.wire_format() == format {
209 Some(&config.default_backend)
210 } else {
211 config
212 .other_backends
213 .as_ref()
214 .and_then(|backends| backends.iter().find(|b| b.wire_format() == format))
215 }
216 })
217 }
218
219 pub fn provider_model<'a>(&'a self, target_name: &'a ModelId) -> &'a ModelId {
221 self.configs_by_target
222 .get(target_name)
223 .and_then(|config| config.provider_model.as_ref())
224 .unwrap_or(target_name)
225 }
226
227 pub fn supports_auxiliary(&self, target_name: &ModelId, operation: AuxiliaryOperation) -> bool {
229 self.backend_for(target_name, operation.wire_format())
230 .is_some()
231 }
232
233 pub async fn get_model_properties(
237 &self,
238 target_name: &ModelId,
239 format: WireFormat,
240 timeout: Duration,
241 ) -> Result<Value> {
242 let backend =
243 self.backend_for(target_name, format)
244 .ok_or_else(|| LlmClientError::Configuration {
245 message: format!("model {target_name} has no backend for {format:?}"),
246 })?;
247 let mut url = reqwest::Url::parse(backend.base_url()).map_err(|error| {
248 LlmClientError::Configuration {
249 message: format!("model {target_name} has an invalid backend URL: {error}"),
250 }
251 })?;
252 let root = url.path().trim_end_matches('/');
253 let root = root
254 .strip_suffix("/chat/completions")
255 .or_else(|| root.strip_suffix("/responses"))
256 .unwrap_or(root);
257 let path = format!(
258 "{}/props",
259 root.strip_suffix("/v1")
260 .unwrap_or(root)
261 .trim_end_matches('/')
262 );
263 url.set_path(&path);
264 url.set_query(None);
265
266 let builder = self.forward_auth_client.get(url).timeout(timeout);
267 let builder = apply_extra_headers(builder, backend);
268 let response = backend
269 .apply_auth(builder)
270 .send()
271 .await
272 .map_err(convert_reqwest_error)?
273 .error_for_status()
274 .map_err(convert_reqwest_error)?;
275 response
276 .json()
277 .await
278 .map_err(|source| LlmClientError::InvalidResponse {
279 source: Box::new(source),
280 })
281 }
282
283 pub async fn call_auxiliary(
288 &self,
289 target_name: &ModelId,
290 request: Request,
291 operation: AuxiliaryOperation,
292 ) -> Result<Value> {
293 let wire_format = operation.wire_format();
294 let backend = self.backend_for(target_name, wire_format).ok_or_else(|| {
295 LlmClientError::Configuration {
296 message: format!("model {target_name} has no backend for {operation:?}"),
297 }
298 })?;
299 let Request {
300 mut llm_request,
301 metadata,
302 ..
303 } = request;
304 llm_request.model = Some(self.provider_model(target_name).to_string());
305 let http_response = self
306 .send_encoded(
307 backend,
308 wire_format,
309 llm_request,
310 metadata.as_ref(),
311 target_name,
312 UpstreamEndpoint::Auxiliary(operation),
313 )
314 .await?;
315 let EncodedResponse::Buffered { body, .. } = http_response else {
316 return Err(LlmClientError::InvalidRequest {
317 message: "auxiliary endpoints do not support streaming".to_string(),
318 });
319 };
320 serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {
321 source: Box::new(error),
322 })
323 }
324
325 async fn send_encoded(
337 &self,
338 backend: &Backend,
339 wire_format: WireFormat,
340 llm_request: LlmRequest,
341 metadata: Option<&Metadata>,
342 target_name: &ModelId,
343 endpoint: UpstreamEndpoint,
344 ) -> Result<EncodedResponse> {
345 let cooldown = backend.failure_cooldown();
346 let unavailable_until =
347 if matches!(endpoint, UpstreamEndpoint::Completion) && !cooldown.is_zero() {
348 self.unavailable_until.get(target_name)
349 } else {
350 None
351 };
352 if let Some(until) = unavailable_until
353 && until.load(Ordering::Relaxed) > duration_millis(self.cooldown_epoch.elapsed())
354 {
355 tracing::debug!(model = %target_name, wire_format = %wire_format, "skipping backend during failure cooldown");
356 return Err(LlmClientError::TemporarilyUnavailable);
357 }
358
359 let mut body = observed_translation::encode_request(&llm_request, wire_format)
360 .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
361 let provider_model_id = self.provider_model(target_name);
364 set_json_model(&mut body, provider_model_id);
365 if matches!(backend, Backend::OpenAiResponses(_)) {
366 sanitize_openai_responses_provider_body(&mut body);
367 }
368 if matches!(backend, Backend::Anthropic(_)) {
371 strip_anthropic_incompatible_fields(&mut body);
372 strip_unsigned_thinking_blocks(&mut body);
373 }
374 omit_configured_body_fields(&mut body, backend.omit_body_fields());
375 let target_config = self.configs_by_target.get(target_name);
376 if matches!(endpoint, UpstreamEndpoint::Completion)
377 && backend.reasoning_effort().is_none()
378 && let Some(config) = target_config
379 {
380 remap_reasoning_effort(
381 &mut body,
382 wire_format.as_str(),
383 &config.reasoning_effort_map,
384 );
385 }
386 merge_extra_body(&mut body, backend.extra_body());
387 if matches!(backend, Backend::OpenAiResponses(_)) {
388 let outbound_model_id = body
389 .get("model")
390 .and_then(Value::as_str)
391 .unwrap_or(provider_model_id)
392 .to_owned();
393 target_config
394 .map(|config| config.responses_reasoning)
395 .unwrap_or_default()
396 .normalize(&mut body, backend.base_url(), &outbound_model_id);
397 }
398 if matches!(endpoint, UpstreamEndpoint::Completion) {
402 apply_reasoning_effort(&mut body, backend);
403 }
404 if matches!(backend, Backend::OpenAiResponses(_)) {
405 normalize_codex_string_input(&backend.url(), &mut body);
406 }
407 if matches!(backend, Backend::Anthropic(_)) {
408 if nvidia_inference_api_backend(backend) {
409 strip_anthropic_tool_strict(&mut body);
410 }
411 enable_anthropic_prompt_caching(&mut body, llm_request.temporary_note_count);
412 }
413 if matches!(
414 backend,
415 Backend::OpenAiChat(_) | Backend::OpenAiResponses(_)
416 ) {
417 enable_openai_guidance_caching(
418 &mut body,
419 wire_format,
420 provider_model_id,
421 llm_request.temporary_note_count,
422 );
423 }
424 if matches!(backend, Backend::OpenAiChat(_)) {
425 ensure_openai_stream_usage(&mut body);
426 }
427 let streaming = endpoint.allows_streaming()
428 && body.get("stream").and_then(Value::as_bool).unwrap_or(false);
429 let url = endpoint.url(backend);
430 record_gen_ai_request(&url, provider_model_id, streaming);
431
432 let result = self
433 .send_with_retries(&url, backend, &body, metadata, target_name, streaming)
434 .await;
435 if let Some(until) = unavailable_until
437 && let Err(error) = &result
438 && is_transient_failure(error)
439 && !((backend.is_forwarding_auth() || !self.is_rate_limit_cooldown_enabled)
440 && matches!(error, LlmClientError::UpstreamHttp { status, .. } if *status == StatusCode::TOO_MANY_REQUESTS))
441 {
442 let deadline = duration_millis(self.cooldown_epoch.elapsed().saturating_add(cooldown));
443 until.fetch_max(deadline, Ordering::Relaxed);
444 }
445 result
447 }
448
449 async fn send_with_retries(
451 &self,
452 url: &str,
453 backend: &Backend,
454 body: &Value,
455 metadata: Option<&Metadata>,
456 target_name: &ModelId,
457 streaming: bool,
458 ) -> Result<EncodedResponse> {
459 let max_retries = u64::from(backend.max_retries());
460 let max_attempts = max_retries
461 + 1
462 + u64::from(
463 matches!(backend, Backend::OpenAiChat(_))
464 && body.get("chat_template_kwargs").is_some(),
465 );
466 let mut body = Cow::Borrowed(body);
467 let mut template_retry = false;
468 let expires_at = backend
469 .timeout()
470 .map(|timeout| (tokio::time::sleep(timeout).deadline(), timeout));
471 let deadline = async {
472 match expires_at {
473 Some((at, timeout)) => {
474 tokio::time::sleep_until(at).await;
475 timeout
476 }
477 None => std::future::pending().await,
478 }
479 };
480 tokio::pin!(deadline);
481 let mut attempt = 0_u64;
482 loop {
483 if let Some((at, timeout)) = expires_at
484 && tokio::time::Instant::now() >= at
485 {
486 return Err(deadline_error(timeout));
487 }
488 let span = tracing::debug_span!(
489 target: "libsy",
490 "libsy.upstream_attempt",
491 model = %target_name,
492 wire_format = %backend.wire_format(),
493 attempt = attempt + 1 + u64::from(template_retry),
494 max_attempts,
495 retry = attempt > 0 || template_retry,
496 openinference.span.kind = "CHAIN",
497 outcome = tracing::field::Empty,
498 status_code = tracing::field::Empty,
499 will_retry = tracing::field::Empty,
500 retry_delay_ms = tracing::field::Empty,
501 );
502 let mut attempt_started = false;
503 let result = tokio::select! {
504 biased;
505 timeout = &mut deadline => {
506 if attempt_started {
507 metrics::record_upstream_attempt(None);
508 }
509 span.record("outcome", "error");
510 span.record("will_retry", false);
511 return Err(deadline_error(timeout));
512 }
513 result = async {
514 attempt_started = true;
515 self.send_once(url, backend, body.as_ref(), metadata, target_name, streaming).await
516 }.instrument(span.clone()) => result,
517 };
518 match result {
520 Ok(response) => {
521 span.record("outcome", "ok");
522 span.record("status_code", response.status());
523 span.record("will_retry", false);
524 if attempt > 0 || template_retry {
525 metrics::record_retry_recovered();
526 }
527 return Ok(match response {
528 EncodedResponse::Streaming {
529 status,
530 chunks,
531 upstream_headers,
532 } => EncodedResponse::Streaming {
533 status,
534 chunks: stream_with_deadline(chunks, expires_at),
535 upstream_headers,
536 },
537 buffered => buffered,
538 });
539 }
540 Err(failure) => {
541 let retry_without_template = matches!(backend, Backend::OpenAiChat(_))
542 && body.get("chat_template_kwargs")
543 == Some(&json!({"enable_thinking": false}))
544 && rejects_template_options(&failure.error);
545 let will_retry =
546 retry_without_template || (attempt < max_retries && failure.is_retryable());
547 span.record("outcome", "error");
548 if let Some(status) = failure.status {
549 span.record("status_code", status.as_u16());
550 }
551 span.record("will_retry", will_retry);
552 if !will_retry {
553 return Err(failure.error);
554 }
555
556 if retry_without_template {
557 if let Some(object) = body.to_mut().as_object_mut() {
558 object.remove("chat_template_kwargs");
559 }
560 template_retry = true;
561 span.record("retry_delay_ms", 0_u64);
562 continue;
563 }
564 let delay = retry_delay(attempt, failure.retry_after);
565 span.record("retry_delay_ms", duration_millis(delay));
566 drop(span);
568 tokio::select! {
569 biased;
570 timeout = &mut deadline => return Err(deadline_error(timeout)),
571 _ = tokio::time::sleep(delay) => {}
572 }
573 attempt += 1;
574 }
575 }
576 }
577 }
578
579 async fn send_once(
581 &self,
582 url: &str,
583 backend: &Backend,
584 body: &Value,
585 metadata: Option<&Metadata>,
586 target_name: &ModelId,
587 streaming: bool,
588 ) -> std::result::Result<EncodedResponse, AttemptFailure> {
589 let client = if backend.is_forwarding_auth() {
590 &self.forward_auth_client
591 } else {
592 &self.client
593 };
594 let builder = client.post(url).json(body);
595 let builder = forward_metadata_headers(builder, metadata, backend);
596 let builder = backend.apply_forwarded_auth(builder, metadata);
597 let builder = apply_extra_headers(builder, backend);
598 let builder = backend.apply_auth(builder);
599
600 let response = match async {
601 let request = builder.build()?;
602 if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) {
603 crate::payload_capture::record("request", || String::from_utf8_lossy(body));
604 }
605 client.execute(request).await
606 }
607 .await
608 {
609 Ok(response) => response,
610 Err(error) => {
611 metrics::record_upstream_attempt(None);
612 return Err(AttemptFailure {
613 error: convert_reqwest_error(error),
614 status: None,
615 retry_after: None,
616 });
617 }
618 };
619 let status = response.status();
620 if status.is_success() {
621 if streaming {
622 let upstream_headers = response.headers().clone();
623 let chunks = match prepare_response_stream(response, backend, target_name).await {
624 Ok(chunks) => chunks,
625 Err(error) => {
626 metrics::record_upstream_attempt(None);
627 return Err(AttemptFailure {
628 error,
629 status: Some(status),
630 retry_after: None,
631 });
632 }
633 };
634 metrics::record_upstream_attempt(Some(status.as_u16()));
635 return Ok(EncodedResponse::Streaming {
636 status: status.as_u16(),
637 chunks,
638 upstream_headers,
639 });
640 }
641 let upstream_headers = response.headers().clone();
642 let body = match response.bytes().await {
643 Ok(body) => body,
644 Err(error) => {
645 metrics::record_upstream_attempt(None);
646 return Err(AttemptFailure {
647 error: convert_reqwest_error(error),
648 status: Some(status),
649 retry_after: None,
650 });
651 }
652 };
653 metrics::record_upstream_attempt(Some(status.as_u16()));
654 crate::payload_capture::record("response", || String::from_utf8_lossy(&body));
655 return Ok(EncodedResponse::Buffered {
656 status: status.as_u16(),
657 body: body.to_vec(),
658 upstream_headers,
659 });
660 }
661
662 let retry_after = retry_after_delay(response.headers());
663 let headers = Box::new(response.headers().clone());
664 let body = match response.text().await {
665 Ok(body) => body,
666 Err(error) => {
667 metrics::record_upstream_attempt(None);
668 return Err(AttemptFailure {
669 error: convert_reqwest_error(error),
670 status: Some(status),
671 retry_after,
672 });
673 }
674 };
675 let body = redact_forwarded_headers(body, metadata, backend.is_forwarding_auth());
676 metrics::record_upstream_attempt(Some(status.as_u16()));
677 let error =
678 if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
679 LlmClientError::ContextWindowExceeded {
680 model: target_name.clone(),
681 message: body,
682 }
683 } else {
684 LlmClientError::UpstreamHttp {
685 status,
686 body,
687 headers,
688 }
689 };
690 Err(AttemptFailure {
691 error,
692 status: Some(status),
693 retry_after,
694 })
695 }
696
697 pub async fn call_rewrite_model(
701 &self,
702 request: Request,
703 target_name: Option<&ModelId>,
704 ) -> Result<Response> {
705 let Request {
706 mut llm_request,
707 metadata,
708 ..
709 } = request;
710
711 let selected_target = target_name
712 .cloned()
713 .or_else(|| llm_request.model.map(ModelId::from))
714 .ok_or_else(|| LlmClientError::InvalidRequest {
715 message: "no model given".to_string(),
716 })?;
717 llm_request.model = Some(self.provider_model(&selected_target).to_string());
718
719 let orig_format = metadata.as_ref().and_then(|m| m.wire_format);
720 let wire_format = orig_format.unwrap_or(
721 self.configs_by_target
722 .get(&selected_target)
723 .map(|config| config.default_backend.wire_format())
724 .ok_or_else(|| LlmClientError::Configuration {
725 message: format!("no backend configured for model {selected_target:?}"),
726 })?,
727 );
728 let backend = self
729 .backend_for(&selected_target, wire_format)
730 .ok_or_else(|| LlmClientError::Configuration {
731 message: format!(
732 "model {selected_target:?} has no backend for format {wire_format}"
733 ),
734 })?;
735
736 let http_response = self
737 .send_encoded(
738 backend,
739 wire_format,
740 llm_request,
741 metadata.as_ref(),
742 &selected_target,
743 UpstreamEndpoint::Completion,
744 )
745 .await?;
746
747 let (llm_response, upstream_headers) = match http_response {
748 EncodedResponse::Streaming {
749 chunks,
750 upstream_headers,
751 ..
752 } => (LlmResponse::Stream(chunks), upstream_headers),
753 EncodedResponse::Buffered {
754 body,
755 upstream_headers,
756 ..
757 } => {
758 let body = serde_json::from_slice::<Value>(&body).map_err(|error| {
759 LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))
760 })?;
761 let mut agg =
764 observed_translation::decode_response(&body, wire_format).map_err(|error| {
765 match error {
766 TranslationError::UpstreamFailure { error } => {
767 LlmClientError::UpstreamHttp {
768 status: StatusCode::BAD_GATEWAY,
769 body: redact_forwarded_headers(
770 json!({ "error": error }).to_string(),
771 metadata.as_ref(),
772 backend.is_forwarding_auth(),
773 ),
774 headers: Box::new(upstream_headers.clone()),
775 }
776 }
777 error => LlmClientError::ResponseTranslation(error.to_string()),
778 }
779 })?;
780 if agg.usage.provider_cost.is_none() {
782 agg.usage.provider_cost = upstream_headers
783 .get("x-litellm-response-cost")
784 .and_then(|value| value.to_str().ok())
785 .and_then(|value| value.trim().parse::<f64>().ok())
786 .filter(|usd| usd.is_finite() && *usd >= 0.0);
787 }
788 (LlmResponse::Agg(agg), upstream_headers)
789 }
790 };
791
792 Ok(Response {
793 llm_response,
794 metadata,
795 upstream_headers,
796 })
797 }
798}
799
800#[async_trait]
801impl RoutedLlmClient for TranslatingLlmClient {
802 fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
803 TranslatingLlmClient::provider_model(self, target)
804 }
805
806 async fn call(&self, request: Request) -> Result<Response> {
807 self.call_rewrite_model(request, None).await
808 }
809}
810
811#[derive(Clone, Copy)]
812enum UpstreamEndpoint {
813 Completion,
814 Auxiliary(AuxiliaryOperation),
815}
816
817impl UpstreamEndpoint {
818 fn url(self, backend: &Backend) -> String {
819 match self {
820 UpstreamEndpoint::Completion => backend.url(),
821 UpstreamEndpoint::Auxiliary(operation) => operation.url(backend),
822 }
823 }
824
825 fn allows_streaming(self) -> bool {
826 matches!(self, UpstreamEndpoint::Completion)
827 }
828}
829
830enum EncodedResponse {
831 Buffered {
832 status: u16,
833 body: Vec<u8>,
834 upstream_headers: HeaderMap,
835 },
836 Streaming {
837 status: u16,
838 chunks: LlmResponseStream,
839 upstream_headers: HeaderMap,
840 },
841}
842
843impl EncodedResponse {
844 fn status(&self) -> u16 {
845 match self {
846 EncodedResponse::Buffered { status, .. }
847 | EncodedResponse::Streaming { status, .. } => *status,
848 }
849 }
850}
851
852struct AttemptFailure {
855 error: LlmClientError,
856 status: Option<StatusCode>,
857 retry_after: Option<Duration>,
858}
859
860fn deadline_error(timeout: Duration) -> LlmClientError {
861 LlmClientError::Timeout {
862 source: Box::new(std::io::Error::new(
863 std::io::ErrorKind::TimedOut,
864 format!(
865 "response did not finish within {} ms, retries included",
866 duration_millis(timeout)
867 ),
868 )),
869 }
870}
871
872impl AttemptFailure {
873 fn is_retryable(&self) -> bool {
874 is_transient_failure(&self.error)
875 }
876}
877
878fn is_transient_failure(error: &LlmClientError) -> bool {
879 match error {
880 LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
881 LlmClientError::UpstreamHttp { status, .. } => {
882 metrics::is_retryable_http_status(status.as_u16())
883 }
884 _ => false,
885 }
886}
887
888fn stream_with_deadline(
889 chunks: LlmResponseStream,
890 deadline: Option<(tokio::time::Instant, Duration)>,
891) -> LlmResponseStream {
892 let Some((at, timeout)) = deadline else {
893 return chunks;
894 };
895 stream::unfold(Some(chunks), move |chunks| async move {
896 let mut chunks = chunks?;
897 let chunk = tokio::select! {
898 biased;
899 _ = tokio::time::sleep_until(at) => return Some((Err(deadline_error(timeout)), None)),
900 chunk = chunks.next() => chunk?,
901 };
902 let next = chunk.is_ok().then_some(chunks);
903 Some((chunk, next))
904 })
905 .boxed()
906}
907
908async fn prepare_response_stream(
909 response: reqwest::Response,
910 backend: &Backend,
911 target_name: &ModelId,
912) -> Result<LlmResponseStream> {
913 let bytes = response.bytes_stream().map(|chunk| {
914 chunk
915 .map(|bytes| bytes.to_vec())
916 .map_err(convert_reqwest_error)
917 });
918 let mut chunks = decode_stream(bytes, backend.wire_format())?;
919 match chunks.next().await {
920 None => Ok(stream::empty().boxed()),
921 Some(Err(error @ (LlmClientError::Transport { .. } | LlmClientError::Timeout { .. }))) => {
923 Err(error)
924 }
925 Some(first) => {
926 if let Some(message) = first_event_overflow(&first, backend) {
928 return Err(LlmClientError::ContextWindowExceeded {
929 model: target_name.clone(),
930 message,
931 });
932 }
933 Ok(stream::once(ready(first)).chain(chunks).boxed())
934 }
935 }
936}
937
938fn first_event_overflow(
941 first: &Result<LlmResponseStreamEvent>,
942 backend: &Backend,
943) -> Option<String> {
944 first
945 .as_ref()
946 .ok()?
947 .normalized()
948 .iter()
949 .find_map(|chunk| match chunk {
950 LlmResponseChunk::StreamError { message } if backend.is_context_overflow(message) => {
951 Some(message.clone())
952 }
953 _ => None,
954 })
955}
956
957fn retry_after_delay(headers: &HeaderMap) -> Option<Duration> {
959 let value = headers.get(RETRY_AFTER)?.to_str().ok()?;
960 let delay = if let Ok(seconds) = value.parse::<u64>() {
961 Duration::from_secs(seconds)
962 } else {
963 let retry_at = httpdate::parse_http_date(value).ok()?;
964 retry_at
965 .duration_since(SystemTime::now())
966 .unwrap_or(Duration::ZERO)
967 };
968 Some(delay.min(MAX_RETRY_AFTER))
969}
970
971fn retry_delay(retry_number: u64, retry_after: Option<Duration>) -> Duration {
972 retry_after.unwrap_or_else(|| {
974 let multiplier = 1_u32 << retry_number.min(3);
975 INITIAL_RETRY_DELAY
976 .saturating_mul(multiplier)
977 .min(MAX_RETRY_BACKOFF)
978 })
979}
980
981fn duration_millis(duration: Duration) -> u64 {
982 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
983}
984
985fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {
986 let span = tracing::Span::current();
987 span.record("gen_ai.request.model", model);
988 if streaming {
989 span.record("gen_ai.request.stream", true);
990 }
991 if let Ok(url) = reqwest::Url::parse(url) {
992 if let Some(host) = url.host_str() {
993 span.record("server.address", host);
994 }
995 if let Some(port) = url.port_or_known_default() {
996 span.record("server.port", i64::from(port));
997 }
998 }
999}
1000
1001pub(crate) fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
1002 let error = error.without_url();
1005 if error.is_connect() {
1009 LlmClientError::Transport {
1010 source: Box::new(error),
1011 }
1012 } else if error.is_timeout() {
1013 LlmClientError::Timeout {
1014 source: Box::new(error),
1015 }
1016 } else if error.is_builder() {
1017 LlmClientError::Configuration {
1018 message: format!("failed to build upstream request: {error}"),
1019 }
1020 } else {
1021 LlmClientError::Transport {
1022 source: Box::new(error),
1023 }
1024 }
1025}
1026
1027fn forward_metadata_headers(
1029 builder: RequestBuilder,
1030 metadata: Option<&Metadata>,
1031 backend: &Backend,
1032) -> RequestBuilder {
1033 let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
1034 return builder;
1035 };
1036 let mut forwarded = HeaderMap::with_capacity(headers.len());
1037 for (name, value) in headers {
1038 let is_allowed = if backend.is_forwarding_auth() {
1039 !is_non_forwardable_header(name.as_str(), headers)
1040 && !backend.is_provider_owned_header(name.as_str())
1041 } else {
1042 is_allowed_metadata_header(name.as_str())
1043 };
1044 if !is_allowed {
1045 continue;
1046 }
1047 forwarded.append(name, value.clone());
1048 }
1049 builder.headers(forwarded)
1050}
1051
1052fn apply_extra_headers(builder: RequestBuilder, backend: &Backend) -> RequestBuilder {
1054 let mut headers = HeaderMap::with_capacity(backend.extra_headers().len());
1055 for (name, value) in backend.extra_headers() {
1057 let name = HeaderName::from_bytes(name.as_bytes()).expect("validated header name");
1058 let value = HeaderValue::from_bytes(value.as_bytes()).expect("validated header value");
1059 headers.insert(name, value);
1060 }
1061 builder.headers(headers)
1062}
1063
1064fn set_json_model(body: &mut Value, model: &str) {
1066 if let Value::Object(object) = body {
1067 object.insert("model".to_string(), Value::String(model.to_string()));
1068 }
1069}
1070
1071fn normalize_codex_string_input(url: &str, body: &mut Value) {
1073 let codex = reqwest::Url::parse(url).ok().is_some_and(|url| {
1074 url.path()
1075 .trim_end_matches('/')
1076 .ends_with("/backend-api/codex/responses")
1077 });
1078 if codex && let Some(input @ Value::String(_)) = body.get_mut("input") {
1079 let text = input.take();
1080 *input = json!([{"type": "message", "role": "user", "content": text}]);
1081 }
1082}
1083
1084fn rejects_template_options(error: &LlmClientError) -> bool {
1085 let LlmClientError::UpstreamHttp { status, body, .. } = error else {
1086 return false;
1087 };
1088 if !matches!(
1089 *status,
1090 StatusCode::BAD_REQUEST | StatusCode::UNPROCESSABLE_ENTITY
1091 ) {
1092 return false;
1093 }
1094 let parsed = serde_json::from_str::<Value>(body).ok();
1095 let error = parsed.as_ref().and_then(|value| value.get("error"));
1096 let message = match &parsed {
1097 Some(_) => error
1098 .and_then(|error| error.get("message"))
1099 .and_then(Value::as_str)
1100 .unwrap_or_default(),
1101 None => body,
1102 }
1103 .to_ascii_lowercase();
1104 let field = error
1105 .and_then(|error| error.get("param"))
1106 .and_then(Value::as_str);
1107 let code = error
1108 .and_then(|error| error.get("code"))
1109 .and_then(Value::as_str);
1110 (field == Some("chat_template_kwargs") || message.contains("chat_template_kwargs"))
1111 && (matches!(code, Some("unknown_parameter" | "unsupported_parameter"))
1112 || [
1113 "unrecognized request argument",
1114 "unknown parameter",
1115 "unsupported parameter",
1116 "unexpected keyword argument",
1117 "unknown field",
1118 "unsupported field",
1119 "extra inputs are not permitted",
1120 ]
1121 .iter()
1122 .any(|phrase| message.contains(phrase)))
1123}
1124
1125const CODEX_NAMESPACE_SEPARATOR: &str = "__";
1126
1127fn sanitize_openai_responses_provider_body(body: &mut Value) {
1132 let Value::Object(object) = body else {
1133 return;
1134 };
1135 sanitize_openai_responses_input_for_provider(object.get_mut("input"));
1136 sanitize_openai_responses_tools_for_provider(object.get_mut("tools"));
1137 sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice"));
1138}
1139
1140fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) {
1141 let Some(Value::Array(items)) = input else {
1142 return;
1143 };
1144 for item in items {
1145 let Some(object) = item.as_object_mut() else {
1146 continue;
1147 };
1148 if object.get("type").and_then(Value::as_str) == Some("function_call") {
1149 qualify_responses_function_name(object);
1150 }
1151 }
1152}
1153
1154fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) {
1155 let Some(Value::Array(tools)) = tools else {
1156 return;
1157 };
1158 let mut flat_tools = Vec::with_capacity(tools.len());
1159 for tool in std::mem::take(tools) {
1160 push_sanitized_openai_responses_tool(&mut flat_tools, tool);
1161 }
1162 *tools = flat_tools;
1163}
1164
1165fn push_sanitized_openai_responses_tool(out: &mut Vec<Value>, tool: Value) {
1166 let Value::Object(mut object) = tool else {
1167 out.push(tool);
1168 return;
1169 };
1170 if object.get("type").and_then(Value::as_str) == Some("namespace") {
1171 let namespace = object
1172 .get("name")
1173 .and_then(Value::as_str)
1174 .unwrap_or_default()
1175 .to_string();
1176 let Some(Value::Array(children)) = object.remove("tools") else {
1177 out.push(Value::Object(object));
1178 return;
1179 };
1180 for child in children {
1181 push_sanitized_namespaced_tool(out, &namespace, child);
1182 }
1183 return;
1184 }
1185 ensure_responses_function_tool_description(&mut object);
1186 out.push(Value::Object(object));
1187}
1188
1189fn push_sanitized_namespaced_tool(out: &mut Vec<Value>, namespace: &str, tool: Value) {
1190 let Value::Object(mut object) = tool else {
1191 out.push(tool);
1192 return;
1193 };
1194 if object.get("type").and_then(Value::as_str) == Some("function") {
1195 qualify_responses_function_name_with_namespace(&mut object, namespace);
1196 ensure_responses_function_tool_description(&mut object);
1197 }
1198 out.push(Value::Object(object));
1199}
1200
1201fn sanitize_openai_responses_tool_choice_for_provider(tool_choice: Option<&mut Value>) {
1202 let Some(Value::Object(object)) = tool_choice else {
1203 return;
1204 };
1205 if object.get("type").and_then(Value::as_str) == Some("function") {
1206 qualify_responses_function_name(object);
1207 }
1208}
1209
1210fn qualify_responses_function_name(object: &mut Map<String, Value>) {
1211 let namespace = object
1212 .remove("namespace")
1213 .and_then(|value| value.as_str().map(ToOwned::to_owned));
1214 let Some(namespace) = namespace.as_deref() else {
1215 return;
1216 };
1217 qualify_responses_function_name_with_namespace(object, namespace);
1218}
1219
1220fn qualify_responses_function_name_with_namespace(
1221 object: &mut Map<String, Value>,
1222 namespace: &str,
1223) {
1224 if namespace.is_empty() {
1225 return;
1226 }
1227 let Some(name) = object.get("name").and_then(Value::as_str) else {
1228 return;
1229 };
1230 let prefix = format!("{namespace}{CODEX_NAMESPACE_SEPARATOR}");
1231 if name.starts_with(&prefix) {
1232 return;
1233 }
1234 object.insert("name".to_string(), Value::String(format!("{prefix}{name}")));
1235}
1236
1237fn ensure_responses_function_tool_description(object: &mut Map<String, Value>) {
1238 if object.get("type").and_then(Value::as_str) == Some("function")
1239 && !matches!(object.get("description"), Some(Value::String(_)))
1240 {
1241 object.insert("description".to_string(), Value::String(String::new()));
1242 }
1243}
1244
1245fn strip_anthropic_incompatible_fields(body: &mut Value) {
1253 if let Value::Object(object) = body {
1254 object.remove("reasoning_effort");
1255 object.remove("context_management");
1256 if let Some(Value::Array(tools)) = object.get_mut("tools") {
1258 for tool in tools {
1259 if let Value::Object(tool) = tool
1260 && tool.get("strict") == Some(&Value::Bool(false))
1261 {
1262 tool.remove("strict");
1263 }
1264 }
1265 }
1266 }
1267}
1268
1269fn nvidia_inference_api_backend(backend: &Backend) -> bool {
1272 reqwest::Url::parse(backend.base_url())
1273 .ok()
1274 .is_some_and(|url| {
1275 url.host_str()
1276 .is_some_and(|host| host.eq_ignore_ascii_case("inference-api.nvidia.com"))
1277 })
1278}
1279
1280fn strip_anthropic_tool_strict(body: &mut Value) {
1281 let Some(tools) = body.get_mut("tools").and_then(Value::as_array_mut) else {
1282 return;
1283 };
1284 for tool in tools.iter_mut().filter_map(Value::as_object_mut) {
1285 tool.remove("strict");
1286 }
1287}
1288
1289fn strip_unsigned_thinking_blocks(body: &mut Value) {
1297 let Value::Object(object) = body else {
1298 return;
1299 };
1300 let Some(Value::Array(messages)) = object.get_mut("messages") else {
1301 return;
1302 };
1303 for message in messages {
1304 strip_unsigned_thinking_from_message(message);
1305 }
1306}
1307
1308fn strip_unsigned_thinking_from_message(message: &mut Value) {
1311 let Value::Object(message) = message else {
1312 return;
1313 };
1314 let Some(Value::Array(blocks)) = message.get("content") else {
1315 return;
1316 };
1317 if !blocks.iter().any(is_unsigned_thinking_block) {
1318 return;
1319 }
1320 let Some(Value::Array(blocks)) = message.get_mut("content") else {
1321 return;
1322 };
1323 blocks.retain(|block| !is_unsigned_thinking_block(block));
1324 if blocks.is_empty() {
1325 message.insert("content".to_string(), Value::String(String::new()));
1326 }
1327}
1328
1329fn is_unsigned_thinking_block(block: &Value) -> bool {
1331 if block.get("type").and_then(Value::as_str) != Some("thinking") {
1332 return false;
1333 }
1334 !matches!(
1335 block.get("signature").and_then(Value::as_str),
1336 Some(signature) if !signature.is_empty()
1337 )
1338}
1339
1340fn apply_reasoning_effort(body: &mut Value, backend: &Backend) {
1344 let Some(effort) = backend.reasoning_effort() else {
1345 return;
1346 };
1347 let Value::Object(object) = body else {
1348 return;
1349 };
1350 match backend {
1351 Backend::OpenAiResponses(_) => set_nested_effort(object, "reasoning", effort),
1353 Backend::Anthropic(_) => set_nested_effort(object, "output_config", effort),
1354 Backend::OpenAiChat(_) => {
1355 object.insert(
1357 "reasoning_effort".to_string(),
1358 Value::String(effort.to_string()),
1359 );
1360 }
1361 }
1362}
1363
1364fn set_nested_effort(object: &mut serde_json::Map<String, Value>, key: &str, effort: &str) {
1367 let container = object
1368 .entry(key.to_string())
1369 .or_insert_with(|| Value::Object(serde_json::Map::new()));
1370 if !container.is_object() {
1371 *container = Value::Object(serde_json::Map::new());
1372 }
1373 if let Value::Object(container) = container {
1374 container.insert("effort".to_string(), Value::String(effort.to_string()));
1375 }
1376}
1377
1378fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {
1380 let Value::Object(object) = body else {
1381 return;
1382 };
1383 for (key, value) in extra_body {
1384 object.entry(key.clone()).or_insert_with(|| value.clone());
1385 }
1386}
1387
1388fn omit_configured_body_fields(body: &mut Value, omit_body_fields: &BTreeSet<String>) {
1389 let Value::Object(object) = body else {
1390 return;
1391 };
1392 for key in omit_body_fields {
1393 object.remove(key);
1394 }
1395}
1396
1397const MAX_CACHE_CONTROL_BLOCKS: usize = 4;
1400
1401fn count_cache_control_blocks(value: &Value) -> usize {
1404 match value {
1405 Value::Object(map) => {
1406 usize::from(map.contains_key("cache_control"))
1407 + map.values().map(count_cache_control_blocks).sum::<usize>()
1408 }
1409 Value::Array(items) => items.iter().map(count_cache_control_blocks).sum(),
1410 _ => 0,
1411 }
1412}
1413
1414fn enable_anthropic_prompt_caching(body: &mut Value, temporary_note_count: usize) {
1416 if count_cache_control_blocks(body) >= MAX_CACHE_CONTROL_BLOCKS {
1419 return;
1420 }
1421 let marker = body
1424 .get("cache_control")
1425 .cloned()
1426 .unwrap_or_else(|| serde_json::json!({"type":"ephemeral"}));
1427 let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
1428 return;
1429 };
1430 let mut remaining = temporary_note_count;
1431 for message in messages.iter_mut().rev() {
1433 match message.get_mut("content") {
1434 Some(content @ Value::String(_)) if remaining == 0 => {
1435 let text = std::mem::take(content);
1436 *content = serde_json::json!([{
1437 "type": "text", "text": text, "cache_control": marker
1438 }]);
1439 return;
1440 }
1441 Some(Value::String(_)) => remaining = remaining.saturating_sub(1),
1442 Some(Value::Array(blocks)) => {
1443 if remaining >= blocks.len() {
1444 remaining -= blocks.len();
1445 continue;
1446 }
1447 let index = blocks.len() - remaining - 1;
1448 if let Some(block) = blocks[index].as_object_mut() {
1449 block.entry("cache_control".to_string()).or_insert(marker);
1450 }
1451 return;
1452 }
1453 _ => {}
1454 }
1455 }
1456}
1457
1458fn enable_openai_guidance_caching(
1461 body: &mut Value,
1462 format: WireFormat,
1463 model: &ModelId,
1464 suffix: usize,
1465) {
1466 let name = model.as_str().rsplit('/').next().unwrap_or_default();
1467 let supported = name == "gpt-5.6"
1468 || name.starts_with("gpt-5.6-")
1469 || name == "gpt-6"
1470 || name.starts_with("gpt-6-")
1471 || name.starts_with("gpt-6.");
1472 if suffix == 0
1475 || !supported
1476 || body
1477 .pointer("/prompt_cache_options/mode")
1478 .and_then(Value::as_str)
1479 == Some("explicit")
1480 {
1481 return;
1482 }
1483 let key = if format == WireFormat::OpenAiResponses {
1484 "input"
1485 } else {
1486 "messages"
1487 };
1488 let Some(items) = body.get_mut(key).and_then(Value::as_array_mut) else {
1489 return;
1490 };
1491 let marker_count = items
1492 .iter()
1493 .map(|item| {
1494 ["content", "output"]
1495 .iter()
1496 .filter_map(|key| item.get(key).and_then(Value::as_array))
1497 .flatten()
1498 .filter(|block| block.get("prompt_cache_breakpoint").is_some())
1499 .count()
1500 })
1501 .sum::<usize>();
1502 if marker_count >= 3 {
1504 return;
1505 }
1506 let Some(prefix_len) = items.len().checked_sub(suffix) else {
1507 return;
1508 };
1509 for item in items[..prefix_len].iter_mut().rev() {
1510 let field = if item.get("type").and_then(Value::as_str) == Some("function_call_output") {
1511 "output"
1512 } else if matches!(
1513 item.get("role").and_then(Value::as_str),
1514 Some("user" | "tool" | "developer" | "system")
1515 ) {
1516 "content"
1517 } else {
1518 continue;
1519 };
1520 let Some(content) = item.get_mut(field) else {
1521 continue;
1522 };
1523 if let Value::String(text) = content {
1524 let text = std::mem::take(text);
1525 let kind = if format == WireFormat::OpenAiResponses {
1526 "input_text"
1527 } else {
1528 "text"
1529 };
1530 *content = serde_json::json!([{"type":kind,"text":text}]);
1531 }
1532 if let Some(block) = content
1534 .as_array_mut()
1535 .and_then(|blocks| blocks.last_mut())
1536 .and_then(Value::as_object_mut)
1537 {
1538 block
1539 .entry("prompt_cache_breakpoint".to_string())
1540 .or_insert_with(|| serde_json::json!({"mode":"explicit"}));
1541 return;
1542 }
1543 }
1544}
1545
1546fn ensure_openai_stream_usage(body: &mut Value) {
1548 let Value::Object(object) = body else {
1549 return;
1550 };
1551 if object.get("stream").and_then(Value::as_bool) != Some(true) {
1552 return;
1553 }
1554
1555 match object.get_mut("stream_options") {
1556 Some(Value::Object(options)) => {
1557 options
1558 .entry("include_usage".to_string())
1559 .or_insert(Value::Bool(true));
1560 }
1561 _ => {
1562 let mut options = Map::new();
1563 options.insert("include_usage".to_string(), Value::Bool(true));
1564 object.insert("stream_options".to_string(), Value::Object(options));
1565 }
1566 }
1567}
1568
1569fn is_allowed_metadata_header(name: &str) -> bool {
1570 ALLOWED_METADATA_HEADERS
1571 .iter()
1572 .any(|allowed| name.eq_ignore_ascii_case(allowed))
1573}
1574
1575fn is_non_forwardable_header(name: &str, headers: &HeaderMap) -> bool {
1576 NON_FORWARDABLE_HEADERS
1577 .iter()
1578 .any(|blocked| name.eq_ignore_ascii_case(blocked))
1579 || headers.get_all("connection").iter().any(|value| {
1580 value
1581 .as_bytes()
1582 .split(|byte| *byte == b',')
1583 .any(|option| option.trim_ascii().eq_ignore_ascii_case(name.as_bytes()))
1584 })
1585}
1586
1587fn redact_forwarded_headers(
1589 mut body: String,
1590 metadata: Option<&Metadata>,
1591 is_forwarding_auth: bool,
1592) -> String {
1593 if !is_forwarding_auth {
1594 return body;
1595 }
1596 let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
1597 return body;
1598 };
1599 for (name, value) in headers {
1600 if is_non_forwardable_header(name.as_str(), headers) {
1601 continue;
1602 }
1603 let Ok(value) = value.to_str() else {
1604 continue;
1605 };
1606 if !value.is_empty() {
1607 body = body.replace(value, "[REDACTED]");
1608 }
1609 }
1610 body
1611}
1612
1613#[cfg(test)]
1614mod tests {
1615 use std::collections::BTreeMap;
1616 use std::error::Error;
1617 use std::io::{Read, Write};
1618 use std::sync::Arc;
1619 use std::sync::atomic::{AtomicUsize, Ordering};
1620 use std::thread::JoinHandle;
1621
1622 use serde_json::json;
1623 use switchyard_protocol::{completion_text, text_request};
1624 use wiremock::matchers::{method, path};
1625 use wiremock::{Mock, MockServer, ResponseTemplate};
1626
1627 use super::*;
1628 use crate::backend::HttpBackendConfig;
1629
1630 fn wire_request(
1632 body: Value,
1633 http_headers: Option<http::HeaderMap>,
1634 wire_format: WireFormat,
1635 ) -> switchyard_translation::Result<Request> {
1636 Ok(Request {
1637 llm_request: switchyard_translation::decode_request(wire_format, &body)?,
1638 raw_request: None,
1639 metadata: Some(Metadata {
1640 http_headers,
1641 ..Default::default()
1642 }),
1643 })
1644 }
1645
1646 fn assert_tool_call_name(agg: &switchyard_protocol::AggLlmResponse, name: &str) {
1647 let Some(switchyard_protocol::ContentBlock::ToolCall(call)) =
1648 agg.outputs[0].content.first()
1649 else {
1650 panic!("expected a tool call, got {:?}", agg.outputs[0].content);
1651 };
1652 assert_eq!(call.name, name);
1653 }
1654
1655 fn config(base_url: &str) -> HttpBackendConfig {
1656 HttpBackendConfig {
1657 base_url: base_url.to_string(),
1658 api_key: Some("secret".to_string()),
1659 forward_auth: false,
1660 extra_headers: BTreeMap::new(),
1661 extra_body: BTreeMap::new(),
1662 omit_body_fields: BTreeSet::new(),
1663 reasoning_effort: None,
1664 max_retries: 0,
1665 failure_cooldown: Duration::ZERO,
1666 timeout: None,
1667 }
1668 }
1669
1670 #[test]
1671 fn provider_endpoint_urls_preserve_query_and_fragment() {
1672 let invalid_url = "not a URL/";
1673 let backend = Backend::OpenAiResponses(config(invalid_url));
1674 assert!(openai_url(invalid_url, "/responses").is_err());
1675 assert_eq!(backend.url(), invalid_url);
1676 assert_eq!(
1677 AuxiliaryOperation::ResponsesCompact.url(&backend),
1678 invalid_url
1679 );
1680
1681 let anthropic = Backend::Anthropic(config(invalid_url));
1682 assert_eq!(anthropic.url(), invalid_url);
1683 assert_eq!(anthropic.count_tokens_url(), invalid_url);
1684
1685 for path in ["", "/v1/", "/v1/messages"] {
1686 for tail in [
1687 "",
1688 "?api-version=2026-09-01&value=%2F/",
1689 "#fragment?value=/",
1690 ] {
1691 let base_url = format!("https://example.com{path}{tail}");
1692 let backend = Backend::Anthropic(config(&base_url));
1693 assert_eq!(
1694 backend.url(),
1695 format!("https://example.com/v1/messages{tail}")
1696 );
1697 assert_eq!(
1698 AuxiliaryOperation::AnthropicCountTokens.url(&backend),
1699 format!("https://example.com/v1/messages/count_tokens{tail}")
1700 );
1701 }
1702 }
1703
1704 for path in ["/v1", "/v1/responses/", "/v1/chat/completions"] {
1705 for tail in [
1706 "",
1707 "?api-version=2026-09-01&value=%2F/",
1708 "#fragment?value=/",
1709 ] {
1710 let base_url = format!("https://example.com{path}{tail}");
1711 let responses = Backend::OpenAiResponses(config(&base_url));
1712 let chat = Backend::OpenAiChat(config(&base_url));
1713 for (actual, endpoint) in [
1714 (responses.url(), "/responses"),
1715 (chat.url(), "/chat/completions"),
1716 (
1717 AuxiliaryOperation::ResponsesCompact.url(&responses),
1718 "/responses/compact",
1719 ),
1720 (
1721 AuxiliaryOperation::ResponsesInputTokens.url(&responses),
1722 "/responses/input_tokens",
1723 ),
1724 ] {
1725 assert_eq!(actual, format!("https://example.com/v1{endpoint}{tail}"));
1726 }
1727 }
1728 }
1729 }
1730
1731 fn config_with_retries(base_url: &str, max_retries: u32) -> HttpBackendConfig {
1732 HttpBackendConfig {
1733 max_retries,
1734 ..config(base_url)
1735 }
1736 }
1737
1738 fn forwarding_config(base_url: &str) -> HttpBackendConfig {
1739 HttpBackendConfig {
1740 api_key: None,
1741 forward_auth: true,
1742 ..config(base_url)
1743 }
1744 }
1745
1746 fn chat_map(base_url: &str) -> Vec<ModelConfig> {
1748 vec![ModelConfig::new(
1749 "gpt",
1750 Backend::OpenAiChat(config(base_url)),
1751 None,
1752 )]
1753 }
1754
1755 fn responses_map(base_url: &str) -> Vec<ModelConfig> {
1756 vec![ModelConfig::new(
1757 "gpt",
1758 Backend::OpenAiResponses(config(base_url)),
1759 None,
1760 )]
1761 }
1762
1763 fn chat_map_with_extra_body(
1764 base_url: &str,
1765 extra_body: BTreeMap<String, Value>,
1766 ) -> Vec<ModelConfig> {
1767 let mut backend = config(base_url);
1768 backend.extra_body = extra_body;
1769 vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
1770 }
1771
1772 fn chat_map_with_omit_body_fields(
1773 base_url: &str,
1774 omit_body_fields: BTreeSet<String>,
1775 ) -> Vec<ModelConfig> {
1776 let mut backend = config(base_url);
1777 backend.omit_body_fields = omit_body_fields;
1778 vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
1779 }
1780
1781 fn chat_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
1782 let mut backend = config(base_url);
1783 backend.reasoning_effort = Some(effort.to_string());
1784 vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
1785 }
1786
1787 fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
1788 let mut backend = config(base_url);
1789 backend.reasoning_effort = Some(effort.to_string());
1790 vec![ModelConfig::new(
1791 "gpt",
1792 Backend::OpenAiResponses(backend),
1793 None,
1794 )]
1795 }
1796
1797 fn anthropic_map(base_url: &str) -> Vec<ModelConfig> {
1798 vec![ModelConfig::new(
1799 "claude",
1800 Backend::Anthropic(config(base_url)),
1801 None,
1802 )]
1803 }
1804
1805 #[tokio::test]
1806 async fn model_properties_uses_configured_auth_and_backend_path()
1807 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
1808 for (base_path, props_path) in [
1809 ("/v1", "/props"),
1810 ("/v1/", "/props"),
1811 ("/v1/chat/completions", "/props"),
1812 ("/v1/responses/", "/props"),
1813 ("/prefix/v1", "/prefix/props"),
1814 ("/prefix/v1/chat/completions", "/prefix/props"),
1815 ("/prefix/v1/responses", "/prefix/props"),
1816 ("/prefix/", "/prefix/props"),
1817 ] {
1818 let server = MockServer::start().await;
1819 Mock::given(method("GET"))
1820 .and(path(props_path))
1821 .and(wiremock::matchers::header("authorization", "Bearer secret"))
1822 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
1823 "default_generation_settings": {"n_ctx": 65_536}
1824 })))
1825 .expect(1)
1826 .mount(&server)
1827 .await;
1828 let client =
1829 TranslatingLlmClient::new(&chat_map(&format!("{}{base_path}", server.uri())))?;
1830 let properties = client
1831 .get_model_properties(
1832 &ModelId::from("gpt"),
1833 WireFormat::OpenAiChat,
1834 Duration::from_secs(1),
1835 )
1836 .await?;
1837 assert_eq!(properties["default_generation_settings"]["n_ctx"], 65_536);
1838 }
1839 Ok(())
1840 }
1841
1842 fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec<ModelConfig> {
1843 vec![ModelConfig::new(
1844 "gpt",
1845 Backend::OpenAiChat(config_with_retries(base_url, max_retries)),
1846 None,
1847 )]
1848 }
1849
1850 fn chat_success_response() -> ResponseTemplate {
1851 ResponseTemplate::new(200).set_body_json(json!({
1852 "id": "chatcmpl-1",
1853 "model": "gpt",
1854 "choices": [{
1855 "index": 0,
1856 "message": {"role": "assistant", "content": "recovered"},
1857 "finish_reason": "stop"
1858 }],
1859 "usage": {}
1860 }))
1861 }
1862
1863 fn truncated_response_server(
1864 content_type: &str,
1865 body: &str,
1866 ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
1867 response_sequence_server(vec![format!(
1868 "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\n\
1869 Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
1870 body.len() + 100
1871 )])
1872 }
1873
1874 fn response_sequence_server(
1875 responses: Vec<String>,
1876 ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
1877 let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
1878 let address = listener.local_addr()?;
1879 let handle = std::thread::spawn(move || {
1880 for response in responses {
1881 let (mut stream, _) = listener.accept()?;
1882 let mut request = [0_u8; 1024];
1883 if stream.read(&mut request)? == 0 {
1884 return Err(std::io::Error::new(
1885 std::io::ErrorKind::UnexpectedEof,
1886 "client closed before sending a request",
1887 ));
1888 }
1889 stream.write_all(response.as_bytes())?;
1890 }
1891 Ok(())
1892 });
1893 Ok((format!("http://{address}/v1"), handle))
1894 }
1895
1896 fn raw_chat_success_response() -> String {
1897 let body = r#"{"id":"chatcmpl-1","model":"gpt","choices":[{"index":0,"message":{"role":"assistant","content":"recovered"},"finish_reason":"stop"}],"usage":{}}"#;
1898 format!(
1899 "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
1900 Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
1901 body.len()
1902 )
1903 }
1904
1905 fn request_for(model: Option<&str>, stream: bool) -> Request {
1906 let mut llm_request = text_request(model.map(str::to_string), "hi");
1907 llm_request.stream = stream;
1908 Request {
1909 llm_request,
1910 raw_request: None,
1911 metadata: None,
1912 }
1913 }
1914
1915 fn request_with_headers(model: &str, headers: HeaderMap) -> Request {
1916 let mut request = request_for(Some(model), false);
1917 request.metadata = Some(Metadata {
1918 http_headers: Some(headers),
1919 ..Default::default()
1920 });
1921 request
1922 }
1923
1924 #[tokio::test]
1925 async fn transport_errors_drop_the_upstream_url() {
1926 let error = reqwest::Client::new()
1927 .post("http://127.0.0.1:1/v1?key=CANARY")
1928 .send()
1929 .await
1930 .expect_err("closed port");
1931
1932 assert!(!convert_reqwest_error(error).to_string().contains("CANARY"));
1933 }
1934
1935 #[tokio::test]
1936 async fn captured_request_build_failure_remains_a_configuration_error() {
1937 let client = TranslatingLlmClient::new(&chat_map("://invalid")).unwrap();
1938 let result =
1939 crate::with_payload_capture(client.call(request_for(Some("gpt"), false))).await;
1940 assert!(matches!(result, Err(LlmClientError::Configuration { .. })));
1941 }
1942
1943 #[test]
1944 fn nvidia_anthropic_tools_drop_unsupported_strict_field() {
1945 let backend = Backend::Anthropic(config("https://inference-api.nvidia.com"));
1946 assert!(nvidia_inference_api_backend(&backend));
1947 assert!(!nvidia_inference_api_backend(&Backend::Anthropic(config(
1948 "https://inference-api.nvidia.com.example.test"
1949 ))));
1950 let mut body = json!({"tools": [{"name": "a", "input_schema": {}, "strict": true}]});
1951 strip_anthropic_tool_strict(&mut body);
1952 assert!(body["tools"][0].get("strict").is_none());
1953 }
1954
1955 #[test]
1956 fn anthropic_prompt_caching_marks_final_message() {
1957 let caller_managed = json!({
1958 "cache_control": {"type": "ephemeral", "ttl": "1h"},
1959 "messages": [{"role": "user", "content": "hello"}]
1960 });
1961 let mut body = caller_managed.clone();
1962 enable_anthropic_prompt_caching(&mut body, 0);
1963 assert_eq!(body["cache_control"], caller_managed["cache_control"]);
1964 assert_eq!(
1965 body["messages"][0]["content"],
1966 json!([{
1967 "type": "text", "text": "hello",
1968 "cache_control": {"type": "ephemeral", "ttl": "1h"}
1969 }])
1970 );
1971
1972 let mut body = json!({
1973 "messages": [{"role": "user", "content": "hello"}]
1974 });
1975
1976 enable_anthropic_prompt_caching(&mut body, 0);
1977
1978 assert_eq!(
1979 body["messages"][0]["content"][0]["cache_control"],
1980 json!({"type": "ephemeral"})
1981 );
1982 }
1983
1984 fn cache_control_blocks(value: &Value) -> usize {
1986 count_cache_control_blocks(value)
1987 }
1988
1989 fn body_at_the_cache_control_limit() -> Value {
1992 json!({
1993 "system": [
1994 {"type": "text", "text": "a", "cache_control": {"type": "ephemeral"}},
1995 {"type": "text", "text": "b", "cache_control": {"type": "ephemeral"}}
1996 ],
1997 "tools": [
1998 {"name": "t", "input_schema": {"type": "object"},
1999 "cache_control": {"type": "ephemeral"}}
2000 ],
2001 "messages": [
2002 {"role": "user", "content": [
2003 {"type": "text", "text": "old", "cache_control": {"type": "ephemeral"}}
2004 ]},
2005 {"role": "assistant", "content": [{"type": "text", "text": "ok"}]},
2006 {"role": "user", "content": [{"type": "text", "text": "new"}]}
2007 ]
2008 })
2009 }
2010
2011 #[test]
2012 fn anthropic_prompt_caching_abstains_at_the_cache_control_limit() {
2013 let mut body = body_at_the_cache_control_limit();
2014 assert_eq!(cache_control_blocks(&body), 4);
2015
2016 enable_anthropic_prompt_caching(&mut body, 0);
2017
2018 assert_eq!(cache_control_blocks(&body), 4);
2021 assert!(
2022 body["messages"][2]["content"][0]
2023 .get("cache_control")
2024 .is_none()
2025 );
2026 }
2027
2028 #[test]
2029 fn anthropic_prompt_caching_still_marks_one_below_the_limit() {
2030 let mut body = body_at_the_cache_control_limit();
2031 body["system"].as_array_mut().unwrap().pop();
2033 assert_eq!(cache_control_blocks(&body), 3);
2034
2035 enable_anthropic_prompt_caching(&mut body, 0);
2036
2037 assert_eq!(cache_control_blocks(&body), 4);
2038 assert_eq!(
2039 body["messages"][2]["content"][0]["cache_control"],
2040 json!({"type": "ephemeral"})
2041 );
2042 }
2043
2044 fn request_with_wire_format(model: &str, format: WireFormat) -> Request {
2047 let mut request = request_for(Some(model), false);
2048 request.metadata = Some(Metadata {
2049 session_id: None,
2050 agent_id: None,
2051 task_id: None,
2052 correlation_id: None,
2053 extra_metadata: None,
2054 http_headers: None,
2055 wire_format: Some(format),
2056 ..Default::default()
2057 });
2058 request
2059 }
2060
2061 #[tokio::test]
2062 async fn missing_model_errors()
2063 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2064 let client = TranslatingLlmClient::new(&[])?;
2065 let Err(error) = client
2066 .call_rewrite_model(request_for(None, false), None)
2067 .await
2068 else {
2069 panic!("expected an error");
2070 };
2071 assert!(matches!(
2072 error,
2073 LlmClientError::InvalidRequest { message } if message == "no model given"
2074 ));
2075 Ok(())
2076 }
2077
2078 #[tokio::test]
2079 async fn unknown_model_errors()
2080 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2081 let client = TranslatingLlmClient::new(&[])?;
2082 let Err(error) = client
2083 .call_rewrite_model(request_for(Some("gpt"), false), None)
2084 .await
2085 else {
2086 panic!("expected an error");
2087 };
2088 assert!(matches!(
2089 error,
2090 LlmClientError::Configuration { message }
2091 if message.contains("gpt")
2092 ));
2093 Ok(())
2094 }
2095
2096 #[tokio::test]
2097 async fn unknown_model_format_errors()
2098 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2099 let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
2101 let Err(error) = client
2102 .call_rewrite_model(
2103 request_with_wire_format("gpt", WireFormat::AnthropicMessages),
2104 None,
2105 )
2106 .await
2107 else {
2108 panic!("expected an error");
2109 };
2110 assert!(matches!(
2111 error,
2112 LlmClientError::Configuration { message }
2113 if message.contains("gpt")
2114 && message.contains(&WireFormat::AnthropicMessages.to_string())
2115 ));
2116 Ok(())
2117 }
2118
2119 #[test]
2120 fn backend_for_resolves_configured_format()
2121 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2122 let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
2123 assert!(
2125 client
2126 .backend_for(&ModelId::from("gpt"), WireFormat::OpenAiChat)
2127 .is_some()
2128 );
2129 assert!(
2130 client
2131 .backend_for(&ModelId::from("gpt"), WireFormat::AnthropicMessages)
2132 .is_none()
2133 );
2134 assert!(
2135 client
2136 .backend_for(&ModelId::from("missing"), WireFormat::OpenAiChat)
2137 .is_none()
2138 );
2139 Ok(())
2140 }
2141
2142 #[tokio::test]
2143 async fn target_name_arg_wins_over_request_model()
2144 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2145 let client = TranslatingLlmClient::new(&[])?;
2146 let Err(error) = client
2148 .call_rewrite_model(request_for(Some("a"), false), Some(&ModelId::from("b")))
2149 .await
2150 else {
2151 panic!("expected an error");
2152 };
2153 assert!(matches!(
2154 error,
2155 LlmClientError::Configuration { message }
2156 if message.contains("\"b\"")
2157 ));
2158 Ok(())
2159 }
2160
2161 #[tokio::test]
2162 async fn buffered_openai_chat_round_trips()
2163 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2164 let server = MockServer::start().await;
2165 Mock::given(method("POST"))
2166 .and(path("/v1/chat/completions"))
2167 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2168 "id": "chatcmpl-1",
2169 "model": "gpt",
2170 "choices": [{
2171 "index": 0,
2172 "message": {"role": "assistant", "content": "Hi there"},
2173 "finish_reason": "stop"
2174 }],
2175 "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
2176 })))
2177 .mount(&server)
2178 .await;
2179
2180 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
2181
2182 let response = client
2183 .call_rewrite_model(request_for(Some("gpt"), false), None)
2184 .await?;
2185 let agg = response.llm_response.into_agg().await?;
2186 assert_eq!(completion_text(&agg), "Hi there");
2187
2188 Ok(())
2189 }
2190
2191 #[tokio::test]
2192 async fn invalid_json_is_a_response_translation_error()
2193 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2194 let server = MockServer::start().await;
2195 let calls = Arc::new(AtomicUsize::new(0));
2196 let observed_calls = Arc::clone(&calls);
2197 Mock::given(method("POST"))
2198 .respond_with(move |_: &wiremock::Request| {
2199 observed_calls.fetch_add(1, Ordering::SeqCst);
2200 ResponseTemplate::new(200).set_body_raw("not json", "application/json")
2201 })
2202 .mount(&server)
2203 .await;
2204
2205 let client =
2206 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
2207 let Err(error) = client
2208 .call_rewrite_model(request_for(Some("gpt"), false), None)
2209 .await
2210 else {
2211 panic!("expected invalid JSON to fail");
2212 };
2213
2214 assert!(matches!(
2215 error,
2216 LlmClientError::ResponseTranslation(message)
2217 if message.contains("invalid upstream JSON")
2218 ));
2219 assert_eq!(calls.load(Ordering::SeqCst), 1);
2220 Ok(())
2221 }
2222
2223 #[tokio::test]
2224 async fn response_body_io_failure_is_a_transport_error()
2225 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2226 let (base_url, server) = truncated_response_server("application/json", "{}")?;
2227 let client = TranslatingLlmClient::new(&chat_map(&base_url))?;
2228 let result = client
2229 .call_rewrite_model(request_for(Some("gpt"), false), None)
2230 .await;
2231 server
2232 .join()
2233 .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2234
2235 let Err(error) = result else {
2236 panic!("expected the truncated response body to fail");
2237 };
2238 let LlmClientError::Transport { source } = error else {
2239 panic!("expected a transport error");
2240 };
2241 let Some(source) = source.downcast_ref::<reqwest::Error>() else {
2242 panic!("expected the reqwest transport source");
2243 };
2244 assert!(source.is_decode());
2245 assert!(
2246 !std::error::Error::source(&source)
2247 .is_some_and(|source| source.is::<serde_json::Error>())
2248 );
2249 Ok(())
2250 }
2251
2252 #[tokio::test]
2253 async fn response_body_transport_failures_are_retried()
2254 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2255 let truncated_responses = [
2256 "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
2257 Content-Length: 102\r\nConnection: close\r\n\r\n{}",
2258 "HTTP/1.1 401 Unauthorized\r\nContent-Type: text/plain\r\n\
2259 Content-Length: 100\r\nConnection: close\r\n\r\nbad",
2260 ];
2261
2262 for truncated in truncated_responses {
2263 let (base_url, server) =
2264 response_sequence_server(vec![truncated.to_string(), raw_chat_success_response()])?;
2265 let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
2266 let response = client
2267 .call_rewrite_model(request_for(Some("gpt"), false), None)
2268 .await?;
2269 server
2270 .join()
2271 .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2272
2273 assert_eq!(
2274 completion_text(&response.llm_response.into_agg().await?),
2275 "recovered"
2276 );
2277 }
2278 Ok(())
2279 }
2280
2281 #[tokio::test]
2282 async fn pre_first_event_transport_failure_uses_retry_budget()
2283 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2284 let truncated = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\
2285 Content-Length: 100\r\nConnection: close\r\n\r\n";
2286 let body = "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"recovered\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
2287 let success = format!(
2288 "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\
2289 Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
2290 body.len()
2291 );
2292
2293 for second in [success, truncated.to_string()] {
2294 let is_exhausted = second == truncated;
2295 let (base_url, server) = response_sequence_server(vec![truncated.to_string(), second])?;
2296 let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
2297 let result = client
2298 .call_rewrite_model(request_for(Some("gpt"), true), None)
2299 .await;
2300 if is_exhausted {
2301 assert!(matches!(result, Err(LlmClientError::Transport { .. })));
2302 } else {
2303 assert_eq!(
2304 completion_text(&result?.llm_response.into_agg().await?),
2305 "recovered"
2306 );
2307 }
2308 server
2309 .join()
2310 .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2311 }
2312 Ok(())
2313 }
2314
2315 #[tokio::test]
2316 async fn streaming_body_io_failure_preserves_transport_error()
2317 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2318 let body = "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n";
2319 let (base_url, server) = truncated_response_server("text/event-stream", body)?;
2320 let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 2))?;
2321 let response = client
2322 .call_rewrite_model(request_for(Some("gpt"), true), None)
2323 .await?;
2324 let result = response.llm_response.into_agg().await;
2325 server
2326 .join()
2327 .map_err(|_| std::io::Error::other("response server thread panicked"))??;
2328
2329 let Err(error) = result else {
2330 panic!("expected the truncated stream body to fail");
2331 };
2332
2333 assert!(matches!(error, LlmClientError::Transport { .. }));
2334 Ok(())
2335 }
2336
2337 #[tokio::test]
2338 async fn rewrites_model_to_resolved_upstream_id()
2339 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2340 let server = MockServer::start().await;
2342 Mock::given(method("POST"))
2343 .and(path("/v1/chat/completions"))
2344 .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"})))
2345 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2346 "id": "1", "model": "gpt",
2347 "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
2348 "usage": {}
2349 })))
2350 .mount(&server)
2351 .await;
2352
2353 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
2354 client
2356 .call_rewrite_model(
2357 request_for(Some("switchyard"), false),
2358 Some(&ModelId::from("gpt")),
2359 )
2360 .await?;
2361 Ok(())
2363 }
2364
2365 #[tokio::test]
2368 async fn reasoning_effort_override_replaces_the_callers_effort()
2369 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2370 let server = MockServer::start().await;
2371 Mock::given(method("POST"))
2372 .and(path("/v1/chat/completions"))
2373 .and(wiremock::matchers::body_partial_json(json!({
2374 "model": "gpt",
2375 "reasoning_effort": "max"
2376 })))
2377 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2378 "id": "1",
2379 "model": "gpt",
2380 "choices": [{
2381 "index": 0,
2382 "message": {"role": "assistant", "content": "ok"},
2383 "finish_reason": "stop"
2384 }],
2385 "usage": {}
2386 })))
2387 .mount(&server)
2388 .await;
2389 let client = TranslatingLlmClient::new(&chat_map_with_effort(
2390 &format!("{}/v1", server.uri()),
2391 "max",
2392 ))?;
2393 client
2394 .call_rewrite_model(
2395 wire_request(
2396 json!({
2397 "model": "client-facing",
2398 "messages": [{"role": "user", "content": "hi"}],
2399 "reasoning_effort": "high"
2400 }),
2401 None,
2402 WireFormat::OpenAiChat,
2403 )?,
2404 Some(&ModelId::from("gpt")),
2405 )
2406 .await?;
2407
2408 let server = MockServer::start().await;
2409 Mock::given(method("POST"))
2410 .and(path("/v1/responses"))
2411 .and(wiremock::matchers::body_partial_json(json!({
2412 "model": "gpt",
2413 "reasoning": {"effort": "max", "summary": "auto"}
2414 })))
2415 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2416 "id": "resp_1",
2417 "object": "response",
2418 "model": "gpt",
2419 "status": "completed",
2420 "output": [{
2421 "type": "message",
2422 "id": "msg_1",
2423 "role": "assistant",
2424 "status": "completed",
2425 "content": [{"type": "output_text", "text": "ok"}]
2426 }],
2427 "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
2428 })))
2429 .mount(&server)
2430 .await;
2431 let client = TranslatingLlmClient::new(&responses_map_with_effort(
2432 &format!("{}/v1", server.uri()),
2433 "max",
2434 ))?;
2435 client
2436 .call_rewrite_model(
2437 wire_request(
2438 json!({
2439 "model": "client-facing",
2440 "input": [{"role": "user", "content": "hi"}],
2441 "reasoning": {"effort": "high", "summary": "auto"}
2442 }),
2443 None,
2444 WireFormat::OpenAiResponses,
2445 )?,
2446 Some(&ModelId::from("gpt")),
2447 )
2448 .await?;
2449 Ok(())
2450 }
2451
2452 #[tokio::test]
2453 async fn reasoning_effort_map_only_rewrites_matching_effort_for_the_selected_target()
2454 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2455 let server = MockServer::start().await;
2456 Mock::given(method("POST"))
2457 .and(path("/v1/chat/completions"))
2458 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2459 "id": "1", "model": "gpt",
2460 "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
2461 "usage": {}
2462 })))
2463 .expect(4)
2464 .mount(&server)
2465 .await;
2466 let backend = Backend::OpenAiChat(config(&format!("{}/v1", server.uri())));
2467 let models = [
2468 ModelConfig::new("mapped", backend.clone(), None)
2469 .with_provider_model("gpt")
2470 .with_reasoning_effort_map(BTreeMap::from([("minimal".into(), "low".into())])),
2471 ModelConfig::new("unmapped", backend, None).with_provider_model("gpt"),
2472 ];
2473 let client = TranslatingLlmClient::new(&models)?;
2474 let cases = [
2475 ("mapped", Some("minimal"), Some("low")),
2476 ("mapped", Some("high"), Some("high")),
2477 ("mapped", None, None),
2478 ("unmapped", Some("minimal"), Some("minimal")),
2479 ];
2480 for (target, effort, _) in cases {
2481 let mut body = json!({
2482 "model": "client-facing",
2483 "messages": [{"role": "user", "content": "hi"}]
2484 });
2485 if let Some(effort) = effort {
2486 body["reasoning_effort"] = json!(effort);
2487 }
2488 client
2489 .call_rewrite_model(
2490 wire_request(body, None, WireFormat::OpenAiChat)?,
2491 Some(&ModelId::from(target)),
2492 )
2493 .await?;
2494 }
2495 let calls = server.received_requests().await.ok_or("missing requests")?;
2496 for (call, (_, _, expected)) in calls.iter().zip(cases) {
2497 let body: Value = serde_json::from_slice(&call.body)?;
2498 assert_eq!(body["model"], "gpt");
2499 assert_eq!(
2500 body.get("reasoning_effort"),
2501 expected.map(Value::from).as_ref()
2502 );
2503 }
2504 Ok(())
2505 }
2506
2507 #[tokio::test]
2509 async fn anthropic_requests_drop_tool_strict_false()
2510 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2511 let server = MockServer::start().await;
2512 Mock::given(method("POST"))
2513 .and(path("/v1/messages"))
2514 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2515 "id": "msg_1",
2516 "type": "message",
2517 "role": "assistant",
2518 "model": "claude",
2519 "content": [{"type": "text", "text": "ok"}],
2520 "stop_reason": "end_turn",
2521 "usage": {"input_tokens": 1, "output_tokens": 1}
2522 })))
2523 .expect(2)
2524 .mount(&server)
2525 .await;
2526 let client = TranslatingLlmClient::new(&[ModelConfig::new(
2527 "claude",
2528 Backend::Anthropic(config(&server.uri())),
2529 None,
2530 )])?;
2531 let schema = json!({"type": "object"});
2532 let requests = [
2533 (
2534 WireFormat::OpenAiChat,
2535 json!({
2536 "model": "client-facing",
2537 "messages": [{"role": "user", "content": "hi"}],
2538 "tools": [
2539 {"type": "function", "function": {"name": "loose", "parameters": schema, "strict": false}},
2540 {"type": "function", "function": {"name": "exact", "parameters": schema, "strict": true}}
2541 ]
2542 }),
2543 ),
2544 (
2545 WireFormat::AnthropicMessages,
2546 json!({
2547 "model": "client-facing",
2548 "max_tokens": 8,
2549 "messages": [{"role": "user", "content": "hi"}],
2550 "tools": [
2551 {"name": "loose", "input_schema": schema, "strict": false},
2552 {"name": "exact", "input_schema": schema, "strict": true}
2553 ]
2554 }),
2555 ),
2556 ];
2557 for (format, request) in requests {
2558 client
2559 .call_rewrite_model(
2560 wire_request(request, None, format)?,
2561 Some(&ModelId::from("claude")),
2562 )
2563 .await?;
2564 }
2565 for received in server.received_requests().await.unwrap_or_default() {
2566 let body: Value = serde_json::from_slice(&received.body)?;
2567 assert!(body["tools"][0].get("strict").is_none(), "{body}");
2568 assert_eq!(body["tools"][1]["strict"], true, "{body}");
2569 }
2570 Ok(())
2571 }
2572
2573 #[tokio::test]
2574 async fn template_rejection_retries_once_without_changing_other_fields()
2575 -> std::result::Result<(), Box<dyn Error + Sync + Send>> {
2576 for (message, reject_retry, omit) in [
2577 (
2578 "Unrecognized request argument supplied: chat_template_kwargs",
2579 false,
2580 false,
2581 ),
2582 (
2583 "Unrecognized request argument supplied: chat_template_kwargs",
2584 true,
2585 false,
2586 ),
2587 (
2588 "Invalid temperature for chat_template_kwargs request",
2589 false,
2590 false,
2591 ),
2592 (
2593 "Unrecognized request argument supplied: chat_template_kwargs",
2594 false,
2595 true,
2596 ),
2597 ] {
2598 let server = MockServer::start().await;
2599 Mock::given(method("POST"))
2600 .respond_with(move |request: &wiremock::Request| {
2601 let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
2602 if body.get("chat_template_kwargs").is_some() || reject_retry {
2603 ResponseTemplate::new(400).set_body_json(json!({"error": {"message": message}}))
2604 } else {
2605 ResponseTemplate::new(200).set_body_json(json!({
2606 "id": "1", "model": "gpt", "choices": [{"index": 0,
2607 "message": {"role": "assistant", "content": "yes"}, "finish_reason": "stop"}],
2608 "usage": {}
2609 }))
2610 }
2611 }).mount(&server).await;
2612 let fields = if omit {
2613 BTreeSet::from(["chat_template_kwargs".into()])
2614 } else {
2615 BTreeSet::new()
2616 };
2617 let client = TranslatingLlmClient::new(&chat_map_with_omit_body_fields(
2618 &format!("{}/v1", server.uri()),
2619 fields,
2620 ))?;
2621 let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "judge"}],
2622 "reasoning_effort": "none", "logprobs": true, "top_logprobs": 8,
2623 "chat_template_kwargs": {"enable_thinking": false}});
2624 let result = client
2625 .call_rewrite_model(
2626 wire_request(raw.clone(), None, WireFormat::OpenAiChat)?,
2627 Some(&ModelId::from("gpt")),
2628 )
2629 .await;
2630 let rejects_field = message.starts_with("Unrecognized");
2631 assert_eq!(result.is_ok(), omit || (rejects_field && !reject_retry));
2632 let calls = server.received_requests().await.ok_or("missing requests")?;
2633 assert_eq!(calls.len(), if !omit && rejects_field { 2 } else { 1 });
2634 for (index, call) in calls.iter().enumerate() {
2635 let mut expected = raw.clone();
2636 if omit || index > 0 {
2637 expected
2638 .as_object_mut()
2639 .unwrap()
2640 .remove("chat_template_kwargs");
2641 }
2642 assert_eq!(serde_json::from_slice::<Value>(&call.body)?, expected);
2643 }
2644 }
2645 Ok(())
2646 }
2647
2648 #[tokio::test]
2649 async fn template_compatibility_preserves_the_transient_retry_budget()
2650 -> std::result::Result<(), Box<dyn Error + Sync + Send>> {
2651 let server = MockServer::start().await;
2652 let count = Arc::new(AtomicUsize::new(0));
2653 let observed = Arc::clone(&count);
2654 Mock::given(method("POST"))
2655 .respond_with(move |request: &wiremock::Request| {
2656 let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
2657 match observed.fetch_add(1, Ordering::SeqCst) {
2658 0 => {
2659 assert!(body.get("chat_template_kwargs").is_some());
2660 ResponseTemplate::new(422).set_body_json(json!({"error": {
2661 "code": "unsupported_parameter", "param": "chat_template_kwargs"
2662 }}))
2663 }
2664 1 => {
2665 assert!(body.get("chat_template_kwargs").is_none());
2666 ResponseTemplate::new(503).insert_header("retry-after", "0")
2667 }
2668 _ => ResponseTemplate::new(200).set_body_json(json!({
2669 "id": "1", "model": "gpt", "choices": [{"index": 0,
2670 "message": {"role": "assistant", "content": "yes"}, "finish_reason": "stop"}],
2671 "usage": {}
2672 })),
2673 }
2674 }).mount(&server).await;
2675 let client =
2676 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
2677 client
2678 .call_rewrite_model(
2679 wire_request(
2680 json!({"model": "gpt", "messages": [{"role": "user", "content": "judge"}],
2681 "chat_template_kwargs": {"enable_thinking": false}}),
2682 None,
2683 WireFormat::OpenAiChat,
2684 )?,
2685 Some(&ModelId::from("gpt")),
2686 )
2687 .await?;
2688 assert_eq!(count.load(Ordering::SeqCst), 3);
2689 Ok(())
2690 }
2691
2692 #[tokio::test]
2693 async fn preserved_codex_string_input_is_normalized_at_the_client_boundary()
2694 -> std::result::Result<(), Box<dyn Error + Sync + Send>> {
2695 for root in ["/backend-api/codex", "/v1"] {
2696 for input in [
2697 json!("hello"),
2698 json!([{"role": "user", "content": "hello"}]),
2699 ] {
2700 let server = MockServer::start().await;
2701 Mock::given(method("POST"))
2702 .and(path(format!("{root}/responses")))
2703 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2704 "id": "resp_1", "object": "response", "model": "gpt", "status": "completed", "output": [], "usage": {}
2705 }))).expect(1).mount(&server).await;
2706 let client =
2707 TranslatingLlmClient::new(&responses_map(&format!("{}{root}", server.uri())))?;
2708 let raw = json!({"model": "client-name", "input": input,
2709 "instructions": "Be brief.", "store": false, "metadata": {"task": "judge"},
2710 "custom_field": {"keep": true}});
2711 client
2712 .call_rewrite_model(
2713 wire_request(raw.clone(), None, WireFormat::OpenAiResponses)?,
2714 Some(&ModelId::from("gpt")),
2715 )
2716 .await?;
2717 let mut expected = raw;
2718 expected["model"] = json!("gpt");
2719 if root == "/backend-api/codex" && input.is_string() {
2720 expected["input"] =
2721 json!([{"type": "message", "role": "user", "content": input}]);
2722 }
2723 let calls = server.received_requests().await.ok_or("missing requests")?;
2724 assert_eq!(serde_json::from_slice::<Value>(&calls[0].body)?, expected);
2725 }
2726 }
2727 Ok(())
2728 }
2729
2730 #[tokio::test]
2731 async fn token_counting_does_not_send_the_effort_override()
2732 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2733 let server = MockServer::start().await;
2734 Mock::given(method("POST"))
2735 .and(path("/v1/messages/count_tokens"))
2736 .respond_with(ResponseTemplate::new(200).set_body_json(json!({"input_tokens": 3})))
2737 .expect(1)
2738 .mount(&server)
2739 .await;
2740 let mut backend = config(&server.uri());
2741 backend.reasoning_effort = Some("max".to_string());
2742 let client = TranslatingLlmClient::new(&[ModelConfig::new(
2743 "claude",
2744 Backend::Anthropic(backend),
2745 None,
2746 )])?;
2747 let llm_request = switchyard_translation::decode_request(
2748 WireFormat::AnthropicMessages,
2749 &json!({"model": "claude", "messages": [{"role": "user", "content": "hi"}]}),
2750 )?;
2751 let request = Request {
2752 llm_request,
2753 raw_request: None,
2754 metadata: None,
2755 };
2756 client
2757 .call_auxiliary(
2758 &ModelId::from("claude"),
2759 request,
2760 AuxiliaryOperation::AnthropicCountTokens,
2761 )
2762 .await?;
2763 let received = server.received_requests().await.unwrap_or_default();
2764 let body: Value = serde_json::from_slice(&received[0].body)?;
2765 assert!(body.get("output_config").is_none(), "{body}");
2766 Ok(())
2767 }
2768
2769 #[tokio::test]
2770 async fn reasoning_effort_override_sets_anthropic_output_config_effort()
2771 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2772 let server = MockServer::start().await;
2773 let format = json!({"type": "json_schema", "schema": {"type": "object"}});
2774 Mock::given(method("POST"))
2775 .and(path("/v1/messages"))
2776 .and(wiremock::matchers::body_partial_json(json!({
2777 "model": "claude",
2778 "output_config": {"effort": "max", "format": format}
2779 })))
2780 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2781 "id": "msg_1",
2782 "type": "message",
2783 "role": "assistant",
2784 "model": "claude",
2785 "content": [{"type": "text", "text": "ok"}],
2786 "stop_reason": "end_turn",
2787 "usage": {"input_tokens": 1, "output_tokens": 1}
2788 })))
2789 .expect(1)
2790 .mount(&server)
2791 .await;
2792 let mut backend = config(&server.uri());
2793 backend.reasoning_effort = Some("max".to_string());
2794 let client = TranslatingLlmClient::new(&[ModelConfig::new(
2795 "claude",
2796 Backend::Anthropic(backend),
2797 None,
2798 )])?;
2799 client
2800 .call_rewrite_model(
2801 wire_request(
2802 json!({
2803 "model": "client-facing",
2804 "max_tokens": 7,
2805 "messages": [{"role": "user", "content": "hi"}],
2806 "output_config": {"effort": "high", "format": format}
2807 }),
2808 None,
2809 WireFormat::AnthropicMessages,
2810 )?,
2811 Some(&ModelId::from("claude")),
2812 )
2813 .await?;
2814 Ok(())
2815 }
2816
2817 #[tokio::test]
2818 async fn extra_body_adds_defaults_without_overriding_the_request()
2819 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2820 let server = MockServer::start().await;
2821 Mock::given(method("POST"))
2822 .and(path("/v1/chat/completions"))
2823 .and(wiremock::matchers::body_partial_json(json!({
2824 "model": "gpt",
2825 "max_tokens": 7,
2826 "service_tier": "priority"
2827 })))
2828 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2829 "id": "1",
2830 "model": "gpt",
2831 "choices": [{
2832 "index": 0,
2833 "message": {"role": "assistant", "content": "ok"},
2834 "finish_reason": "stop"
2835 }],
2836 "usage": {}
2837 })))
2838 .mount(&server)
2839 .await;
2840
2841 let extra_body = BTreeMap::from([
2842 ("max_tokens".to_string(), json!(999)),
2843 ("service_tier".to_string(), json!("priority")),
2844 ]);
2845 let client = TranslatingLlmClient::new(&chat_map_with_extra_body(
2846 &format!("{}/v1", server.uri()),
2847 extra_body,
2848 ))?;
2849 let raw = json!({
2850 "model": "client-facing",
2851 "messages": [{"role": "user", "content": "hi"}],
2852 "max_tokens": 7
2853 });
2854
2855 client
2856 .call_rewrite_model(
2857 wire_request(raw, None, WireFormat::OpenAiChat)?,
2858 Some(&ModelId::from("gpt")),
2859 )
2860 .await?;
2861 Ok(())
2862 }
2863
2864 #[tokio::test]
2865 async fn omit_body_fields_strips_a_field_extra_body_cannot_override()
2866 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2867 let server = MockServer::start().await;
2868 Mock::given(method("POST"))
2869 .and(path("/v1/chat/completions"))
2870 .and(|request: &wiremock::Request| {
2871 let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
2872 body.get("max_tokens").is_none()
2873 })
2874 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2875 "id": "1",
2876 "model": "gpt",
2877 "choices": [{
2878 "index": 0,
2879 "message": {"role": "assistant", "content": "ok"},
2880 "finish_reason": "stop"
2881 }],
2882 "usage": {}
2883 })))
2884 .mount(&server)
2885 .await;
2886
2887 let client = TranslatingLlmClient::new(&chat_map_with_omit_body_fields(
2888 &format!("{}/v1", server.uri()),
2889 BTreeSet::from(["max_tokens".to_string()]),
2890 ))?;
2891 let raw = json!({
2892 "model": "client-facing",
2893 "messages": [{"role": "user", "content": "hi"}],
2894 "max_tokens": 7
2895 });
2896
2897 client
2898 .call_rewrite_model(
2899 wire_request(raw, None, WireFormat::OpenAiChat)?,
2900 Some(&ModelId::from("gpt")),
2901 )
2902 .await?;
2903 Ok(())
2904 }
2905
2906 #[tokio::test]
2910 async fn anthropic_requests_drop_unsigned_thinking_blocks()
2911 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2912 let server = MockServer::start().await;
2913 Mock::given(method("POST"))
2914 .and(path("/v1/messages"))
2915 .and(|request: &wiremock::Request| {
2916 let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
2917 let messages = body.get("messages").and_then(Value::as_array).cloned();
2918 let Some(messages) = messages else {
2919 return false;
2920 };
2921 let blocks: Vec<&Value> = messages
2925 .iter()
2926 .filter_map(|message| message.get("content"))
2927 .filter_map(Value::as_array)
2928 .flatten()
2929 .collect();
2930 let thinking: Vec<&&Value> = blocks
2931 .iter()
2932 .filter(|block| block.get("type").and_then(Value::as_str) == Some("thinking"))
2933 .collect();
2934 thinking.len() == 1
2935 && thinking[0].get("signature").and_then(Value::as_str) == Some("sig-abc")
2936 && messages
2937 .iter()
2938 .all(|message| message.get("content") != Some(&json!([])))
2939 })
2940 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
2941 "id": "msg_1",
2942 "type": "message",
2943 "role": "assistant",
2944 "model": "claude",
2945 "content": [{"type": "text", "text": "ok"}],
2946 "stop_reason": "end_turn",
2947 "usage": {"input_tokens": 1, "output_tokens": 1}
2948 })))
2949 .mount(&server)
2950 .await;
2951
2952 let client = TranslatingLlmClient::new(&anthropic_map(&server.uri()))?;
2953 let raw = json!({
2954 "model": "client-facing",
2955 "max_tokens": 7,
2956 "messages": [
2957 {"role": "user", "content": "fix the build"},
2958 {"role": "assistant", "content": [
2959 {"type": "thinking", "thinking": "weak tier reasoning", "signature": ""}
2960 ]},
2961 {"role": "assistant", "content": [
2962 {"type": "thinking", "thinking": "signed reasoning", "signature": "sig-abc"},
2963 {"type": "text", "text": "here goes"}
2964 ]},
2965 {"role": "user", "content": "continue"}
2966 ]
2967 });
2968
2969 client
2970 .call_rewrite_model(
2971 wire_request(raw, None, WireFormat::AnthropicMessages)?,
2972 Some(&ModelId::from("claude")),
2973 )
2974 .await?;
2975 Ok(())
2976 }
2977
2978 #[tokio::test]
2979 async fn responses_reasoning_policy_preserves_replay_history()
2980 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
2981 use crate::ResponsesReasoningPolicy::{Auto, Drop, PreserveEncrypted, PreservePlaintext};
2982 let summary = json!([{"type": "summary_text", "text": "Check the parser."}]);
2983 let history = json!([
2984 {"type": "message", "role": "user", "content": "continue"},
2985 {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]},
2986 {"type": "reasoning", "id": "rs_stored", "status": "completed", "summary": []},
2987 {"type": "reasoning", "id": "rs_glm", "summary": summary, "encrypted_content": null},
2989 {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "Run the tests."}], "encrypted_content": ""},
2990 {"type": "reasoning"},
2991 {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"},
2992 {"type": "function_call_output", "call_id": "call_1", "output": "ok"}
2993 ]);
2994 let cases = [
2996 (Auto, "openai/gpt-5", PreserveEncrypted),
2997 (Auto, "nvidia/zai-org/glm-5.3-flash", PreservePlaintext),
2998 (
2999 PreserveEncrypted,
3000 "nvidia/zai-org/glm-5.3-flash",
3001 PreserveEncrypted,
3002 ),
3003 (Drop, "openai/gpt-5", Drop),
3004 ];
3005 for store in [Some(true), Some(false), None] {
3006 for (policy, model, behavior) in cases {
3007 for has_input_override in [false, true] {
3008 let server = MockServer::start().await;
3009 Mock::given(method("POST"))
3010 .and(path("/v1/responses"))
3011 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3012 "id": "resp_1", "object": "response", "model": "gpt",
3013 "status": "completed", "output": [],
3014 "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
3015 })))
3016 .expect(1)
3017 .mount(&server)
3018 .await;
3019 let mut backend = config(&format!("{}/v1", server.uri()));
3020 if has_input_override {
3021 backend.omit_body_fields.insert("input".to_string());
3022 backend
3023 .extra_body
3024 .insert("input".to_string(), history.clone());
3025 }
3026 let client = TranslatingLlmClient::new(&[ModelConfig::new(
3027 model,
3028 Backend::OpenAiResponses(backend),
3029 None,
3030 )
3031 .with_responses_reasoning(policy)])?;
3032 let mut request = json!({"model": "switchyard", "input": history});
3033 if let Some(store) = store {
3034 request["store"] = json!(store);
3035 }
3036 client
3037 .call_rewrite_model(
3038 wire_request(request, None, WireFormat::OpenAiResponses)?,
3039 Some(&ModelId::from(model)),
3040 )
3041 .await?;
3042 let requests = server.received_requests().await.ok_or("missing requests")?;
3043 let body: Value = serde_json::from_slice(&requests[0].body)?;
3044 let stored = store != Some(false);
3045 let mut expected = vec![history[0].clone()];
3046 match behavior {
3047 PreserveEncrypted => {
3049 let indexes: &[usize] = if stored { &[1, 2, 3] } else { &[1] };
3050 for &index in indexes {
3051 let mut item = history[index].clone();
3052 item["content"] = json!([]);
3053 expected.push(item);
3054 }
3055 }
3056 PreservePlaintext => {
3058 let mut item = history[1].clone();
3059 item.as_object_mut()
3060 .ok_or("object")?
3061 .remove("encrypted_content");
3062 expected.push(item);
3063 if stored {
3064 expected.push(history[2].clone());
3065 }
3066 expected.extend([history[3].clone(), history[4].clone()]);
3067 }
3068 _ => {}
3069 }
3070 expected.extend([history[6].clone(), history[7].clone()]);
3071 assert_eq!(
3072 body["input"],
3073 json!(expected),
3074 "store={store:?}, policy={policy:?}, model={model}, replace={has_input_override}"
3075 );
3076 }
3077 }
3078 }
3079
3080 Ok(())
3081 }
3082
3083 #[tokio::test]
3084 async fn nvidia_anthropic_requests_strip_strict_from_extra_body()
3085 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3086 let server = MockServer::start().await;
3087 Mock::given(method("POST"))
3088 .and(path("/v1/messages"))
3089 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3090 "id": "msg_1",
3091 "type": "message",
3092 "role": "assistant",
3093 "model": "claude",
3094 "content": [{"type": "text", "text": "ok"}],
3095 "stop_reason": "end_turn",
3096 "usage": {"input_tokens": 1, "output_tokens": 1}
3097 })))
3098 .expect(1)
3099 .mount(&server)
3100 .await;
3101
3102 let mut backend = config(&format!(
3103 "http://inference-api.nvidia.com:{}",
3104 server.address().port()
3105 ));
3106 backend.extra_body.insert(
3107 "tools".into(),
3108 json!([{
3109 "name": "lookup",
3110 "input_schema": {"type": "object", "properties": {}},
3111 "strict": true
3112 }]),
3113 );
3114 let mut client = TranslatingLlmClient::new(&[ModelConfig::new(
3115 "claude",
3116 Backend::Anthropic(backend),
3117 None,
3118 )])?;
3119 client.client = reqwest::Client::builder()
3120 .no_proxy()
3121 .resolve("inference-api.nvidia.com", *server.address())
3122 .build()?;
3123 client
3124 .call_rewrite_model(
3125 wire_request(
3126 json!({
3127 "model": "claude",
3128 "max_tokens": 7,
3129 "messages": [{"role": "user", "content": "hi"}]
3130 }),
3131 None,
3132 WireFormat::AnthropicMessages,
3133 )?,
3134 Some(&ModelId::from("claude")),
3135 )
3136 .await?;
3137
3138 let requests = server.received_requests().await.unwrap();
3139 let body: Value = serde_json::from_slice(&requests[0].body)?;
3140 assert_eq!(body["tools"][0]["name"], "lookup");
3141 assert!(body["tools"][0].get("strict").is_none());
3142 Ok(())
3143 }
3144
3145 #[tokio::test]
3149 async fn anthropic_requests_drop_openai_only_fields()
3150 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3151 let server = MockServer::start().await;
3152 Mock::given(method("POST"))
3153 .and(path("/v1/messages"))
3154 .and(|request: &wiremock::Request| {
3155 let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
3156 body.get("context_management").is_none() && body.get("reasoning_effort").is_none()
3157 })
3158 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3159 "id": "msg_1",
3160 "type": "message",
3161 "role": "assistant",
3162 "model": "claude",
3163 "content": [{"type": "text", "text": "ok"}],
3164 "stop_reason": "end_turn",
3165 "usage": {"input_tokens": 1, "output_tokens": 1}
3166 })))
3167 .mount(&server)
3168 .await;
3169
3170 let client = TranslatingLlmClient::new(&anthropic_map(&server.uri()))?;
3171 let raw = json!({
3172 "model": "client-facing",
3173 "messages": [{"role": "user", "content": "hi"}],
3174 "max_tokens": 7,
3175 "reasoning_effort": "high",
3176 "context_management": {
3177 "edits": [{"type": "clear_thinking_20251015"}]
3178 }
3179 });
3180
3181 client
3182 .call_rewrite_model(
3183 wire_request(raw, None, WireFormat::AnthropicMessages)?,
3184 Some(&ModelId::from("claude")),
3185 )
3186 .await?;
3187 Ok(())
3188 }
3189
3190 #[tokio::test]
3191 async fn streaming_openai_chat_aggregates()
3192 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3193 let server = MockServer::start().await;
3194 let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
3195 data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
3196 data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n\
3197 data: [DONE]\n\n";
3198 Mock::given(method("POST"))
3199 .and(path("/v1/chat/completions"))
3200 .and(wiremock::matchers::body_partial_json(json!({
3201 "stream": true,
3202 "stream_options": {"include_usage": true}
3203 })))
3204 .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
3205 .mount(&server)
3206 .await;
3207
3208 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3209
3210 let response = client
3211 .call_rewrite_model(request_for(Some("gpt"), true), None)
3212 .await?;
3213 assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
3214 let agg = response.llm_response.into_agg().await?;
3215 assert_eq!(completion_text(&agg), "Hello world");
3216 assert_eq!(agg.usage.input_tokens, Some(1));
3217 assert_eq!(agg.usage.output_tokens, Some(2));
3218 assert_eq!(agg.usage.total_tokens, Some(3));
3219 Ok(())
3220 }
3221
3222 #[tokio::test]
3223 async fn streaming_openai_chat_preserves_usage_opt_out()
3224 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3225 let server = MockServer::start().await;
3226 Mock::given(method("POST"))
3227 .and(path("/v1/chat/completions"))
3228 .and(wiremock::matchers::body_partial_json(json!({
3229 "stream": true,
3230 "stream_options": {"include_usage": false}
3231 })))
3232 .respond_with(
3233 ResponseTemplate::new(200).set_body_raw("data: [DONE]\n\n", "text/event-stream"),
3234 )
3235 .mount(&server)
3236 .await;
3237
3238 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3239 let raw = json!({
3240 "model": "client-facing",
3241 "messages": [{"role": "user", "content": "hi"}],
3242 "stream": true,
3243 "stream_options": {"include_usage": false}
3244 });
3245
3246 let response = client
3247 .call_rewrite_model(
3248 wire_request(raw, None, WireFormat::OpenAiChat)?,
3249 Some(&ModelId::from("gpt")),
3250 )
3251 .await?;
3252 assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
3253 Ok(())
3254 }
3255
3256 #[tokio::test]
3257 async fn upstream_500_is_upstream_http()
3258 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3259 let server = MockServer::start().await;
3260 Mock::given(method("POST"))
3261 .respond_with(ResponseTemplate::new(500).set_body_string("boom"))
3262 .mount(&server)
3263 .await;
3264
3265 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3266
3267 let Err(error) = client
3268 .call_rewrite_model(request_for(Some("gpt"), false), None)
3269 .await
3270 else {
3271 panic!("expected an error");
3272 };
3273 assert!(matches!(
3274 error,
3275 LlmClientError::UpstreamHttp {
3276 status: StatusCode::INTERNAL_SERVER_ERROR,
3277 ..
3278 }
3279 ));
3280 Ok(())
3281 }
3282
3283 #[tokio::test]
3284 async fn retryable_http_failure_recovers_within_budget()
3285 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3286 let server = MockServer::start().await;
3287 let calls = Arc::new(AtomicUsize::new(0));
3288 let observed_calls = Arc::clone(&calls);
3289 Mock::given(method("POST"))
3290 .respond_with(move |_: &wiremock::Request| {
3291 if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {
3292 ResponseTemplate::new(503)
3293 .insert_header("retry-after", "0")
3294 .set_body_string("temporarily unavailable")
3295 } else {
3296 chat_success_response()
3297 }
3298 })
3299 .mount(&server)
3300 .await;
3301
3302 let client =
3303 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
3304 let response = client
3305 .call_rewrite_model(request_for(Some("gpt"), false), None)
3306 .await?;
3307 let agg = response.llm_response.into_agg().await?;
3308
3309 assert_eq!(completion_text(&agg), "recovered");
3310 assert_eq!(calls.load(Ordering::SeqCst), 2);
3311 Ok(())
3312 }
3313
3314 #[tokio::test]
3315 async fn deterministic_http_failure_is_not_retried()
3316 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3317 let server = MockServer::start().await;
3318 let calls = Arc::new(AtomicUsize::new(0));
3319 let observed_calls = Arc::clone(&calls);
3320 Mock::given(method("POST"))
3321 .respond_with(move |_: &wiremock::Request| {
3322 observed_calls.fetch_add(1, Ordering::SeqCst);
3323 ResponseTemplate::new(401).set_body_string("invalid key")
3324 })
3325 .mount(&server)
3326 .await;
3327
3328 let client =
3329 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
3330 let Err(error) = client
3331 .call_rewrite_model(request_for(Some("gpt"), false), None)
3332 .await
3333 else {
3334 panic!("expected an upstream error");
3335 };
3336
3337 assert!(matches!(
3338 error,
3339 LlmClientError::UpstreamHttp {
3340 status: StatusCode::UNAUTHORIZED,
3341 body,
3342 ..
3343 } if body == "invalid key"
3344 ));
3345 assert_eq!(calls.load(Ordering::SeqCst), 1);
3346 Ok(())
3347 }
3348
3349 #[tokio::test]
3350 async fn rate_limit_cooldown_depends_on_shared_credentials()
3351 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3352 for forward_auth in [true, false] {
3353 let server = MockServer::start().await;
3354 Mock::given(method("POST"))
3355 .respond_with(ResponseTemplate::new(429).insert_header("retry-after", "0"))
3356 .expect(if forward_auth { 6 } else { 3 })
3357 .mount(&server)
3358 .await;
3359 let backend = HttpBackendConfig {
3360 forward_auth,
3361 failure_cooldown: Duration::from_secs(60),
3362 ..config_with_retries(&format!("{}/v1", server.uri()), 2)
3363 };
3364 let client = TranslatingLlmClient::new(&[ModelConfig::new(
3365 "gpt",
3366 Backend::OpenAiChat(backend),
3367 None,
3368 )])?;
3369 for credential in ["Bearer first-user", "Bearer second-user"] {
3370 let mut headers = http::HeaderMap::new();
3371 headers.insert("authorization", http::HeaderValue::from_static(credential));
3372 let result = client
3373 .call_rewrite_model(request_with_headers("gpt", headers), None)
3374 .await;
3375 if !forward_auth && credential == "Bearer second-user" {
3376 assert!(matches!(
3377 result,
3378 Err(LlmClientError::TemporarilyUnavailable)
3379 ));
3380 } else {
3381 assert!(matches!(
3382 result,
3383 Err(LlmClientError::UpstreamHttp {
3384 status: StatusCode::TOO_MANY_REQUESTS,
3385 ..
3386 })
3387 ));
3388 }
3389 }
3390 }
3391 Ok(())
3392 }
3393
3394 #[tokio::test]
3395 async fn retry_exhaustion_returns_the_final_upstream_error()
3396 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3397 let server = MockServer::start().await;
3398 let calls = Arc::new(AtomicUsize::new(0));
3399 let observed_calls = Arc::clone(&calls);
3400 Mock::given(method("POST"))
3401 .respond_with(move |_: &wiremock::Request| {
3402 let attempt = observed_calls.fetch_add(1, Ordering::SeqCst) + 1;
3403 ResponseTemplate::new(500)
3404 .insert_header("retry-after", "0")
3405 .set_body_string(format!("attempt {attempt}"))
3406 })
3407 .mount(&server)
3408 .await;
3409
3410 let client =
3411 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
3412 let Err(error) = client
3413 .call_rewrite_model(request_for(Some("gpt"), false), None)
3414 .await
3415 else {
3416 panic!("expected retry exhaustion");
3417 };
3418
3419 assert!(matches!(
3420 error,
3421 LlmClientError::UpstreamHttp {
3422 status: StatusCode::INTERNAL_SERVER_ERROR,
3423 body,
3424 ..
3425 } if body == "attempt 3"
3426 ));
3427 assert_eq!(calls.load(Ordering::SeqCst), 3);
3428 Ok(())
3429 }
3430
3431 #[tokio::test]
3433 async fn upstream_error_carries_headers_from_the_last_attempt()
3434 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3435 let server = MockServer::start().await;
3436 let reply = |id: &str| {
3438 ResponseTemplate::new(429)
3439 .insert_header("retry-after", "0")
3440 .insert_header("x-request-id", id)
3441 .insert_header("set-cookie", "a=b")
3442 };
3443 Mock::given(method("POST"))
3444 .respond_with(reply("first"))
3445 .up_to_n_times(1)
3446 .mount(&server)
3447 .await;
3448 Mock::given(method("POST"))
3449 .respond_with(reply("second"))
3450 .mount(&server)
3451 .await;
3452 let client =
3453 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
3454 let Err(LlmClientError::UpstreamHttp { headers, .. }) = client
3455 .call_rewrite_model(request_for(Some("gpt"), false), None)
3456 .await
3457 else {
3458 panic!("expected an upstream HTTP error");
3459 };
3460 assert_eq!(headers["x-request-id"], "second");
3461 assert_eq!(headers["retry-after"], "0");
3462 assert_eq!(headers["set-cookie"], "a=b");
3463 Ok(())
3464 }
3465
3466 #[tokio::test]
3467 async fn deadline_expires_before_send_or_stream_poll()
3468 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3469 let server = MockServer::start().await;
3470 Mock::given(method("POST"))
3471 .respond_with(ResponseTemplate::new(200).set_body_raw(
3472 "data: {\"id\":\"test\",\"model\":\"gpt\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
3473 "text/event-stream",
3474 ))
3475 .mount(&server)
3476 .await;
3477 for timeout in [Duration::ZERO, Duration::from_millis(200)] {
3478 let mut backend = config(&server.uri());
3479 backend.timeout = Some(timeout);
3480 let client = TranslatingLlmClient::new(&[ModelConfig::new(
3481 "gpt",
3482 Backend::OpenAiChat(backend),
3483 None,
3484 )])?;
3485 let result = client
3486 .call_rewrite_model(request_for(Some("gpt"), true), None)
3487 .await;
3488 if timeout.is_zero() {
3489 assert!(matches!(result, Err(LlmClientError::Timeout { .. })));
3490 assert!(
3491 server
3492 .received_requests()
3493 .await
3494 .expect("recorded requests")
3495 .is_empty()
3496 );
3497 } else {
3498 let LlmResponse::Stream(mut chunks) = result?.llm_response else {
3499 return Err("expected a streaming response".into());
3500 };
3501 tokio::time::sleep(Duration::from_millis(300)).await;
3502 assert!(matches!(
3503 chunks.next().await,
3504 Some(Err(LlmClientError::Timeout { .. }))
3505 ));
3506 assert!(chunks.next().await.is_none());
3507 }
3508 }
3509 Ok(())
3510 }
3511
3512 #[tokio::test]
3513 async fn timeout_is_retried_before_a_response_is_returned()
3514 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3515 let server = MockServer::start().await;
3516 let calls = Arc::new(AtomicUsize::new(0));
3517 let observed_calls = Arc::clone(&calls);
3518 Mock::given(method("POST"))
3519 .respond_with(move |_: &wiremock::Request| {
3520 if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {
3521 ResponseTemplate::new(200).set_delay(Duration::from_millis(500))
3522 } else {
3523 chat_success_response()
3524 }
3525 })
3526 .mount(&server)
3527 .await;
3528
3529 let mut client =
3530 TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
3531 client.client = reqwest::Client::builder()
3532 .timeout(Duration::from_millis(100))
3533 .build()?;
3534 let response = client
3535 .call_rewrite_model(request_for(Some("gpt"), false), None)
3536 .await?;
3537
3538 assert_eq!(
3539 completion_text(&response.llm_response.into_agg().await?),
3540 "recovered"
3541 );
3542 assert_eq!(calls.load(Ordering::SeqCst), 2);
3543 Ok(())
3544 }
3545
3546 #[test]
3547 fn retryable_error_classes_are_explicit() {
3548 let transport = AttemptFailure {
3549 error: LlmClientError::Transport {
3550 source: std::io::Error::other("disconnected").into(),
3551 },
3552 status: None,
3553 retry_after: None,
3554 };
3555 assert!(transport.is_retryable());
3556
3557 for status in [
3558 StatusCode::REQUEST_TIMEOUT,
3559 StatusCode::TOO_MANY_REQUESTS,
3560 StatusCode::INTERNAL_SERVER_ERROR,
3561 StatusCode::SERVICE_UNAVAILABLE,
3562 StatusCode::from_u16(599)
3563 .expect("599 is a syntactically valid HTTP code in the http crate"),
3564 ] {
3565 let failure = AttemptFailure {
3566 error: LlmClientError::UpstreamHttp {
3567 status,
3568 body: String::new(),
3569 headers: Box::default(),
3570 },
3571 status: Some(status),
3572 retry_after: None,
3573 };
3574 assert!(failure.is_retryable(), "HTTP {status} should retry");
3575 }
3576 for status in [
3577 StatusCode::BAD_REQUEST,
3578 StatusCode::UNAUTHORIZED,
3579 StatusCode::CONFLICT,
3580 StatusCode::from_u16(600)
3581 .expect("600 is a syntactically valid HTTP code in the http crate"),
3582 ] {
3583 let failure = AttemptFailure {
3584 error: LlmClientError::UpstreamHttp {
3585 status,
3586 body: String::new(),
3587 headers: Box::default(),
3588 },
3589 status: Some(status),
3590 retry_after: None,
3591 };
3592 assert!(!failure.is_retryable(), "HTTP {status} should fail fast");
3593 }
3594
3595 let configuration = AttemptFailure {
3596 error: LlmClientError::Configuration {
3597 message: "invalid header".to_string(),
3598 },
3599 status: None,
3600 retry_after: None,
3601 };
3602 assert!(!configuration.is_retryable());
3603
3604 let context_window = AttemptFailure {
3605 error: LlmClientError::ContextWindowExceeded {
3606 model: ModelId::from("gpt"),
3607 message: "too long".to_string(),
3608 },
3609 status: Some(StatusCode::BAD_REQUEST),
3610 retry_after: None,
3611 };
3612 assert!(!context_window.is_retryable());
3613 }
3614
3615 #[test]
3616 fn retry_after_supports_seconds_and_http_dates() {
3617 let mut headers = HeaderMap::new();
3618 headers.insert(RETRY_AFTER, reqwest::header::HeaderValue::from_static("3"));
3619 assert_eq!(retry_after_delay(&headers), Some(Duration::from_secs(3)));
3620
3621 let retry_at = SystemTime::now() + Duration::from_secs(2);
3622 let value = httpdate::fmt_http_date(retry_at);
3623 let Ok(value) = reqwest::header::HeaderValue::from_str(&value) else {
3624 panic!("formatted HTTP date should be a valid header");
3625 };
3626 headers.insert(RETRY_AFTER, value);
3627 let Some(delay) = retry_after_delay(&headers) else {
3628 panic!("HTTP date should produce a retry delay");
3629 };
3630 assert!(delay <= Duration::from_secs(2));
3631 }
3632
3633 #[tokio::test]
3634 async fn routed_llm_client_exposes_timeout_variant()
3635 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3636 let server = MockServer::start().await;
3637 Mock::given(method("POST"))
3638 .respond_with(
3639 ResponseTemplate::new(200).set_delay(std::time::Duration::from_millis(100)),
3640 )
3641 .mount(&server)
3642 .await;
3643
3644 let mut client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3645 client.client = reqwest::Client::builder()
3646 .timeout(std::time::Duration::from_millis(10))
3647 .build()?;
3648
3649 let Err(error) = client.call(request_for(Some("gpt"), false)).await else {
3650 panic!("expected a timeout");
3651 };
3652 let LlmClientError::Timeout { source } = error else {
3653 panic!("expected the protocol timeout variant");
3654 };
3655 let Some(source) = source.downcast_ref::<reqwest::Error>() else {
3656 panic!("expected the reqwest timeout source");
3657 };
3658 assert!(source.is_timeout());
3659 Ok(())
3660 }
3661
3662 #[cfg(target_os = "linux")]
3664 #[tokio::test]
3665 async fn connect_timeout_is_a_transport_error()
3666 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3667 let socket = tokio::net::TcpSocket::new_v4()?;
3670 socket.bind("127.0.0.1:0".parse()?)?;
3671 let listener = socket.listen(0)?;
3672 let addr = listener.local_addr()?;
3673 let _queued: Vec<_> = (0..2)
3674 .map(|_| {
3675 std::net::TcpStream::connect_timeout(&addr, std::time::Duration::from_millis(100))
3676 })
3677 .collect();
3678
3679 let mut client = TranslatingLlmClient::new(&chat_map(&format!("http://{addr}/v1")))?;
3680 client.client = reqwest::Client::builder()
3683 .connect_timeout(std::time::Duration::from_millis(100))
3684 .build()?;
3685
3686 let Err(error) = client.call(request_for(Some("gpt"), false)).await else {
3687 panic!("expected the connect to fail");
3688 };
3689 let LlmClientError::Transport { source } = error else {
3690 panic!("expected Transport, got {error:?}");
3691 };
3692 let Some(source) = source.downcast_ref::<reqwest::Error>() else {
3693 panic!("expected the reqwest connect-timeout source");
3694 };
3695 assert!(source.is_connect());
3696 assert!(source.is_timeout());
3697 Ok(())
3698 }
3699
3700 #[tokio::test]
3701 async fn llama_cpp_context_overflow_400_is_mapped()
3702 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3703 let server = MockServer::start().await;
3704 Mock::given(method("POST"))
3705 .respond_with(ResponseTemplate::new(400).set_body_json(json!({
3706 "error": {
3707 "message": "request (6016 tokens) exceeds the available context size (4096 tokens), try increasing it"
3708 }
3709 })))
3710 .mount(&server)
3711 .await;
3712
3713 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3714
3715 let Err(error) = client
3716 .call_rewrite_model(request_for(Some("gpt"), false), None)
3717 .await
3718 else {
3719 panic!("expected an error");
3720 };
3721 assert!(matches!(
3722 error,
3723 LlmClientError::ContextWindowExceeded { model, .. } if model == "gpt"
3724 ));
3725 Ok(())
3726 }
3727
3728 #[tokio::test]
3729 async fn content_policy_400_preserves_upstream_error()
3730 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3731 let server = MockServer::start().await;
3732 let body = json!({
3733 "error": {
3734 "code": "content_policy_violation",
3735 "message": "request blocked by content policy",
3736 "type": "invalid_request_error"
3737 },
3738 "metadata": {"documentation_section": "context window"}
3739 });
3740 Mock::given(method("POST"))
3741 .respond_with(ResponseTemplate::new(400).set_body_json(&body))
3742 .expect(1)
3743 .mount(&server)
3744 .await;
3745 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3746 let Err(error) = client
3747 .call_rewrite_model(request_for(Some("gpt"), false), None)
3748 .await
3749 else {
3750 panic!("expected a policy denial");
3751 };
3752 let LlmClientError::UpstreamHttp {
3753 status,
3754 body: actual,
3755 ..
3756 } = error
3757 else {
3758 panic!("expected the upstream HTTP error, got {error:?}");
3759 };
3760 assert_eq!(status, StatusCode::BAD_REQUEST);
3761 assert_eq!(serde_json::from_str::<serde_json::Value>(&actual)?, body);
3762 Ok(())
3763 }
3764
3765 #[tokio::test]
3766 async fn forwards_only_allowlisted_metadata_headers()
3767 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3768 let server = MockServer::start().await;
3769 Mock::given(method("POST"))
3770 .and(wiremock::matchers::header("x-request-id", "abc"))
3771 .and(wiremock::matchers::header("authorization", "Bearer secret"))
3773 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3774 "id": "1", "model": "gpt",
3775 "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
3776 "usage": {}
3777 })))
3778 .mount(&server)
3779 .await;
3780
3781 let mut headers = http::HeaderMap::new();
3782 headers.insert("x-request-id", http::HeaderValue::from_static("abc"));
3783 headers.insert(
3784 "authorization",
3785 http::HeaderValue::from_static("Bearer client-key"),
3786 );
3787 headers.insert(
3788 "x-goog-api-key",
3789 http::HeaderValue::from_static("client-google-key"),
3790 );
3791 headers.insert(
3792 "x-custom-internal-secret",
3793 http::HeaderValue::from_static("client-custom-key"),
3794 );
3795 let request = request_with_headers("gpt", headers);
3796
3797 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3798
3799 client.call_rewrite_model(request, None).await?;
3802 let received = server
3803 .received_requests()
3804 .await
3805 .ok_or("request recording should be enabled")?;
3806 let received = received.first().ok_or("expected one upstream request")?;
3807 assert!(!received.headers.contains_key("x-goog-api-key"));
3808 assert!(!received.headers.contains_key("x-custom-internal-secret"));
3809 Ok(())
3810 }
3811
3812 #[tokio::test]
3813 async fn forwards_application_headers_with_forward_auth()
3814 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3815 let server = MockServer::start().await;
3816 Mock::given(method("POST"))
3817 .and(wiremock::matchers::header(
3818 "authorization",
3819 "Bearer client-key",
3820 ))
3821 .and(wiremock::matchers::header(
3822 "x-goog-api-key",
3823 "client-google-key",
3824 ))
3825 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3826 "id": "1", "model": "gpt",
3827 "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
3828 "usage": {}
3829 })))
3830 .mount(&server)
3831 .await;
3832
3833 let mut headers = http::HeaderMap::new();
3834 headers.insert(
3835 "authorization",
3836 http::HeaderValue::from_static("Bearer client-key"),
3837 );
3838 headers.insert(
3839 "x-goog-api-key",
3840 http::HeaderValue::from_static("client-google-key"),
3841 );
3842 headers.append("x-custom", http::HeaderValue::from_static("caller-first"));
3843 headers.append("x-custom", http::HeaderValue::from_static("caller-second"));
3844 headers.insert("host", http::HeaderValue::from_static("client.example"));
3845 headers.insert("content-type", http::HeaderValue::from_static("text/plain"));
3846 headers.insert("connection", http::HeaderValue::from_static("x-hop-by-hop"));
3847 headers.insert(
3848 "x-hop-by-hop",
3849 http::HeaderValue::from_static("client-only"),
3850 );
3851 let request = request_with_headers("gpt", headers);
3852 let mut backend_config = forwarding_config(&format!("{}/v1", server.uri()));
3853 backend_config.extra_headers = BTreeMap::from([
3854 ("X-Custom".into(), "configured".into()),
3855 ("X-Configured-Only".into(), "static".into()),
3856 ]);
3857 let client = TranslatingLlmClient::new(&[ModelConfig::new(
3858 "gpt",
3859 Backend::OpenAiChat(backend_config),
3860 None,
3861 )])?;
3862
3863 client.call_rewrite_model(request, None).await?;
3864 let received = server
3865 .received_requests()
3866 .await
3867 .ok_or("request recording should be enabled")?;
3868 let received = received.first().ok_or("expected one upstream request")?;
3869 assert_ne!(received.headers["host"], "client.example");
3870 assert_eq!(received.headers["content-type"], "application/json");
3871 assert!(!received.headers.contains_key("connection"));
3872 assert!(!received.headers.contains_key("x-hop-by-hop"));
3873 assert_eq!(received.headers.get_all("x-custom").iter().count(), 1);
3874 assert_eq!(received.headers["x-custom"], "configured");
3875 assert_eq!(received.headers["x-configured-only"], "static");
3876 Ok(())
3877 }
3878
3879 #[tokio::test]
3880 async fn forward_auth_preserves_anthropic_headers_and_redacts_errors()
3881 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3882 let server = MockServer::start().await;
3883 Mock::given(method("POST"))
3884 .and(path("/v1/messages"))
3885 .and(|request: &wiremock::Request| {
3886 request
3887 .headers
3888 .get("anthropic-beta")
3889 .and_then(|value| value.to_str().ok())
3890 == Some("oauth-2025-04-20")
3891 && request.headers.get_all("anthropic-version").iter().count() == 1
3892 && request
3893 .headers
3894 .get("anthropic-version")
3895 .and_then(|value| value.to_str().ok())
3896 == Some("2023-06-01")
3897 })
3898 .respond_with(ResponseTemplate::new(401).set_body_json(json!({
3899 "error": {"message": "rejected client-google-key"}
3900 })))
3901 .mount(&server)
3902 .await;
3903
3904 let mut headers = http::HeaderMap::new();
3905 headers.insert(
3906 "authorization",
3907 http::HeaderValue::from_static("Bearer client-key"),
3908 );
3909 headers.insert(
3910 "x-goog-api-key",
3911 http::HeaderValue::from_static("client-google-key"),
3912 );
3913 headers.insert(
3914 "anthropic-beta",
3915 http::HeaderValue::from_static("oauth-2025-04-20,prompt-caching-2024-07-31"),
3916 );
3917 headers.insert(
3918 "anthropic-version",
3919 http::HeaderValue::from_static("caller-version"),
3920 );
3921 let backend_config = forwarding_config(&server.uri());
3922 let client = TranslatingLlmClient::new(&[ModelConfig::new(
3923 "claude",
3924 Backend::Anthropic(backend_config),
3925 None,
3926 )])?;
3927 let raw = json!({
3928 "model": "claude",
3929 "max_tokens": 8,
3930 "messages": [{"role": "user", "content": "hello"}]
3931 });
3932
3933 let Err(LlmClientError::UpstreamHttp { body, .. }) = client
3934 .call_rewrite_model(
3935 wire_request(raw, Some(headers), WireFormat::AnthropicMessages)?,
3936 Some(&ModelId::from("claude")),
3937 )
3938 .await
3939 else {
3940 panic!("expected an upstream HTTP error");
3941 };
3942 assert_eq!(body, r#"{"error":{"message":"rejected [REDACTED]"}}"#);
3943 Ok(())
3944 }
3945
3946 #[tokio::test]
3949 async fn routed_llm_client_serves_the_request_model()
3950 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3951 let server = MockServer::start().await;
3952 Mock::given(method("POST"))
3953 .and(path("/v1/chat/completions"))
3954 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3955 "id": "chatcmpl-1",
3956 "model": "gpt",
3957 "choices": [{
3958 "index": 0,
3959 "message": {"role": "assistant", "content": "routed hi"},
3960 "finish_reason": "stop"
3961 }],
3962 "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
3963 })))
3964 .mount(&server)
3965 .await;
3966
3967 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
3968 let response = client.call(request_for(Some("gpt"), false)).await?;
3969 let agg = response.llm_response.into_agg().await?;
3970 assert_eq!(completion_text(&agg), "routed hi");
3971 Ok(())
3972 }
3973
3974 #[tokio::test]
3976 async fn chat_backend_folds_codex_mcp_namespace_into_the_tool_name()
3977 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
3978 let server = MockServer::start().await;
3979 Mock::given(method("POST"))
3980 .and(path("/v1/chat/completions"))
3981 .and(wiremock::matchers::body_partial_json(json!({
3982 "tools": [{
3983 "type": "function",
3984 "function": {"name": "mcp__open_websearch__search"}
3985 }]
3986 })))
3987 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
3988 "id": "chatcmpl-1",
3989 "model": "gpt",
3990 "choices": [{
3991 "index": 0,
3992 "message": {
3993 "role": "assistant",
3994 "content": null,
3995 "tool_calls": [{
3996 "id": "call_1",
3997 "type": "function",
3998 "function": {
3999 "name": "mcp__open_websearch__search",
4000 "arguments": "{\"q\":\"rust\"}"
4001 }
4002 }]
4003 },
4004 "finish_reason": "tool_calls"
4005 }],
4006 "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
4007 })))
4008 .mount(&server)
4009 .await;
4010
4011 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
4012 let raw = json!({
4013 "model": "client-facing",
4014 "input": "Search for Rust.",
4015 "tools": [{
4016 "type": "namespace",
4017 "name": "mcp__open_websearch",
4018 "tools": [{
4019 "type": "function",
4020 "name": "search",
4021 "description": "Search the web",
4022 "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}
4023 }]
4024 }]
4025 });
4026
4027 let response = client
4028 .call_rewrite_model(
4029 wire_request(raw, None, WireFormat::OpenAiResponses)?,
4030 Some(&ModelId::from("gpt")),
4031 )
4032 .await?;
4033 let agg = response.llm_response.into_agg().await?;
4034
4035 assert_tool_call_name(&agg, "mcp__open_websearch__search");
4036 Ok(())
4037 }
4038
4039 #[tokio::test]
4040 async fn openai_responses_backend_flattens_codex_namespaces_before_upstream()
4041 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
4042 let server = MockServer::start().await;
4043 Mock::given(method("POST"))
4044 .and(path("/v1/responses"))
4045 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
4046 "id": "resp_1",
4047 "model": "gpt",
4048 "object": "response",
4049 "created_at": 0,
4050 "status": "completed",
4051 "output": [{
4052 "type": "function_call",
4053 "call_id": "call_1",
4054 "name": "mcp__open_websearch__search",
4055 "arguments": "{\"q\":\"rust\"}"
4056 }],
4057 "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
4058 })))
4059 .mount(&server)
4060 .await;
4061
4062 let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?;
4063 let parameters = json!({"type": "object", "properties": {"q": {"type": "string"}}});
4064 let raw = json!({
4065 "model": "client-facing",
4066 "input": [{
4067 "type": "function_call",
4068 "call_id": "call_0",
4069 "name": "search",
4070 "namespace": "mcp__open_websearch",
4071 "arguments": "{}"
4072 }],
4073 "tool_choice": {
4074 "type": "function",
4075 "name": "search",
4076 "namespace": "mcp__open_websearch"
4077 },
4078 "tools": [{
4079 "type": "namespace",
4080 "name": "mcp__open_websearch",
4081 "tools": [{
4082 "type": "function",
4083 "name": "search",
4084 "parameters": parameters.clone()
4085 }]
4086 }]
4087 });
4088
4089 let response = client
4090 .call_rewrite_model(
4091 wire_request(raw, None, WireFormat::OpenAiResponses)?,
4092 Some(&ModelId::from("gpt")),
4093 )
4094 .await?;
4095 let agg = response.llm_response.into_agg().await?;
4096
4097 let received = server
4098 .received_requests()
4099 .await
4100 .ok_or("request recording should be enabled")?;
4101 let request_body: Value = serde_json::from_slice(&received[0].body)?;
4102 assert_eq!(request_body["model"], "gpt");
4103 assert_eq!(
4104 request_body["tools"],
4105 json!([{
4106 "type": "function",
4107 "name": "mcp__open_websearch__search",
4108 "description": "",
4109 "parameters": parameters
4110 }])
4111 );
4112 assert_eq!(
4113 request_body["tool_choice"],
4114 json!({"type": "function", "name": "mcp__open_websearch__search"})
4115 );
4116 assert_eq!(
4117 request_body["input"][0]["name"],
4118 "mcp__open_websearch__search"
4119 );
4120 assert!(request_body["input"][0].get("namespace").is_none());
4121 assert_tool_call_name(&agg, "mcp__open_websearch__search");
4122 Ok(())
4123 }
4124
4125 #[tokio::test]
4132 async fn openai_responses_backend_drops_reasoning_the_provider_cannot_read()
4133 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
4134 let server = MockServer::start().await;
4135 Mock::given(method("POST"))
4136 .and(path("/v1/responses"))
4137 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
4138 "id": "resp_1",
4139 "model": "gpt",
4140 "object": "response",
4141 "created_at": 0,
4142 "status": "completed",
4143 "output": [{
4144 "type": "message",
4145 "role": "assistant",
4146 "content": [{"type": "output_text", "text": "ok"}]
4147 }],
4148 "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
4149 })))
4150 .mount(&server)
4151 .await;
4152
4153 let mut claude_reasoning = json!({
4154 "type": "reasoning",
4155 "id": "rs_b99feb9a-763d-4f7f-b6b2-2c0199b47ded",
4156 "summary": [{"type": "summary_text", "text": "Read the files first."}],
4157 "encrypted_content": null
4158 });
4159 let claude_reply = switchyard_translation::TranslationEngine::default()
4162 .translate_response(
4163 WireFormat::AnthropicMessages,
4164 WireFormat::OpenAiResponses,
4165 &json!({
4166 "id": "msg_1",
4167 "type": "message",
4168 "role": "assistant",
4169 "model": "claude",
4170 "content": [
4171 {"type": "thinking", "thinking": "Plan the edit.", "signature": "sig-abc"},
4172 {"type": "text", "text": "ok"}
4173 ],
4174 "stop_reason": "end_turn",
4175 "usage": {"input_tokens": 1, "output_tokens": 1}
4176 }),
4177 &switchyard_translation::TranslationPolicy::default(),
4178 )?
4179 .body;
4180 let switchyard_reasoning = claude_reply["output"]
4181 .as_array()
4182 .and_then(|output| output.iter().find(|item| item["type"] == "reasoning"))
4183 .cloned()
4184 .ok_or("translated Claude reply should contain a reasoning item")?;
4185 assert!(switchyard_translation::is_anthropic_thinking_envelope(
4186 switchyard_reasoning["encrypted_content"]
4187 .as_str()
4188 .unwrap_or_default()
4189 ));
4190 let mut gpt_reasoning = json!({
4191 "type": "reasoning",
4192 "id": "rs_gpt",
4193 "summary": [],
4194 "encrypted_content": "opaque-encrypted-reasoning"
4195 });
4196 let mut empty_reasoning =
4197 json!({"type": "reasoning", "id": "rs_empty", "summary": [], "encrypted_content": ""});
4198 let user = json!({"type": "message", "role": "user", "content": "fix the build"});
4199 let call = json!({"type": "function_call", "call_id": "call_1", "name": "exec", "arguments": "{}"});
4200 let output = json!({"type": "function_call_output", "call_id": "call_1", "output": "ok"});
4201 let input = json!([
4202 user,
4203 claude_reasoning,
4204 switchyard_reasoning,
4205 call,
4206 output,
4207 gpt_reasoning,
4208 empty_reasoning
4209 ]);
4210
4211 let models = ["openai/gpt-5", "nvidia/zai-org/glm-5.3-flash"].map(|model| {
4212 ModelConfig::new(
4213 model,
4214 Backend::OpenAiResponses(config(&format!("{}/v1", server.uri()))),
4215 None,
4216 )
4217 });
4218 let client = TranslatingLlmClient::new(&models)?;
4219 for model in ["openai/gpt-5", "nvidia/zai-org/glm-5.3-flash"] {
4220 for store in [json!(false), json!(true)] {
4221 let raw = json!({"model": "client-facing", "store": store, "input": input.clone()});
4222 client
4223 .call_rewrite_model(
4224 wire_request(raw, None, WireFormat::OpenAiResponses)?,
4225 Some(&ModelId::from(model)),
4226 )
4227 .await?;
4228 }
4229 }
4230
4231 let received = server
4232 .received_requests()
4233 .await
4234 .ok_or("request recording should be enabled")?;
4235 let sent: Vec<Value> = received
4236 .iter()
4237 .map(|request| serde_json::from_slice::<Value>(&request.body))
4238 .collect::<std::result::Result<_, _>>()?;
4239 assert_eq!(sent.len(), 4);
4240 for request in &sent {
4241 assert!(
4242 request["input"]
4243 .as_array()
4244 .ok_or("expected input array")?
4245 .iter()
4246 .all(|item| !item["encrypted_content"]
4247 .as_str()
4248 .is_some_and(switchyard_translation::is_anthropic_thinking_envelope)),
4249 "provider cannot read translated Anthropic reasoning"
4250 );
4251 }
4252
4253 let mut switchyard_plaintext = switchyard_reasoning.clone();
4255 switchyard_plaintext
4256 .as_object_mut()
4257 .ok_or("object")?
4258 .remove("encrypted_content");
4259 assert_eq!(
4260 sent[2]["input"],
4261 json!([user, claude_reasoning, switchyard_plaintext, call, output])
4262 );
4263 assert_eq!(
4264 sent[3]["input"],
4265 json!([
4266 user,
4267 claude_reasoning,
4268 switchyard_plaintext,
4269 call,
4270 output,
4271 empty_reasoning
4272 ])
4273 );
4274
4275 for item in [
4276 &mut claude_reasoning,
4277 &mut gpt_reasoning,
4278 &mut empty_reasoning,
4279 ] {
4280 item["content"] = json!([]);
4281 }
4282 assert_eq!(sent[0]["input"], json!([user, call, output, gpt_reasoning]));
4284 assert_eq!(
4287 sent[1]["input"],
4288 json!([
4289 user,
4290 claude_reasoning,
4291 call,
4292 output,
4293 gpt_reasoning,
4294 empty_reasoning
4295 ])
4296 );
4297 Ok(())
4298 }
4299
4300 #[tokio::test]
4302 async fn call_rewrite_model_forwards_caller_headers()
4303 -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
4304 let server = MockServer::start().await;
4305 Mock::given(method("POST"))
4306 .and(wiremock::matchers::header("x-request-id", "abc"))
4307 .and(wiremock::matchers::header("authorization", "Bearer secret"))
4309 .respond_with(ResponseTemplate::new(200).set_body_json(json!({
4310 "id": "1", "model": "gpt",
4311 "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
4312 "usage": {}
4313 })))
4314 .mount(&server)
4315 .await;
4316
4317 let mut headers = http::HeaderMap::new();
4318 headers.insert("x-request-id", http::HeaderValue::from_static("abc"));
4319 headers.insert(
4320 "authorization",
4321 http::HeaderValue::from_static("Bearer client-key"),
4322 );
4323
4324 let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
4325 let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]});
4326 client
4329 .call_rewrite_model(
4330 wire_request(raw, Some(headers), WireFormat::OpenAiChat)?,
4331 Some(&ModelId::from("gpt")),
4332 )
4333 .await?;
4334 Ok(())
4335 }
4336}