1use std::sync::LazyLock;
10
11use futures::Stream;
12use serde_json::Value;
13use switchyard_protocol::LlmClientError;
14
15use crate::{
16 AggLlmResponse, LlmRequest, LlmResponseStream, RawEventStream, Result, TranslationEngine,
17 TranslationPolicy, WireFormat,
18};
19
20static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =
21 LazyLock::new(TranslationPolicy::default);
22static DEFAULT_TRANSLATION_ENGINE: LazyLock<TranslationEngine> =
23 LazyLock::new(TranslationEngine::default);
24
25pub fn decode_request(wire_format: WireFormat, body: &Value) -> Result<LlmRequest> {
27 Ok(DEFAULT_TRANSLATION_ENGINE
28 .decode_request(wire_format, body, &DEFAULT_TRANSLATION_POLICY)?
29 .request)
30}
31
32pub fn encode_request(request: &LlmRequest, wire_format: WireFormat) -> Result<Value> {
34 Ok(DEFAULT_TRANSLATION_ENGINE
35 .encode_request(wire_format, request, &DEFAULT_TRANSLATION_POLICY)?
36 .body)
37}
38
39pub fn decode_aggregated_response(body: &Value, wire_format: WireFormat) -> Result<AggLlmResponse> {
41 Ok(DEFAULT_TRANSLATION_ENGINE
42 .decode_response(wire_format, body, &DEFAULT_TRANSLATION_POLICY)?
43 .response)
44}
45
46pub fn encode_stream(
53 chunks: LlmResponseStream,
54 target: WireFormat,
55 served_model: Option<String>,
56) -> std::result::Result<RawEventStream, LlmClientError> {
57 DEFAULT_TRANSLATION_ENGINE.encode_stream(target, chunks, served_model)
58}
59
60pub fn encode_stream_with_extensions(
65 chunks: LlmResponseStream,
66 target: WireFormat,
67 served_model: Option<String>,
68 request_extensions: &switchyard_protocol::ProviderExtensions,
69) -> std::result::Result<RawEventStream, LlmClientError> {
70 DEFAULT_TRANSLATION_ENGINE.encode_stream_with_extensions(
71 target,
72 chunks,
73 served_model,
74 request_extensions,
75 )
76}
77
78pub fn decode_stream<S>(
89 bytes: S,
90 source: WireFormat,
91) -> std::result::Result<LlmResponseStream, LlmClientError>
92where
93 S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,
94{
95 DEFAULT_TRANSLATION_ENGINE.decode_stream(source, bytes)
96}
97
98#[cfg(test)]
99mod tests {
100 use futures::executor::block_on;
101 use futures::{Stream, StreamExt, stream};
102 use serde_json::{Value, json};
103 use switchyard_protocol::{
104 LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, completion_text,
105 };
106
107 use super::{
108 decode_aggregated_response, decode_request, decode_stream, encode_request, encode_stream,
109 };
110 use crate::{LlmResponseStream, LlmStreamError, WireFormat};
111
112 type BoxError = Box<dyn std::error::Error + Send + Sync>;
114
115 fn decode_all(
117 bytes: impl Stream<Item = Result<Vec<u8>, LlmClientError>> + Send + 'static,
118 source: WireFormat,
119 ) -> Result<Vec<LlmResponseStreamEvent>, LlmClientError> {
120 block_on(decode_stream(bytes, source)?.collect::<Vec<_>>())
121 .into_iter()
122 .collect()
123 }
124
125 fn text_of(events: &[LlmResponseStreamEvent]) -> String {
127 events
128 .iter()
129 .flat_map(LlmResponseStreamEvent::normalized)
130 .filter_map(|chunk| {
131 if let LlmResponseChunk::TextDelta { text, .. } = chunk {
132 Some(text.as_str())
133 } else {
134 None
135 }
136 })
137 .collect()
138 }
139
140 #[test]
141 fn request_round_trips_through_openai_chat() -> Result<(), BoxError> {
142 let body = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]});
143 let request = decode_request(WireFormat::OpenAiChat, &body)?;
144 assert_eq!(request.model.as_deref(), Some("gpt"));
145
146 let encoded = encode_request(&request, WireFormat::OpenAiChat)?;
147 assert_eq!(encoded["model"], "gpt");
148 assert_eq!(encoded["messages"][0]["content"], "hi");
149 Ok(())
150 }
151
152 #[test]
153 fn aggregated_response_decodes_from_openai_chat() -> Result<(), BoxError> {
154 let body = json!({
155 "id": "1",
156 "model": "upstream",
157 "choices": [{
158 "index": 0,
159 "message": {"role": "assistant", "content": "Hi there"},
160 "finish_reason": "stop"
161 }],
162 "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
163 });
164 let agg = decode_aggregated_response(&body, WireFormat::OpenAiChat)?;
165 assert_eq!(completion_text(&agg), "Hi there");
166 Ok(())
167 }
168
169 #[test]
170 fn encode_stream_reassembles_deltas_and_finishes() -> Result<(), BoxError> {
171 let chunks: LlmResponseStream = stream::iter(vec![
172 Ok(LlmResponseChunk::TextDelta {
173 index: 0,
174 text: "Hello".to_string(),
175 }
176 .into()),
177 Ok(LlmResponseChunk::TextDelta {
178 index: 0,
179 text: " world".to_string(),
180 }
181 .into()),
182 Ok(LlmResponseChunk::MessageStop {
183 reason: Some("stop".to_string()),
184 }
185 .into()),
186 ])
187 .boxed();
188
189 let events = block_on(
190 encode_stream(chunks, WireFormat::OpenAiChat, Some("m".to_string()))?
191 .collect::<Vec<_>>(),
192 )
193 .into_iter()
194 .collect::<Result<Vec<Value>, LlmStreamError>>()?;
195
196 let content: String = events
197 .iter()
198 .filter_map(|event| event["choices"][0]["delta"]["content"].as_str())
199 .collect();
200 assert_eq!(content, "Hello world");
201 assert!(
203 events
204 .iter()
205 .any(|event| event["choices"][0]["finish_reason"] == "stop")
206 );
207 Ok(())
208 }
209
210 #[test]
213 fn encode_stream_stamps_the_served_model_on_message_start() -> Result<(), BoxError> {
214 let chunks: LlmResponseStream = stream::iter(vec![
215 Ok(LlmResponseChunk::MessageStart {
216 id: Some("msg_1".to_string()),
217 model: Some("upstream/model".to_string()),
218 }
219 .into()),
220 Ok(LlmResponseChunk::TextDelta {
221 index: 0,
222 text: "hi".to_string(),
223 }
224 .into()),
225 ])
226 .boxed();
227
228 let events = block_on(
229 encode_stream(
230 chunks,
231 WireFormat::AnthropicMessages,
232 Some("served/model".to_string()),
233 )?
234 .collect::<Vec<_>>(),
235 )
236 .into_iter()
237 .collect::<Result<Vec<Value>, LlmStreamError>>()?;
238
239 assert_eq!(events[0]["type"], "message_start");
240 assert_eq!(events[0]["message"]["model"], "served/model");
241 Ok(())
242 }
243
244 #[test]
247 fn encode_stream_falls_back_to_the_source_model() -> Result<(), BoxError> {
248 let chunks: LlmResponseStream = stream::iter(vec![
249 Ok(LlmResponseChunk::MessageStart {
250 id: Some("msg_1".to_string()),
251 model: Some("upstream/model".to_string()),
252 }
253 .into()),
254 Ok(LlmResponseChunk::TextDelta {
255 index: 0,
256 text: "hi".to_string(),
257 }
258 .into()),
259 ])
260 .boxed();
261
262 let events = block_on(
263 encode_stream(chunks, WireFormat::AnthropicMessages, None)?.collect::<Vec<_>>(),
264 )
265 .into_iter()
266 .collect::<Result<Vec<Value>, LlmStreamError>>()?;
267
268 assert_eq!(events[0]["message"]["model"], "upstream/model");
269 Ok(())
270 }
271
272 #[test]
273 fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> {
274 let chunks: LlmResponseStream =
275 stream::iter(vec![Err::<LlmResponseStreamEvent, LlmClientError>(
276 LlmClientError::General("chunk exploded".to_string()),
277 )])
278 .boxed();
279 let results =
280 block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());
281 assert!(results.iter().any(Result::is_err));
282 Ok(())
283 }
284
285 #[test]
290 fn encode_stream_stops_after_an_in_band_error() -> Result<(), BoxError> {
291 for message in [
292 LlmResponseChunk::StreamError {
293 message: "boom".to_string(),
294 },
295 LlmResponseChunk::DecodeError {
296 message: "boom".to_string(),
297 },
298 ] {
299 for target in [
300 WireFormat::OpenAiChat,
301 WireFormat::OpenAiResponses,
302 WireFormat::AnthropicMessages,
303 ] {
304 let chunks: LlmResponseStream = stream::iter(vec![
305 Ok(LlmResponseChunk::TextDelta {
306 index: 0,
307 text: "before".to_string(),
308 }
309 .into()),
310 Ok(message.clone().into()),
311 Ok(LlmResponseChunk::TextDelta {
312 index: 0,
313 text: "after".to_string(),
314 }
315 .into()),
316 ])
317 .boxed();
318 let results = block_on(encode_stream(chunks, target, None)?.collect::<Vec<_>>());
319 let (data, errors): (Vec<_>, Vec<_>) = results.into_iter().partition(Result::is_ok);
320 let data = data.into_iter().flatten().collect::<Vec<Value>>();
321 let body = serde_json::to_string(&data)?;
322
323 assert!(
324 body.contains("before"),
325 "{target:?}: pre-error content missing:\n{body}"
326 );
327 assert!(
328 !body.contains("after"),
329 "{target:?}/{message:?}: content leaked after the error:\n{body}"
330 );
331 let [Err(LlmStreamError::Upstream(event))] = errors.as_slice() else {
334 panic!("{target:?}/{message:?}: expected one in-band error, got {errors:?}");
335 };
336 assert!(
337 serde_json::to_string(event)?.contains("boom"),
338 "{target:?}: error event lost its message: {event}"
339 );
340 }
341 }
342 Ok(())
343 }
344
345 #[test]
348 fn encode_stream_keeps_a_normal_stop_off_the_error_channel() -> Result<(), BoxError> {
349 let chunks: LlmResponseStream = stream::iter(vec![
350 Ok(LlmResponseChunk::TextDelta {
351 index: 0,
352 text: "hi".to_string(),
353 }
354 .into()),
355 Ok(LlmResponseChunk::MessageStop {
356 reason: Some("stop".to_string()),
357 }
358 .into()),
359 ])
360 .boxed();
361
362 let results =
363 block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());
364
365 assert!(
366 results.iter().all(Result::is_ok),
367 "a completed turn reported an error: {results:?}"
368 );
369 Ok(())
370 }
371
372 #[test]
374 fn encode_stream_stops_polling_after_a_replayed_error() -> Result<(), BoxError> {
375 let error = LlmResponseStreamEvent::preserved(
376 WireFormat::OpenAiResponses,
377 json!({"type": "error", "message": "boom"}),
378 vec![LlmResponseChunk::StreamError {
379 message: "boom".to_string(),
380 }],
381 );
382 let chunks: LlmResponseStream = stream::iter([Ok(error)])
383 .chain(stream::poll_fn(|_| {
384 panic!("encode_stream polled the source after an in-band error")
385 }))
386 .boxed();
387
388 let events =
389 block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());
390
391 let [Err(LlmStreamError::Upstream(event))] = events.as_slice() else {
394 return Err(format!("expected one in-band error, got {events:?}").into());
395 };
396 assert_eq!(*event, json!({"type": "error", "message": "boom"}));
397 Ok(())
398 }
399
400 #[test]
403 fn encode_stream_keeps_trailing_usage_after_a_normal_stop() -> Result<(), BoxError> {
404 let chunks: LlmResponseStream = stream::iter(vec![
405 Ok(LlmResponseChunk::TextDelta {
406 index: 0,
407 text: "hi".to_string(),
408 }
409 .into()),
410 Ok(LlmResponseChunk::MessageStop {
411 reason: Some("stop".to_string()),
412 }
413 .into()),
414 Ok(LlmResponseChunk::Usage(switchyard_protocol::llm::Usage {
415 output_tokens: Some(7),
416 ..Default::default()
417 })
418 .into()),
419 ])
420 .boxed();
421 let events =
422 block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
423 .into_iter()
424 .collect::<Result<Vec<Value>, LlmStreamError>>()?;
425 let body = serde_json::to_string(&events)?;
426 assert!(
427 events
428 .iter()
429 .any(|event| event["choices"][0]["finish_reason"] == "stop"),
430 "missing stop terminal:\n{body}"
431 );
432 assert!(
433 body.contains("\"usage\""),
434 "trailing usage dropped after a normal stop:\n{body}"
435 );
436 Ok(())
437 }
438
439 #[test]
440 fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), LlmClientError> {
441 let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
442 data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
443 data: [DONE]\n\n\
444 data: {\"choices\":[{\"delta\":{\"content\":\" ignored\"}}]}\n\n"
445 .to_vec();
446 let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
447 let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
448 assert_eq!(text_of(&chunks), "Hello world");
449 Ok(())
450 }
451
452 #[test]
453 fn stream_helpers_replay_same_format_provider_fields() -> Result<(), BoxError> {
454 let provider_event = json!({
455 "id": "chatcmpl-test",
456 "object": "chat.completion.chunk",
457 "system_fingerprint": "fp_provider_specific",
458 "choices": [{
459 "index": 0,
460 "delta": {"content": "Hello"},
461 "finish_reason": "stop"
462 }]
463 });
464 let bytes = stream::once({
465 let frame = format!("data: {provider_event}\n\n").into_bytes();
466 async move { Ok::<Vec<u8>, LlmClientError>(frame) }
467 });
468 let decoded = decode_stream(bytes, WireFormat::OpenAiChat)?;
469 let replayed =
470 block_on(encode_stream(decoded, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
471 .into_iter()
472 .collect::<Result<Vec<Value>, LlmStreamError>>()?;
473
474 assert_eq!(replayed, vec![provider_event]);
475 Ok(())
476 }
477
478 #[test]
479 fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> {
480 let payload = json!({"choices": [{"delta": {"content": "café"}}]});
483 let sse = format!("data: {payload}\n\ndata: [DONE]\n\n");
484 let bytes = stream::iter(
485 sse.into_bytes()
486 .into_iter()
487 .map(|byte| Ok::<Vec<u8>, LlmClientError>(vec![byte])),
488 );
489 let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
490 assert_eq!(text_of(&chunks), "café");
491 Ok(())
492 }
493
494 #[test]
495 fn decode_stream_decodes_trailing_frame_without_blank_line() -> Result<(), BoxError> {
496 let sse =
499 b"data: {\"choices\":[{\"delta\":{\"content\":\"tail\"}}]}\n\ndata: [DONE]".to_vec();
500 let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
501 let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
502 assert_eq!(text_of(&chunks), "tail");
503 Ok(())
504 }
505
506 #[test]
507 fn decode_stream_rejects_eof_before_a_terminal_event() -> Result<(), BoxError> {
508 let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"},\"finish_reason\":null}]}\n\n".to_vec();
510 let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
511 let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
512
513 let Some(Ok(first)) = results.first() else {
514 return Err("expected the partial event".into());
515 };
516 assert_eq!(text_of(std::slice::from_ref(first)), "partial");
517 let Some(Err(LlmClientError::ResponseTranslation(message))) = results.last() else {
518 panic!("expected incomplete OpenAI stream to fail");
519 };
520 assert_eq!(message, "openai_chat stream ended before a terminal event");
521 Ok(())
522 }
523
524 #[test]
525 fn responses_failed_before_done_remains_a_stream_error() -> Result<(), BoxError> {
526 let sse = b"data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"message\":\"upstream failed\"}}}\n\ndata: [DONE]\n\n".to_vec();
527 let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
528 let events = decode_all(bytes, WireFormat::OpenAiResponses)?;
529
530 let Some(LlmResponseChunk::StreamError { message }) =
531 events.first().and_then(|event| event.normalized().first())
532 else {
533 return Err("expected response.failed to decode as StreamError".into());
534 };
535 assert_eq!(message, "upstream failed");
536
537 let chunks: LlmResponseStream = stream::iter(
538 events
539 .into_iter()
540 .map(Ok::<LlmResponseStreamEvent, LlmClientError>),
541 )
542 .boxed();
543 let replayed =
544 block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());
545
546 let [Err(LlmStreamError::Upstream(event))] = replayed.as_slice() else {
548 return Err(format!("expected one in-band error, got {replayed:?}").into());
549 };
550 assert_eq!(
551 *event,
552 json!({
553 "type": "response.failed",
554 "response": {"error": {"message": "upstream failed"}}
555 })
556 );
557 Ok(())
558 }
559
560 #[test]
561 fn decode_stream_decodes_crlf_delimited_frames() -> Result<(), BoxError> {
562 let sse =
565 b"data: {\"choices\":[{\"delta\":{\"content\":\"crlf\"}}]}\r\n\r\ndata: [DONE]\r\n\r\n"
566 .to_vec();
567 let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
568 let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
569 assert_eq!(text_of(&chunks), "crlf");
570 Ok(())
571 }
572
573 #[test]
574 fn decode_stream_propagates_source_errors() -> Result<(), BoxError> {
575 let bytes = stream::iter(vec![
577 Ok::<Vec<u8>, LlmClientError>(
578 b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n\n".to_vec(),
579 ),
580 Err::<Vec<u8>, LlmClientError>(LlmClientError::Transport {
581 source: Box::new(std::io::Error::other("upstream exploded")),
582 }),
583 ]);
584 let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
585 let Some(Err(error)) = results.last() else {
586 panic!("expected the source error");
587 };
588 assert!(matches!(error, LlmClientError::Transport { .. }));
589 Ok(())
590 }
591
592 #[test]
593 fn decode_stream_classifies_invalid_sse_json() -> Result<(), BoxError> {
594 let bytes =
595 stream::once(async { Ok::<Vec<u8>, LlmClientError>(b"data: {invalid}\n\n".to_vec()) });
596 let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
597 let Some(Err(error)) = results.last() else {
598 panic!("expected invalid SSE JSON to fail");
599 };
600 assert!(matches!(error, LlmClientError::ResponseTranslation(_)));
601 Ok(())
602 }
603}