Skip to main content

switchyard_translation/
helpers.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Convenience wrappers over the default [`TranslationEngine`] — decode a wire
5//! request/response to the neutral IR, encode the IR back, and decode/encode a
6//! streamed response — so callers can translate without threading an engine and
7//! policy through every call.
8
9use std::sync::LazyLock;
10
11use futures::Stream;
12use serde_json::Value;
13use switchyard_protocol::LlmClientError;
14
15use crate::{
16    AggLlmResponse, LlmRequest, LlmResponseStream, RawEventStream, Result, TranslationEngine,
17    TranslationPolicy, WireFormat,
18};
19
20static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =
21    LazyLock::new(TranslationPolicy::default);
22static DEFAULT_TRANSLATION_ENGINE: LazyLock<TranslationEngine> =
23    LazyLock::new(TranslationEngine::default);
24
25/// Decodes a `wire_format` request body into the neutral IR.
26pub fn decode_request(wire_format: WireFormat, body: &Value) -> Result<LlmRequest> {
27    Ok(DEFAULT_TRANSLATION_ENGINE
28        .decode_request(wire_format, body, &DEFAULT_TRANSLATION_POLICY)?
29        .request)
30}
31
32/// Encodes a normalized request into `wire_format`'s JSON body.
33pub fn encode_request(request: &LlmRequest, wire_format: WireFormat) -> Result<Value> {
34    Ok(DEFAULT_TRANSLATION_ENGINE
35        .encode_request(wire_format, request, &DEFAULT_TRANSLATION_POLICY)?
36        .body)
37}
38
39/// Decodes a buffered `wire_format` response body into the neutral aggregate.
40pub fn decode_aggregated_response(body: &Value, wire_format: WireFormat) -> Result<AggLlmResponse> {
41    Ok(DEFAULT_TRANSLATION_ENGINE
42        .decode_response(wire_format, body, &DEFAULT_TRANSLATION_POLICY)?
43        .response)
44}
45
46/// Encodes a stream of IR chunks into a stream of target-format wire events.
47///
48/// `served_model` is exposed as the response model (via the stream state's
49/// `target_model`); `None` falls back to the id observed on the source stream.
50/// The target stream codec is resolved once and reused per chunk; terminal events
51/// (`message_stop` / `response.completed`) come from `finish`.
52pub fn encode_stream(
53    chunks: LlmResponseStream,
54    target: WireFormat,
55    served_model: Option<String>,
56) -> std::result::Result<RawEventStream, LlmClientError> {
57    DEFAULT_TRANSLATION_ENGINE.encode_stream(target, chunks, served_model)
58}
59
60/// Encodes a response stream, honouring request extensions.
61///
62/// Identical to [`encode_stream`] except that Codex tool namespaces recorded on
63/// the request are restored on each encoded event.
64pub fn encode_stream_with_extensions(
65    chunks: LlmResponseStream,
66    target: WireFormat,
67    served_model: Option<String>,
68    request_extensions: &switchyard_protocol::ProviderExtensions,
69) -> std::result::Result<RawEventStream, LlmClientError> {
70    DEFAULT_TRANSLATION_ENGINE.encode_stream_with_extensions(
71        target,
72        chunks,
73        served_model,
74        request_extensions,
75    )
76}
77
78/// Decodes provider SSE bytes into normalized stream events.
79///
80/// Operates on raw bytes, not any HTTP client type: the caller adapts its
81/// transport's body stream into `Stream<Item = Result<Vec<u8>, _>>`. Frames are
82/// buffered across chunks (a partial frame waits for its boundary); the source
83/// stream codec is resolved once and reused for every frame.
84///
85/// The decoder tracks the source format's protocol-specific terminal event. If
86/// EOF arrives without that required event, the stream yields a deferred
87/// [`LlmClientError::ResponseTranslation`] error after any valid decoded events.
88pub fn decode_stream<S>(
89    bytes: S,
90    source: WireFormat,
91) -> std::result::Result<LlmResponseStream, LlmClientError>
92where
93    S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,
94{
95    DEFAULT_TRANSLATION_ENGINE.decode_stream(source, bytes)
96}
97
98#[cfg(test)]
99mod tests {
100    use futures::executor::block_on;
101    use futures::{Stream, StreamExt, stream};
102    use serde_json::{Value, json};
103    use switchyard_protocol::{
104        LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, completion_text,
105    };
106
107    use super::{
108        decode_aggregated_response, decode_request, decode_stream, encode_request, encode_stream,
109    };
110    use crate::{LlmResponseStream, LlmStreamError, WireFormat};
111
112    // A boxed stream item error, matching the streamed IR contract.
113    type BoxError = Box<dyn std::error::Error + Send + Sync>;
114
115    // Collects a decoded IR stream, surfacing the first error instead of panicking.
116    fn decode_all(
117        bytes: impl Stream<Item = Result<Vec<u8>, LlmClientError>> + Send + 'static,
118        source: WireFormat,
119    ) -> Result<Vec<LlmResponseStreamEvent>, LlmClientError> {
120        block_on(decode_stream(bytes, source)?.collect::<Vec<_>>())
121            .into_iter()
122            .collect()
123    }
124
125    // Concatenates the text of every `TextDelta` chunk.
126    fn text_of(events: &[LlmResponseStreamEvent]) -> String {
127        events
128            .iter()
129            .flat_map(LlmResponseStreamEvent::normalized)
130            .filter_map(|chunk| {
131                if let LlmResponseChunk::TextDelta { text, .. } = chunk {
132                    Some(text.as_str())
133                } else {
134                    None
135                }
136            })
137            .collect()
138    }
139
140    #[test]
141    fn request_round_trips_through_openai_chat() -> Result<(), BoxError> {
142        let body = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]});
143        let request = decode_request(WireFormat::OpenAiChat, &body)?;
144        assert_eq!(request.model.as_deref(), Some("gpt"));
145
146        let encoded = encode_request(&request, WireFormat::OpenAiChat)?;
147        assert_eq!(encoded["model"], "gpt");
148        assert_eq!(encoded["messages"][0]["content"], "hi");
149        Ok(())
150    }
151
152    #[test]
153    fn aggregated_response_decodes_from_openai_chat() -> Result<(), BoxError> {
154        let body = json!({
155            "id": "1",
156            "model": "upstream",
157            "choices": [{
158                "index": 0,
159                "message": {"role": "assistant", "content": "Hi there"},
160                "finish_reason": "stop"
161            }],
162            "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
163        });
164        let agg = decode_aggregated_response(&body, WireFormat::OpenAiChat)?;
165        assert_eq!(completion_text(&agg), "Hi there");
166        Ok(())
167    }
168
169    #[test]
170    fn encode_stream_reassembles_deltas_and_finishes() -> Result<(), BoxError> {
171        let chunks: LlmResponseStream = stream::iter(vec![
172            Ok(LlmResponseChunk::TextDelta {
173                index: 0,
174                text: "Hello".to_string(),
175            }
176            .into()),
177            Ok(LlmResponseChunk::TextDelta {
178                index: 0,
179                text: " world".to_string(),
180            }
181            .into()),
182            Ok(LlmResponseChunk::MessageStop {
183                reason: Some("stop".to_string()),
184            }
185            .into()),
186        ])
187        .boxed();
188
189        let events = block_on(
190            encode_stream(chunks, WireFormat::OpenAiChat, Some("m".to_string()))?
191                .collect::<Vec<_>>(),
192        )
193        .into_iter()
194        .collect::<Result<Vec<Value>, LlmStreamError>>()?;
195
196        let content: String = events
197            .iter()
198            .filter_map(|event| event["choices"][0]["delta"]["content"].as_str())
199            .collect();
200        assert_eq!(content, "Hello world");
201        // The terminal chunk carries the stop reason through to the wire events.
202        assert!(
203            events
204                .iter()
205                .any(|event| event["choices"][0]["finish_reason"] == "stop")
206        );
207        Ok(())
208    }
209
210    // The served model must win over the id the source stream announced, so a
211    // routed response never reports the route the caller addressed.
212    #[test]
213    fn encode_stream_stamps_the_served_model_on_message_start() -> Result<(), BoxError> {
214        let chunks: LlmResponseStream = stream::iter(vec![
215            Ok(LlmResponseChunk::MessageStart {
216                id: Some("msg_1".to_string()),
217                model: Some("upstream/model".to_string()),
218            }
219            .into()),
220            Ok(LlmResponseChunk::TextDelta {
221                index: 0,
222                text: "hi".to_string(),
223            }
224            .into()),
225        ])
226        .boxed();
227
228        let events = block_on(
229            encode_stream(
230                chunks,
231                WireFormat::AnthropicMessages,
232                Some("served/model".to_string()),
233            )?
234            .collect::<Vec<_>>(),
235        )
236        .into_iter()
237        .collect::<Result<Vec<Value>, LlmStreamError>>()?;
238
239        assert_eq!(events[0]["type"], "message_start");
240        assert_eq!(events[0]["message"]["model"], "served/model");
241        Ok(())
242    }
243
244    // Without a served model the upstream id survives, so passthrough routes keep
245    // reporting whatever the provider announced.
246    #[test]
247    fn encode_stream_falls_back_to_the_source_model() -> Result<(), BoxError> {
248        let chunks: LlmResponseStream = stream::iter(vec![
249            Ok(LlmResponseChunk::MessageStart {
250                id: Some("msg_1".to_string()),
251                model: Some("upstream/model".to_string()),
252            }
253            .into()),
254            Ok(LlmResponseChunk::TextDelta {
255                index: 0,
256                text: "hi".to_string(),
257            }
258            .into()),
259        ])
260        .boxed();
261
262        let events = block_on(
263            encode_stream(chunks, WireFormat::AnthropicMessages, None)?.collect::<Vec<_>>(),
264        )
265        .into_iter()
266        .collect::<Result<Vec<Value>, LlmStreamError>>()?;
267
268        assert_eq!(events[0]["message"]["model"], "upstream/model");
269        Ok(())
270    }
271
272    #[test]
273    fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> {
274        let chunks: LlmResponseStream =
275            stream::iter(vec![Err::<LlmResponseStreamEvent, LlmClientError>(
276                LlmClientError::General("chunk exploded".to_string()),
277            )])
278            .boxed();
279        let results =
280            block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());
281        assert!(results.iter().any(Result::is_err));
282        Ok(())
283    }
284
285    // An in-band error is terminal for every target format: the encoder emits the pre-error
286    // content, surfaces the error event through the error channel, and drops any later chunk.
287    // Production truncates the source before the encoder, so this contract is only observable
288    // by driving encode_stream directly.
289    #[test]
290    fn encode_stream_stops_after_an_in_band_error() -> Result<(), BoxError> {
291        for message in [
292            LlmResponseChunk::StreamError {
293                message: "boom".to_string(),
294            },
295            LlmResponseChunk::DecodeError {
296                message: "boom".to_string(),
297            },
298        ] {
299            for target in [
300                WireFormat::OpenAiChat,
301                WireFormat::OpenAiResponses,
302                WireFormat::AnthropicMessages,
303            ] {
304                let chunks: LlmResponseStream = stream::iter(vec![
305                    Ok(LlmResponseChunk::TextDelta {
306                        index: 0,
307                        text: "before".to_string(),
308                    }
309                    .into()),
310                    Ok(message.clone().into()),
311                    Ok(LlmResponseChunk::TextDelta {
312                        index: 0,
313                        text: "after".to_string(),
314                    }
315                    .into()),
316                ])
317                .boxed();
318                let results = block_on(encode_stream(chunks, target, None)?.collect::<Vec<_>>());
319                let (data, errors): (Vec<_>, Vec<_>) = results.into_iter().partition(Result::is_ok);
320                let data = data.into_iter().flatten().collect::<Vec<Value>>();
321                let body = serde_json::to_string(&data)?;
322
323                assert!(
324                    body.contains("before"),
325                    "{target:?}: pre-error content missing:\n{body}"
326                );
327                assert!(
328                    !body.contains("after"),
329                    "{target:?}/{message:?}: content leaked after the error:\n{body}"
330                );
331                // The error rides the error channel, carrying the codec's own encoded event
332                // so the consumer can forward the upstream payload rather than rebuild it.
333                let [Err(LlmStreamError::Upstream(event))] = errors.as_slice() else {
334                    panic!("{target:?}/{message:?}: expected one in-band error, got {errors:?}");
335                };
336                assert!(
337                    serde_json::to_string(event)?.contains("boom"),
338                    "{target:?}: error event lost its message: {event}"
339                );
340            }
341        }
342        Ok(())
343    }
344
345    // A clean stop stays entirely on the data channel, so a consumer that appends a
346    // format-level success sentinel only does so for a turn that actually completed.
347    #[test]
348    fn encode_stream_keeps_a_normal_stop_off_the_error_channel() -> Result<(), BoxError> {
349        let chunks: LlmResponseStream = stream::iter(vec![
350            Ok(LlmResponseChunk::TextDelta {
351                index: 0,
352                text: "hi".to_string(),
353            }
354            .into()),
355            Ok(LlmResponseChunk::MessageStop {
356                reason: Some("stop".to_string()),
357            }
358            .into()),
359        ])
360        .boxed();
361
362        let results =
363            block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());
364
365        assert!(
366            results.iter().all(Result::is_ok),
367            "a completed turn reported an error: {results:?}"
368        );
369        Ok(())
370    }
371
372    // A replayed provider error ends the stream before the encoder polls the source again.
373    #[test]
374    fn encode_stream_stops_polling_after_a_replayed_error() -> Result<(), BoxError> {
375        let error = LlmResponseStreamEvent::preserved(
376            WireFormat::OpenAiResponses,
377            json!({"type": "error", "message": "boom"}),
378            vec![LlmResponseChunk::StreamError {
379                message: "boom".to_string(),
380            }],
381        );
382        let chunks: LlmResponseStream = stream::iter([Ok(error)])
383            .chain(stream::poll_fn(|_| {
384                panic!("encode_stream polled the source after an in-band error")
385            }))
386            .boxed();
387
388        let events =
389            block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());
390
391        // The replayed provider event is the terminal frame, so it arrives verbatim on the
392        // error channel and nothing follows it.
393        let [Err(LlmStreamError::Upstream(event))] = events.as_slice() else {
394            return Err(format!("expected one in-band error, got {events:?}").into());
395        };
396        assert_eq!(*event, json!({"type": "error", "message": "boom"}));
397        Ok(())
398    }
399
400    // The guard keys on `errored`, not `finished`, so a normal completion still emits the
401    // trailing usage chunk the OpenAI chat codec reports only after `finished` is set.
402    #[test]
403    fn encode_stream_keeps_trailing_usage_after_a_normal_stop() -> Result<(), BoxError> {
404        let chunks: LlmResponseStream = stream::iter(vec![
405            Ok(LlmResponseChunk::TextDelta {
406                index: 0,
407                text: "hi".to_string(),
408            }
409            .into()),
410            Ok(LlmResponseChunk::MessageStop {
411                reason: Some("stop".to_string()),
412            }
413            .into()),
414            Ok(LlmResponseChunk::Usage(switchyard_protocol::llm::Usage {
415                output_tokens: Some(7),
416                ..Default::default()
417            })
418            .into()),
419        ])
420        .boxed();
421        let events =
422            block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
423                .into_iter()
424                .collect::<Result<Vec<Value>, LlmStreamError>>()?;
425        let body = serde_json::to_string(&events)?;
426        assert!(
427            events
428                .iter()
429                .any(|event| event["choices"][0]["finish_reason"] == "stop"),
430            "missing stop terminal:\n{body}"
431        );
432        assert!(
433            body.contains("\"usage\""),
434            "trailing usage dropped after a normal stop:\n{body}"
435        );
436        Ok(())
437    }
438
439    #[test]
440    fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), LlmClientError> {
441        let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
442             data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
443             data: [DONE]\n\n\
444             data: {\"choices\":[{\"delta\":{\"content\":\" ignored\"}}]}\n\n"
445            .to_vec();
446        let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
447        let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
448        assert_eq!(text_of(&chunks), "Hello world");
449        Ok(())
450    }
451
452    #[test]
453    fn stream_helpers_replay_same_format_provider_fields() -> Result<(), BoxError> {
454        let provider_event = json!({
455            "id": "chatcmpl-test",
456            "object": "chat.completion.chunk",
457            "system_fingerprint": "fp_provider_specific",
458            "choices": [{
459                "index": 0,
460                "delta": {"content": "Hello"},
461                "finish_reason": "stop"
462            }]
463        });
464        let bytes = stream::once({
465            let frame = format!("data: {provider_event}\n\n").into_bytes();
466            async move { Ok::<Vec<u8>, LlmClientError>(frame) }
467        });
468        let decoded = decode_stream(bytes, WireFormat::OpenAiChat)?;
469        let replayed =
470            block_on(encode_stream(decoded, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
471                .into_iter()
472                .collect::<Result<Vec<Value>, LlmStreamError>>()?;
473
474        assert_eq!(replayed, vec![provider_event]);
475        Ok(())
476    }
477
478    #[test]
479    fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> {
480        // A multi-byte codepoint and the frame boundaries are split across
481        // one-byte chunks; the BufReader must reassemble them losslessly.
482        let payload = json!({"choices": [{"delta": {"content": "café"}}]});
483        let sse = format!("data: {payload}\n\ndata: [DONE]\n\n");
484        let bytes = stream::iter(
485            sse.into_bytes()
486                .into_iter()
487                .map(|byte| Ok::<Vec<u8>, LlmClientError>(vec![byte])),
488        );
489        let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
490        assert_eq!(text_of(&chunks), "café");
491        Ok(())
492    }
493
494    #[test]
495    fn decode_stream_decodes_trailing_frame_without_blank_line() -> Result<(), BoxError> {
496        // A non-standard upstream omits the final blank line; the last frame
497        // must still be decoded rather than dropped.
498        let sse =
499            b"data: {\"choices\":[{\"delta\":{\"content\":\"tail\"}}]}\n\ndata: [DONE]".to_vec();
500        let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
501        let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
502        assert_eq!(text_of(&chunks), "tail");
503        Ok(())
504    }
505
506    #[test]
507    fn decode_stream_rejects_eof_before_a_terminal_event() -> Result<(), BoxError> {
508        // Preserve valid content before surfacing premature EOF as the terminal stream error.
509        let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"},\"finish_reason\":null}]}\n\n".to_vec();
510        let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
511        let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
512
513        let Some(Ok(first)) = results.first() else {
514            return Err("expected the partial event".into());
515        };
516        assert_eq!(text_of(std::slice::from_ref(first)), "partial");
517        let Some(Err(LlmClientError::ResponseTranslation(message))) = results.last() else {
518            panic!("expected incomplete OpenAI stream to fail");
519        };
520        assert_eq!(message, "openai_chat stream ended before a terminal event");
521        Ok(())
522    }
523
524    #[test]
525    fn responses_failed_before_done_remains_a_stream_error() -> Result<(), BoxError> {
526        let sse = b"data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"message\":\"upstream failed\"}}}\n\ndata: [DONE]\n\n".to_vec();
527        let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
528        let events = decode_all(bytes, WireFormat::OpenAiResponses)?;
529
530        let Some(LlmResponseChunk::StreamError { message }) =
531            events.first().and_then(|event| event.normalized().first())
532        else {
533            return Err("expected response.failed to decode as StreamError".into());
534        };
535        assert_eq!(message, "upstream failed");
536
537        let chunks: LlmResponseStream = stream::iter(
538            events
539                .into_iter()
540                .map(Ok::<LlmResponseStreamEvent, LlmClientError>),
541        )
542        .boxed();
543        let replayed =
544            block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());
545
546        // `response.failed` stays a terminal error, replayed with its payload intact.
547        let [Err(LlmStreamError::Upstream(event))] = replayed.as_slice() else {
548            return Err(format!("expected one in-band error, got {replayed:?}").into());
549        };
550        assert_eq!(
551            *event,
552            json!({
553                "type": "response.failed",
554                "response": {"error": {"message": "upstream failed"}}
555            })
556        );
557        Ok(())
558    }
559
560    #[test]
561    fn decode_stream_decodes_crlf_delimited_frames() -> Result<(), BoxError> {
562        // CRLF framing: blank lines are `\r\n\r\n` and the bare `\r` must not
563        // block the frame boundary.
564        let sse =
565            b"data: {\"choices\":[{\"delta\":{\"content\":\"crlf\"}}]}\r\n\r\ndata: [DONE]\r\n\r\n"
566                .to_vec();
567        let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
568        let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
569        assert_eq!(text_of(&chunks), "crlf");
570        Ok(())
571    }
572
573    #[test]
574    fn decode_stream_propagates_source_errors() -> Result<(), BoxError> {
575        // A transport error mid-stream surfaces as an error item, not a panic.
576        let bytes = stream::iter(vec![
577            Ok::<Vec<u8>, LlmClientError>(
578                b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n\n".to_vec(),
579            ),
580            Err::<Vec<u8>, LlmClientError>(LlmClientError::Transport {
581                source: Box::new(std::io::Error::other("upstream exploded")),
582            }),
583        ]);
584        let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
585        let Some(Err(error)) = results.last() else {
586            panic!("expected the source error");
587        };
588        assert!(matches!(error, LlmClientError::Transport { .. }));
589        Ok(())
590    }
591
592    #[test]
593    fn decode_stream_classifies_invalid_sse_json() -> Result<(), BoxError> {
594        let bytes =
595            stream::once(async { Ok::<Vec<u8>, LlmClientError>(b"data: {invalid}\n\n".to_vec()) });
596        let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
597        let Some(Err(error)) = results.last() else {
598            panic!("expected invalid SSE JSON to fail");
599        };
600        assert!(matches!(error, LlmClientError::ResponseTranslation(_)));
601        Ok(())
602    }
603}