Skip to main content

switchyard_translation/codecs/openai_chat/
stream.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Streaming codec for OpenAI Chat Completions chunks.
5
6use serde_json::{Map, Value, json};
7
8use super::buffered::{decode_openai_usage, encode_openai_usage};
9use crate::LlmResponseChunk;
10use crate::codecs::common::{first_nonempty_string, reasoning_text_from_details};
11use crate::codecs::stream::{
12    StreamCodec, StreamTranslationState, record_source_identity, state_source_is, string_field,
13    target_model_or_source_model,
14};
15use crate::format::{FormatId, WireFormat};
16
17/// Stream codec for OpenAI Chat Completions chunks.
18pub struct OpenAiChatStreamCodec;
19
20impl StreamCodec for OpenAiChatStreamCodec {
21    fn format(&self) -> FormatId {
22        WireFormat::OpenAiChat.into()
23    }
24
25    fn is_terminal_event(&self, event: &Value) -> bool {
26        event
27            .get("choices")
28            .and_then(Value::as_array)
29            .into_iter()
30            .flatten()
31            .any(|choice| {
32                choice
33                    .get("finish_reason")
34                    .and_then(Value::as_str)
35                    .is_some()
36            })
37    }
38
39    fn sse_done_is_terminal(&self) -> bool {
40        true
41    }
42
43    fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {
44        if let Some(event) = event.as_object_mut() {
45            event.insert("model".to_string(), Value::String(served_model.to_string()));
46        }
47    }
48
49    fn decode_event(
50        &self,
51        state: &mut StreamTranslationState,
52        event: &Value,
53    ) -> Vec<LlmResponseChunk> {
54        decode_openai_chat_stream(state, event)
55    }
56
57    fn encode_event(
58        &self,
59        state: &mut StreamTranslationState,
60        event: LlmResponseChunk,
61    ) -> Vec<Value> {
62        encode_openai_chat_stream(state, event)
63    }
64
65    fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value> {
66        finish_openai_chat_stream(state)
67    }
68}
69
70// Decodes one OpenAI Chat chunk into neutral streaming events.
71fn decode_openai_chat_stream(
72    state: &mut StreamTranslationState,
73    event: &Value,
74) -> Vec<LlmResponseChunk> {
75    let Some(object) = event.as_object() else {
76        return vec![LlmResponseChunk::DecodeError {
77            message: "OpenAI stream event is not an object".to_string(),
78        }];
79    };
80
81    // Upstream error frames carry no choices, so without this they would decode to a bare
82    // `MessageStart` and the error text would be dropped.
83    if let Some(error) = object.get("error") {
84        return vec![LlmResponseChunk::StreamError {
85            message: error
86                .get("message")
87                .and_then(Value::as_str)
88                .unwrap_or("unknown OpenAI stream error")
89                .to_string(),
90        }];
91    }
92
93    let mut out = Vec::new();
94    let mut identity_changed = false;
95    if state.model.is_none() {
96        state.model = string_field(object, "model");
97        identity_changed |= state.model.is_some();
98    }
99    if state.message_id.is_none() {
100        state.message_id = string_field(object, "id");
101        identity_changed |= state.message_id.is_some();
102    }
103    if !state.saw_message_start || identity_changed {
104        state.saw_message_start = true;
105        out.push(LlmResponseChunk::MessageStart {
106            id: state.message_id.clone(),
107            model: state.model.clone(),
108        });
109    }
110
111    if let Some(usage) = object.get("usage").filter(|usage| usage.is_object()) {
112        let usage = decode_openai_usage(Some(usage));
113        state.usage = usage.clone();
114        state.saw_backend_usage = true;
115        out.push(LlmResponseChunk::Usage(usage));
116    }
117
118    for choice in object
119        .get("choices")
120        .and_then(Value::as_array)
121        .into_iter()
122        .flatten()
123    {
124        let Some(choice) = choice.as_object() else {
125            continue;
126        };
127        let choice_index = choice.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
128        if let Some(delta) = choice.get("delta").and_then(Value::as_object) {
129            if let Some(details) = delta
130                .get("reasoning_details")
131                .and_then(Value::as_array)
132                .filter(|details| !details.is_empty())
133            {
134                let text = reasoning_text_from_details(details)
135                    .or_else(|| {
136                        first_nonempty_string(delta, &["reasoning_content", "reasoning"])
137                            .map(ToOwned::to_owned)
138                    })
139                    .unwrap_or_default();
140                out.push(LlmResponseChunk::ReasoningDetailsDelta {
141                    index: 0,
142                    details: details.clone(),
143                    text,
144                });
145            } else {
146                for reasoning_key in ["reasoning_content", "reasoning"] {
147                    if let Some(text) = delta.get(reasoning_key).and_then(Value::as_str)
148                        && !text.is_empty()
149                    {
150                        out.push(LlmResponseChunk::ReasoningDelta {
151                            index: 0,
152                            text: text.to_string(),
153                        });
154                    }
155                }
156            }
157            if let Some(text) = delta.get("content").and_then(Value::as_str)
158                && !text.is_empty()
159            {
160                out.push(LlmResponseChunk::TextDelta {
161                    index: 0,
162                    text: text.to_string(),
163                });
164            }
165            if let Some(text) = delta.get("refusal").and_then(Value::as_str) {
166                state.chat_refusal_choices.insert(choice_index);
167                out.push(LlmResponseChunk::TextDelta {
168                    index: choice_index,
169                    text: text.to_string(),
170                });
171            }
172            if let Some(tool_calls) = delta.get("tool_calls").and_then(Value::as_array) {
173                for tool_call in tool_calls {
174                    if let Some(tool_call) = tool_call.as_object() {
175                        let function = tool_call.get("function").and_then(Value::as_object);
176                        let index =
177                            tool_call.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
178                        if let Some(name) = function
179                            .and_then(|function| function.get("name"))
180                            .and_then(Value::as_str)
181                            .filter(|name| !name.is_empty())
182                        {
183                            state
184                                .pending_chat_tool_names
185                                .entry(index)
186                                .or_default()
187                                .push_str(name);
188                        }
189                        let arguments_delta = function
190                            .and_then(|function| function.get("arguments"))
191                            .and_then(Value::as_str)
192                            .map(ToOwned::to_owned);
193                        out.push(LlmResponseChunk::ToolCallDelta {
194                            index,
195                            id: tool_call
196                                .get("id")
197                                .and_then(Value::as_str)
198                                .map(ToOwned::to_owned),
199                            // Names may continue after arguments begin; emit them at finish_reason.
200                            name: None,
201                            arguments_delta,
202                        });
203                    }
204                }
205            }
206        }
207        if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) {
208            for (index, name) in std::mem::take(&mut state.pending_chat_tool_names) {
209                out.push(LlmResponseChunk::ToolCallDelta {
210                    index,
211                    id: None,
212                    name: Some(name),
213                    arguments_delta: None,
214                });
215            }
216            let saw_refusal = state.chat_refusal_choices.remove(&choice_index);
217            let reason = if saw_refusal && reason == "stop" {
218                "refusal"
219            } else {
220                reason
221            };
222            state.stop_reason = Some(reason.to_string());
223            out.push(LlmResponseChunk::MessageStop {
224                reason: Some(reason.to_string()),
225            });
226        }
227    }
228    out
229}
230
231// Encodes neutral streaming events into OpenAI Chat chunks.
232fn encode_openai_chat_stream(
233    state: &mut StreamTranslationState,
234    event: LlmResponseChunk,
235) -> Vec<Value> {
236    // An in-band error is terminal: once the error is emitted, drop every later chunk.
237    if state.errored {
238        return Vec::new();
239    }
240    match event {
241        LlmResponseChunk::MessageStart { id, model } => {
242            record_source_identity(state, id, model);
243            // `finish_openai_chat_stream` uses this flag to decide whether a clean EOF needs a
244            // synthesized terminal chunk. Set it on the encode path as well as the decode path.
245            state.saw_message_start = true;
246            if state.emitted_message_start
247                || (!state_source_is(state, WireFormat::AnthropicMessages)
248                    && !state_source_is(state, WireFormat::OpenAiResponses))
249            {
250                Vec::new()
251            } else {
252                state.emitted_message_start = true;
253                vec![openai_stream_chunk(
254                    state,
255                    json!({"role": "assistant"}),
256                    None,
257                    None,
258                )]
259            }
260        }
261        LlmResponseChunk::TextDelta { text, .. } => {
262            vec![openai_stream_chunk(
263                state,
264                json!({"content": text}),
265                None,
266                None,
267            )]
268        }
269        LlmResponseChunk::ReasoningDelta { text, .. } => {
270            vec![openai_stream_chunk(
271                state,
272                json!({"reasoning_content": text}),
273                None,
274                None,
275            )]
276        }
277        LlmResponseChunk::ReasoningDetailsDelta { details, text, .. } => {
278            // Chat cannot replay Anthropic signature fragments or an encrypted item's
279            // id announcement without its payload.
280            let details: Vec<Value> = details
281                .into_iter()
282                .filter(|detail| match detail.get("type").and_then(Value::as_str) {
283                    Some("anthropic.signature_delta") => false,
284                    Some("reasoning.encrypted") => detail.get("data").is_some(),
285                    _ => true,
286                })
287                .collect();
288            if details.is_empty() && text.is_empty() {
289                return Vec::new();
290            }
291            let details_text = reasoning_text_from_details(&details);
292            let mut delta = json!({"reasoning_details": details});
293            if !text.is_empty() && details_text.as_deref() != Some(text.as_str()) {
294                delta["reasoning"] = Value::String(text);
295            }
296            vec![openai_stream_chunk(state, delta, None, None)]
297        }
298        LlmResponseChunk::ToolCallDelta {
299            index,
300            id,
301            name,
302            arguments_delta,
303        } => {
304            let tool = state.tool_states.entry(index).or_default();
305            // Repeated full names would be concatenated by Chat clients as new fragments.
306            let name = name.filter(|name| tool.name.as_ref() != Some(name));
307            if name.is_some() {
308                tool.name = name.clone();
309            }
310            // The source index counts every content block (Anthropic) or output item
311            // (Responses), so text ahead of the first tool call shifts it. Chat clients use
312            // the index as a subscript into `tool_calls`, so number calls in that array
313            // instead, in order of first appearance.
314            let chat_index = match tool.chat_tool_index {
315                Some(chat_index) => chat_index,
316                None => {
317                    let chat_index = state.next_chat_tool_index;
318                    state.next_chat_tool_index += 1;
319                    state.tool_states.entry(index).or_default().chat_tool_index = Some(chat_index);
320                    chat_index
321                }
322            };
323            vec![openai_tool_call_chunk(
324                state,
325                chat_index,
326                id,
327                name,
328                arguments_delta,
329            )]
330        }
331        LlmResponseChunk::Usage(usage) => {
332            state.usage = usage;
333            state.saw_backend_usage = true;
334            if state.finished {
335                vec![openai_usage_chunk(state)]
336            } else {
337                Vec::new()
338            }
339        }
340        LlmResponseChunk::MessageStop { reason } => {
341            if state.finished {
342                return Vec::new();
343            }
344            let mut out = Vec::new();
345            if reason.as_deref() == Some("refusal") {
346                // Text was already emitted. Preserve the refusal signal without repeating it.
347                out.push(openai_stream_chunk(
348                    state,
349                    json!({"refusal": ""}),
350                    None,
351                    None,
352                ));
353            }
354            state.finished = true;
355            out.push(openai_stream_chunk(
356                state,
357                json!({}),
358                Some(openai_finish_reason(reason.as_deref())),
359                state
360                    .saw_backend_usage
361                    .then(|| encode_openai_usage(&state.usage)),
362            ));
363            out
364        }
365        LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {
366            // An in-band error is terminal: emit the error, then nothing further.
367            state.finished = true; // finish() adds no success events
368            state.errored = true; // the entry guard drops any later chunk
369            vec![json!({"error": {"message": message}})]
370        }
371    }
372}
373
374// Emits a terminal chunk if the source stream ended before a stop event arrived.
375fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec<Value> {
376    if state.finished || !state.saw_message_start {
377        return Vec::new();
378    }
379    encode_openai_chat_stream(
380        state,
381        LlmResponseChunk::MessageStop {
382            reason: state.stop_reason.clone(),
383        },
384    )
385}
386
387// Builds a single OpenAI Chat stream chunk payload.
388fn openai_stream_chunk(
389    state: &StreamTranslationState,
390    delta: Value,
391    finish_reason: Option<String>,
392    usage: Option<Value>,
393) -> Value {
394    let mut payload = json!({
395        "id": openai_stream_id(state),
396        "object": "chat.completion.chunk",
397        "created": 0,
398        "model": target_model_or_source_model(state),
399        "choices": [{
400            "index": 0,
401            "delta": delta,
402            "finish_reason": finish_reason,
403        }],
404    });
405    if let Some(usage) = usage {
406        payload["usage"] = usage;
407    }
408    payload
409}
410
411// Builds the usage-only chunk OpenAI emits after the terminal choices chunk.
412fn openai_usage_chunk(state: &StreamTranslationState) -> Value {
413    let mut payload = openai_stream_chunk(
414        state,
415        json!({}),
416        None,
417        Some(encode_openai_usage(&state.usage)),
418    );
419    payload["choices"] = json!([]);
420    payload
421}
422
423// Builds an OpenAI Chat tool-call delta chunk.
424fn openai_tool_call_chunk(
425    state: &StreamTranslationState,
426    index: usize,
427    id: Option<String>,
428    name: Option<String>,
429    arguments: Option<String>,
430) -> Value {
431    let mut function = Map::new();
432    if let Some(name) = name {
433        function.insert("name".to_string(), Value::String(name));
434    }
435    if let Some(arguments) = arguments {
436        function.insert("arguments".to_string(), Value::String(arguments));
437    }
438    let mut tool_call = Map::new();
439    tool_call.insert("index".to_string(), json!(index));
440    tool_call.insert("type".to_string(), json!("function"));
441    tool_call.insert("function".to_string(), Value::Object(function));
442    if let Some(id) = id {
443        tool_call.insert("id".to_string(), Value::String(id));
444    }
445    openai_stream_chunk(
446        state,
447        json!({"tool_calls": [Value::Object(tool_call)]}),
448        None,
449        None,
450    )
451}
452
453// Converts any upstream message ID into an OpenAI-looking stream ID.
454fn openai_stream_id(state: &StreamTranslationState) -> String {
455    let Some(id) = state.message_id.as_deref() else {
456        return "chatcmpl_switchyard".to_string();
457    };
458    if id.starts_with("chatcmpl") {
459        id.to_string()
460    } else if let Some(rest) = id.strip_prefix("msg_").or_else(|| id.strip_prefix("resp_")) {
461        format!("chatcmpl_{rest}")
462    } else {
463        format!("chatcmpl_{id}")
464    }
465}
466
467// Maps provider stop reasons into OpenAI's finish-reason vocabulary.
468fn openai_finish_reason(reason: Option<&str>) -> String {
469    match reason {
470        Some("end_turn") | Some("stop_sequence") | None => "stop".to_string(),
471        Some("max_tokens") => "length".to_string(),
472        Some("tool_use") => "tool_calls".to_string(),
473        Some("refusal") => "stop".to_string(),
474        Some(other) => other.to_string(),
475    }
476}
477
478#[cfg(test)]
479mod tests {
480    use super::*;
481
482    #[test]
483    fn recognizes_stream_completion() {
484        let codec = OpenAiChatStreamCodec;
485        assert!(codec.sse_done_is_terminal());
486        assert!(codec.is_terminal_event(&json!({"choices": [{"finish_reason": "stop"}]})));
487        assert!(!codec.is_terminal_event(&json!({"choices": [{"finish_reason": null}]})));
488    }
489
490    #[test]
491    fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {
492        let mut event = json!({
493            "choices": [{"delta": {"content": "Hello"}}],
494            "system_fingerprint": "fp_provider_specific",
495        });
496
497        OpenAiChatStreamCodec.stamp_streamed_response_model(&mut event, "served/model");
498
499        assert_eq!(event["model"], "served/model");
500        assert_eq!(event["system_fingerprint"], "fp_provider_specific");
501    }
502}