switchyard_translation/codecs/openai_chat/
stream.rs1use serde_json::{Map, Value, json};
7
8use super::buffered::{decode_openai_usage, encode_openai_usage};
9use crate::LlmResponseChunk;
10use crate::codecs::common::{first_nonempty_string, reasoning_text_from_details};
11use crate::codecs::stream::{
12 StreamCodec, StreamTranslationState, record_source_identity, state_source_is, string_field,
13 target_model_or_source_model,
14};
15use crate::format::{FormatId, WireFormat};
16
17pub struct OpenAiChatStreamCodec;
19
20impl StreamCodec for OpenAiChatStreamCodec {
21 fn format(&self) -> FormatId {
22 WireFormat::OpenAiChat.into()
23 }
24
25 fn is_terminal_event(&self, event: &Value) -> bool {
26 event
27 .get("choices")
28 .and_then(Value::as_array)
29 .into_iter()
30 .flatten()
31 .any(|choice| {
32 choice
33 .get("finish_reason")
34 .and_then(Value::as_str)
35 .is_some()
36 })
37 }
38
39 fn sse_done_is_terminal(&self) -> bool {
40 true
41 }
42
43 fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {
44 if let Some(event) = event.as_object_mut() {
45 event.insert("model".to_string(), Value::String(served_model.to_string()));
46 }
47 }
48
49 fn decode_event(
50 &self,
51 state: &mut StreamTranslationState,
52 event: &Value,
53 ) -> Vec<LlmResponseChunk> {
54 decode_openai_chat_stream(state, event)
55 }
56
57 fn encode_event(
58 &self,
59 state: &mut StreamTranslationState,
60 event: LlmResponseChunk,
61 ) -> Vec<Value> {
62 encode_openai_chat_stream(state, event)
63 }
64
65 fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value> {
66 finish_openai_chat_stream(state)
67 }
68}
69
70fn decode_openai_chat_stream(
72 state: &mut StreamTranslationState,
73 event: &Value,
74) -> Vec<LlmResponseChunk> {
75 let Some(object) = event.as_object() else {
76 return vec![LlmResponseChunk::DecodeError {
77 message: "OpenAI stream event is not an object".to_string(),
78 }];
79 };
80
81 if let Some(error) = object.get("error") {
84 return vec![LlmResponseChunk::StreamError {
85 message: error
86 .get("message")
87 .and_then(Value::as_str)
88 .unwrap_or("unknown OpenAI stream error")
89 .to_string(),
90 }];
91 }
92
93 let mut out = Vec::new();
94 let mut identity_changed = false;
95 if state.model.is_none() {
96 state.model = string_field(object, "model");
97 identity_changed |= state.model.is_some();
98 }
99 if state.message_id.is_none() {
100 state.message_id = string_field(object, "id");
101 identity_changed |= state.message_id.is_some();
102 }
103 if !state.saw_message_start || identity_changed {
104 state.saw_message_start = true;
105 out.push(LlmResponseChunk::MessageStart {
106 id: state.message_id.clone(),
107 model: state.model.clone(),
108 });
109 }
110
111 if let Some(usage) = object.get("usage").filter(|usage| usage.is_object()) {
112 let usage = decode_openai_usage(Some(usage));
113 state.usage = usage.clone();
114 state.saw_backend_usage = true;
115 out.push(LlmResponseChunk::Usage(usage));
116 }
117
118 for choice in object
119 .get("choices")
120 .and_then(Value::as_array)
121 .into_iter()
122 .flatten()
123 {
124 let Some(choice) = choice.as_object() else {
125 continue;
126 };
127 let choice_index = choice.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
128 if let Some(delta) = choice.get("delta").and_then(Value::as_object) {
129 if let Some(details) = delta
130 .get("reasoning_details")
131 .and_then(Value::as_array)
132 .filter(|details| !details.is_empty())
133 {
134 let text = reasoning_text_from_details(details)
135 .or_else(|| {
136 first_nonempty_string(delta, &["reasoning_content", "reasoning"])
137 .map(ToOwned::to_owned)
138 })
139 .unwrap_or_default();
140 out.push(LlmResponseChunk::ReasoningDetailsDelta {
141 index: 0,
142 details: details.clone(),
143 text,
144 });
145 } else {
146 for reasoning_key in ["reasoning_content", "reasoning"] {
147 if let Some(text) = delta.get(reasoning_key).and_then(Value::as_str)
148 && !text.is_empty()
149 {
150 out.push(LlmResponseChunk::ReasoningDelta {
151 index: 0,
152 text: text.to_string(),
153 });
154 }
155 }
156 }
157 if let Some(text) = delta.get("content").and_then(Value::as_str)
158 && !text.is_empty()
159 {
160 out.push(LlmResponseChunk::TextDelta {
161 index: 0,
162 text: text.to_string(),
163 });
164 }
165 if let Some(text) = delta.get("refusal").and_then(Value::as_str) {
166 state.chat_refusal_choices.insert(choice_index);
167 out.push(LlmResponseChunk::TextDelta {
168 index: choice_index,
169 text: text.to_string(),
170 });
171 }
172 if let Some(tool_calls) = delta.get("tool_calls").and_then(Value::as_array) {
173 for tool_call in tool_calls {
174 if let Some(tool_call) = tool_call.as_object() {
175 let function = tool_call.get("function").and_then(Value::as_object);
176 let index =
177 tool_call.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
178 if let Some(name) = function
179 .and_then(|function| function.get("name"))
180 .and_then(Value::as_str)
181 .filter(|name| !name.is_empty())
182 {
183 state
184 .pending_chat_tool_names
185 .entry(index)
186 .or_default()
187 .push_str(name);
188 }
189 let arguments_delta = function
190 .and_then(|function| function.get("arguments"))
191 .and_then(Value::as_str)
192 .map(ToOwned::to_owned);
193 out.push(LlmResponseChunk::ToolCallDelta {
194 index,
195 id: tool_call
196 .get("id")
197 .and_then(Value::as_str)
198 .map(ToOwned::to_owned),
199 name: None,
201 arguments_delta,
202 });
203 }
204 }
205 }
206 }
207 if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) {
208 for (index, name) in std::mem::take(&mut state.pending_chat_tool_names) {
209 out.push(LlmResponseChunk::ToolCallDelta {
210 index,
211 id: None,
212 name: Some(name),
213 arguments_delta: None,
214 });
215 }
216 let saw_refusal = state.chat_refusal_choices.remove(&choice_index);
217 let reason = if saw_refusal && reason == "stop" {
218 "refusal"
219 } else {
220 reason
221 };
222 state.stop_reason = Some(reason.to_string());
223 out.push(LlmResponseChunk::MessageStop {
224 reason: Some(reason.to_string()),
225 });
226 }
227 }
228 out
229}
230
231fn encode_openai_chat_stream(
233 state: &mut StreamTranslationState,
234 event: LlmResponseChunk,
235) -> Vec<Value> {
236 if state.errored {
238 return Vec::new();
239 }
240 match event {
241 LlmResponseChunk::MessageStart { id, model } => {
242 record_source_identity(state, id, model);
243 state.saw_message_start = true;
246 if state.emitted_message_start
247 || (!state_source_is(state, WireFormat::AnthropicMessages)
248 && !state_source_is(state, WireFormat::OpenAiResponses))
249 {
250 Vec::new()
251 } else {
252 state.emitted_message_start = true;
253 vec![openai_stream_chunk(
254 state,
255 json!({"role": "assistant"}),
256 None,
257 None,
258 )]
259 }
260 }
261 LlmResponseChunk::TextDelta { text, .. } => {
262 vec![openai_stream_chunk(
263 state,
264 json!({"content": text}),
265 None,
266 None,
267 )]
268 }
269 LlmResponseChunk::ReasoningDelta { text, .. } => {
270 vec![openai_stream_chunk(
271 state,
272 json!({"reasoning_content": text}),
273 None,
274 None,
275 )]
276 }
277 LlmResponseChunk::ReasoningDetailsDelta { details, text, .. } => {
278 let details: Vec<Value> = details
281 .into_iter()
282 .filter(|detail| match detail.get("type").and_then(Value::as_str) {
283 Some("anthropic.signature_delta") => false,
284 Some("reasoning.encrypted") => detail.get("data").is_some(),
285 _ => true,
286 })
287 .collect();
288 if details.is_empty() && text.is_empty() {
289 return Vec::new();
290 }
291 let details_text = reasoning_text_from_details(&details);
292 let mut delta = json!({"reasoning_details": details});
293 if !text.is_empty() && details_text.as_deref() != Some(text.as_str()) {
294 delta["reasoning"] = Value::String(text);
295 }
296 vec![openai_stream_chunk(state, delta, None, None)]
297 }
298 LlmResponseChunk::ToolCallDelta {
299 index,
300 id,
301 name,
302 arguments_delta,
303 } => {
304 let tool = state.tool_states.entry(index).or_default();
305 let name = name.filter(|name| tool.name.as_ref() != Some(name));
307 if name.is_some() {
308 tool.name = name.clone();
309 }
310 let chat_index = match tool.chat_tool_index {
315 Some(chat_index) => chat_index,
316 None => {
317 let chat_index = state.next_chat_tool_index;
318 state.next_chat_tool_index += 1;
319 state.tool_states.entry(index).or_default().chat_tool_index = Some(chat_index);
320 chat_index
321 }
322 };
323 vec![openai_tool_call_chunk(
324 state,
325 chat_index,
326 id,
327 name,
328 arguments_delta,
329 )]
330 }
331 LlmResponseChunk::Usage(usage) => {
332 state.usage = usage;
333 state.saw_backend_usage = true;
334 if state.finished {
335 vec![openai_usage_chunk(state)]
336 } else {
337 Vec::new()
338 }
339 }
340 LlmResponseChunk::MessageStop { reason } => {
341 if state.finished {
342 return Vec::new();
343 }
344 let mut out = Vec::new();
345 if reason.as_deref() == Some("refusal") {
346 out.push(openai_stream_chunk(
348 state,
349 json!({"refusal": ""}),
350 None,
351 None,
352 ));
353 }
354 state.finished = true;
355 out.push(openai_stream_chunk(
356 state,
357 json!({}),
358 Some(openai_finish_reason(reason.as_deref())),
359 state
360 .saw_backend_usage
361 .then(|| encode_openai_usage(&state.usage)),
362 ));
363 out
364 }
365 LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {
366 state.finished = true; state.errored = true; vec![json!({"error": {"message": message}})]
370 }
371 }
372}
373
374fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec<Value> {
376 if state.finished || !state.saw_message_start {
377 return Vec::new();
378 }
379 encode_openai_chat_stream(
380 state,
381 LlmResponseChunk::MessageStop {
382 reason: state.stop_reason.clone(),
383 },
384 )
385}
386
387fn openai_stream_chunk(
389 state: &StreamTranslationState,
390 delta: Value,
391 finish_reason: Option<String>,
392 usage: Option<Value>,
393) -> Value {
394 let mut payload = json!({
395 "id": openai_stream_id(state),
396 "object": "chat.completion.chunk",
397 "created": 0,
398 "model": target_model_or_source_model(state),
399 "choices": [{
400 "index": 0,
401 "delta": delta,
402 "finish_reason": finish_reason,
403 }],
404 });
405 if let Some(usage) = usage {
406 payload["usage"] = usage;
407 }
408 payload
409}
410
411fn openai_usage_chunk(state: &StreamTranslationState) -> Value {
413 let mut payload = openai_stream_chunk(
414 state,
415 json!({}),
416 None,
417 Some(encode_openai_usage(&state.usage)),
418 );
419 payload["choices"] = json!([]);
420 payload
421}
422
423fn openai_tool_call_chunk(
425 state: &StreamTranslationState,
426 index: usize,
427 id: Option<String>,
428 name: Option<String>,
429 arguments: Option<String>,
430) -> Value {
431 let mut function = Map::new();
432 if let Some(name) = name {
433 function.insert("name".to_string(), Value::String(name));
434 }
435 if let Some(arguments) = arguments {
436 function.insert("arguments".to_string(), Value::String(arguments));
437 }
438 let mut tool_call = Map::new();
439 tool_call.insert("index".to_string(), json!(index));
440 tool_call.insert("type".to_string(), json!("function"));
441 tool_call.insert("function".to_string(), Value::Object(function));
442 if let Some(id) = id {
443 tool_call.insert("id".to_string(), Value::String(id));
444 }
445 openai_stream_chunk(
446 state,
447 json!({"tool_calls": [Value::Object(tool_call)]}),
448 None,
449 None,
450 )
451}
452
453fn openai_stream_id(state: &StreamTranslationState) -> String {
455 let Some(id) = state.message_id.as_deref() else {
456 return "chatcmpl_switchyard".to_string();
457 };
458 if id.starts_with("chatcmpl") {
459 id.to_string()
460 } else if let Some(rest) = id.strip_prefix("msg_").or_else(|| id.strip_prefix("resp_")) {
461 format!("chatcmpl_{rest}")
462 } else {
463 format!("chatcmpl_{id}")
464 }
465}
466
467fn openai_finish_reason(reason: Option<&str>) -> String {
469 match reason {
470 Some("end_turn") | Some("stop_sequence") | None => "stop".to_string(),
471 Some("max_tokens") => "length".to_string(),
472 Some("tool_use") => "tool_calls".to_string(),
473 Some("refusal") => "stop".to_string(),
474 Some(other) => other.to_string(),
475 }
476}
477
478#[cfg(test)]
479mod tests {
480 use super::*;
481
482 #[test]
483 fn recognizes_stream_completion() {
484 let codec = OpenAiChatStreamCodec;
485 assert!(codec.sse_done_is_terminal());
486 assert!(codec.is_terminal_event(&json!({"choices": [{"finish_reason": "stop"}]})));
487 assert!(!codec.is_terminal_event(&json!({"choices": [{"finish_reason": null}]})));
488 }
489
490 #[test]
491 fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {
492 let mut event = json!({
493 "choices": [{"delta": {"content": "Hello"}}],
494 "system_fingerprint": "fp_provider_specific",
495 });
496
497 OpenAiChatStreamCodec.stamp_streamed_response_model(&mut event, "served/model");
498
499 assert_eq!(event["model"], "served/model");
500 assert_eq!(event["system_fingerprint"], "fp_provider_specific");
501 }
502}