switchyard_translation/codecs/
stream.rs1use std::collections::BTreeMap;
7use std::sync::Arc;
8
9use serde::{Deserialize, Serialize};
10use serde_json::{Map, Value, json};
11
12use crate::LlmResponseChunk;
13use crate::codecs::anthropic::AnthropicMessagesStreamCodec;
14use crate::codecs::openai_chat::OpenAiChatStreamCodec;
15use crate::codecs::responses::OpenAiResponsesStreamCodec;
16use crate::error::{Result, TranslationError};
17use crate::format::{FormatId, WireFormat};
18use crate::llm::Usage;
19
20#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
22pub struct StreamTranslationState {
23 pub source: Option<FormatId>,
24 pub target: Option<FormatId>,
25 pub model: Option<String>,
27 pub message_id: Option<String>,
29 pub target_model: Option<String>,
32 pub target_message_id: Option<String>,
34 pub saw_message_start: bool,
35 pub emitted_message_start: bool,
36 pub finished: bool,
37 pub errored: bool,
39 pub usage: Usage,
40
41 pub(crate) output_tokens_seen: u64,
42 pub(crate) saw_backend_usage: bool,
43 pub(crate) stop_reason: Option<String>,
44 pub(crate) emitted_message_delta: bool,
45
46 pub(crate) next_content_index: usize,
47 pub(crate) text_block_index: Option<usize>,
48 pub(crate) text_block_started: bool,
49 pub(crate) emitted_content_block: bool,
50 pub(crate) tool_states: BTreeMap<usize, StreamToolState>,
51 #[serde(default)]
52 pub(crate) pending_chat_tool_names: BTreeMap<usize, String>,
53 #[serde(default)]
55 pub(crate) chat_refusal_choices: std::collections::BTreeSet<usize>,
56 #[serde(default)]
57 pub(crate) active_anthropic_tool: Option<usize>,
58 #[serde(default)]
59 pub(crate) deferred_anthropic_tools: Vec<usize>,
60 #[serde(default)]
62 pub(crate) empty_anthropic_tool_inputs: std::collections::BTreeSet<usize>,
63 pub(crate) decoded_reasoning: BTreeMap<usize, String>,
66 pub(crate) decoded_reasoning_encrypted: std::collections::BTreeSet<usize>,
68 pub(crate) decoded_tool_call: bool,
71 #[serde(default)]
73 pub(crate) decoded_response_text: BTreeMap<usize, BTreeMap<usize, String>>,
74
75 pub(crate) response_created: bool,
76 pub(crate) response_text_started: bool,
77 pub(crate) response_text_output_index: Option<usize>,
78 pub(crate) response_text: String,
79 pub(crate) response_reasoning: BTreeMap<usize, ResponseReasoningState>,
83 pub(crate) decoded_reasoning_ids: std::collections::BTreeSet<usize>,
85 pub(crate) next_response_output_index: usize,
86 pub(crate) response_sequence_number: u64,
87 pub(crate) next_chat_tool_index: usize,
88
89 pub(crate) reasoning_block_index: Option<usize>,
90 pub(crate) reasoning_block_started: bool,
91}
92
93#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
95pub(crate) struct ResponseReasoningState {
96 pub(crate) started: bool,
98 pub(crate) summary_started: bool,
101 pub(crate) output_index: Option<usize>,
102 pub(crate) item_id: Option<String>,
105 pub(crate) text: String,
106 pub(crate) encrypted: Option<String>,
109 #[serde(default)]
110 pub(crate) anthropic_signature: Option<String>,
111 #[serde(default)]
114 pub(crate) pending_text: String,
115}
116
117#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
119pub(crate) struct StreamToolState {
120 pub(crate) id: Option<String>,
121 pub(crate) name: Option<String>,
122 pub(crate) arguments: String,
123 pub(crate) decoded_arguments: String,
131 #[serde(default)]
132 pub(crate) has_decoded_identity: bool,
133 #[serde(default)]
137 pub(crate) decoded_custom_call: bool,
138 pub(crate) pending_arguments: String,
139 pub(crate) started: bool,
140 pub(crate) content_index: Option<usize>,
141 pub(crate) chat_tool_index: Option<usize>,
145 pub(crate) response_output_index: Option<usize>,
146 pub(crate) response_item_id: Option<String>,
147}
148
149impl StreamTranslationState {
150 pub fn new(source: impl Into<FormatId>, target: impl Into<FormatId>) -> Self {
152 Self {
153 source: Some(source.into()),
154 target: Some(target.into()),
155 ..Self::default()
156 }
157 }
158}
159
160pub trait StreamCodec: Send + Sync {
162 fn format(&self) -> FormatId;
164
165 fn is_terminal_event(&self, event: &Value) -> bool;
167
168 fn stops_at_terminal_event(&self) -> bool {
172 false
173 }
174
175 fn sse_done_is_terminal(&self) -> bool;
180
181 fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str);
185
186 fn decode_event(
188 &self,
189 state: &mut StreamTranslationState,
190 event: &Value,
191 ) -> Vec<LlmResponseChunk>;
192
193 fn encode_event(
195 &self,
196 state: &mut StreamTranslationState,
197 event: LlmResponseChunk,
198 ) -> Vec<Value>;
199
200 fn observe_replayed_event(
208 &self,
209 state: &mut StreamTranslationState,
210 _raw: &Value,
211 normalized: Vec<LlmResponseChunk>,
212 ) {
213 let replayed_terminal = normalized
214 .iter()
215 .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }));
216 for chunk in normalized {
217 drop(self.encode_event(state, chunk));
218 }
219 if replayed_terminal {
220 state.finished = true;
221 }
222 }
223
224 fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value>;
233}
234
235#[derive(Default)]
237pub struct StreamCodecRegistry {
238 codecs: BTreeMap<FormatId, Arc<dyn StreamCodec>>,
239}
240
241impl StreamCodecRegistry {
242 pub fn new() -> Self {
244 Self::default()
245 }
246
247 pub fn with_builtins() -> Self {
249 let mut registry = Self::new();
250 registry.register(OpenAiChatStreamCodec);
251 registry.register(AnthropicMessagesStreamCodec);
252 registry.register(OpenAiResponsesStreamCodec);
253 registry
254 }
255
256 pub fn register(&mut self, codec: impl StreamCodec + 'static) {
258 self.codecs.insert(codec.format(), Arc::new(codec));
259 }
260
261 pub fn codec(&self, format: impl Into<FormatId>) -> Result<Arc<dyn StreamCodec>> {
263 let format = format.into();
264 self.codecs.get(&format).cloned().ok_or_else(|| {
265 TranslationError::Other(format!("no stream codec registered for {format}"))
266 })
267 }
268}
269
270pub fn decode_stream_event(
272 state: &mut StreamTranslationState,
273 source: impl Into<FormatId>,
274 event: &Value,
275) -> Vec<LlmResponseChunk> {
276 let source = source.into();
277 StreamCodecRegistry::with_builtins()
278 .codec(source)
279 .map(|codec| codec.decode_event(state, event))
280 .unwrap_or_else(|error| {
281 vec![LlmResponseChunk::DecodeError {
284 message: error.to_string(),
285 }]
286 })
287}
288
289pub(crate) fn encode_response_stream_event(
290 state: &mut StreamTranslationState,
291 target_codec: &dyn StreamCodec,
292 target: &FormatId,
293 event: crate::LlmResponseStreamEvent,
294) -> Vec<Value> {
295 if state.errored {
296 return Vec::new();
297 }
298 let (preservation, normalized) = event.into_parts();
299 if let Some(preservation) = preservation {
300 let (source, raw) = preservation.into_parts();
301 if let Err(error) = super::responses::validate_stream_output(&source, target, &raw) {
302 return target_codec.encode_event(
303 state,
304 LlmResponseChunk::DecodeError {
305 message: error.to_string(),
306 },
307 );
308 }
309 let has_decode_error = normalized
311 .iter()
312 .any(|chunk| matches!(chunk, LlmResponseChunk::DecodeError { .. }));
313 if &source == target && !has_decode_error {
314 target_codec.observe_replayed_event(state, &raw, normalized);
319 return vec![raw];
320 }
321 }
322
323 normalized
324 .into_iter()
325 .flat_map(|chunk| target_codec.encode_event(state, chunk))
326 .collect()
327}
328
329pub fn encode_stream_event(
331 state: &mut StreamTranslationState,
332 target: impl Into<FormatId>,
333 event: LlmResponseChunk,
334) -> Vec<Value> {
335 StreamCodecRegistry::with_builtins()
336 .codec(target)
337 .map(|codec| codec.encode_event(state, event))
338 .unwrap_or_else(|error| vec![json!({"error": {"message": error.to_string()}})])
339}
340
341pub(crate) fn record_source_identity(
343 state: &mut StreamTranslationState,
344 id: Option<String>,
345 model: Option<String>,
346) {
347 if id.is_some() {
348 state.message_id = id;
349 }
350 if model.is_some() {
351 state.model = model;
352 }
353}
354
355pub(crate) fn target_model_or_source_model(state: &StreamTranslationState) -> String {
357 state
358 .target_model
359 .clone()
360 .or_else(|| state.model.clone())
361 .unwrap_or_else(|| "unknown".to_string())
362}
363
364pub(crate) fn target_message_id_or_source_message_id(
366 state: &StreamTranslationState,
367) -> Option<&str> {
368 state
369 .target_message_id
370 .as_deref()
371 .or(state.message_id.as_deref())
372}
373
374pub(crate) fn state_source_is(state: &StreamTranslationState, format: WireFormat) -> bool {
376 let format_id: FormatId = format.into();
377 match &state.source {
378 Some(source) => source == &format_id,
379 None => false,
380 }
381}
382
383pub(crate) fn string_field(object: &Map<String, Value>, key: &str) -> Option<String> {
385 object
386 .get(key)
387 .and_then(Value::as_str)
388 .filter(|value| !value.is_empty())
389 .map(ToOwned::to_owned)
390}