Skip to main content

switchyard_translation/codecs/
stream.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Shared streaming codec contracts, registry, and stream state.
5
6use std::collections::BTreeMap;
7use std::sync::Arc;
8
9use serde::{Deserialize, Serialize};
10use serde_json::{Map, Value, json};
11
12use crate::LlmResponseChunk;
13use crate::codecs::anthropic::AnthropicMessagesStreamCodec;
14use crate::codecs::openai_chat::OpenAiChatStreamCodec;
15use crate::codecs::responses::OpenAiResponsesStreamCodec;
16use crate::error::{Result, TranslationError};
17use crate::format::{FormatId, WireFormat};
18use crate::llm::Usage;
19
20/// Mutable state accumulated while translating one streaming response.
21#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
22pub struct StreamTranslationState {
23    pub source: Option<FormatId>,
24    pub target: Option<FormatId>,
25    /// Model name observed on the source provider stream.
26    pub model: Option<String>,
27    /// Message/response ID observed on the source provider stream.
28    pub message_id: Option<String>,
29    /// Model that served the call, exposed to the client in place of the id the
30    /// source stream reported. `None` falls back to [`Self::model`].
31    pub target_model: Option<String>,
32    /// Optional message/response ID the target stream should expose to the client.
33    pub target_message_id: Option<String>,
34    pub saw_message_start: bool,
35    pub emitted_message_start: bool,
36    pub finished: bool,
37    /// Set once an in-band error event was emitted; the encoder then emits nothing further.
38    pub errored: bool,
39    pub usage: Usage,
40
41    pub(crate) output_tokens_seen: u64,
42    pub(crate) saw_backend_usage: bool,
43    pub(crate) stop_reason: Option<String>,
44    pub(crate) emitted_message_delta: bool,
45
46    pub(crate) next_content_index: usize,
47    pub(crate) text_block_index: Option<usize>,
48    pub(crate) text_block_started: bool,
49    pub(crate) emitted_content_block: bool,
50    pub(crate) tool_states: BTreeMap<usize, StreamToolState>,
51    #[serde(default)]
52    pub(crate) pending_chat_tool_names: BTreeMap<usize, String>,
53    /// Choices that decoded a refusal, including an empty refusal string.
54    #[serde(default)]
55    pub(crate) chat_refusal_choices: std::collections::BTreeSet<usize>,
56    #[serde(default)]
57    pub(crate) active_anthropic_tool: Option<usize>,
58    #[serde(default)]
59    pub(crate) deferred_anthropic_tools: Vec<usize>,
60    /// Empty initial inputs remain placeholders until their blocks close without arguments.
61    #[serde(default)]
62    pub(crate) empty_anthropic_tool_inputs: std::collections::BTreeSet<usize>,
63    /// Reasoning text observed while DECODING, per output index, so a completed item
64    /// that repeats already-streamed text is not decoded twice.
65    pub(crate) decoded_reasoning: BTreeMap<usize, String>,
66    /// Output indexes whose encrypted reasoning payload was already decoded.
67    pub(crate) decoded_reasoning_encrypted: std::collections::BTreeSet<usize>,
68    /// Set once a tool call was observed while DECODING, so a terminal event that names no
69    /// stop reason can still report tool use.
70    pub(crate) decoded_tool_call: bool,
71    /// Text decoded from Responses, keyed by output index and then content index.
72    #[serde(default)]
73    pub(crate) decoded_response_text: BTreeMap<usize, BTreeMap<usize, String>>,
74
75    pub(crate) response_created: bool,
76    pub(crate) response_text_started: bool,
77    pub(crate) response_text_output_index: Option<usize>,
78    pub(crate) response_text: String,
79    /// Reasoning items being ENCODED, keyed by source content index. A response can carry
80    /// several reasoning items (GPT-5 emits one ahead of each tool call); each becomes its
81    /// own output item so none of their encrypted payloads is lost.
82    pub(crate) response_reasoning: BTreeMap<usize, ResponseReasoningState>,
83    /// Source indexes whose provider reasoning item id was already announced while DECODING.
84    pub(crate) decoded_reasoning_ids: std::collections::BTreeSet<usize>,
85    pub(crate) next_response_output_index: usize,
86    pub(crate) response_sequence_number: u64,
87    pub(crate) next_chat_tool_index: usize,
88
89    pub(crate) reasoning_block_index: Option<usize>,
90    pub(crate) reasoning_block_started: bool,
91}
92
93// One Responses reasoning output item under construction by the encoder.
94#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
95pub(crate) struct ResponseReasoningState {
96    /// Set once the item's `added` event was emitted.
97    pub(crate) started: bool,
98    /// Set once the item's summary part opened, which happens on the first text delta. An
99    /// encrypted-only item never opens one, so it never has to close one either.
100    pub(crate) summary_started: bool,
101    pub(crate) output_index: Option<usize>,
102    /// Provider item id the encrypted reasoning was issued under. Used as the emitted item id
103    /// so the client's replay verifies upstream; `None` falls back to a synthesized id.
104    pub(crate) item_id: Option<String>,
105    pub(crate) text: String,
106    /// Opaque `encrypted_content` carried by a Responses reasoning item. Kept verbatim so
107    /// the emitted item stays replayable by the client even when no plaintext streamed.
108    pub(crate) encrypted: Option<String>,
109    #[serde(default)]
110    pub(crate) anthropic_signature: Option<String>,
111    /// Summary text held back while the item's provider id is still unknown. The item opens
112    /// once that id arrives, or once later output shows none is coming.
113    #[serde(default)]
114    pub(crate) pending_text: String,
115}
116
117// Tracks an in-progress streamed tool call across provider-specific deltas.
118#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
119pub(crate) struct StreamToolState {
120    pub(crate) id: Option<String>,
121    pub(crate) name: Option<String>,
122    pub(crate) arguments: String,
123    /// Arguments observed while DECODING the source stream.
124    ///
125    /// Separate from `arguments`, which encoders accumulate. A decoder that
126    /// deduplicates against `arguments` only works when one state performs
127    /// both halves of the translation; when a caller buffers a stream with its
128    /// own state and encodes later, the field is empty and the duplicate is
129    /// emitted.
130    pub(crate) decoded_arguments: String,
131    #[serde(default)]
132    pub(crate) has_decoded_identity: bool,
133    /// True when the source announced a `custom_tool_call`. The decoder then reads the call's
134    /// input only from the completed item, because some providers also stream argument deltas
135    /// whose JSON does not match that input.
136    #[serde(default)]
137    pub(crate) decoded_custom_call: bool,
138    pub(crate) pending_arguments: String,
139    pub(crate) started: bool,
140    pub(crate) content_index: Option<usize>,
141    /// Position in the OpenAI Chat `tool_calls` array, assigned when ENCODING. Chat numbers
142    /// tool calls on their own, while Anthropic and Responses index the whole content array,
143    /// so the source index cannot be reused.
144    pub(crate) chat_tool_index: Option<usize>,
145    pub(crate) response_output_index: Option<usize>,
146    pub(crate) response_item_id: Option<String>,
147}
148
149impl StreamTranslationState {
150    /// Creates stream state with source and target formats already attached.
151    pub fn new(source: impl Into<FormatId>, target: impl Into<FormatId>) -> Self {
152        Self {
153            source: Some(source.into()),
154            target: Some(target.into()),
155            ..Self::default()
156        }
157    }
158}
159
160/// Codec contract for one provider streaming event format.
161pub trait StreamCodec: Send + Sync {
162    /// Returns the stream format handled by this codec.
163    fn format(&self) -> FormatId;
164
165    /// Returns whether a JSON event establishes stream completion.
166    fn is_terminal_event(&self, event: &Value) -> bool;
167
168    /// Returns whether decoding stops after yielding a terminal JSON event.
169    ///
170    /// The default keeps reading so codecs can emit trailing events, such as usage.
171    fn stops_at_terminal_event(&self) -> bool {
172        false
173    }
174
175    /// Returns whether the SSE `[DONE]` marker establishes stream completion.
176    ///
177    /// The marker ends consumption even when this returns `false`; a terminal JSON event
178    /// must then have been observed to avoid a premature-EOF error.
179    fn sse_done_is_terminal(&self) -> bool;
180
181    /// Overrides the model reported by an encoded or preserved event.
182    ///
183    /// Called only when a served model was supplied, including for events emitted by `finish`.
184    fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str);
185
186    /// Decodes one provider event into zero or more neutral events.
187    fn decode_event(
188        &self,
189        state: &mut StreamTranslationState,
190        event: &Value,
191    ) -> Vec<LlmResponseChunk>;
192
193    /// Encodes one neutral event into zero or more provider events.
194    fn encode_event(
195        &self,
196        state: &mut StreamTranslationState,
197        event: LlmResponseChunk,
198    ) -> Vec<Value>;
199
200    /// Advances encoder state after an exact same-format event replay.
201    ///
202    /// Exact replay returns the preserved provider JSON instead of the JSON emitted by
203    /// [`Self::encode_event`]. The encoder must nevertheless observe the normalized chunks so
204    /// [`Self::finish`] can close an incomplete stream without duplicating an already replayed
205    /// terminal event. Codecs whose terminal state cannot be inferred from `MessageStop` alone
206    /// may override this hook and inspect `raw`.
207    fn observe_replayed_event(
208        &self,
209        state: &mut StreamTranslationState,
210        _raw: &Value,
211        normalized: Vec<LlmResponseChunk>,
212    ) {
213        let replayed_terminal = normalized
214            .iter()
215            .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }));
216        for chunk in normalized {
217            drop(self.encode_event(state, chunk));
218        }
219        if replayed_terminal {
220            state.finished = true;
221        }
222    }
223
224    /// Emits any terminal provider events needed after the source stream ends.
225    ///
226    /// This is intentionally required on every codec. Some target formats
227    /// need explicit terminal events after the source closes (for example,
228    /// Anthropic ``message_delta``/``message_stop`` or Responses
229    /// ``response.completed``). Formats that have no source-close work should
230    /// return an empty vector explicitly so the no-op behavior is a conscious
231    /// codec-level choice.
232    fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value>;
233}
234
235/// Registry mapping stream wire formats to stream codecs.
236#[derive(Default)]
237pub struct StreamCodecRegistry {
238    codecs: BTreeMap<FormatId, Arc<dyn StreamCodec>>,
239}
240
241impl StreamCodecRegistry {
242    /// Creates an empty stream codec registry.
243    pub fn new() -> Self {
244        Self::default()
245    }
246
247    /// Creates a registry populated with built-in stream codecs.
248    pub fn with_builtins() -> Self {
249        let mut registry = Self::new();
250        registry.register(OpenAiChatStreamCodec);
251        registry.register(AnthropicMessagesStreamCodec);
252        registry.register(OpenAiResponsesStreamCodec);
253        registry
254    }
255
256    /// Registers or replaces a stream codec for its declared format.
257    pub fn register(&mut self, codec: impl StreamCodec + 'static) {
258        self.codecs.insert(codec.format(), Arc::new(codec));
259    }
260
261    /// Looks up a stream codec by format identifier.
262    pub fn codec(&self, format: impl Into<FormatId>) -> Result<Arc<dyn StreamCodec>> {
263        let format = format.into();
264        self.codecs.get(&format).cloned().ok_or_else(|| {
265            TranslationError::Other(format!("no stream codec registered for {format}"))
266        })
267    }
268}
269
270/// Decodes one provider stream event with the built-in codec registry.
271pub fn decode_stream_event(
272    state: &mut StreamTranslationState,
273    source: impl Into<FormatId>,
274    event: &Value,
275) -> Vec<LlmResponseChunk> {
276    let source = source.into();
277    StreamCodecRegistry::with_builtins()
278        .codec(source)
279        .map(|codec| codec.decode_event(state, event))
280        .unwrap_or_else(|error| {
281            // A missing codec is a translation-side failure, not something the
282            // upstream sent, so it decodes to `DecodeError` rather than `StreamError`.
283            vec![LlmResponseChunk::DecodeError {
284                message: error.to_string(),
285            }]
286        })
287}
288
289pub(crate) fn encode_response_stream_event(
290    state: &mut StreamTranslationState,
291    target_codec: &dyn StreamCodec,
292    target: &FormatId,
293    event: crate::LlmResponseStreamEvent,
294) -> Vec<Value> {
295    if state.errored {
296        return Vec::new();
297    }
298    let (preservation, normalized) = event.into_parts();
299    if let Some(preservation) = preservation {
300        let (source, raw) = preservation.into_parts();
301        if let Err(error) = super::responses::validate_stream_output(&source, target, &raw) {
302            return target_codec.encode_event(
303                state,
304                LlmResponseChunk::DecodeError {
305                    message: error.to_string(),
306                },
307            );
308        }
309        // Invalid protocol data must become an error frame, not be replayed as ordinary data.
310        let has_decode_error = normalized
311            .iter()
312            .any(|chunk| matches!(chunk, LlmResponseChunk::DecodeError { .. }));
313        if &source == target && !has_decode_error {
314            // Exact replay bypasses the target encoder's emitted JSON, but the encoder must
315            // still observe every normalized chunk. Otherwise `finish` starts from empty state:
316            // a clean EOF after a nonterminal provider event can omit or synthesize malformed
317            // terminal events, while a replayed terminal can be emitted twice.
318            target_codec.observe_replayed_event(state, &raw, normalized);
319            return vec![raw];
320        }
321    }
322
323    normalized
324        .into_iter()
325        .flat_map(|chunk| target_codec.encode_event(state, chunk))
326        .collect()
327}
328
329/// Encodes one neutral stream event with the built-in codec registry.
330pub fn encode_stream_event(
331    state: &mut StreamTranslationState,
332    target: impl Into<FormatId>,
333    event: LlmResponseChunk,
334) -> Vec<Value> {
335    StreamCodecRegistry::with_builtins()
336        .codec(target)
337        .map(|codec| codec.encode_event(state, event))
338        .unwrap_or_else(|error| vec![json!({"error": {"message": error.to_string()}})])
339}
340
341// Records source-provider identity carried by decoded stream events.
342pub(crate) fn record_source_identity(
343    state: &mut StreamTranslationState,
344    id: Option<String>,
345    model: Option<String>,
346) {
347    if id.is_some() {
348        state.message_id = id;
349    }
350    if model.is_some() {
351        state.model = model;
352    }
353}
354
355// Returns the served model when supplied, otherwise the upstream model.
356pub(crate) fn target_model_or_source_model(state: &StreamTranslationState) -> String {
357    state
358        .target_model
359        .clone()
360        .or_else(|| state.model.clone())
361        .unwrap_or_else(|| "unknown".to_string())
362}
363
364// Returns the target/client ID when supplied, otherwise the upstream ID.
365pub(crate) fn target_message_id_or_source_message_id(
366    state: &StreamTranslationState,
367) -> Option<&str> {
368    state
369        .target_message_id
370        .as_deref()
371        .or(state.message_id.as_deref())
372}
373
374// Checks whether the current source format matches a built-in format.
375pub(crate) fn state_source_is(state: &StreamTranslationState, format: WireFormat) -> bool {
376    let format_id: FormatId = format.into();
377    match &state.source {
378        Some(source) => source == &format_id,
379        None => false,
380    }
381}
382
383// Reads a non-empty string field from an event object.
384pub(crate) fn string_field(object: &Map<String, Value>, key: &str) -> Option<String> {
385    object
386        .get(key)
387        .and_then(Value::as_str)
388        .filter(|value| !value.is_empty())
389        .map(ToOwned::to_owned)
390}