1use std::collections::BTreeMap;
9use std::pin::Pin;
10
11use futures::{Stream, StreamExt};
12use http::StatusCode;
13use serde::{Deserialize, Serialize};
14use serde_json::Value;
15use thiserror::Error;
16
17use crate::{
18 LlmClientError,
19 format::FormatId,
20 llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage},
21};
22
23const MID_STREAM_UPSTREAM_STATUS: StatusCode = StatusCode::BAD_GATEWAY;
27
28#[derive(Debug, Error)]
30pub enum LlmStreamError {
31 #[error("upstream stream error: {0}")]
33 Upstream(Value),
34
35 #[error(transparent)]
37 Client(#[from] LlmClientError),
38}
39
40pub type LlmResponseStream =
42 Pin<Box<dyn Stream<Item = Result<LlmResponseStreamEvent, LlmClientError>> + Send>>;
43
44#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
46pub struct ProviderStreamEvent {
47 source: FormatId,
48 raw: Value,
49}
50
51impl ProviderStreamEvent {
52 pub fn source(&self) -> &FormatId {
54 &self.source
55 }
56
57 pub fn raw(&self) -> &Value {
59 &self.raw
60 }
61
62 pub fn into_parts(self) -> (FormatId, Value) {
64 (self.source, self.raw)
65 }
66}
67
68#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
73pub struct LlmResponseStreamEvent {
74 preservation: Option<ProviderStreamEvent>,
75 normalized: Vec<LlmResponseChunk>,
76}
77
78impl LlmResponseStreamEvent {
79 pub fn new(normalized: Vec<LlmResponseChunk>) -> Self {
81 Self {
82 preservation: None,
83 normalized,
84 }
85 }
86
87 pub fn preserved(
89 source: impl Into<FormatId>,
90 raw: Value,
91 normalized: Vec<LlmResponseChunk>,
92 ) -> Self {
93 Self {
94 preservation: Some(ProviderStreamEvent {
95 source: source.into(),
96 raw,
97 }),
98 normalized,
99 }
100 }
101
102 pub fn preservation(&self) -> Option<&ProviderStreamEvent> {
104 self.preservation.as_ref()
105 }
106
107 pub fn normalized(&self) -> &[LlmResponseChunk] {
109 &self.normalized
110 }
111
112 pub fn into_parts(self) -> (Option<ProviderStreamEvent>, Vec<LlmResponseChunk>) {
114 (self.preservation, self.normalized)
115 }
116
117 pub fn replace_normalized(self, normalized: Vec<LlmResponseChunk>) -> Self {
119 Self::new(normalized)
120 }
121}
122
123impl From<LlmResponseChunk> for LlmResponseStreamEvent {
124 fn from(chunk: LlmResponseChunk) -> Self {
125 Self::new(vec![chunk])
126 }
127}
128
129#[allow(clippy::large_enum_variant)]
136pub enum LlmResponse {
137 Stream(LlmResponseStream),
139 Agg(AggLlmResponse),
141}
142
143impl LlmResponse {
144 pub fn as_agg(&self) -> Option<&AggLlmResponse> {
146 match self {
147 LlmResponse::Agg(agg) => Some(agg),
148 LlmResponse::Stream(_) => None,
149 }
150 }
151
152 pub async fn into_agg(self) -> Result<AggLlmResponse, LlmClientError> {
158 match self {
159 LlmResponse::Agg(agg) => Ok(agg),
160 LlmResponse::Stream(mut stream) => {
161 let mut accumulator = ResponseAccumulator::new();
162 while let Some(item) = stream.next().await {
163 for chunk in item?.normalized {
164 push_checked_chunk(&mut accumulator, chunk)?;
165 }
166 }
167 Ok(accumulator.finish())
168 }
169 }
170 }
171
172 pub fn selected_model(&self) -> Option<&str> {
177 match self {
178 LlmResponse::Agg(agg) => agg.model.as_deref(),
179 LlmResponse::Stream(_) => None,
181 }
182 }
183}
184
185fn is_reasoning_id_announcement(detail: &Value) -> bool {
187 detail.get("type").and_then(Value::as_str) == Some("reasoning.encrypted")
188 && detail.get("data").is_none()
189}
190
191fn push_reasoning_detail(details: &mut Vec<Value>, detail: Value) {
195 if detail.get("type").and_then(Value::as_str) == Some("reasoning.encrypted")
196 && let Some(id) = detail.get("id").and_then(Value::as_str)
197 && let Some(announcement) = details.iter_mut().find(|existing| {
198 is_reasoning_id_announcement(existing)
199 && existing.get("id").and_then(Value::as_str) == Some(id)
200 })
201 {
202 *announcement = detail;
203 return;
204 }
205 details.push(detail);
206}
207
208impl AggLlmResponse {
209 pub fn into_stream(self) -> LlmResponseStream {
219 let mut chunks: Vec<LlmResponseChunk> = Vec::new();
220 chunks.push(LlmResponseChunk::MessageStart {
221 id: self.id,
222 model: self.model,
223 });
224 let mut tool_call_index = 0usize;
225 for (output_index, output) in self.outputs.into_iter().enumerate() {
226 for block in output.content {
227 match block {
228 ContentBlock::Text { text } => {
229 chunks.push(LlmResponseChunk::TextDelta {
230 index: output_index,
231 text,
232 });
233 }
234 ContentBlock::Reasoning { text, details, .. } => {
235 if !details.is_empty() {
236 chunks.push(LlmResponseChunk::ReasoningDetailsDelta {
237 index: output_index,
238 details,
239 text,
240 });
241 } else {
242 chunks.push(LlmResponseChunk::ReasoningDelta {
243 index: output_index,
244 text,
245 });
246 }
247 }
248 ContentBlock::ToolCall(tool) => {
249 let args = serde_json::to_string(&tool.arguments).unwrap_or_default();
250 chunks.push(LlmResponseChunk::ToolCallDelta {
251 index: tool_call_index,
252 id: Some(tool.id),
253 name: Some(tool.name),
254 arguments_delta: Some(args),
255 });
256 tool_call_index += 1;
257 }
258 _ => {}
260 }
261 }
262 chunks.push(LlmResponseChunk::MessageStop {
263 reason: output.stop_reason.and_then(|r| {
264 serde_json::to_value(r)
265 .ok()
266 .and_then(|v| v.as_str().map(String::from))
267 }),
268 });
269 }
270 chunks.push(LlmResponseChunk::Usage(self.usage));
271 Box::pin(futures::stream::iter(
272 chunks.into_iter().map(|chunk| Ok(chunk.into())),
273 ))
274 }
275}
276
277fn push_checked_chunk(
278 accumulator: &mut ResponseAccumulator,
279 chunk: LlmResponseChunk,
280) -> Result<(), LlmClientError> {
281 match chunk {
282 LlmResponseChunk::DecodeError { message } => {
283 Err(LlmClientError::ResponseTranslation(message))
284 }
285 LlmResponseChunk::StreamError { message } => Err(LlmClientError::UpstreamHttp {
286 status: MID_STREAM_UPSTREAM_STATUS,
287 body: message,
288 headers: Box::default(),
289 }),
290 chunk => {
291 accumulator.push(chunk);
292 Ok(())
293 }
294 }
295}
296
297#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
299pub enum LlmResponseChunk {
300 MessageStart {
302 id: Option<String>,
304 model: Option<String>,
306 },
307 TextDelta {
309 index: usize,
311 text: String,
313 },
314 ReasoningDelta {
316 index: usize,
318 text: String,
320 },
321 ReasoningDetailsDelta {
323 index: usize,
325 details: Vec<Value>,
327 text: String,
329 },
330 ToolCallDelta {
332 index: usize,
334 id: Option<String>,
336 name: Option<String>,
338 arguments_delta: Option<String>,
340 },
341 Usage(Usage),
343 MessageStop {
345 reason: Option<String>,
347 },
348 DecodeError {
350 message: String,
352 },
353 StreamError {
355 message: String,
357 },
358}
359
360#[derive(Default)]
375pub struct ResponseAccumulator {
376 id: Option<String>,
377 model: Option<String>,
378 text: String,
379 reasoning: Option<String>,
380 reasoning_details: Vec<Value>,
381 tool_calls: BTreeMap<usize, PartialToolCall>,
382 usage: Usage,
383 stop_reason: Option<StopReason>,
384}
385
386#[derive(Default)]
388struct PartialToolCall {
389 id: Option<String>,
390 name: Option<String>,
391 arguments: String,
392}
393
394impl ResponseAccumulator {
395 pub fn new() -> Self {
397 Self::default()
398 }
399
400 pub fn push(&mut self, chunk: LlmResponseChunk) {
403 match chunk {
404 LlmResponseChunk::MessageStart { id, model } => {
405 if id.is_some() {
406 self.id = id;
407 }
408 if model.is_some() {
409 self.model = model;
410 }
411 }
412 LlmResponseChunk::TextDelta { text, .. } => self.text.push_str(&text),
413 LlmResponseChunk::ReasoningDelta { text, .. } => {
414 self.reasoning
415 .get_or_insert_with(String::new)
416 .push_str(&text);
417 }
418 LlmResponseChunk::ReasoningDetailsDelta { details, text, .. } => {
419 for detail in details {
420 push_reasoning_detail(&mut self.reasoning_details, detail);
421 }
422 if !text.is_empty() {
423 self.reasoning
424 .get_or_insert_with(String::new)
425 .push_str(&text);
426 }
427 }
428 LlmResponseChunk::ToolCallDelta {
429 index,
430 id,
431 name,
432 arguments_delta,
433 } => {
434 let call = self.tool_calls.entry(index).or_default();
435 if id.is_some() {
436 call.id = id;
437 }
438 if name.is_some() {
439 call.name = name;
440 }
441 if let Some(delta) = arguments_delta {
442 call.arguments.push_str(&delta);
443 }
444 }
445 LlmResponseChunk::Usage(usage) => self.usage = usage,
446 LlmResponseChunk::MessageStop { reason } => {
447 self.stop_reason = Some(stop_reason_from_str(reason.as_deref()));
448 }
449 LlmResponseChunk::DecodeError { .. } | LlmResponseChunk::StreamError { .. } => {}
450 }
451 }
452
453 pub fn finish(self) -> AggLlmResponse {
456 let mut content = Vec::new();
457 if self.reasoning.is_some() || !self.reasoning_details.is_empty() {
458 content.push(ContentBlock::Reasoning {
459 text: self.reasoning.unwrap_or_default(),
460 signature: None,
461 details: self
463 .reasoning_details
464 .into_iter()
465 .filter(|detail| !is_reasoning_id_announcement(detail))
466 .collect(),
467 });
468 }
469 if !self.text.is_empty() {
470 content.push(ContentBlock::Text { text: self.text });
471 }
472 for call in self.tool_calls.into_values() {
473 content.push(ContentBlock::ToolCall(ToolCall {
474 id: call.id.unwrap_or_default(),
475 name: call.name.unwrap_or_default(),
476 arguments: parse_tool_arguments(&call.arguments),
477 }));
478 }
479 AggLlmResponse {
480 id: self.id,
481 model: self.model,
482 outputs: vec![ResponseOutput {
483 role: Role::Assistant,
484 content,
485 url_citations: Vec::new(),
486 stop_reason: self.stop_reason,
487 }],
488 usage: self.usage,
489 ..AggLlmResponse::default()
490 }
491 }
492}
493
494fn parse_tool_arguments(arguments: &str) -> Value {
497 if arguments.is_empty() {
498 return Value::Object(serde_json::Map::new());
499 }
500 serde_json::from_str(arguments).unwrap_or_else(|_| Value::String(arguments.to_string()))
501}
502
503fn stop_reason_from_str(reason: Option<&str>) -> StopReason {
506 match reason {
507 Some("length" | "max_tokens") => StopReason::MaxTokens,
508 Some("tool_calls" | "function_call" | "tool_use") => StopReason::ToolUse,
509 Some("content_filter" | "refusal") => StopReason::ContentFilter,
512 Some("stop" | "end_turn" | "stop_sequence") | None => StopReason::EndTurn,
513 Some(_) => StopReason::Unknown,
514 }
515}
516
517#[cfg(test)]
518mod tests {
519 use futures::executor::block_on;
520 use futures::stream;
521
522 use super::*;
523 use serde_json::json;
524
525 fn fold(chunks: Vec<LlmResponseChunk>) -> AggLlmResponse {
526 let mut accumulator = ResponseAccumulator::new();
527 for chunk in chunks {
528 accumulator.push(chunk);
529 }
530 accumulator.finish()
531 }
532
533 #[test]
534 fn folds_text_usage_and_stop_reason() {
535 let agg = fold(vec![
536 LlmResponseChunk::MessageStart {
537 id: Some("id1".to_string()),
538 model: Some("m".to_string()),
539 },
540 LlmResponseChunk::TextDelta {
541 index: 0,
542 text: "Hel".to_string(),
543 },
544 LlmResponseChunk::TextDelta {
545 index: 0,
546 text: "lo".to_string(),
547 },
548 LlmResponseChunk::Usage(Usage {
549 output_tokens: Some(2),
550 ..Usage::default()
551 }),
552 LlmResponseChunk::MessageStop {
553 reason: Some("length".to_string()),
554 },
555 ]);
556 assert_eq!(agg.id.as_deref(), Some("id1"));
557 assert_eq!(agg.model.as_deref(), Some("m"));
558 assert_eq!(agg.usage.output_tokens, Some(2));
559 assert_eq!(agg.outputs[0].stop_reason, Some(StopReason::MaxTokens));
560 assert_eq!(
561 agg.outputs[0].content,
562 vec![ContentBlock::Text {
563 text: "Hello".to_string()
564 }]
565 );
566 }
567
568 #[test]
569 fn stop_reason_spellings_normalize_the_same_streamed_as_buffered() {
570 for (reason, expected) in [
574 ("length", StopReason::MaxTokens),
575 ("max_tokens", StopReason::MaxTokens),
576 ("tool_calls", StopReason::ToolUse),
577 ("tool_use", StopReason::ToolUse),
578 ("content_filter", StopReason::ContentFilter),
579 ("refusal", StopReason::ContentFilter),
580 ("stop", StopReason::EndTurn),
581 ("end_turn", StopReason::EndTurn),
582 ] {
583 let agg = fold(vec![LlmResponseChunk::MessageStop {
584 reason: Some(reason.to_string()),
585 }]);
586
587 assert_eq!(
588 agg.outputs.first().and_then(|output| output.stop_reason),
589 Some(expected),
590 "stop reason {reason:?}"
591 );
592 }
593 }
594
595 #[test]
596 fn aggregates_normalized_chunks_inside_stream_event() {
597 let response = LlmResponse::Stream(Box::pin(stream::iter([Ok(
598 LlmResponseStreamEvent::preserved(
599 crate::WireFormat::OpenAiChat,
600 json!({
601 "choices": [{"delta": {"content": "hello"}}],
602 "system_fingerprint": "fp_exact"
603 }),
604 vec![LlmResponseChunk::TextDelta {
605 index: 0,
606 text: "hello".to_string(),
607 }],
608 ),
609 )])));
610 let aggregate = block_on(response.into_agg()).expect("stream event should aggregate");
611
612 assert_eq!(
613 aggregate.outputs[0].content,
614 vec![ContentBlock::Text {
615 text: "hello".to_string()
616 }]
617 );
618 }
619
620 #[test]
621 fn replacing_normalized_content_drops_preservation() {
622 let event = LlmResponseStreamEvent::preserved(
623 crate::WireFormat::OpenAiChat,
624 json!({"choices": [{"delta": {"content": "old"}}]}),
625 vec![LlmResponseChunk::TextDelta {
626 index: 0,
627 text: "old".to_string(),
628 }],
629 )
630 .replace_normalized(vec![LlmResponseChunk::TextDelta {
631 index: 0,
632 text: "new".to_string(),
633 }]);
634
635 assert!(event.preservation().is_none());
636 assert_eq!(
637 event.normalized(),
638 &[LlmResponseChunk::TextDelta {
639 index: 0,
640 text: "new".to_string(),
641 }]
642 );
643 }
644
645 #[test]
646 fn stream_errors_inside_preserved_events_remain_typed() {
647 let response = LlmResponse::Stream(Box::pin(stream::iter([Ok(
648 LlmResponseStreamEvent::preserved(
649 crate::WireFormat::OpenAiChat,
650 json!({"error": {"message": "provider failed"}}),
651 vec![LlmResponseChunk::StreamError {
652 message: "provider failed".to_string(),
653 }],
654 ),
655 )])));
656
657 let error = block_on(response.into_agg()).err();
658 assert!(matches!(
659 error,
660 Some(LlmClientError::UpstreamHttp {
661 status: MID_STREAM_UPSTREAM_STATUS,
662 ..
663 })
664 ));
665 }
666
667 #[test]
668 fn assembles_tool_calls_by_index() {
669 let agg = fold(vec![
671 LlmResponseChunk::ToolCallDelta {
672 index: 0,
673 id: Some("call_1".to_string()),
674 name: Some("lookup".to_string()),
675 arguments_delta: Some("{\"q\":".to_string()),
676 },
677 LlmResponseChunk::ToolCallDelta {
678 index: 0,
679 id: None,
680 name: None,
681 arguments_delta: Some("\"rust\"}".to_string()),
682 },
683 LlmResponseChunk::MessageStop {
684 reason: Some("tool_calls".to_string()),
685 },
686 ]);
687 assert_eq!(agg.outputs[0].stop_reason, Some(StopReason::ToolUse));
688 assert_eq!(
689 agg.outputs[0].content,
690 vec![ContentBlock::ToolCall(ToolCall {
691 id: "call_1".to_string(),
692 name: "lookup".to_string(),
693 arguments: json!({"q": "rust"}),
694 })]
695 );
696 }
697
698 #[test]
699 fn reasoning_precedes_text_in_content() {
700 let agg = fold(vec![
701 LlmResponseChunk::ReasoningDelta {
702 index: 0,
703 text: "think".to_string(),
704 },
705 LlmResponseChunk::TextDelta {
706 index: 0,
707 text: "answer".to_string(),
708 },
709 ]);
710 assert_eq!(
711 agg.outputs[0].content,
712 vec![
713 ContentBlock::Reasoning {
714 text: "think".to_string(),
715 signature: None,
716 details: Vec::new(),
717 },
718 ContentBlock::Text {
719 text: "answer".to_string(),
720 },
721 ]
722 );
723 }
724
725 #[test]
726 fn into_stream_round_trips_through_into_agg() {
727 let original = AggLlmResponse {
728 id: Some("id1".to_string()),
729 model: Some("m".to_string()),
730 outputs: vec![ResponseOutput {
731 role: Role::Assistant,
732 content: vec![ContentBlock::Text {
733 text: "hello".to_string(),
734 }],
735 url_citations: Vec::new(),
736 stop_reason: Some(StopReason::EndTurn),
737 }],
738 usage: Usage {
739 output_tokens: Some(3),
740 ..Usage::default()
741 },
742 ..AggLlmResponse::default()
743 };
744 let stream = LlmResponse::Stream(original.clone().into_stream());
745 let recovered = block_on(stream.into_agg()).expect("into_agg failed");
746 assert_eq!(recovered.id, original.id);
747 assert_eq!(recovered.model, original.model);
748 assert_eq!(recovered.usage.output_tokens, original.usage.output_tokens);
749 assert_eq!(
750 recovered.outputs[0].stop_reason,
751 original.outputs[0].stop_reason
752 );
753 assert_eq!(recovered.outputs[0].content, original.outputs[0].content);
754 }
755
756 #[test]
757 fn into_stream_retains_encrypted_reasoning_and_text() {
758 let details = vec![json!({
759 "type": "reasoning.encrypted",
760 "data": "opaque-encrypted-reasoning"
761 })];
762 let original = AggLlmResponse {
763 outputs: vec![ResponseOutput {
764 role: Role::Assistant,
765 content: vec![ContentBlock::Reasoning {
766 text: "fallback reasoning".to_string(),
767 signature: None,
768 details: details.clone(),
769 }],
770 url_citations: Vec::new(),
771 stop_reason: Some(StopReason::EndTurn),
772 }],
773 ..AggLlmResponse::default()
774 };
775
776 let recovered = block_on(LlmResponse::Stream(original.into_stream()).into_agg())
777 .expect("into_agg failed");
778 let ContentBlock::Reasoning {
779 text,
780 details: recovered_details,
781 ..
782 } = &recovered.outputs[0].content[0]
783 else {
784 panic!("expected reasoning block");
785 };
786 assert_eq!(text, "fallback reasoning");
787 assert_eq!(recovered_details, &details);
788 }
789
790 #[test]
791 fn into_agg_preserves_stream_item_error() {
792 let response = LlmResponse::Stream(Box::pin(stream::once(async {
793 Err(LlmClientError::Timeout {
794 source: Box::new(std::io::Error::other("timed out")),
795 })
796 })));
797
798 let Err(error) = block_on(response.into_agg()) else {
799 panic!("expected stream aggregation to fail");
800 };
801 assert!(matches!(error, LlmClientError::Timeout { .. }));
802 }
803}