switchyard_protocol/
llm.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Provider-neutral conversation types shared by routing, clients, and translation.
5//!
6//! Debug output redacts MCP URLs and authorization tokens and hides raw provider tool definitions,
7//! extensions, and preserved bodies. Conversation content, including unknown provider
8//! blocks, remains visible and may contain secrets. Serialization retains the stored values.
9
10use std::collections::BTreeMap;
11use std::fmt;
12
13use serde::{Deserialize, Serialize};
14use serde_json::{Map, Value};
15
16use crate::format::FormatId;
17
18/// Actor role normalized across provider APIs.
19#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, strum_macros::Display)]
20#[serde(rename_all = "snake_case")]
21#[strum(serialize_all = "snake_case")]
22pub enum Role {
23    /// System-level instructions.
24    System,
25    /// Developer-level instructions supported by some provider APIs.
26    Developer,
27    /// End-user input.
28    User,
29    /// Model-generated output.
30    Assistant,
31    /// Tool execution output.
32    Tool,
33}
34
35/// Instruction content separated from normal conversation messages.
36#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
37pub struct InstructionBlock {
38    /// Instruction authority, normally [`Role::System`] or [`Role::Developer`].
39    pub role: Role,
40    /// Ordered instruction content.
41    pub content: Vec<ContentBlock>,
42}
43
44/// One normalized conversation message.
45#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
46pub struct Message {
47    /// Actor that produced the message.
48    pub role: Role,
49    /// Ordered message content.
50    pub content: Vec<ContentBlock>,
51    /// This metadata retains native Responses fields outside `role` and `content`.
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    pub responses: Option<ResponsesItemMetadata>,
54}
55
56/// This enum records how a Responses item represented its content or tool output.
57#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
58#[serde(rename_all = "snake_case")]
59pub enum ResponsesContentKind {
60    /// The item omitted its `content` or `output` field.
61    #[default]
62    Missing,
63    /// The item supplied an explicit null.
64    Null,
65    /// The item supplied one string.
66    String,
67    /// The item supplied an ordered array of content blocks.
68    Array,
69    /// The request supplied its entire `input` as one string.
70    InputString,
71}
72
73/// Native item fields stay with their message when the conversation is edited.
74#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
75#[serde(default)]
76pub struct ResponsesItemMetadata {
77    /// These fields retain item data outside the decoded role, content, and tool call.
78    pub fields: Map<String, Value>,
79    /// This value records the original content or tool output representation.
80    pub content_kind: ResponsesContentKind,
81}
82
83impl fmt::Debug for ResponsesItemMetadata {
84    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
85        f.debug_struct("ResponsesItemMetadata")
86            .field("fields", &self.fields.keys())
87            .field("content_kind", &self.content_kind)
88            .finish()
89    }
90}
91
92/// Native text-block fields stay attached when blocks are edited or reordered.
93#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
94#[serde(transparent)]
95pub struct ResponsesTextMetadata {
96    /// These fields retain block data outside the decoded text value.
97    pub fields: Map<String, Value>,
98}
99
100impl fmt::Debug for ResponsesTextMetadata {
101    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
102        f.debug_struct("ResponsesTextMetadata")
103            .field("fields", &self.fields.keys())
104            .finish()
105    }
106}
107
108impl Message {
109    /// Creates a text-only message for the given role.
110    pub fn text(role: Role, text: impl Into<String>) -> Self {
111        Self {
112            role,
113            content: vec![ContentBlock::text(text)],
114            responses: None,
115        }
116    }
117
118    /// Concatenates text-like content blocks when the message has any.
119    pub fn text_content(&self, separator: &str) -> Option<String> {
120        let parts = self
121            .content
122            .iter()
123            .filter_map(|block| match block {
124                ContentBlock::Text { text, .. } => Some(text.as_str()),
125                ContentBlock::Refusal { text } => Some(text.as_str()),
126                _ => None,
127            })
128            .collect::<Vec<_>>();
129        if parts.is_empty() {
130            None
131        } else {
132            Some(parts.join(separator))
133        }
134    }
135}
136
137/// Normalized content block variants carried by messages and tool results.
138#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
139#[serde(tag = "type", rename_all = "snake_case")]
140pub enum ContentBlock {
141    /// Plain text content.
142    Text {
143        /// Text value.
144        text: String,
145        /// This metadata retains native Responses fields, including the block type.
146        #[serde(default, skip_serializing_if = "Option::is_none")]
147        responses: Option<ResponsesTextMetadata>,
148    },
149    /// Model reasoning or thinking content.
150    Reasoning {
151        /// Reasoning text.
152        text: String,
153        /// Provider signature used to validate or continue the reasoning block.
154        signature: Option<String>,
155        /// Structured reasoning details, such as an encrypted `{ "type":
156        /// "reasoning.encrypted", "data": "..." }` object, replayed without modification.
157        #[serde(default, skip_serializing_if = "Vec::is_empty")]
158        details: Vec<Value>,
159    },
160    /// Image content.
161    Image {
162        /// Image location or inline payload.
163        source: ImageSource,
164    },
165    /// Audio content.
166    Audio {
167        /// Audio location or inline payload.
168        source: MediaSource,
169    },
170    /// Video content.
171    Video {
172        /// Video location or inline payload.
173        source: MediaSource,
174    },
175    /// File content.
176    File {
177        /// File identifier or inline payload.
178        source: FileSource,
179    },
180    /// Tool invocation requested by the assistant.
181    ToolCall(ToolCall),
182    /// Result of an earlier tool invocation.
183    ToolResult(ToolResult),
184    /// MCP activity executed by the backend, rather than by the calling agent.
185    Mcp(McpContent),
186    /// Provider refusal content.
187    Refusal {
188        /// Human-readable refusal text.
189        text: String,
190    },
191    /// Provider block that has no normalized representation.
192    Unknown {
193        /// Wire format that supplied the block.
194        provider: FormatId,
195        /// Exact provider block.
196        raw: Value,
197    },
198}
199
200impl ContentBlock {
201    /// This constructor creates plain text without provider metadata.
202    pub fn text(text: impl Into<String>) -> Self {
203        Self::Text {
204            text: text.into(),
205            responses: None,
206        }
207    }
208}
209
210/// Image payload forms supported by the conversation model.
211#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
212#[serde(tag = "type", content = "data", rename_all = "snake_case")]
213pub enum ImageSource {
214    /// Image fetched from a URL.
215    Url {
216        /// Image URL.
217        url: String,
218        /// Optional provider-specific image detail level.
219        detail: Option<String>,
220    },
221    /// Base64-encoded image bytes.
222    Base64 {
223        /// MIME type, when supplied.
224        media_type: Option<String>,
225        /// Base64-encoded bytes.
226        data: String,
227    },
228    /// Provider image source with no normalized representation.
229    Raw(Value),
230}
231
232/// File payload forms supported by the conversation model.
233#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
234#[serde(tag = "type", content = "data", rename_all = "snake_case")]
235pub enum FileSource {
236    /// Provider-managed file identifier.
237    FileId(String),
238    /// Inline file payload.
239    FileData {
240        /// Encoded or textual file data as supplied by the provider format.
241        data: String,
242        /// Original filename, when supplied.
243        filename: Option<String>,
244    },
245    /// Provider file source with no normalized representation.
246    Raw(Value),
247}
248
249/// Audio and video payload forms supported by the conversation model.
250#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
251#[serde(tag = "type", content = "data", rename_all = "snake_case")]
252pub enum MediaSource {
253    /// Media fetched from a URL.
254    Url {
255        /// Media URL.
256        url: String,
257        /// MIME type, when supplied.
258        media_type: Option<String>,
259    },
260    /// Base64-encoded media bytes.
261    Base64 {
262        /// MIME type, when supplied.
263        media_type: Option<String>,
264        /// Base64-encoded bytes.
265        data: String,
266    },
267    /// Provider media source with no normalized representation.
268    Raw(Value),
269}
270
271/// Normalized assistant tool call.
272#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
273pub struct ToolCall {
274    /// Provider tool-call identifier used to pair the result.
275    pub id: String,
276    /// Tool name.
277    pub name: String,
278    /// Parsed tool arguments.
279    pub arguments: Value,
280}
281
282/// Normalized tool result message content.
283#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
284pub struct ToolResult {
285    /// Identifier of the [`ToolCall`] this result answers.
286    pub tool_call_id: String,
287    /// Ordered tool output content.
288    pub content: Vec<ContentBlock>,
289    /// Whether tool execution failed, when the provider reports it.
290    pub is_error: Option<bool>,
291}
292
293/// Normalized tool definition.
294#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
295pub struct ToolDefinition {
296    /// Tool name exposed to the model.
297    pub name: String,
298    /// Human-readable tool description.
299    pub description: Option<String>,
300    /// JSON Schema describing accepted arguments.
301    pub parameters: Value,
302    /// Whether the provider should enforce the schema strictly.
303    pub strict: Option<bool>,
304}
305
306/// A tool available to the model, in catalog order.
307#[derive(Clone, PartialEq, Serialize)]
308#[serde(tag = "type", rename_all = "snake_case")]
309pub enum Tool {
310    /// A remote MCP server and the tools it exposes.
311    Mcp(McpToolset),
312    /// A definition without a supported neutral representation.
313    Provider {
314        /// Source wire format; this definition is only replayed in that format.
315        provider: FormatId,
316        /// Complete provider definition.
317        raw: Value,
318    },
319    /// An agent-executed function. Its JSON retains the existing flat definition shape.
320    #[serde(untagged)]
321    Function(ToolDefinition),
322}
323
324impl<'de> Deserialize<'de> for Tool {
325    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
326    where
327        D: serde::Deserializer<'de>,
328    {
329        // Keep these variants and the conversion below in sync with Tool.
330        #[derive(Deserialize)]
331        #[serde(tag = "type", rename_all = "snake_case")]
332        enum TaggedTool {
333            Mcp(McpToolset),
334            Provider { provider: FormatId, raw: Value },
335            Function(ToolDefinition),
336        }
337
338        let value = Value::deserialize(deserializer)?;
339        // Only definitions without a type use the legacy function shape. A malformed
340        // tagged definition must fail instead of falling back to a function.
341        if value.get("type").is_none() {
342            return serde_json::from_value(value)
343                .map(Self::Function)
344                .map_err(serde::de::Error::custom);
345        }
346        let tool = serde_json::from_value(value).map_err(serde::de::Error::custom)?;
347        Ok(match tool {
348            TaggedTool::Mcp(mcp) => Self::Mcp(mcp),
349            TaggedTool::Provider { provider, raw } => Self::Provider { provider, raw },
350            TaggedTool::Function(function) => Self::Function(function),
351        })
352    }
353}
354
355impl fmt::Debug for Tool {
356    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
357        match self {
358            Self::Function(definition) => f.debug_tuple("Function").field(definition).finish(),
359            Self::Mcp(mcp) => f.debug_tuple("Mcp").field(mcp).finish(),
360            // Raw provider definitions can contain credentials without normalized fields.
361            Self::Provider { provider, .. } => f
362                .debug_struct("Provider")
363                .field("provider", provider)
364                .finish_non_exhaustive(),
365        }
366    }
367}
368
369impl From<ToolDefinition> for Tool {
370    fn from(definition: ToolDefinition) -> Self {
371        Self::Function(definition)
372    }
373}
374
375impl Tool {
376    /// Returns the function definition, if this is an agent-executed function.
377    pub fn function(&self) -> Option<&ToolDefinition> {
378        match self {
379            Self::Function(definition) => Some(definition),
380            _ => None,
381        }
382    }
383}
384
385/// Configuration for a URL-based remote MCP toolset.
386///
387/// This type covers the supported neutral settings. Definitions with additional settings,
388/// such as Responses headers, server descriptions, or conditional approval rules, belong
389/// in [`Tool::Provider`] with the complete original definition.
390#[derive(Clone, PartialEq, Serialize, Deserialize)]
391pub struct McpToolset {
392    /// Server identity used by MCP calls.
393    pub name: String,
394    /// Remote MCP endpoint.
395    pub url: String,
396    /// Credential sent to the MCP server, when supplied.
397    #[serde(default, skip_serializing_if = "Option::is_none")]
398    pub authorization_token: Option<String>,
399    /// Defaults for tools without a per-tool setting in `configs`.
400    #[serde(default, skip_serializing_if = "McpToolConfig::is_empty")]
401    pub default_config: McpToolConfig,
402    /// Per-tool settings, keyed by tool name. Each omitted setting inherits
403    /// from `default_config`, then falls back to the provider's default.
404    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
405    pub configs: BTreeMap<String, McpToolConfig>,
406    /// Whether all calls require approval (`true`) or none do (`false`).
407    /// Absence retains provider defaults. Definitions with conditional approval rules
408    /// are stored in full as [`Tool::Provider`], rather than as this type.
409    #[serde(default, skip_serializing_if = "Option::is_none")]
410    pub require_approval: Option<bool>,
411}
412
413impl fmt::Debug for McpToolset {
414    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
415        let Self {
416            name,
417            url: _,
418            authorization_token,
419            default_config,
420            configs,
421            require_approval,
422        } = self;
423        f.debug_struct("McpToolset")
424            .field("name", name)
425            // Credentials can appear in any part of the URL, including its path.
426            .field("url", &"[REDACTED]")
427            .field(
428                "authorization_token",
429                &authorization_token.as_ref().map(|_| "[REDACTED]"),
430            )
431            .field("default_config", default_config)
432            .field("configs", configs)
433            .field("require_approval", require_approval)
434            .finish()
435    }
436}
437
438/// Settings for an individual MCP tool or the defaults for a toolset.
439#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
440pub struct McpToolConfig {
441    /// Whether the tool is available for the model to call.
442    #[serde(default, skip_serializing_if = "Option::is_none")]
443    pub enabled: Option<bool>,
444    /// Whether the tool definition is loaded through tool search instead of initially.
445    #[serde(default, skip_serializing_if = "Option::is_none")]
446    pub defer_loading: Option<bool>,
447}
448
449impl McpToolConfig {
450    fn is_empty(&self) -> bool {
451        self.enabled.is_none() && self.defer_loading.is_none()
452    }
453}
454
455/// A tool discovered on an MCP server.
456#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
457pub struct McpListedTool {
458    /// Tool name as exposed by the server.
459    pub name: String,
460    /// Human-readable tool description, when supplied.
461    #[serde(default, skip_serializing_if = "Option::is_none")]
462    pub description: Option<String>,
463    /// JSON Schema describing accepted arguments.
464    pub input_schema: Value,
465    /// Additional tool metadata reported by the server.
466    #[serde(default, skip_serializing_if = "Option::is_none")]
467    pub annotations: Option<Value>,
468}
469
470/// MCP discovery and execution history. Results reference the matching call's server identity.
471#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
472#[serde(tag = "kind", rename_all = "snake_case")]
473pub enum McpContent {
474    /// Tools discovered on a server at this point in the conversation.
475    Listing {
476        /// Identity of the MCP server.
477        server_name: String,
478        /// Listing identifier, when supplied by the provider.
479        #[serde(default, skip_serializing_if = "Option::is_none")]
480        id: Option<String>,
481        /// Discovered tools in the order reported by the server.
482        tools: Vec<McpListedTool>,
483        /// Discovery failure, when reported. An empty tool list alone is not a failure.
484        #[serde(default, skip_serializing_if = "Option::is_none")]
485        error: Option<String>,
486    },
487    /// A call executed by the backend's MCP client.
488    Call {
489        /// Identity of the MCP server.
490        server_name: String,
491        /// Ordinary call ID, name, and arguments.
492        call: ToolCall,
493    },
494    /// Output from a backend-executed MCP call.
495    Result(ToolResult),
496}
497
498/// Normalized tool choice policy.
499#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
500#[serde(tag = "type", content = "data", rename_all = "snake_case")]
501pub enum ToolChoice {
502    /// Let the model decide whether to call a tool.
503    Auto,
504    /// Require at least one tool call.
505    Required,
506    /// Disallow tool calls.
507    None,
508    /// Require a specific tool.
509    Tool {
510        /// Required tool name.
511        name: String,
512    },
513    /// Provider tool-choice value with no normalized representation.
514    Raw(Value),
515}
516
517/// Provider sampling parameters with common cross-provider names.
518#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
519pub struct SamplingParams {
520    /// Sampling temperature.
521    pub temperature: Option<f64>,
522    /// Nucleus-sampling probability mass.
523    pub top_p: Option<f64>,
524    /// Maximum number of candidate tokens considered at each step.
525    pub top_k: Option<i64>,
526}
527
528/// Output budget and structured-output options.
529#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
530pub struct OutputParams {
531    /// Maximum tokens the model may generate.
532    pub max_output_tokens: Option<u64>,
533    /// Provider-neutral or provider-specific structured-output configuration.
534    pub response_format: Option<Value>,
535    /// Whether the provider must enforce the response schema, overriding native
536    /// flags when set and retaining native behavior when `None`.
537    #[serde(default, skip_serializing_if = "Option::is_none")]
538    pub is_schema_enforced: Option<bool>,
539}
540
541/// Provider reasoning controls preserved by translation.
542#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
543pub struct ReasoningParams {
544    /// Requested reasoning effort or level.
545    pub effort: Option<String>,
546    /// Provider reasoning controls without a normalized field.
547    pub raw: Option<Value>,
548}
549
550/// Provider-specific fields that do not have first-class conversation fields.
551#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
552#[serde(default)]
553pub struct ProviderExtensions {
554    /// Provider fields keyed by their original wire names.
555    ///
556    /// Codecs use these fields when translating to another format. Exact
557    /// same-format replay is controlled separately by [`PreservationMetadata`].
558    pub fields: Map<String, Value>,
559}
560
561// Extension values can include credentials or embedded original request bodies.
562impl fmt::Debug for ProviderExtensions {
563    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
564        f.debug_struct("ProviderExtensions")
565            .field("field_count", &self.fields.len())
566            .finish_non_exhaustive()
567    }
568}
569
570/// Exact source payloads retained for lossless same-format round trips.
571///
572/// Translation's default preservation policy prefers a stored same-format body
573/// over reconstructing one from normalized fields. A caller that mutates the IR
574/// must clear the corresponding entry or use a policy with preservation disabled
575/// when those mutations must be encoded.
576#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
577#[serde(default)]
578pub struct PreservationMetadata {
579    /// Original request bodies keyed by source format.
580    pub requests: BTreeMap<FormatId, Value>,
581    /// Original response bodies keyed by source format.
582    pub responses: BTreeMap<FormatId, Value>,
583}
584
585// Original payloads can contain credentials even when normalized fields redact them.
586impl fmt::Debug for PreservationMetadata {
587    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
588        f.debug_struct("PreservationMetadata")
589            .field("request_formats", &self.requests.keys())
590            .field("response_formats", &self.responses.keys())
591            .finish_non_exhaustive()
592    }
593}
594
595/// Normalized request representation shared by Switchyard components.
596#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
597#[serde(default)]
598pub struct LlmRequest {
599    /// Model currently addressed by the request.
600    ///
601    /// This initially contains the name supplied by the inbound client. A routing host may
602    /// replace it with the selected target before serving the request.
603    pub model: Option<String>,
604    /// System and developer instructions separated from conversation turns.
605    pub instructions: Vec<InstructionBlock>,
606    /// Ordered conversation messages.
607    pub messages: Vec<Message>,
608    /// This request-local count tracks temporary notes appended after the caller's conversation.
609    /// The HTTP client uses it to choose a cache breakpoint before the notes, subject to
610    /// provider support, caller policy, marker limits, and an eligible prefix.
611    /// Rust moves and clones retain the count.
612    /// Serialization omits the count, and deserialization resets it to zero.
613    /// Note text and preserved provider bodies remain serialized.
614    #[serde(skip)]
615    pub temporary_note_count: usize,
616    /// Whether this request performs compaction or resumes from a summary.
617    #[serde(skip_serializing_if = "std::ops::Not::not")]
618    pub is_compaction: bool,
619    /// Whether the history contains a recognized compaction marker.
620    #[serde(skip_serializing_if = "std::ops::Not::not")]
621    pub has_compaction: bool,
622    /// Tools available to the model.
623    pub tools: Vec<Tool>,
624    /// Policy controlling model tool selection.
625    pub tool_choice: Option<ToolChoice>,
626    /// Common sampling controls.
627    pub sampling: SamplingParams,
628    /// Output budget and shape controls.
629    pub output: OutputParams,
630    /// Reasoning controls.
631    pub reasoning: ReasoningParams,
632    /// Whether the caller requested a streamed response.
633    pub stream: bool,
634    /// Provider fields without first-class normalized equivalents.
635    pub extensions: ProviderExtensions,
636    /// Exact provider bodies used by codecs for lossless same-format round trips.
637    /// This is separate from a host's optional
638    /// [`Request::raw_request`](crate::Request::raw_request).
639    pub preservation: PreservationMetadata,
640}
641
642/// Normalized token usage counts.
643#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
644pub struct Usage {
645    /// The upstream provider reports this monetary charge in USD.
646    #[serde(default, skip_serializing_if = "Option::is_none")]
647    pub provider_cost: Option<f64>,
648    /// Non-cached input tokens. Provider codecs normalize aggregate OpenAI
649    /// input counts by subtracting the cache detail fields.
650    pub input_tokens: Option<u64>,
651    /// Cache-read and cache-creation token detail, when reported.
652    #[serde(flatten)]
653    pub cache: Option<Box<InputCacheUsage>>,
654    /// The output count includes reasoning tokens when the provider counts them.
655    pub output_tokens: Option<u64>,
656    /// Provider-reported or codec-computed total token count.
657    ///
658    /// Codecs normalize OpenAI aggregate input counts before computing totals, so
659    /// this may equal non-cached input plus cache detail plus output.
660    pub total_tokens: Option<u64>,
661    /// Reasoning output tokens, when reported separately.
662    pub reasoning_tokens: Option<u64>,
663}
664
665/// Optional cache-token detail kept out of the common, cache-free usage allocation.
666#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
667pub struct InputCacheUsage {
668    /// The provider reports these cache-write tokens at the one-hour rate.
669    #[serde(default, skip_serializing_if = "Option::is_none")]
670    pub cache_creation_1h_input_tokens: Option<u64>,
671    /// Input tokens read from a provider cache.
672    pub cached_input_tokens: Option<u64>,
673    /// Input tokens written into a provider cache.
674    pub cache_creation_input_tokens: Option<u64>,
675}
676
677impl Usage {
678    /// Builds cache detail when at least one count is present.
679    pub fn cache_details(
680        cached_input_tokens: Option<u64>,
681        cache_creation_input_tokens: Option<u64>,
682    ) -> Option<Box<InputCacheUsage>> {
683        if cached_input_tokens.is_none() && cache_creation_input_tokens.is_none() {
684            return None;
685        }
686        Some(Box::new(InputCacheUsage {
687            cache_creation_1h_input_tokens: None,
688            cached_input_tokens,
689            cache_creation_input_tokens,
690        }))
691    }
692
693    /// Returns the cache-read input-token count.
694    pub fn cached_input_tokens(&self) -> Option<u64> {
695        self.cache
696            .as_ref()
697            .and_then(|cache| cache.cached_input_tokens)
698    }
699
700    /// Returns the cache-creation input-token count.
701    pub fn cache_creation_input_tokens(&self) -> Option<u64> {
702        self.cache
703            .as_ref()
704            .and_then(|cache| cache.cache_creation_input_tokens)
705    }
706
707    /// Sets the cache-read count, allocating cache detail when needed.
708    pub fn set_cached_input_tokens(&mut self, value: u64) {
709        self.cache
710            .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
711            .cached_input_tokens = Some(value);
712    }
713
714    /// Sets the cache-creation count, allocating cache detail when needed.
715    pub fn set_cache_creation_input_tokens(&mut self, value: u64) {
716        self.cache
717            .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
718            .cache_creation_input_tokens = Some(value);
719    }
720}
721
722/// Normalized reason a model stopped producing output.
723#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
724#[serde(rename_all = "snake_case")]
725pub enum StopReason {
726    /// Model completed its turn normally.
727    EndTurn,
728    /// Model reached the configured output limit.
729    MaxTokens,
730    /// Model stopped to request a tool call.
731    ToolUse,
732    /// Provider safety or content filtering stopped generation.
733    ContentFilter,
734    /// Generation terminated because of an error.
735    Error,
736    /// Provider stop reason with no normalized equivalent.
737    Unknown,
738}
739
740/// A source URL cited by generated text.
741#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
742pub struct UrlCitation {
743    /// Start character offset in the concatenated text and refusal blocks.
744    pub start_index: usize,
745    /// Exclusive end character offset in the same text.
746    pub end_index: usize,
747    /// Source URL.
748    pub url: String,
749    /// Source title.
750    pub title: String,
751}
752
753/// One assistant output item in a normalized response.
754#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
755pub struct ResponseOutput {
756    /// Actor that produced the output, normally [`Role::Assistant`].
757    pub role: Role,
758    /// Ordered output content.
759    pub content: Vec<ContentBlock>,
760    /// URL citations in the concatenated text and refusal blocks.
761    #[serde(default, skip_serializing_if = "Vec::is_empty")]
762    pub url_citations: Vec<UrlCitation>,
763    /// Why this output stopped, when known.
764    pub stop_reason: Option<StopReason>,
765}
766
767/// Normalized, fully-buffered response — the aggregate of a completed generation.
768/// This is the terminal form of a streamed [`LlmResponse`](crate::LlmResponse).
769#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
770#[serde(default)]
771pub struct AggLlmResponse {
772    /// Provider response identifier.
773    pub id: Option<String>,
774    /// Model reported by the provider.
775    pub model: Option<String>,
776    /// Ordered response output items.
777    pub outputs: Vec<ResponseOutput>,
778    /// Normalized token usage.
779    pub usage: Usage,
780    /// Provider response fields without normalized equivalents.
781    pub extensions: ProviderExtensions,
782    /// Exact provider bodies retained for lossless round trips.
783    pub preservation: PreservationMetadata,
784}
785
786impl AggLlmResponse {
787    /// Returns the first output item when a response has any output.
788    pub fn first_output(&self) -> Option<&ResponseOutput> {
789        self.outputs.first()
790    }
791}
792
793#[cfg(test)]
794mod tests {
795    use serde_json::json;
796
797    use super::*;
798
799    #[test]
800    fn serde_uses_python_friendly_dictionary_shapes() -> Result<(), serde_json::Error> {
801        let request: LlmRequest = serde_json::from_value(json!({
802            "model": "auto",
803            "messages": [{
804                "role": "user",
805                "content": [{"type": "text", "text": "hello"}]
806            }]
807        }))?;
808        assert_eq!(request.messages[0], Message::text(Role::User, "hello"));
809
810        let tool_call = ContentBlock::ToolCall(ToolCall {
811            id: "call-1".to_string(),
812            name: "lookup".to_string(),
813            arguments: json!({"query": "rust"}),
814        });
815        assert_eq!(
816            serde_json::to_value(tool_call)?,
817            json!({
818                "type": "tool_call",
819                "id": "call-1",
820                "name": "lookup",
821                "arguments": {"query": "rust"}
822            })
823        );
824        Ok(())
825    }
826
827    #[test]
828    fn tool_deserialization_rejects_invalid_tagged_definitions() {
829        for definition in [
830            json!({"type": "mcp", "name": "inv", "parameters": {}}),
831            json!({"type": "provider", "provider": "x", "name": "n", "parameters": {}}),
832            json!({"type": "unknown", "name": "n", "parameters": {}}),
833            json!({"type": null, "name": "n", "parameters": {}}),
834        ] {
835            assert!(serde_json::from_value::<Tool>(definition).is_err());
836        }
837        let error = serde_json::from_value::<Tool>(json!({
838            "type": "mcp", "name": "inv", "url": "https://example.invalid/mcp",
839            "authorization_token": 5, "parameters": {}
840        }))
841        .expect_err("an invalid credential must not become a function")
842        .to_string();
843        assert!(error.contains("string"), "field error was lost: {error}");
844    }
845
846    #[test]
847    fn tool_catalog_preserves_function_json_and_backend_history() -> Result<(), serde_json::Error> {
848        let function = json!({"name": "lookup", "description": null, "parameters": {"type": "object"}, "strict": null});
849        let definitions = json!([
850            function,
851            {"type": "mcp", "name": "inventory", "url": "https://example.invalid/mcp", "require_approval": false},
852            {"type": "provider", "provider": "openai_responses", "raw": {"type": "mcp", "connector_id": "connector_example", "server_label": "example"}},
853            {
854                "type": "mcp", "name": "configured", "url": "https://example.invalid/mcp",
855                "default_config": {"enabled": false, "defer_loading": true},
856                "configs": {"lookup": {"enabled": true}, "search": {"defer_loading": false}}
857            }
858        ]);
859        let tools: Vec<Tool> = serde_json::from_value(definitions.clone())?;
860        assert!(matches!(&tools[0], Tool::Function(_)));
861        assert!(matches!(&tools[1], Tool::Mcp(_)));
862        assert!(matches!(&tools[2], Tool::Provider { .. }));
863        assert_eq!(serde_json::to_value(&tools)?, definitions);
864        let explicit_function: Tool = serde_json::from_value(json!({
865            "type": "function", "name": "lookup", "parameters": {"type": "object"}
866        }))?;
867        assert_eq!(explicit_function, tools[0]);
868
869        let history = json!([
870            {
871                "type": "mcp", "kind": "listing", "server_name": "inventory",
872                "id": "listing_1",
873                "tools": [{
874                    "name": "lookup", "description": "Look up stock",
875                    "input_schema": {"type": "object"},
876                    "annotations": {"readOnlyHint": true}
877                }]
878            },
879            {"type": "mcp", "kind": "listing", "server_name": "empty", "tools": []},
880            {
881                "type": "mcp", "kind": "listing", "server_name": "unavailable",
882                "tools": [], "error": "Server unavailable"
883            },
884            {
885                "type": "mcp", "kind": "call", "server_name": "inventory",
886                "call": {"id": "call_1", "name": "lookup", "arguments": {}}
887            },
888            {
889                "type": "mcp", "kind": "result", "tool_call_id": "call_1",
890                "content": [{"type": "text", "text": "in stock"}], "is_error": false
891            }
892        ]);
893        let content: Vec<ContentBlock> = serde_json::from_value(history.clone())?;
894        assert_eq!(serde_json::to_value(&content)?, history);
895        Ok(())
896    }
897
898    #[test]
899    fn mcp_credentials_are_omitted_from_debug() -> Result<(), serde_json::Error> {
900        let token = "synthetic-mcp-debug-secret";
901        let url_secret = "synthetic-mcp-url-secret";
902        let url = format!("https://example.invalid/mcp?api_key={url_secret}");
903        let mcp: McpToolset = serde_json::from_value(json!({
904            "name": "inventory", "url": url,
905            "authorization_token": token
906        }))?;
907        let raw = json!({"mcp_servers": [{"authorization_token": token}]});
908        let mut request = LlmRequest {
909            tools: vec![
910                Tool::Mcp(mcp.clone()),
911                Tool::Provider {
912                    provider: "openai_responses".into(),
913                    raw: json!({"type": "mcp", "authorization": token}),
914                },
915            ],
916            ..LlmRequest::default()
917        };
918        request
919            .preservation
920            .requests
921            .insert("anthropic_messages".into(), raw.clone());
922        request.extensions.fields.insert("metadata".into(), raw);
923        let serialized = serde_json::to_value(&request)?;
924        for debug in [
925            format!("{mcp:?}"),
926            format!("{mcp:#?}"),
927            format!("{:?}", request.tools),
928            format!("{:?}", request.extensions),
929            format!("{:?}", request.preservation),
930            format!("{request:?}"),
931            format!("{request:#?}"),
932        ] {
933            assert!(!debug.contains(token), "Debug exposed an MCP credential");
934            assert!(
935                !debug.contains(url_secret),
936                "Debug exposed an MCP URL secret"
937            );
938        }
939        assert_eq!(serde_json::to_value(&request)?, serialized);
940        assert_eq!(serialized["tools"][0]["url"], url);
941        assert_eq!(serialized["tools"][0]["authorization_token"], token);
942        assert_eq!(serialized["tools"][1]["raw"]["authorization"], token);
943        Ok(())
944    }
945}
switchyard_protocol/llm.rs

switchyard_protocol/
llm.rs
