switchyard_protocol/
llm.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Provider-neutral conversation types shared by routing, clients, and translation.
5
6use std::collections::BTreeMap;
7
8use serde::{Deserialize, Serialize};
9use serde_json::{Map, Value};
10
11use crate::format::FormatId;
12
13/// Actor role normalized across provider APIs.
14#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum Role {
17    /// System-level instructions.
18    System,
19    /// Developer-level instructions supported by some provider APIs.
20    Developer,
21    /// End-user input.
22    User,
23    /// Model-generated output.
24    Assistant,
25    /// Tool execution output.
26    Tool,
27}
28
29/// Instruction content separated from normal conversation messages.
30#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
31pub struct InstructionBlock {
32    /// Instruction authority, normally [`Role::System`] or [`Role::Developer`].
33    pub role: Role,
34    /// Ordered instruction content.
35    pub content: Vec<ContentBlock>,
36}
37
38/// One normalized conversation message.
39#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
40pub struct Message {
41    /// Actor that produced the message.
42    pub role: Role,
43    /// Ordered message content.
44    pub content: Vec<ContentBlock>,
45}
46
47impl Message {
48    /// Creates a text-only message for the given role.
49    pub fn text(role: Role, text: impl Into<String>) -> Self {
50        Self {
51            role,
52            content: vec![ContentBlock::Text { text: text.into() }],
53        }
54    }
55
56    /// Concatenates text-like content blocks when the message has any.
57    pub fn text_content(&self, separator: &str) -> Option<String> {
58        let parts = self
59            .content
60            .iter()
61            .filter_map(|block| match block {
62                ContentBlock::Text { text } => Some(text.as_str()),
63                ContentBlock::Refusal { text } => Some(text.as_str()),
64                _ => None,
65            })
66            .collect::<Vec<_>>();
67        if parts.is_empty() {
68            None
69        } else {
70            Some(parts.join(separator))
71        }
72    }
73}
74
75/// Normalized content block variants carried by messages and tool results.
76#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
77#[serde(tag = "type", rename_all = "snake_case")]
78pub enum ContentBlock {
79    /// Plain text content.
80    Text {
81        /// Text value.
82        text: String,
83    },
84    /// Model reasoning or thinking content.
85    Reasoning {
86        /// Reasoning text.
87        text: String,
88        /// Provider signature used to validate or continue the reasoning block.
89        signature: Option<String>,
90        /// Structured reasoning details, such as an encrypted `{ "type":
91        /// "reasoning.encrypted", "data": "..." }` object, replayed without modification.
92        #[serde(default, skip_serializing_if = "Vec::is_empty")]
93        details: Vec<Value>,
94    },
95    /// Image content.
96    Image {
97        /// Image location or inline payload.
98        source: ImageSource,
99    },
100    /// Audio content.
101    Audio {
102        /// Audio location or inline payload.
103        source: MediaSource,
104    },
105    /// Video content.
106    Video {
107        /// Video location or inline payload.
108        source: MediaSource,
109    },
110    /// File content.
111    File {
112        /// File identifier or inline payload.
113        source: FileSource,
114    },
115    /// Tool invocation requested by the assistant.
116    ToolCall(ToolCall),
117    /// Result of an earlier tool invocation.
118    ToolResult(ToolResult),
119    /// Provider refusal content.
120    Refusal {
121        /// Human-readable refusal text.
122        text: String,
123    },
124    /// Provider block that has no normalized representation.
125    Unknown {
126        /// Wire format that supplied the block.
127        provider: FormatId,
128        /// Exact provider block.
129        raw: Value,
130    },
131}
132
133/// Image payload forms supported by the conversation model.
134#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
135#[serde(tag = "type", content = "data", rename_all = "snake_case")]
136pub enum ImageSource {
137    /// Image fetched from a URL.
138    Url {
139        /// Image URL.
140        url: String,
141        /// Optional provider-specific image detail level.
142        detail: Option<String>,
143    },
144    /// Base64-encoded image bytes.
145    Base64 {
146        /// MIME type, when supplied.
147        media_type: Option<String>,
148        /// Base64-encoded bytes.
149        data: String,
150    },
151    /// Provider image source with no normalized representation.
152    Raw(Value),
153}
154
155/// File payload forms supported by the conversation model.
156#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
157#[serde(tag = "type", content = "data", rename_all = "snake_case")]
158pub enum FileSource {
159    /// Provider-managed file identifier.
160    FileId(String),
161    /// Inline file payload.
162    FileData {
163        /// Encoded or textual file data as supplied by the provider format.
164        data: String,
165        /// Original filename, when supplied.
166        filename: Option<String>,
167    },
168    /// Provider file source with no normalized representation.
169    Raw(Value),
170}
171
172/// Audio and video payload forms supported by the conversation model.
173#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
174#[serde(tag = "type", content = "data", rename_all = "snake_case")]
175pub enum MediaSource {
176    /// Media fetched from a URL.
177    Url {
178        /// Media URL.
179        url: String,
180        /// MIME type, when supplied.
181        media_type: Option<String>,
182    },
183    /// Base64-encoded media bytes.
184    Base64 {
185        /// MIME type, when supplied.
186        media_type: Option<String>,
187        /// Base64-encoded bytes.
188        data: String,
189    },
190    /// Provider media source with no normalized representation.
191    Raw(Value),
192}
193
194/// Normalized assistant tool call.
195#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
196pub struct ToolCall {
197    /// Provider tool-call identifier used to pair the result.
198    pub id: String,
199    /// Tool name.
200    pub name: String,
201    /// Parsed tool arguments.
202    pub arguments: Value,
203}
204
205/// Normalized tool result message content.
206#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
207pub struct ToolResult {
208    /// Identifier of the [`ToolCall`] this result answers.
209    pub tool_call_id: String,
210    /// Ordered tool output content.
211    pub content: Vec<ContentBlock>,
212    /// Whether tool execution failed, when the provider reports it.
213    pub is_error: Option<bool>,
214}
215
216/// Normalized tool definition.
217#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
218pub struct ToolDefinition {
219    /// Tool name exposed to the model.
220    pub name: String,
221    /// Human-readable tool description.
222    pub description: Option<String>,
223    /// JSON Schema describing accepted arguments.
224    pub parameters: Value,
225    /// Whether the provider should enforce the schema strictly.
226    pub strict: Option<bool>,
227}
228
229/// Normalized tool choice policy.
230#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
231#[serde(tag = "type", content = "data", rename_all = "snake_case")]
232pub enum ToolChoice {
233    /// Let the model decide whether to call a tool.
234    Auto,
235    /// Require at least one tool call.
236    Required,
237    /// Disallow tool calls.
238    None,
239    /// Require a specific tool.
240    Tool {
241        /// Required tool name.
242        name: String,
243    },
244    /// Provider tool-choice value with no normalized representation.
245    Raw(Value),
246}
247
248/// Provider sampling parameters with common cross-provider names.
249#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
250pub struct SamplingParams {
251    /// Sampling temperature.
252    pub temperature: Option<f64>,
253    /// Nucleus-sampling probability mass.
254    pub top_p: Option<f64>,
255    /// Maximum number of candidate tokens considered at each step.
256    pub top_k: Option<i64>,
257}
258
259/// Output budget and structured-output options.
260#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
261pub struct OutputParams {
262    /// Maximum tokens the model may generate.
263    pub max_output_tokens: Option<u64>,
264    /// Provider-neutral or provider-specific structured-output configuration.
265    pub response_format: Option<Value>,
266    /// Whether the provider must enforce the response schema, overriding native
267    /// flags when set and retaining native behavior when `None`.
268    #[serde(default, skip_serializing_if = "Option::is_none")]
269    pub is_schema_enforced: Option<bool>,
270}
271
272/// Provider reasoning controls preserved by translation.
273#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
274pub struct ReasoningParams {
275    /// Requested reasoning effort or level.
276    pub effort: Option<String>,
277    /// Provider reasoning controls without a normalized field.
278    pub raw: Option<Value>,
279}
280
281/// Provider-specific fields that do not have first-class conversation fields.
282#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
283#[serde(default)]
284pub struct ProviderExtensions {
285    /// Provider fields keyed by their original wire names.
286    ///
287    /// Codecs use these fields when translating to another format. Exact
288    /// same-format replay is controlled separately by [`PreservationMetadata`].
289    pub fields: Map<String, Value>,
290}
291
292/// Exact source payloads retained for lossless same-format round trips.
293///
294/// Translation's default preservation policy prefers a stored same-format body
295/// over reconstructing one from normalized fields. A caller that mutates the IR
296/// must clear the corresponding entry or use a policy with preservation disabled
297/// when those mutations must be encoded.
298#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
299#[serde(default)]
300pub struct PreservationMetadata {
301    /// Original request bodies keyed by source format.
302    pub requests: BTreeMap<FormatId, Value>,
303    /// Original response bodies keyed by source format.
304    pub responses: BTreeMap<FormatId, Value>,
305}
306
307/// Normalized request representation shared by Switchyard components.
308#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
309#[serde(default)]
310pub struct LlmRequest {
311    /// Model currently addressed by the request.
312    ///
313    /// This initially contains the name supplied by the inbound client. A routing host may
314    /// replace it with the selected target before serving the request.
315    pub model: Option<String>,
316    /// System and developer instructions separated from conversation turns.
317    pub instructions: Vec<InstructionBlock>,
318    /// Ordered conversation messages.
319    pub messages: Vec<Message>,
320    /// Tools available to the model.
321    pub tools: Vec<ToolDefinition>,
322    /// Policy controlling model tool selection.
323    pub tool_choice: Option<ToolChoice>,
324    /// Common sampling controls.
325    pub sampling: SamplingParams,
326    /// Output budget and shape controls.
327    pub output: OutputParams,
328    /// Reasoning controls.
329    pub reasoning: ReasoningParams,
330    /// Whether the caller requested a streamed response.
331    pub stream: bool,
332    /// Provider fields without first-class normalized equivalents.
333    pub extensions: ProviderExtensions,
334    /// Exact provider bodies used by codecs for lossless same-format round trips.
335    /// This is separate from a host's optional
336    /// [`Request::raw_request`](crate::Request::raw_request).
337    pub preservation: PreservationMetadata,
338}
339
340/// Normalized token usage counts.
341#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
342pub struct Usage {
343    /// Non-cached input tokens. Provider codecs normalize aggregate OpenAI
344    /// input counts by subtracting the cache detail fields.
345    pub input_tokens: Option<u64>,
346    /// Cache-read and cache-creation token detail, when reported.
347    #[serde(flatten)]
348    pub cache: Option<Box<InputCacheUsage>>,
349    /// Generated output tokens, excluding reasoning detail when the provider reports it separately.
350    pub output_tokens: Option<u64>,
351    /// Provider-reported or codec-computed total token count.
352    ///
353    /// Codecs normalize OpenAI aggregate input counts before computing totals, so
354    /// this may equal non-cached input plus cache detail plus output.
355    pub total_tokens: Option<u64>,
356    /// Reasoning output tokens, when reported separately.
357    pub reasoning_tokens: Option<u64>,
358}
359
360/// Optional cache-token detail kept out of the common, cache-free usage allocation.
361#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
362pub struct InputCacheUsage {
363    /// Input tokens read from a provider cache.
364    pub cached_input_tokens: Option<u64>,
365    /// Input tokens written into a provider cache.
366    pub cache_creation_input_tokens: Option<u64>,
367}
368
369impl Usage {
370    /// Builds cache detail when at least one count is present.
371    pub fn cache_details(
372        cached_input_tokens: Option<u64>,
373        cache_creation_input_tokens: Option<u64>,
374    ) -> Option<Box<InputCacheUsage>> {
375        if cached_input_tokens.is_none() && cache_creation_input_tokens.is_none() {
376            return None;
377        }
378        Some(Box::new(InputCacheUsage {
379            cached_input_tokens,
380            cache_creation_input_tokens,
381        }))
382    }
383
384    /// Returns the cache-read input-token count.
385    pub fn cached_input_tokens(&self) -> Option<u64> {
386        self.cache
387            .as_ref()
388            .and_then(|cache| cache.cached_input_tokens)
389    }
390
391    /// Returns the cache-creation input-token count.
392    pub fn cache_creation_input_tokens(&self) -> Option<u64> {
393        self.cache
394            .as_ref()
395            .and_then(|cache| cache.cache_creation_input_tokens)
396    }
397
398    /// Sets the cache-read count, allocating cache detail when needed.
399    pub fn set_cached_input_tokens(&mut self, value: u64) {
400        self.cache
401            .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
402            .cached_input_tokens = Some(value);
403    }
404
405    /// Sets the cache-creation count, allocating cache detail when needed.
406    pub fn set_cache_creation_input_tokens(&mut self, value: u64) {
407        self.cache
408            .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
409            .cache_creation_input_tokens = Some(value);
410    }
411}
412
413/// Normalized reason a model stopped producing output.
414#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
415#[serde(rename_all = "snake_case")]
416pub enum StopReason {
417    /// Model completed its turn normally.
418    EndTurn,
419    /// Model reached the configured output limit.
420    MaxTokens,
421    /// Model stopped to request a tool call.
422    ToolUse,
423    /// Provider safety or content filtering stopped generation.
424    ContentFilter,
425    /// Generation terminated because of an error.
426    Error,
427    /// Provider stop reason with no normalized equivalent.
428    Unknown,
429}
430
431/// A source URL cited by generated text.
432#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
433pub struct UrlCitation {
434    /// Start character offset in the concatenated text and refusal blocks.
435    pub start_index: usize,
436    /// Exclusive end character offset in the same text.
437    pub end_index: usize,
438    /// Source URL.
439    pub url: String,
440    /// Source title.
441    pub title: String,
442}
443
444/// One assistant output item in a normalized response.
445#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
446pub struct ResponseOutput {
447    /// Actor that produced the output, normally [`Role::Assistant`].
448    pub role: Role,
449    /// Ordered output content.
450    pub content: Vec<ContentBlock>,
451    /// URL citations in the concatenated text and refusal blocks.
452    #[serde(default, skip_serializing_if = "Vec::is_empty")]
453    pub url_citations: Vec<UrlCitation>,
454    /// Why this output stopped, when known.
455    pub stop_reason: Option<StopReason>,
456}
457
458/// Normalized, fully-buffered response — the aggregate of a completed generation.
459/// This is the terminal form of a streamed [`LlmResponse`](crate::LlmResponse).
460#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
461#[serde(default)]
462pub struct AggLlmResponse {
463    /// Provider response identifier.
464    pub id: Option<String>,
465    /// Model reported by the provider.
466    pub model: Option<String>,
467    /// Ordered response output items.
468    pub outputs: Vec<ResponseOutput>,
469    /// Normalized token usage.
470    pub usage: Usage,
471    /// Provider response fields without normalized equivalents.
472    pub extensions: ProviderExtensions,
473    /// Exact provider bodies retained for lossless round trips.
474    pub preservation: PreservationMetadata,
475}
476
477impl AggLlmResponse {
478    /// Returns the first output item when a response has any output.
479    pub fn first_output(&self) -> Option<&ResponseOutput> {
480        self.outputs.first()
481    }
482}
483
484#[cfg(test)]
485mod tests {
486    use serde_json::json;
487
488    use super::*;
489
490    #[test]
491    fn serde_uses_python_friendly_dictionary_shapes() -> Result<(), serde_json::Error> {
492        let request: LlmRequest = serde_json::from_value(json!({
493            "model": "auto",
494            "messages": [{
495                "role": "user",
496                "content": [{"type": "text", "text": "hello"}]
497            }]
498        }))?;
499        assert_eq!(request.messages[0], Message::text(Role::User, "hello"));
500
501        let tool_call = ContentBlock::ToolCall(ToolCall {
502            id: "call-1".to_string(),
503            name: "lookup".to_string(),
504            arguments: json!({"query": "rust"}),
505        });
506        assert_eq!(
507            serde_json::to_value(tool_call)?,
508            json!({
509                "type": "tool_call",
510                "id": "call-1",
511                "name": "lookup",
512                "arguments": {"query": "rust"}
513            })
514        );
515        Ok(())
516    }
517}
switchyard_protocol/llm.rs

switchyard_protocol/
llm.rs
