switchyard_protocol/
llm.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Provider-neutral conversation types shared by routing, clients, and translation.
5
6use std::collections::BTreeMap;
7
8use serde::{Deserialize, Serialize};
9use serde_json::{Map, Value};
10
11use crate::format::FormatId;
12
13/// Actor role normalized across provider APIs.
14#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum Role {
17    /// System-level instructions.
18    System,
19    /// Developer-level instructions supported by some provider APIs.
20    Developer,
21    /// End-user input.
22    User,
23    /// Model-generated output.
24    Assistant,
25    /// Tool execution output.
26    Tool,
27}
28
29/// Instruction content separated from normal conversation messages.
30#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
31pub struct InstructionBlock {
32    /// Instruction authority, normally [`Role::System`] or [`Role::Developer`].
33    pub role: Role,
34    /// Ordered instruction content.
35    pub content: Vec<ContentBlock>,
36}
37
38/// One normalized conversation message.
39#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
40pub struct Message {
41    /// Actor that produced the message.
42    pub role: Role,
43    /// Ordered message content.
44    pub content: Vec<ContentBlock>,
45}
46
47impl Message {
48    /// Creates a text-only message for the given role.
49    pub fn text(role: Role, text: impl Into<String>) -> Self {
50        Self {
51            role,
52            content: vec![ContentBlock::Text { text: text.into() }],
53        }
54    }
55
56    /// Concatenates text-like content blocks when the message has any.
57    pub fn text_content(&self, separator: &str) -> Option<String> {
58        let parts = self
59            .content
60            .iter()
61            .filter_map(|block| match block {
62                ContentBlock::Text { text } => Some(text.as_str()),
63                ContentBlock::Refusal { text } => Some(text.as_str()),
64                _ => None,
65            })
66            .collect::<Vec<_>>();
67        if parts.is_empty() {
68            None
69        } else {
70            Some(parts.join(separator))
71        }
72    }
73}
74
75/// Normalized content block variants carried by messages and tool results.
76#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
77#[serde(tag = "type", rename_all = "snake_case")]
78pub enum ContentBlock {
79    /// Plain text content.
80    Text {
81        /// Text value.
82        text: String,
83    },
84    /// Model reasoning or thinking content.
85    Reasoning {
86        /// Reasoning text.
87        text: String,
88        /// Provider signature used to validate or continue the reasoning block.
89        signature: Option<String>,
90    },
91    /// Image content.
92    Image {
93        /// Image location or inline payload.
94        source: ImageSource,
95    },
96    /// Audio content.
97    Audio {
98        /// Audio location or inline payload.
99        source: MediaSource,
100    },
101    /// Video content.
102    Video {
103        /// Video location or inline payload.
104        source: MediaSource,
105    },
106    /// File content.
107    File {
108        /// File identifier or inline payload.
109        source: FileSource,
110    },
111    /// Tool invocation requested by the assistant.
112    ToolCall(ToolCall),
113    /// Result of an earlier tool invocation.
114    ToolResult(ToolResult),
115    /// Provider refusal content.
116    Refusal {
117        /// Human-readable refusal text.
118        text: String,
119    },
120    /// Provider block that has no normalized representation.
121    Unknown {
122        /// Wire format that supplied the block.
123        provider: FormatId,
124        /// Exact provider block.
125        raw: Value,
126    },
127}
128
129/// Image payload forms supported by the conversation model.
130#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
131#[serde(tag = "type", content = "data", rename_all = "snake_case")]
132pub enum ImageSource {
133    /// Image fetched from a URL.
134    Url {
135        /// Image URL.
136        url: String,
137        /// Optional provider-specific image detail level.
138        detail: Option<String>,
139    },
140    /// Base64-encoded image bytes.
141    Base64 {
142        /// MIME type, when supplied.
143        media_type: Option<String>,
144        /// Base64-encoded bytes.
145        data: String,
146    },
147    /// Provider image source with no normalized representation.
148    Raw(Value),
149}
150
151/// File payload forms supported by the conversation model.
152#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
153#[serde(tag = "type", content = "data", rename_all = "snake_case")]
154pub enum FileSource {
155    /// Provider-managed file identifier.
156    FileId(String),
157    /// Inline file payload.
158    FileData {
159        /// Encoded or textual file data as supplied by the provider format.
160        data: String,
161        /// Original filename, when supplied.
162        filename: Option<String>,
163    },
164    /// Provider file source with no normalized representation.
165    Raw(Value),
166}
167
168/// Audio and video payload forms supported by the conversation model.
169#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
170#[serde(tag = "type", content = "data", rename_all = "snake_case")]
171pub enum MediaSource {
172    /// Media fetched from a URL.
173    Url {
174        /// Media URL.
175        url: String,
176        /// MIME type, when supplied.
177        media_type: Option<String>,
178    },
179    /// Base64-encoded media bytes.
180    Base64 {
181        /// MIME type, when supplied.
182        media_type: Option<String>,
183        /// Base64-encoded bytes.
184        data: String,
185    },
186    /// Provider media source with no normalized representation.
187    Raw(Value),
188}
189
190/// Normalized assistant tool call.
191#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
192pub struct ToolCall {
193    /// Provider tool-call identifier used to pair the result.
194    pub id: String,
195    /// Tool name.
196    pub name: String,
197    /// Parsed tool arguments.
198    pub arguments: Value,
199}
200
201/// Normalized tool result message content.
202#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
203pub struct ToolResult {
204    /// Identifier of the [`ToolCall`] this result answers.
205    pub tool_call_id: String,
206    /// Ordered tool output content.
207    pub content: Vec<ContentBlock>,
208    /// Whether tool execution failed, when the provider reports it.
209    pub is_error: Option<bool>,
210}
211
212/// Normalized tool definition.
213#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
214pub struct ToolDefinition {
215    /// Tool name exposed to the model.
216    pub name: String,
217    /// Human-readable tool description.
218    pub description: Option<String>,
219    /// JSON Schema describing accepted arguments.
220    pub parameters: Value,
221    /// Whether the provider should enforce the schema strictly.
222    pub strict: Option<bool>,
223}
224
225/// Normalized tool choice policy.
226#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
227#[serde(tag = "type", content = "data", rename_all = "snake_case")]
228pub enum ToolChoice {
229    /// Let the model decide whether to call a tool.
230    Auto,
231    /// Require at least one tool call.
232    Required,
233    /// Disallow tool calls.
234    None,
235    /// Require a specific tool.
236    Tool {
237        /// Required tool name.
238        name: String,
239    },
240    /// Provider tool-choice value with no normalized representation.
241    Raw(Value),
242}
243
244/// Provider sampling parameters with common cross-provider names.
245#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
246pub struct SamplingParams {
247    /// Sampling temperature.
248    pub temperature: Option<f64>,
249    /// Nucleus-sampling probability mass.
250    pub top_p: Option<f64>,
251    /// Maximum number of candidate tokens considered at each step.
252    pub top_k: Option<i64>,
253}
254
255/// Output budget and structured-output options.
256#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
257pub struct OutputParams {
258    /// Maximum tokens the model may generate.
259    pub max_output_tokens: Option<u64>,
260    /// Provider-neutral or provider-specific structured-output configuration.
261    pub response_format: Option<Value>,
262}
263
264/// Provider reasoning controls preserved by translation.
265#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
266pub struct ReasoningParams {
267    /// Requested reasoning effort or level.
268    pub effort: Option<String>,
269    /// Provider reasoning controls without a normalized field.
270    pub raw: Option<Value>,
271}
272
273/// Provider-specific fields that do not have first-class conversation fields.
274#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
275#[serde(default)]
276pub struct ProviderExtensions {
277    /// Provider fields keyed by their original wire names.
278    ///
279    /// Codecs use these fields when translating to another format. Exact
280    /// same-format replay is controlled separately by [`PreservationMetadata`].
281    pub fields: Map<String, Value>,
282}
283
284/// Exact source payloads retained for lossless same-format round trips.
285///
286/// Translation's default preservation policy prefers a stored same-format body
287/// over reconstructing one from normalized fields, which is what makes a
288/// same-format hop lossless.
289///
290/// A stored body is only a faithful stand-in while the IR still matches it.
291/// [`LlmRequest::seal_preservation`] records what the IR looked like when the
292/// body was captured, and [`LlmRequest::preserved_request_is_current`] reports
293/// whether it still does. Callers that mutate the IR — adding a system prompt,
294/// appending a handoff note — need do nothing: the seal stops matching and
295/// codecs re-encode from normalized fields on their own.
296#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
297#[serde(default)]
298pub struct PreservationMetadata {
299    /// Original request bodies keyed by source format.
300    pub requests: BTreeMap<FormatId, Value>,
301    /// Original response bodies keyed by source format.
302    pub responses: BTreeMap<FormatId, Value>,
303    /// Fingerprint of the request IR at the moment the bodies above were
304    /// captured. `None` means unsealed, which reads as "not current".
305    pub request_seal: Option<u64>,
306}
307
308/// Normalized request representation shared by Switchyard components.
309#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
310#[serde(default)]
311pub struct LlmRequest {
312    /// Model requested by the inbound client.
313    pub model: Option<String>,
314    /// System and developer instructions separated from conversation turns.
315    pub instructions: Vec<InstructionBlock>,
316    /// Ordered conversation messages.
317    pub messages: Vec<Message>,
318    /// Tools available to the model.
319    pub tools: Vec<ToolDefinition>,
320    /// Policy controlling model tool selection.
321    pub tool_choice: Option<ToolChoice>,
322    /// Common sampling controls.
323    pub sampling: SamplingParams,
324    /// Output budget and shape controls.
325    pub output: OutputParams,
326    /// Reasoning controls.
327    pub reasoning: ReasoningParams,
328    /// Whether the caller requested a streamed response.
329    pub stream: bool,
330    /// Provider fields without first-class normalized equivalents.
331    pub extensions: ProviderExtensions,
332    /// Exact provider bodies used by codecs for lossless same-format round trips.
333    /// This is separate from a host's optional
334    /// [`Request::raw_request`](crate::Request::raw_request).
335    pub preservation: PreservationMetadata,
336}
337
338impl LlmRequest {
339    /// Records the current shape of this request against its preserved bodies.
340    ///
341    /// Codecs call this once decoding is complete. Until it is called the
342    /// preserved bodies are treated as stale, so an un-sealed request always
343    /// re-encodes from normalized fields.
344    pub fn seal_preservation(&mut self) {
345        self.preservation.request_seal = None;
346        self.preservation.request_seal = Some(self.shape_fingerprint());
347    }
348
349    /// Whether the preserved bodies still describe this request.
350    ///
351    /// Returns `false` once anything has changed since [`Self::seal_preservation`]
352    /// — a routing algorithm inserting a tier system prompt, appending a handoff
353    /// note, rewriting the model — which is what stops a same-format hop from
354    /// replaying a body that predates the change.
355    pub fn preserved_request_is_current(&self) -> bool {
356        self.preservation.request_seal == Some(self.shape_fingerprint())
357    }
358
359    /// Hashes everything except the seal itself, so sealing is idempotent.
360    ///
361    /// `model` is deliberately excluded. Routing rewrites it on every hop and the
362    /// client stamps the resolved name onto the encoded body afterwards, so a
363    /// replayed body is never wrong about the model — unlike a prompt or a note,
364    /// which only exist in the IR.
365    fn shape_fingerprint(&self) -> u64 {
366        use std::hash::{Hash, Hasher};
367
368        let mut unsealed = self.clone();
369        unsealed.preservation.request_seal = None;
370        unsealed.model = None;
371        let mut hasher = std::collections::hash_map::DefaultHasher::new();
372        // Serialization gives a total order over the IR without requiring `Hash`
373        // on every nested provider value; `Value` maps are ordered.
374        serde_json::to_string(&unsealed)
375            .unwrap_or_default()
376            .hash(&mut hasher);
377        hasher.finish()
378    }
379}
380
381/// Normalized token usage counts.
382#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
383pub struct Usage {
384    /// Non-cached input tokens. Provider codecs normalize aggregate OpenAI
385    /// input counts by subtracting the cache detail fields.
386    pub input_tokens: Option<u64>,
387    /// Cache-read and cache-creation token detail, when reported.
388    #[serde(flatten)]
389    pub cache: Option<Box<InputCacheUsage>>,
390    /// Generated output tokens, excluding reasoning detail when the provider reports it separately.
391    pub output_tokens: Option<u64>,
392    /// Provider-reported or codec-computed total token count.
393    ///
394    /// Codecs normalize OpenAI aggregate input counts before computing totals, so
395    /// this may equal non-cached input plus cache detail plus output.
396    pub total_tokens: Option<u64>,
397    /// Reasoning output tokens, when reported separately.
398    pub reasoning_tokens: Option<u64>,
399}
400
401/// Optional cache-token detail kept out of the common, cache-free usage allocation.
402#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
403pub struct InputCacheUsage {
404    /// Input tokens read from a provider cache.
405    pub cached_input_tokens: Option<u64>,
406    /// Input tokens written into a provider cache.
407    pub cache_creation_input_tokens: Option<u64>,
408}
409
410impl Usage {
411    /// Builds cache detail when at least one count is present.
412    pub fn cache_details(
413        cached_input_tokens: Option<u64>,
414        cache_creation_input_tokens: Option<u64>,
415    ) -> Option<Box<InputCacheUsage>> {
416        if cached_input_tokens.is_none() && cache_creation_input_tokens.is_none() {
417            return None;
418        }
419        Some(Box::new(InputCacheUsage {
420            cached_input_tokens,
421            cache_creation_input_tokens,
422        }))
423    }
424
425    /// Returns the cache-read input-token count.
426    pub fn cached_input_tokens(&self) -> Option<u64> {
427        self.cache
428            .as_ref()
429            .and_then(|cache| cache.cached_input_tokens)
430    }
431
432    /// Returns the cache-creation input-token count.
433    pub fn cache_creation_input_tokens(&self) -> Option<u64> {
434        self.cache
435            .as_ref()
436            .and_then(|cache| cache.cache_creation_input_tokens)
437    }
438
439    /// Sets the cache-read count, allocating cache detail when needed.
440    pub fn set_cached_input_tokens(&mut self, value: u64) {
441        self.cache
442            .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
443            .cached_input_tokens = Some(value);
444    }
445
446    /// Sets the cache-creation count, allocating cache detail when needed.
447    pub fn set_cache_creation_input_tokens(&mut self, value: u64) {
448        self.cache
449            .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
450            .cache_creation_input_tokens = Some(value);
451    }
452}
453
454/// Normalized reason a model stopped producing output.
455#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
456#[serde(rename_all = "snake_case")]
457pub enum StopReason {
458    /// Model completed its turn normally.
459    EndTurn,
460    /// Model reached the configured output limit.
461    MaxTokens,
462    /// Model stopped to request a tool call.
463    ToolUse,
464    /// Provider safety or content filtering stopped generation.
465    ContentFilter,
466    /// Generation terminated because of an error.
467    Error,
468    /// Provider stop reason with no normalized equivalent.
469    Unknown,
470}
471
472/// One assistant output item in a normalized response.
473#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
474pub struct ResponseOutput {
475    /// Actor that produced the output, normally [`Role::Assistant`].
476    pub role: Role,
477    /// Ordered output content.
478    pub content: Vec<ContentBlock>,
479    /// Why this output stopped, when known.
480    pub stop_reason: Option<StopReason>,
481}
482
483/// Normalized, fully-buffered response — the aggregate of a completed generation.
484/// This is the terminal form of a streamed [`LlmResponse`](crate::LlmResponse).
485#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
486#[serde(default)]
487pub struct AggLlmResponse {
488    /// Provider response identifier.
489    pub id: Option<String>,
490    /// Model reported by the provider.
491    pub model: Option<String>,
492    /// Ordered response output items.
493    pub outputs: Vec<ResponseOutput>,
494    /// Normalized token usage.
495    pub usage: Usage,
496    /// Provider response fields without normalized equivalents.
497    pub extensions: ProviderExtensions,
498    /// Exact provider bodies retained for lossless round trips.
499    pub preservation: PreservationMetadata,
500}
501
502impl AggLlmResponse {
503    /// Returns the first output item when a response has any output.
504    pub fn first_output(&self) -> Option<&ResponseOutput> {
505        self.outputs.first()
506    }
507}
508
509#[cfg(test)]
510mod tests {
511    use serde_json::json;
512
513    use super::*;
514
515    #[test]
516    fn serde_uses_python_friendly_dictionary_shapes() -> Result<(), serde_json::Error> {
517        let request: LlmRequest = serde_json::from_value(json!({
518            "model": "auto",
519            "messages": [{
520                "role": "user",
521                "content": [{"type": "text", "text": "hello"}]
522            }]
523        }))?;
524        assert_eq!(request.messages[0], Message::text(Role::User, "hello"));
525
526        let tool_call = ContentBlock::ToolCall(ToolCall {
527            id: "call-1".to_string(),
528            name: "lookup".to_string(),
529            arguments: json!({"query": "rust"}),
530        });
531        assert_eq!(
532            serde_json::to_value(tool_call)?,
533            json!({
534                "type": "tool_call",
535                "id": "call-1",
536                "name": "lookup",
537                "arguments": {"query": "rust"}
538            })
539        );
540        Ok(())
541    }
542}
switchyard_protocol/llm.rs

switchyard_protocol/
llm.rs
