switchyard_protocol/
client.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! The routed-call server trait and its shared error types.
5//!
6//! [`RoutedLlmClient`] is the one piece of I/O the protocol does not own: a host
7//! implements it to actually perform a model call. It lives here — rather than in
8//! libsy's orchestration crate — so a client crate that depends only on the protocol
9//! can serve routed calls without pulling in the orchestrator.
10
11use async_trait::async_trait;
12use thiserror::Error;
13
14use crate::{ModelId, Request, Response};
15
16/// A boxed client-specific error preserved as the source of a routed call failure.
17pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
18
19/// Failures a routed LLM client can surface to its caller.
20///
21/// The variants classify failures that routing hosts commonly need to handle,
22/// while boxed sources preserve implementation-specific detail. `General` is the
23/// escape hatch for failures that do not fit a shared category.
24#[non_exhaustive]
25#[derive(Debug, Error)]
26pub enum LlmClientError {
27    /// The request cannot be served as supplied.
28    #[error("invalid request: {message}")]
29    InvalidRequest {
30        /// Human-readable request validation failure.
31        message: String,
32    },
33
34    /// Decoding the inbound request failed in the translation engine.
35    #[error("request translation failed: {0}")]
36    RequestTranslation(String),
37
38    /// Encoding the request for the upstream failed in the translation engine.
39    #[error("outbound request encoding failed: {0}")]
40    RequestEncoding(String),
41
42    /// Decoding or encoding the response failed in the translation engine.
43    #[error("response translation failed: {0}")]
44    ResponseTranslation(String),
45
46    /// The client is not configured to serve the selected target.
47    #[error("client configuration error: {message}")]
48    Configuration {
49        /// Human-readable configuration failure.
50        message: String,
51    },
52
53    /// The route cannot record another provider-owned response or conversation ID.
54    #[error(
55        "Responses state tracking reached its limit of {limit} IDs; no existing records were removed"
56    )]
57    ResponseStateLimitExceeded {
58        /// Maximum number of IDs retained by this route.
59        limit: usize,
60    },
61
62    /// Two configured models reported the same saved response or conversation ID.
63    #[error("Responses state ID is already recorded for another model; its owner was not changed")]
64    ResponseStateConflict,
65
66    /// The upstream could not be reached or the request could not be sent.
67    #[error("upstream transport error: {source}")]
68    Transport {
69        /// Client-specific transport failure.
70        #[source]
71        source: BoxError,
72    },
73
74    /// The upstream request exceeded its timeout.
75    #[error("upstream request timed out: {source}")]
76    Timeout {
77        /// Client-specific timeout failure.
78        #[source]
79        source: BoxError,
80    },
81
82    /// The upstream rejected the request because it exceeds the model's context window.
83    #[error("context window exceeded for model {model}: {message}")]
84    ContextWindowExceeded {
85        /// Model whose context window was exceeded.
86        model: ModelId,
87        /// Upstream error message.
88        message: String,
89    },
90
91    /// The upstream returned a non-success HTTP response.
92    #[error("upstream returned HTTP {status}: {body}")]
93    UpstreamHttp {
94        /// Upstream HTTP status code.
95        status: http::StatusCode,
96        /// Raw upstream error body.
97        body: String,
98    },
99
100    /// The upstream returned a response the client could not decode.
101    #[error("invalid upstream response: {source}")]
102    InvalidResponse {
103        /// Client-specific decoding or validation failure.
104        #[source]
105        source: BoxError,
106    },
107
108    /// A call across a foreign-function boundary (e.g. a Python-implemented client)
109    /// failed. The boxed source is the foreign error itself.
110    #[error("foreign function interface error: {source}")]
111    Ffi {
112        /// Foreign-language failure, preserved verbatim.
113        #[source]
114        source: BoxError,
115    },
116
117    /// A string message. Useful in testing, but prefer adding variants over using this.
118    #[error("{0}")]
119    General(String),
120}
121
122/// Why routing replaced a selected target with another eligible target.
123#[derive(Clone, Copy, Debug, Eq, PartialEq)]
124pub enum RoutingFallbackReason {
125    /// The selected target rejected the request because its context window was too small.
126    ContextWindow,
127    /// The selected target was unavailable after its client retries finished.
128    Unavailable,
129}
130
131impl RoutingFallbackReason {
132    /// Stable value used when logging a routing fallback.
133    pub const fn as_str(self) -> &'static str {
134        match self {
135            Self::ContextWindow => "context_window",
136            Self::Unavailable => "unavailable",
137        }
138    }
139}
140
141/// Performs the actual model call for a target. This is the one piece of I/O the
142/// library does not own — a host implements it over its own transport (HTTP SDK,
143/// in-process model, mock). It serves a call the stream consumer chose not to
144/// override, reached as a routed request's `default_client`.
145///
146/// # Concurrency
147///
148/// A client may be shared by many targets and concurrent algorithm runs. Calls may
149/// overlap, so implementations must synchronize mutable state internally and should
150/// not serialize requests unless their transport requires it.
151#[async_trait]
152pub trait RoutedLlmClient: Send + Sync {
153    /// Make a request
154    async fn call(&self, request: Request) -> Result<Response, LlmClientError>;
155}
switchyard_protocol/client.rs

switchyard_protocol/
client.rs
