Skip to main content

switchyard_protocol/
client.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Routed-call client contracts and shared error types.
5//!
6//! Hosts implement these traits to perform model calls. Keeping the contracts here
7//! lets clients depend on the protocol without pulling in libsy's orchestration.
8
9use async_trait::async_trait;
10use thiserror::Error;
11
12use crate::{DecisionRequest, DecisionResponse, ModelId, Request, Response};
13
14/// A boxed client-specific error preserved as the source of a routed call failure.
15pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
16
17/// Failures a routed LLM client can surface to its caller.
18///
19/// Boxed sources preserve client-specific errors; `General` keeps only a message.
20#[non_exhaustive]
21#[derive(Debug, Error)]
22pub enum LlmClientError {
23    /// The request cannot be served as supplied.
24    #[error("invalid request: {message}")]
25    InvalidRequest {
26        /// Human-readable request validation failure.
27        message: String,
28    },
29
30    /// Decoding the inbound request failed in the translation engine.
31    #[error("request translation failed: {0}")]
32    RequestTranslation(String),
33
34    /// Encoding the request for the upstream failed in the translation engine.
35    #[error("outbound request encoding failed: {0}")]
36    RequestEncoding(String),
37
38    /// Decoding or encoding the response failed in the translation engine.
39    #[error("response translation failed: {0}")]
40    ResponseTranslation(String),
41
42    /// The client is not configured to serve the selected target.
43    #[error("client configuration error: {message}")]
44    Configuration {
45        /// Human-readable configuration failure.
46        message: String,
47    },
48
49    /// The route cannot record another provider-owned response or conversation ID.
50    #[error(
51        "Responses state tracking reached its limit of {limit} IDs; no existing records were removed"
52    )]
53    ResponseStateLimitExceeded {
54        /// Maximum number of IDs retained by this route.
55        limit: usize,
56    },
57
58    /// Two configured models reported the same saved response or conversation ID.
59    #[error("Responses state ID is already recorded for another model; its owner was not changed")]
60    ResponseStateConflict,
61
62    /// The selected backend is temporarily unavailable. Another candidate may serve the request.
63    #[error("backend temporarily unavailable")]
64    TemporarilyUnavailable,
65
66    /// The upstream could not be reached or the request could not be sent.
67    #[error("upstream transport error: {source}")]
68    Transport {
69        /// Client-specific transport failure.
70        #[source]
71        source: BoxError,
72    },
73
74    /// The upstream request exceeded its timeout.
75    #[error("upstream request timed out: {source}")]
76    Timeout {
77        /// Client-specific timeout failure.
78        #[source]
79        source: BoxError,
80    },
81
82    /// The upstream rejected the request because it exceeds the model's context window.
83    #[error("context window exceeded for model {model}: {message}")]
84    ContextWindowExceeded {
85        /// Model whose context window was exceeded.
86        model: ModelId,
87        /// Upstream error message.
88        message: String,
89    },
90
91    /// The upstream returned a non-success HTTP response.
92    #[error("upstream returned HTTP {status}: {body}")]
93    UpstreamHttp {
94        /// Upstream HTTP status code.
95        status: http::StatusCode,
96        /// Raw upstream error body.
97        body: String,
98        /// Upstream HTTP response headers from the last attempt. Consumers such as
99        /// switchyard-server may forward an allowed subset to the downstream client.
100        /// Boxed to keep the error type small.
101        headers: Box<http::HeaderMap>,
102    },
103
104    /// The upstream returned a response the client could not decode.
105    #[error("invalid upstream response: {source}")]
106    InvalidResponse {
107        /// Client-specific decoding or validation failure.
108        #[source]
109        source: BoxError,
110    },
111
112    /// A native host failure that does not fit another category.
113    /// The default completion runner treats it as terminal.
114    #[error("host client error: {source}")]
115    Host {
116        /// Original error, retained for the host.
117        #[source]
118        source: BoxError,
119    },
120
121    /// A call across a foreign-function boundary (e.g. a Python-implemented client)
122    /// failed. The boxed source is the foreign error itself.
123    #[error("foreign function interface error: {source}")]
124    Ffi {
125        /// Foreign-language failure, preserved verbatim.
126        #[source]
127        source: BoxError,
128    },
129
130    /// A string-only failure. Useful in testing; prefer a typed variant when possible.
131    #[error("{0}")]
132    General(String),
133}
134
135/// Why routing replaced a selected target with another eligible target.
136#[derive(Clone, Copy, Debug, Eq, PartialEq)]
137pub enum RoutingFallbackReason {
138    /// The selected target rejected the request because its context window was too small.
139    ContextWindow,
140    /// The selected target was unavailable after its client retries finished.
141    Unavailable,
142}
143
144impl RoutingFallbackReason {
145    /// Stable value used when logging a routing fallback.
146    pub const fn as_str(self) -> &'static str {
147        match self {
148            Self::ContextWindow => "context_window",
149            Self::Unavailable => "unavailable",
150        }
151    }
152}
153
154/// Performs the actual model call for a target. This is the one piece of I/O the
155/// library does not own — a host implements it over its own transport (HTTP SDK,
156/// in-process model, mock). It serves a call the stream consumer chose not to
157/// override, reached as a routed request's `default_client`.
158///
159/// # Concurrency
160///
161/// A client may be shared by many targets and concurrent algorithm runs. Calls may
162/// overlap, so implementations must synchronize mutable state internally and should
163/// not serialize requests unless their transport requires it.
164#[async_trait]
165pub trait RoutedLlmClient: Send + Sync {
166    /// Resolves a routing key for telemetry. Direct model clients keep the key unchanged.
167    fn provider_model<'a>(&'a self, target: &'a ModelId) -> &'a ModelId {
168        target
169    }
170
171    /// Make a request
172    async fn call(&self, request: Request) -> Result<Response, LlmClientError>;
173}
174
175/// Serves typed decision calls over a host-owned transport.
176/// Implementations may be shared across targets and called concurrently.
177#[async_trait]
178pub trait RoutedDecisionClient: Send + Sync {
179    /// Evaluate the request using its selected model.
180    async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse, LlmClientError>;
181}