switchyard_protocol/client.rs
1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Routed-call client contracts and shared error types.
5//!
6//! Hosts implement these traits to perform model calls. Keeping the contracts here
7//! lets clients depend on the protocol without pulling in libsy's orchestration.
8
9use async_trait::async_trait;
10use thiserror::Error;
11
12use crate::{DecisionRequest, DecisionResponse, ModelId, Request, Response};
13
14/// A boxed client-specific error preserved as the source of a routed call failure.
15pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
16
17/// Failures a routed LLM client can surface to its caller.
18///
19/// The variants classify failures that routing hosts commonly need to handle,
20/// while boxed sources preserve implementation-specific detail. `General` is the
21/// escape hatch for failures that do not fit a shared category.
22#[non_exhaustive]
23#[derive(Debug, Error)]
24pub enum LlmClientError {
25 /// The request cannot be served as supplied.
26 #[error("invalid request: {message}")]
27 InvalidRequest {
28 /// Human-readable request validation failure.
29 message: String,
30 },
31
32 /// Decoding the inbound request failed in the translation engine.
33 #[error("request translation failed: {0}")]
34 RequestTranslation(String),
35
36 /// Encoding the request for the upstream failed in the translation engine.
37 #[error("outbound request encoding failed: {0}")]
38 RequestEncoding(String),
39
40 /// Decoding or encoding the response failed in the translation engine.
41 #[error("response translation failed: {0}")]
42 ResponseTranslation(String),
43
44 /// The client is not configured to serve the selected target.
45 #[error("client configuration error: {message}")]
46 Configuration {
47 /// Human-readable configuration failure.
48 message: String,
49 },
50
51 /// The route cannot record another provider-owned response or conversation ID.
52 #[error(
53 "Responses state tracking reached its limit of {limit} IDs; no existing records were removed"
54 )]
55 ResponseStateLimitExceeded {
56 /// Maximum number of IDs retained by this route.
57 limit: usize,
58 },
59
60 /// Two configured models reported the same saved response or conversation ID.
61 #[error("Responses state ID is already recorded for another model; its owner was not changed")]
62 ResponseStateConflict,
63
64 /// The selected backend is temporarily unavailable. Another candidate may serve the request.
65 #[error("backend temporarily unavailable")]
66 TemporarilyUnavailable,
67
68 /// The upstream could not be reached or the request could not be sent.
69 #[error("upstream transport error: {source}")]
70 Transport {
71 /// Client-specific transport failure.
72 #[source]
73 source: BoxError,
74 },
75
76 /// The upstream request exceeded its timeout.
77 #[error("upstream request timed out: {source}")]
78 Timeout {
79 /// Client-specific timeout failure.
80 #[source]
81 source: BoxError,
82 },
83
84 /// The upstream rejected the request because it exceeds the model's context window.
85 #[error("context window exceeded for model {model}: {message}")]
86 ContextWindowExceeded {
87 /// Model whose context window was exceeded.
88 model: ModelId,
89 /// Upstream error message.
90 message: String,
91 },
92
93 /// The upstream returned a non-success HTTP response.
94 #[error("upstream returned HTTP {status}: {body}")]
95 UpstreamHttp {
96 /// Upstream HTTP status code.
97 status: http::StatusCode,
98 /// Raw upstream error body.
99 body: String,
100 },
101
102 /// The upstream returned a response the client could not decode.
103 #[error("invalid upstream response: {source}")]
104 InvalidResponse {
105 /// Client-specific decoding or validation failure.
106 #[source]
107 source: BoxError,
108 },
109
110 /// A call across a foreign-function boundary (e.g. a Python-implemented client)
111 /// failed. The boxed source is the foreign error itself.
112 #[error("foreign function interface error: {source}")]
113 Ffi {
114 /// Foreign-language failure, preserved verbatim.
115 #[source]
116 source: BoxError,
117 },
118
119 /// A string message. Useful in testing, but prefer adding variants over using this.
120 #[error("{0}")]
121 General(String),
122}
123
124/// Why routing replaced a selected target with another eligible target.
125#[derive(Clone, Copy, Debug, Eq, PartialEq)]
126pub enum RoutingFallbackReason {
127 /// The selected target rejected the request because its context window was too small.
128 ContextWindow,
129 /// The selected target was unavailable after its client retries finished.
130 Unavailable,
131}
132
133impl RoutingFallbackReason {
134 /// Stable value used when logging a routing fallback.
135 pub const fn as_str(self) -> &'static str {
136 match self {
137 Self::ContextWindow => "context_window",
138 Self::Unavailable => "unavailable",
139 }
140 }
141}
142
143/// Performs the actual model call for a target. This is the one piece of I/O the
144/// library does not own — a host implements it over its own transport (HTTP SDK,
145/// in-process model, mock). It serves a call the stream consumer chose not to
146/// override, reached as a routed request's `default_client`.
147///
148/// # Concurrency
149///
150/// A client may be shared by many targets and concurrent algorithm runs. Calls may
151/// overlap, so implementations must synchronize mutable state internally and should
152/// not serialize requests unless their transport requires it.
153#[async_trait]
154pub trait RoutedLlmClient: Send + Sync {
155 /// Make a request
156 async fn call(&self, request: Request) -> Result<Response, LlmClientError>;
157}
158
159/// Serves typed decision calls over a host-owned transport.
160/// Implementations may be shared across targets and called concurrently.
161#[async_trait]
162pub trait RoutedDecisionClient: Send + Sync {
163 /// Evaluate the request using its selected model.
164 async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse, LlmClientError>;
165}