switchyard_protocol/client.rs
1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! The routed-call server trait and its shared error types.
5//!
6//! [`RoutedLlmClient`] is the one piece of I/O the protocol does not own: a host
7//! implements it to actually perform a model call. It lives here — rather than in
8//! libsy's orchestration crate — so a client crate that depends only on the protocol
9//! can serve routed calls without pulling in the orchestrator.
10
11use async_trait::async_trait;
12use thiserror::Error;
13
14use crate::{ModelId, Request, Response};
15
16/// A boxed client-specific error preserved as the source of a routed call failure.
17pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
18
19/// Failures a routed LLM client can surface to its caller.
20///
21/// The variants classify failures that routing hosts commonly need to handle,
22/// while boxed sources preserve implementation-specific detail. `General` is the
23/// escape hatch for failures that do not fit a shared category.
24#[non_exhaustive]
25#[derive(Debug, Error)]
26pub enum LlmClientError {
27 /// The request cannot be served as supplied.
28 #[error("invalid request: {message}")]
29 InvalidRequest {
30 /// Human-readable request validation failure.
31 message: String,
32 },
33
34 /// Decoding the inbound request failed in the translation engine.
35 #[error("request translation failed: {0}")]
36 RequestTranslation(String),
37
38 /// Encoding the request for the upstream failed in the translation engine.
39 #[error("outbound request encoding failed: {0}")]
40 RequestEncoding(String),
41
42 /// Decoding or encoding the response failed in the translation engine.
43 #[error("response translation failed: {0}")]
44 ResponseTranslation(String),
45
46 /// The client is not configured to serve the selected target.
47 #[error("client configuration error: {message}")]
48 Configuration {
49 /// Human-readable configuration failure.
50 message: String,
51 },
52
53 /// The route cannot record another provider-owned response or conversation ID.
54 #[error(
55 "Responses state tracking reached its limit of {limit} IDs; no existing records were removed"
56 )]
57 ResponseStateLimitExceeded {
58 /// Maximum number of IDs retained by this route.
59 limit: usize,
60 },
61
62 /// Two configured models reported the same saved response or conversation ID.
63 #[error("Responses state ID is already recorded for another model; its owner was not changed")]
64 ResponseStateConflict,
65
66 /// The selected backend is temporarily unavailable. Another candidate may serve the request.
67 #[error("backend temporarily unavailable")]
68 TemporarilyUnavailable,
69
70 /// The upstream could not be reached or the request could not be sent.
71 #[error("upstream transport error: {source}")]
72 Transport {
73 /// Client-specific transport failure.
74 #[source]
75 source: BoxError,
76 },
77
78 /// The upstream request exceeded its timeout.
79 #[error("upstream request timed out: {source}")]
80 Timeout {
81 /// Client-specific timeout failure.
82 #[source]
83 source: BoxError,
84 },
85
86 /// The upstream rejected the request because it exceeds the model's context window.
87 #[error("context window exceeded for model {model}: {message}")]
88 ContextWindowExceeded {
89 /// Model whose context window was exceeded.
90 model: ModelId,
91 /// Upstream error message.
92 message: String,
93 },
94
95 /// The upstream returned a non-success HTTP response.
96 #[error("upstream returned HTTP {status}: {body}")]
97 UpstreamHttp {
98 /// Upstream HTTP status code.
99 status: http::StatusCode,
100 /// Raw upstream error body.
101 body: String,
102 },
103
104 /// The upstream returned a response the client could not decode.
105 #[error("invalid upstream response: {source}")]
106 InvalidResponse {
107 /// Client-specific decoding or validation failure.
108 #[source]
109 source: BoxError,
110 },
111
112 /// A call across a foreign-function boundary (e.g. a Python-implemented client)
113 /// failed. The boxed source is the foreign error itself.
114 #[error("foreign function interface error: {source}")]
115 Ffi {
116 /// Foreign-language failure, preserved verbatim.
117 #[source]
118 source: BoxError,
119 },
120
121 /// A string message. Useful in testing, but prefer adding variants over using this.
122 #[error("{0}")]
123 General(String),
124}
125
126/// Why routing replaced a selected target with another eligible target.
127#[derive(Clone, Copy, Debug, Eq, PartialEq)]
128pub enum RoutingFallbackReason {
129 /// The selected target rejected the request because its context window was too small.
130 ContextWindow,
131 /// The selected target was unavailable after its client retries finished.
132 Unavailable,
133}
134
135impl RoutingFallbackReason {
136 /// Stable value used when logging a routing fallback.
137 pub const fn as_str(self) -> &'static str {
138 match self {
139 Self::ContextWindow => "context_window",
140 Self::Unavailable => "unavailable",
141 }
142 }
143}
144
145/// Performs the actual model call for a target. This is the one piece of I/O the
146/// library does not own — a host implements it over its own transport (HTTP SDK,
147/// in-process model, mock). It serves a call the stream consumer chose not to
148/// override, reached as a routed request's `default_client`.
149///
150/// # Concurrency
151///
152/// A client may be shared by many targets and concurrent algorithm runs. Calls may
153/// overlap, so implementations must synchronize mutable state internally and should
154/// not serialize requests unless their transport requires it.
155#[async_trait]
156pub trait RoutedLlmClient: Send + Sync {
157 /// Make a request
158 async fn call(&self, request: Request) -> Result<Response, LlmClientError>;
159}