Skip to main content

switchyard_llm_client/
system_one.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Buffered System One calls using the provider-neutral decision IR.
5
6use std::collections::BTreeMap;
7use std::time::Duration;
8
9use async_trait::async_trait;
10use reqwest::{Url, header::HeaderValue};
11use serde::Deserialize;
12use serde_json::{Value, json};
13use switchyard_protocol::{
14    BooleanEstimate, DecisionAnswer, DecisionKind, DecisionRequest, DecisionResponse,
15    DecisionValue, ModelId, Probability, ProviderConfidence, RoutedDecisionClient, ScoreValue,
16    Usage,
17};
18
19use crate::client::convert_reqwest_error;
20use crate::{LlmClientError, Result, metrics};
21
22/// Serves decision requests through a System One endpoint, such as TypeSafe's Jev API.
23pub struct SystemOneClient {
24    client: reqwest::Client,
25    endpoint: Url,
26    api_key: String,
27}
28
29impl SystemOneClient {
30    /// `endpoint` is the full URL, including `/v1/systemone`. Each call makes one
31    /// attempt; `timeout` covers sending the request and reading the response body.
32    pub fn new(endpoint: Url, api_key: String, timeout: Duration) -> Result<Self> {
33        if HeaderValue::try_from(format!("Bearer {api_key}")).is_err() {
34            return Err(LlmClientError::Configuration {
35                message: "System One API key cannot be encoded as an HTTP header".into(),
36            });
37        }
38        let client = reqwest::Client::builder()
39            .timeout(timeout)
40            .redirect(reqwest::redirect::Policy::none())
41            .build()
42            .map_err(convert_reqwest_error)?;
43        Ok(Self {
44            client,
45            endpoint,
46            api_key,
47        })
48    }
49}
50
51#[async_trait]
52impl RoutedDecisionClient for SystemOneClient {
53    async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {
54        let builder = self
55            .client
56            .post(self.endpoint.clone())
57            .bearer_auth(&self.api_key)
58            .json(&encode(&request)?);
59        let mut response = async {
60            let request = builder.build()?;
61            if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) {
62                crate::payload_capture::record("request", || String::from_utf8_lossy(body));
63            }
64            self.client.execute(request).await
65        }
66        .await
67        .inspect_err(|_| metrics::record_upstream_attempt(None))
68        .map_err(convert_reqwest_error)?;
69        let status = response.status();
70        let headers = std::mem::take(response.headers_mut());
71        let body = response
72            .bytes()
73            .await
74            .inspect_err(|_| metrics::record_upstream_attempt(None))
75            .map_err(convert_reqwest_error)?;
76        metrics::record_upstream_attempt(Some(status.as_u16()));
77        if !status.is_success() {
78            return Err(LlmClientError::UpstreamHttp {
79                status,
80                body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),
81                headers: Box::new(headers),
82            });
83        }
84        crate::payload_capture::record("response", || String::from_utf8_lossy(&body));
85        let response: WireResponse =
86            serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {
87                source: Box::new(source),
88            })?;
89        if !response.answers.keys().eq(request.questions.keys()) {
90            return Err(LlmClientError::ResponseTranslation(
91                "System One answer keys do not match the requested question keys".into(),
92            ));
93        }
94        let answers = response
95            .answers
96            .into_iter()
97            .map(|(id, answer)| {
98                let value = match (request.questions.get(&id).map(|q| &q.kind), answer.value) {
99                    (Some(DecisionKind::Boolean { .. }), WireValue::Noul { noul }) => {
100                        DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(noul))
101                    }
102                    (
103                        Some(DecisionKind::Choice { options }),
104                        WireValue::Choice {
105                            choice,
106                            probabilities,
107                        },
108                    ) if options.iter().any(|option| option.id == choice) => {
109                        DecisionValue::Choice {
110                            selected: choice,
111                            probabilities,
112                        }
113                    }
114                    (
115                        Some(DecisionKind::Score { levels }),
116                        WireValue::Score {
117                            score,
118                            probabilities,
119                        },
120                    ) if !levels.is_empty()
121                        && (0.0..=(levels.len() - 1) as f64).contains(&score.0) =>
122                    {
123                        let probabilities = probabilities
124                            .map(|mut probabilities| {
125                                let invalid_rubric = || {
126                                    LlmClientError::ResponseTranslation(format!(
127                                        "score probabilities for {id:?} do not match its rubric"
128                                    ))
129                                };
130                                if probabilities.len() != levels.len() {
131                                    return Err(invalid_rubric());
132                                }
133                                // JSON keys are strings; order probabilities by the request's rubric.
134                                (0..levels.len())
135                                    .map(|index| {
136                                        probabilities
137                                            .remove(&index.to_string())
138                                            .ok_or_else(invalid_rubric)
139                                    })
140                                    .collect()
141                            })
142                            .transpose()?;
143                        DecisionValue::Score {
144                            value: score,
145                            probabilities,
146                        }
147                    }
148                    _ => {
149                        return Err(LlmClientError::ResponseTranslation(format!(
150                            "answer {id:?} has an unexpected question ID, kind, or value"
151                        )));
152                    }
153                };
154                Ok((
155                    id,
156                    DecisionAnswer {
157                        value,
158                        provider_confidence: answer.confidence,
159                    },
160                ))
161            })
162            .collect::<Result<_>>()?;
163        Ok(DecisionResponse {
164            id: response.id,
165            model: response.model,
166            answers,
167            usage: response.usage,
168        })
169    }
170}
171
172fn encode(request: &DecisionRequest) -> Result<Value> {
173    let model = request
174        .model
175        .as_ref()
176        .ok_or_else(|| LlmClientError::InvalidRequest {
177            message: "System One requires a selected model".into(),
178        })?;
179    let mut questions = serde_json::Map::new();
180    for (id, question) in &request.questions {
181        let (kind, criteria) = match &question.kind {
182            DecisionKind::Boolean {
183                true_description,
184                false_description,
185            } => {
186                let mut criteria = serde_json::Map::new();
187                for (key, description) in [("true", true_description), ("false", false_description)]
188                {
189                    if let Some(description) = description {
190                        criteria.insert(key.into(), description.clone());
191                    }
192                }
193                ("noul", Value::Object(criteria))
194            }
195            DecisionKind::Choice { options } => {
196                let mut criteria = serde_json::Map::new();
197                for option in options {
198                    let description = option.description.as_ref().unwrap_or(&Value::Null);
199                    if criteria
200                        .insert(option.id.clone(), description.clone())
201                        .is_some()
202                    {
203                        return Err(LlmClientError::InvalidRequest {
204                            message: format!("duplicate option {:?} in questions.{id}", option.id),
205                        });
206                    }
207                }
208                ("choice", Value::Object(criteria))
209            }
210            DecisionKind::Score { levels } => ("score", json!(levels)),
211        };
212        questions.insert(
213            id.clone(),
214            json!({
215                "type": kind, "instructions": question.instructions, "criteria": criteria,
216            }),
217        );
218    }
219    Ok(json!({"model": model, "state": request.input, "questions": questions}))
220}
221
222#[derive(Deserialize)]
223struct WireResponse {
224    id: Option<String>,
225    model: Option<ModelId>,
226    answers: BTreeMap<String, WireAnswer>,
227    #[serde(default)]
228    usage: Usage,
229}
230
231#[derive(Deserialize)]
232struct WireAnswer {
233    #[serde(flatten)]
234    value: WireValue,
235    confidence: Option<ProviderConfidence>,
236}
237
238#[derive(Deserialize)]
239#[serde(tag = "type", rename_all = "snake_case")]
240enum WireValue {
241    Noul {
242        noul: Probability,
243    },
244    Choice {
245        choice: String,
246        probabilities: Option<BTreeMap<String, Probability>>,
247    },
248    Score {
249        score: ScoreValue,
250        probabilities: Option<BTreeMap<String, Probability>>,
251    },
252}