Skip to main content

switchyard_llm_client/
system_one.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Buffered System One calls using the provider-neutral decision IR.
5
6use std::collections::BTreeMap;
7use std::time::Duration;
8
9use async_trait::async_trait;
10use reqwest::{Url, header::HeaderValue};
11use serde::Deserialize;
12use serde_json::{Value, json};
13use switchyard_protocol::{
14    BooleanEstimate, DecisionAnswer, DecisionKind, DecisionRequest, DecisionResponse,
15    DecisionValue, ModelId, Probability, ProviderConfidence, RoutedDecisionClient, ScoreValue,
16    Usage,
17};
18
19use crate::client::convert_reqwest_error;
20use crate::{LlmClientError, Result, metrics};
21
22/// Serves decision requests through a System One endpoint, such as TypeSafe's Jev API.
23pub struct SystemOneClient {
24    client: reqwest::Client,
25    endpoint: Url,
26    api_key: String,
27}
28
29impl SystemOneClient {
30    /// `endpoint` is the full URL, including `/v1/systemone`. Each call makes one
31    /// attempt; `timeout` covers sending the request and reading the response body.
32    pub fn new(endpoint: Url, api_key: String, timeout: Duration) -> Result<Self> {
33        if HeaderValue::try_from(format!("Bearer {api_key}")).is_err() {
34            return Err(LlmClientError::Configuration {
35                message: "System One API key cannot be encoded as an HTTP header".into(),
36            });
37        }
38        let client = reqwest::Client::builder()
39            .timeout(timeout)
40            .redirect(reqwest::redirect::Policy::none())
41            .build()
42            .map_err(convert_reqwest_error)?;
43        Ok(Self {
44            client,
45            endpoint,
46            api_key,
47        })
48    }
49}
50
51#[async_trait]
52impl RoutedDecisionClient for SystemOneClient {
53    async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {
54        let builder = self
55            .client
56            .post(self.endpoint.clone())
57            .bearer_auth(&self.api_key)
58            .json(&encode(&request)?);
59        let response = async {
60            let request = builder.build()?;
61            if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) {
62                crate::payload_capture::record("request", || String::from_utf8_lossy(body));
63            }
64            self.client.execute(request).await
65        }
66        .await
67        .inspect_err(|_| metrics::record_upstream_attempt(None))
68        .map_err(convert_reqwest_error)?;
69        let status = response.status();
70        let body = response
71            .bytes()
72            .await
73            .inspect_err(|_| metrics::record_upstream_attempt(None))
74            .map_err(convert_reqwest_error)?;
75        metrics::record_upstream_attempt(Some(status.as_u16()));
76        if !status.is_success() {
77            return Err(LlmClientError::UpstreamHttp {
78                status,
79                body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),
80            });
81        }
82        crate::payload_capture::record("response", || String::from_utf8_lossy(&body));
83        let response: WireResponse =
84            serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {
85                source: Box::new(source),
86            })?;
87        if !response.answers.keys().eq(request.questions.keys()) {
88            return Err(LlmClientError::ResponseTranslation(
89                "System One answer keys do not match the requested question keys".into(),
90            ));
91        }
92        let answers = response
93            .answers
94            .into_iter()
95            .map(|(id, answer)| {
96                let value = match (request.questions.get(&id).map(|q| &q.kind), answer.value) {
97                    (Some(DecisionKind::Boolean { .. }), WireValue::Noul { noul }) => {
98                        DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(noul))
99                    }
100                    (
101                        Some(DecisionKind::Choice { options }),
102                        WireValue::Choice {
103                            choice,
104                            probabilities,
105                        },
106                    ) if options.iter().any(|option| option.id == choice) => {
107                        DecisionValue::Choice {
108                            selected: choice,
109                            probabilities,
110                        }
111                    }
112                    (
113                        Some(DecisionKind::Score { levels }),
114                        WireValue::Score {
115                            score,
116                            probabilities,
117                        },
118                    ) if !levels.is_empty()
119                        && (0.0..=(levels.len() - 1) as f64).contains(&score.0) =>
120                    {
121                        let probabilities = probabilities
122                            .map(|mut probabilities| {
123                                let invalid_rubric = || {
124                                    LlmClientError::ResponseTranslation(format!(
125                                        "score probabilities for {id:?} do not match its rubric"
126                                    ))
127                                };
128                                if probabilities.len() != levels.len() {
129                                    return Err(invalid_rubric());
130                                }
131                                // JSON keys are strings; order probabilities by the request's rubric.
132                                (0..levels.len())
133                                    .map(|index| {
134                                        probabilities
135                                            .remove(&index.to_string())
136                                            .ok_or_else(invalid_rubric)
137                                    })
138                                    .collect()
139                            })
140                            .transpose()?;
141                        DecisionValue::Score {
142                            value: score,
143                            probabilities,
144                        }
145                    }
146                    _ => {
147                        return Err(LlmClientError::ResponseTranslation(format!(
148                            "answer {id:?} has an unexpected question ID, kind, or value"
149                        )));
150                    }
151                };
152                Ok((
153                    id,
154                    DecisionAnswer {
155                        value,
156                        provider_confidence: answer.confidence,
157                    },
158                ))
159            })
160            .collect::<Result<_>>()?;
161        Ok(DecisionResponse {
162            id: response.id,
163            model: response.model,
164            answers,
165            usage: response.usage,
166        })
167    }
168}
169
170fn encode(request: &DecisionRequest) -> Result<Value> {
171    let model = request
172        .model
173        .as_ref()
174        .ok_or_else(|| LlmClientError::InvalidRequest {
175            message: "System One requires a selected model".into(),
176        })?;
177    let mut questions = serde_json::Map::new();
178    for (id, question) in &request.questions {
179        let (kind, criteria) = match &question.kind {
180            DecisionKind::Boolean {
181                true_description,
182                false_description,
183            } => {
184                let mut criteria = serde_json::Map::new();
185                for (key, description) in [("true", true_description), ("false", false_description)]
186                {
187                    if let Some(description) = description {
188                        criteria.insert(key.into(), description.clone());
189                    }
190                }
191                ("noul", Value::Object(criteria))
192            }
193            DecisionKind::Choice { options } => {
194                let mut criteria = serde_json::Map::new();
195                for option in options {
196                    let description = option.description.as_ref().unwrap_or(&Value::Null);
197                    if criteria
198                        .insert(option.id.clone(), description.clone())
199                        .is_some()
200                    {
201                        return Err(LlmClientError::InvalidRequest {
202                            message: format!("duplicate option {:?} in questions.{id}", option.id),
203                        });
204                    }
205                }
206                ("choice", Value::Object(criteria))
207            }
208            DecisionKind::Score { levels } => ("score", json!(levels)),
209        };
210        questions.insert(
211            id.clone(),
212            json!({
213                "type": kind, "instructions": question.instructions, "criteria": criteria,
214            }),
215        );
216    }
217    Ok(json!({"model": model, "state": request.input, "questions": questions}))
218}
219
220#[derive(Deserialize)]
221struct WireResponse {
222    id: Option<String>,
223    model: Option<ModelId>,
224    answers: BTreeMap<String, WireAnswer>,
225    #[serde(default)]
226    usage: Usage,
227}
228
229#[derive(Deserialize)]
230struct WireAnswer {
231    #[serde(flatten)]
232    value: WireValue,
233    confidence: Option<ProviderConfidence>,
234}
235
236#[derive(Deserialize)]
237#[serde(tag = "type", rename_all = "snake_case")]
238enum WireValue {
239    Noul {
240        noul: Probability,
241    },
242    Choice {
243        choice: String,
244        probabilities: Option<BTreeMap<String, Probability>>,
245    },
246    Score {
247        score: ScoreValue,
248        probabilities: Option<BTreeMap<String, Probability>>,
249    },
250}