switchyard_llm_client/
system_one.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Buffered System One calls using the provider-neutral decision IR.
5
6use std::collections::BTreeMap;
7use std::time::Duration;
8
9use async_trait::async_trait;
10use reqwest::{Url, header::HeaderValue};
11use serde::Deserialize;
12use serde_json::{Value, json};
13use switchyard_protocol::{
14    BooleanEstimate, DecisionAnswer, DecisionKind, DecisionRequest, DecisionResponse,
15    DecisionValue, ModelId, Probability, ProviderConfidence, RoutedDecisionClient, ScoreValue,
16    Usage,
17};
18
19use crate::client::convert_reqwest_error;
20use crate::{LlmClientError, Result, metrics};
21
22/// Serves decision requests through a System One endpoint, such as TypeSafe's Jev API.
23pub struct SystemOneClient {
24    client: reqwest::Client,
25    endpoint: Url,
26    api_key: String,
27}
28
29impl SystemOneClient {
30    /// `endpoint` is the full URL, including `/v1/systemone`. Each call makes one
31    /// attempt; `timeout` covers sending the request and reading the response body.
32    pub fn new(endpoint: Url, api_key: String, timeout: Duration) -> Result<Self> {
33        if HeaderValue::try_from(format!("Bearer {api_key}")).is_err() {
34            return Err(LlmClientError::Configuration {
35                message: "System One API key cannot be encoded as an HTTP header".into(),
36            });
37        }
38        let client = reqwest::Client::builder()
39            .timeout(timeout)
40            .redirect(reqwest::redirect::Policy::none())
41            .build()
42            .map_err(convert_reqwest_error)?;
43        Ok(Self {
44            client,
45            endpoint,
46            api_key,
47        })
48    }
49}
50
51#[async_trait]
52impl RoutedDecisionClient for SystemOneClient {
53    async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {
54        let response = self
55            .client
56            .post(self.endpoint.clone())
57            .bearer_auth(&self.api_key)
58            .json(&encode(&request)?)
59            .send()
60            .await
61            .inspect_err(|_| metrics::record_upstream_attempt(None))
62            .map_err(convert_reqwest_error)?;
63        let status = response.status();
64        let body = response
65            .bytes()
66            .await
67            .inspect_err(|_| metrics::record_upstream_attempt(None))
68            .map_err(convert_reqwest_error)?;
69        metrics::record_upstream_attempt(Some(status.as_u16()));
70        if !status.is_success() {
71            return Err(LlmClientError::UpstreamHttp {
72                status,
73                body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),
74            });
75        }
76        let response: WireResponse =
77            serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {
78                source: Box::new(source),
79            })?;
80        if !response.answers.keys().eq(request.questions.keys()) {
81            return Err(LlmClientError::ResponseTranslation(
82                "System One answer keys do not match the requested question keys".into(),
83            ));
84        }
85        let answers = response
86            .answers
87            .into_iter()
88            .map(|(id, answer)| {
89                let value = match (request.questions.get(&id).map(|q| &q.kind), answer.value) {
90                    (Some(DecisionKind::Boolean { .. }), WireValue::Noul { noul }) => {
91                        DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(noul))
92                    }
93                    (
94                        Some(DecisionKind::Choice { options }),
95                        WireValue::Choice {
96                            choice,
97                            probabilities,
98                        },
99                    ) if options.iter().any(|option| option.id == choice) => {
100                        DecisionValue::Choice {
101                            selected: choice,
102                            probabilities,
103                        }
104                    }
105                    (
106                        Some(DecisionKind::Score { levels }),
107                        WireValue::Score {
108                            score,
109                            probabilities,
110                        },
111                    ) if !levels.is_empty()
112                        && (0.0..=(levels.len() - 1) as f64).contains(&score.0) =>
113                    {
114                        let probabilities = probabilities
115                            .map(|mut probabilities| {
116                                let invalid_rubric = || {
117                                    LlmClientError::ResponseTranslation(format!(
118                                        "score probabilities for {id:?} do not match its rubric"
119                                    ))
120                                };
121                                if probabilities.len() != levels.len() {
122                                    return Err(invalid_rubric());
123                                }
124                                // JSON keys are strings; order probabilities by the request's rubric.
125                                (0..levels.len())
126                                    .map(|index| {
127                                        probabilities
128                                            .remove(&index.to_string())
129                                            .ok_or_else(invalid_rubric)
130                                    })
131                                    .collect()
132                            })
133                            .transpose()?;
134                        DecisionValue::Score {
135                            value: score,
136                            probabilities,
137                        }
138                    }
139                    _ => {
140                        return Err(LlmClientError::ResponseTranslation(format!(
141                            "answer {id:?} has an unexpected question ID, kind, or value"
142                        )));
143                    }
144                };
145                Ok((
146                    id,
147                    DecisionAnswer {
148                        value,
149                        provider_confidence: answer.confidence,
150                    },
151                ))
152            })
153            .collect::<Result<_>>()?;
154        Ok(DecisionResponse {
155            id: response.id,
156            model: response.model,
157            answers,
158            usage: response.usage,
159        })
160    }
161}
162
163fn encode(request: &DecisionRequest) -> Result<Value> {
164    let model = request
165        .model
166        .as_ref()
167        .ok_or_else(|| LlmClientError::InvalidRequest {
168            message: "System One requires a selected model".into(),
169        })?;
170    let mut questions = serde_json::Map::new();
171    for (id, question) in &request.questions {
172        let (kind, criteria) = match &question.kind {
173            DecisionKind::Boolean {
174                true_description,
175                false_description,
176            } => {
177                let mut criteria = serde_json::Map::new();
178                for (key, description) in [("true", true_description), ("false", false_description)]
179                {
180                    if let Some(description) = description {
181                        criteria.insert(key.into(), description.clone());
182                    }
183                }
184                ("noul", Value::Object(criteria))
185            }
186            DecisionKind::Choice { options } => {
187                let mut criteria = serde_json::Map::new();
188                for option in options {
189                    let description = option.description.as_ref().unwrap_or(&Value::Null);
190                    if criteria
191                        .insert(option.id.clone(), description.clone())
192                        .is_some()
193                    {
194                        return Err(LlmClientError::InvalidRequest {
195                            message: format!("duplicate option {:?} in questions.{id}", option.id),
196                        });
197                    }
198                }
199                ("choice", Value::Object(criteria))
200            }
201            DecisionKind::Score { levels } => ("score", json!(levels)),
202        };
203        questions.insert(
204            id.clone(),
205            json!({
206                "type": kind, "instructions": question.instructions, "criteria": criteria,
207            }),
208        );
209    }
210    Ok(json!({"model": model, "state": request.context, "questions": questions}))
211}
212
213#[derive(Deserialize)]
214struct WireResponse {
215    id: Option<String>,
216    model: Option<ModelId>,
217    answers: BTreeMap<String, WireAnswer>,
218    #[serde(default)]
219    usage: Usage,
220}
221
222#[derive(Deserialize)]
223struct WireAnswer {
224    #[serde(flatten)]
225    value: WireValue,
226    confidence: Option<ProviderConfidence>,
227}
228
229#[derive(Deserialize)]
230#[serde(tag = "type", rename_all = "snake_case")]
231enum WireValue {
232    Noul {
233        noul: Probability,
234    },
235    Choice {
236        choice: String,
237        probabilities: Option<BTreeMap<String, Probability>>,
238    },
239    Score {
240        score: ScoreValue,
241        probabilities: Option<BTreeMap<String, Probability>>,
242    },
243}
switchyard_llm_client/system_one.rs

switchyard_llm_client/
system_one.rs
