switchyard_libsy/algorithms/util/
classifier_contract.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Prompt and structured-output contracts shared by LLM classifiers.
5
6use jsonschema::Validator;
7use serde::Deserialize;
8use serde_json::{Value, json};
9
10use crate::{LibsyError, Result};
11
12/// Provider-side structured-output mode used by a classifier judge.
13#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)]
14#[serde(rename_all = "snake_case")]
15pub enum ClassifierResponseFormat {
16    /// Send the verdict schema through the provider's strict JSON Schema wrapper.
17    #[default]
18    JsonSchema,
19    /// Request a JSON object and enforce the verdict schema locally.
20    JsonObject,
21}
22
23/// User-configurable parts of a classifier's prompt and verdict contract.
24///
25/// Fields are private so new contract settings can be added without breaking Rust struct literals.
26#[derive(Clone, Debug, Default, Deserialize)]
27pub struct ClassifierContractConfig {
28    #[serde(default)]
29    prompt: Option<String>,
30    #[serde(default)]
31    response_format_type: ClassifierResponseFormat,
32}
33
34impl ClassifierContractConfig {
35    /// Overrides the packaged classifier prompt.
36    pub fn with_prompt(mut self, prompt: impl Into<String>) -> Self {
37        self.prompt = Some(prompt.into());
38        self
39    }
40
41    /// Returns the configured prompt override.
42    pub fn prompt(&self) -> Option<&str> {
43        self.prompt.as_deref()
44    }
45
46    /// Selects the provider-side structured-output mode.
47    pub fn with_response_format_type(
48        mut self,
49        response_format_type: ClassifierResponseFormat,
50    ) -> Self {
51        self.response_format_type = response_format_type;
52        self
53    }
54
55    /// Returns the configured provider-side structured-output mode.
56    pub fn response_format_type(&self) -> ClassifierResponseFormat {
57        self.response_format_type
58    }
59}
60
61/// Rendered prompt and response format for one classifier.
62#[derive(Debug)]
63pub(crate) struct ClassifierContract {
64    system_prompt: String,
65    response_format: Value,
66    validator: Option<Validator>,
67}
68
69impl ClassifierContract {
70    /// Builds a contract from user settings and packaged defaults.
71    ///
72    /// The packaged response format must contain `json_schema.schema`. JSON Schema mode retains
73    /// that wrapper for the model request; JSON Object mode moves the schema into the prompt and
74    /// compiles it for local validation.
75    pub(crate) fn from_config(
76        config: &ClassifierContractConfig,
77        default_prompt: &str,
78        response_format_json: &str,
79    ) -> Result<Self> {
80        let prompt_template = config.prompt().unwrap_or(default_prompt);
81        let response_format: Value =
82            serde_json::from_str(response_format_json).map_err(|error| {
83                LibsyError::AlgorithmError {
84                    message: format!("response schema is invalid: {error}"),
85                }
86            })?;
87        let schema = response_format
88            .pointer("/json_schema/schema")
89            .ok_or_else(|| LibsyError::AlgorithmError {
90                message: "response schema has no json_schema.schema".to_string(),
91            })?;
92        match config.response_format_type() {
93            ClassifierResponseFormat::JsonSchema => {
94                Self::from_response_format(prompt_template, response_format, None)
95            }
96            ClassifierResponseFormat::JsonObject => {
97                validate_prompt(prompt_template)?;
98                let validator = compile_schema(schema)?;
99                let rendered_schema = serde_json::to_string_pretty(schema).map_err(|error| {
100                    algorithm_error(format!("response schema could not be rendered: {error}"))
101                })?;
102                let system_prompt = format!(
103                    "{prompt_template}\n\nReturn exactly one JSON object matching this JSON Schema:\n{rendered_schema}"
104                );
105                Self::from_response_format(
106                    &system_prompt,
107                    json!({"type": "json_object"}),
108                    Some(validator),
109                )
110            }
111        }
112    }
113
114    /// Builds a provider response format around a user-supplied inner JSON Schema.
115    pub(crate) fn from_inner_schema(prompt_template: &str, schema: Value) -> Result<Self> {
116        if schema.get("json_schema").is_some() {
117            return Err(LibsyError::AlgorithmError {
118                message:
119                    "response_schema must be the inner JSON Schema, not a response_format wrapper"
120                        .to_string(),
121            });
122        }
123        let validator = compile_schema(&schema)?;
124        Self::from_response_format(
125            prompt_template,
126            json!({
127                "type": "json_schema",
128                "json_schema": {
129                    "name": "switchyard_classifier_response",
130                    "strict": true,
131                    "schema": schema,
132                }
133            }),
134            Some(validator),
135        )
136    }
137
138    fn from_response_format(
139        prompt_template: &str,
140        response_format: Value,
141        validator: Option<Validator>,
142    ) -> Result<Self> {
143        validate_prompt(prompt_template)?;
144
145        Ok(Self {
146            system_prompt: prompt_template.to_string(),
147            response_format,
148            validator,
149        })
150    }
151
152    pub(crate) fn system_prompt(&self) -> &str {
153        &self.system_prompt
154    }
155
156    pub(crate) fn response_format(&self) -> &Value {
157        &self.response_format
158    }
159
160    /// Whether the provider response must be checked against the compiled schema locally.
161    pub(crate) fn validates_locally(&self) -> bool {
162        self.validator.is_some()
163    }
164
165    /// Validates a dynamic verdict when this contract carries a runtime schema validator.
166    pub(crate) fn validate_verdict(&self, verdict: &Value) -> Result<()> {
167        let Some(validator) = &self.validator else {
168            return Ok(());
169        };
170        validator
171            .validate(verdict)
172            .map_err(|error| LibsyError::AlgorithmError {
173                message: format!("classifier verdict did not match response_schema: {error}"),
174            })
175    }
176}
177
178fn validate_prompt(prompt_template: &str) -> Result<()> {
179    if prompt_template.trim().is_empty() {
180        return Err(algorithm_error("classifier prompt must not be empty"));
181    }
182    if prompt_template.contains("{{RESPONSE_SCHEMA}}") {
183        return Err(algorithm_error(
184            "classifier prompt must not include {{RESPONSE_SCHEMA}}; remove the placeholder because Switchyard supplies the schema automatically",
185        ));
186    }
187    Ok(())
188}
189
190fn compile_schema(schema: &Value) -> Result<Validator> {
191    if !schema.is_object() {
192        return Err(algorithm_error("response_schema must be a JSON object"));
193    }
194    jsonschema::meta::validate(schema).map_err(|error| {
195        algorithm_error(format!(
196            "response_schema is not a valid JSON Schema: {error}"
197        ))
198    })?;
199    jsonschema::validator_for(schema)
200        .map_err(|error| algorithm_error(format!("response_schema could not be compiled: {error}")))
201}
202
203fn algorithm_error(message: impl Into<String>) -> LibsyError {
204    LibsyError::AlgorithmError {
205        message: message.into(),
206    }
207}
208
209#[cfg(test)]
210mod tests {
211    use super::*;
212
213    #[test]
214    fn a_runtime_contract_keeps_its_schema_out_of_the_prompt() -> Result<()> {
215        let schema = r#"{
216            "type": "json_schema",
217            "json_schema": {
218                "name": "RiskDecision",
219                "schema": {
220                    "type": "object",
221                    "properties": {"risk": {"type": "number"}}
222                }
223            }
224        }"#;
225        let config = ClassifierContractConfig::default().with_prompt(
226            "Return a risk verdict matching the response schema supplied with the request.",
227        );
228        let contract = ClassifierContract::from_config(&config, "packaged prompt", schema)?;
229
230        assert_eq!(
231            contract.system_prompt(),
232            "Return a risk verdict matching the response schema supplied with the request."
233        );
234        assert!(!contract.system_prompt().contains("\"risk\""));
235        assert_eq!(
236            contract
237                .response_format()
238                .pointer("/json_schema/name")
239                .and_then(Value::as_str),
240            Some("RiskDecision")
241        );
242        Ok(())
243    }
244
245    #[test]
246    fn a_contract_requires_an_inner_json_schema() {
247        let error = ClassifierContract::from_config(
248            &ClassifierContractConfig::default(),
249            "classify",
250            r#"{"type":"json"}"#,
251        )
252        .expect_err("missing inner schema should be rejected");
253
254        assert!(error.to_string().contains("json_schema.schema"));
255    }
256
257    #[test]
258    fn a_contract_rejects_an_empty_prompt() {
259        let config = ClassifierContractConfig::default().with_prompt("  \n");
260        let error = ClassifierContract::from_config(
261            &config,
262            "packaged prompt",
263            r#"{"json_schema":{"schema":{"type":"object"}}}"#,
264        )
265        .expect_err("empty prompt should be rejected");
266
267        assert!(error.to_string().contains("prompt must not be empty"));
268    }
269
270    #[test]
271    fn a_contract_rejects_a_response_schema_placeholder() {
272        let config = ClassifierContractConfig::default()
273            .with_prompt("Return JSON matching {{RESPONSE_SCHEMA}}");
274        let error = ClassifierContract::from_config(
275            &config,
276            "packaged prompt",
277            r#"{"json_schema":{"schema":{"type":"object"}}}"#,
278        )
279        .expect_err("schema placeholders should be rejected");
280
281        assert!(
282            error
283                .to_string()
284                .contains("Switchyard supplies the schema automatically")
285        );
286    }
287
288    #[test]
289    fn a_custom_contract_wraps_and_validates_its_inner_schema() -> Result<()> {
290        let contract = ClassifierContract::from_inner_schema(
291            "Choose a target matching the response schema supplied with the request.",
292            json!({
293                "type": "object",
294                "$defs": {
295                    "decision": {
296                        "type": "object",
297                        "properties": {
298                            "target": {"type": "string", "enum": ["sonnet", "opus"]}
299                        },
300                        "required": ["target"],
301                        "additionalProperties": false
302                    }
303                },
304                "properties": {
305                    "decision": {"$ref": "#/$defs/decision"}
306                },
307                "required": ["decision"],
308                "additionalProperties": false
309            }),
310        )?;
311
312        assert_eq!(
313            contract
314                .response_format()
315                .pointer("/json_schema/name")
316                .and_then(Value::as_str),
317            Some("switchyard_classifier_response")
318        );
319        assert_eq!(
320            contract
321                .response_format()
322                .pointer("/json_schema/strict")
323                .and_then(Value::as_bool),
324            Some(true)
325        );
326        assert!(!contract.system_prompt().contains("\"target\""));
327        contract.validate_verdict(&json!({"decision": {"target": "sonnet"}}))?;
328        assert!(
329            contract
330                .validate_verdict(&json!({"decision": {"target": "unknown"}}))
331                .is_err()
332        );
333        Ok(())
334    }
335
336    #[test]
337    fn a_provider_wrapper_is_rejected_as_an_inner_schema() {
338        let error = ClassifierContract::from_inner_schema(
339            "classify",
340            json!({"json_schema": {"schema": {"type": "object"}}}),
341        )
342        .expect_err("provider wrapper should be rejected");
343
344        assert!(error.to_string().contains("inner JSON Schema"));
345    }
346}
switchyard_libsy/algorithms/util/classifier_contract.rs

switchyard_libsy/algorithms/util/
classifier_contract.rs
