switchyard_llm_client/
responses_reasoning.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Target-specific replay policy for OpenAI Responses reasoning items.
5
6use serde::Deserialize;
7use serde_json::{Map, Value};
8
9/// Controls which Responses reasoning items are replayed to an upstream.
10#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq)]
11#[serde(rename_all = "snake_case")]
12pub enum ResponsesReasoningPolicy {
13    /// Choose from the target on each request.
14    ///
15    /// OpenAI-hosted targets use [`Self::PreserveEncrypted`]. Every other target uses
16    /// [`Self::PreservePlaintext`].
17    #[default]
18    Auto,
19    /// Keep encrypted reasoning and stored IDs. Remove plaintext reasoning.
20    ///
21    /// OpenAI rejects plaintext reasoning `content` and unresolvable stored IDs.
22    PreserveEncrypted,
23    /// Keep plaintext reasoning. Remove opaque encrypted payloads.
24    ///
25    /// Switchyard cannot tell which provider issued an encrypted payload, so it
26    /// removes the payload and keeps any `summary` or `content` beside it.
27    PreservePlaintext,
28    /// Drop all reasoning items while preserving messages and tool-call history.
29    ///
30    /// Use this for local Responses-compatible servers that cannot consume
31    /// another provider's reasoning representation.
32    Drop,
33}
34
35impl ResponsesReasoningPolicy {
36    /// Normalizes a Responses request body sent to `model` at `base_url`.
37    pub(crate) fn normalize(self, body: &mut Value, base_url: &str, model: &str) {
38        let policy = match self {
39            Self::Auto if is_openai_target(base_url, model) => Self::PreserveEncrypted,
40            Self::Auto => Self::PreservePlaintext,
41            policy => policy,
42        };
43        let has_stored_reasoning = body.get("store").and_then(Value::as_bool) != Some(false);
44        let Some(Value::Array(input)) = body.get_mut("input") else {
45            return;
46        };
47        let mut stats = Stats::default();
48        input.retain_mut(|item| match item.as_object_mut() {
49            Some(item) if item.get("type").and_then(Value::as_str) == Some("reasoning") => {
50                stats.items += 1;
51                let keep = policy.keep(item, has_stored_reasoning, &mut stats.removed_payloads);
52                stats.removed_items += usize::from(!keep);
53                keep
54            }
55            _ => true,
56        });
57        if stats.items > 0 {
58            tracing::trace!(
59                policy = ?policy,
60                reasoning_items = stats.items,
61                removed_items = stats.removed_items,
62                removed_payloads = stats.removed_payloads,
63                "normalized Responses reasoning"
64            );
65        }
66    }
67
68    // Returns whether to keep a reasoning item, after removing parts the target cannot use.
69    // Stored reasoning can be replayed by ID without encrypted content.
70    fn keep(
71        self,
72        item: &mut Map<String, Value>,
73        has_stored_reasoning: bool,
74        removed_payloads: &mut usize,
75    ) -> bool {
76        let encrypted_content = item
77            .get("encrypted_content")
78            .and_then(Value::as_str)
79            .filter(|content| !content.is_empty());
80        let has_stored_id = has_stored_reasoning
81            && item
82                .get("id")
83                .and_then(Value::as_str)
84                .is_some_and(|id| !id.is_empty());
85        match self {
86            Self::PreserveEncrypted => {
87                if encrypted_content
88                    .is_some_and(switchyard_translation::is_anthropic_thinking_envelope)
89                    || (encrypted_content.is_none() && !has_stored_id)
90                {
91                    return false;
92                }
93                item.insert("content".to_string(), Value::Array(Vec::new()));
94                true
95            }
96            Self::PreservePlaintext => {
97                let has_plaintext = ["summary", "content"].into_iter().any(|field| {
98                    item.get(field)
99                        .and_then(Value::as_array)
100                        .is_some_and(|parts| !parts.is_empty())
101                });
102                if encrypted_content.is_none() {
103                    return has_plaintext || has_stored_id;
104                }
105                // The item ID belongs to the payload's issuer, so it cannot stand alone.
106                item.remove("encrypted_content");
107                *removed_payloads += 1;
108                has_plaintext
109            }
110            // `normalize` resolves `Auto` before calling this.
111            Self::Auto | Self::Drop => false,
112        }
113    }
114}
115
116// Counts for one request. They hold no prompt or reasoning text.
117#[derive(Default)]
118struct Stats {
119    items: usize,
120    removed_items: usize,
121    removed_payloads: usize,
122}
123
124// OpenAI and Azure OpenAI reject plaintext reasoning in Responses input. A shared
125// gateway hostname says nothing about the model behind it, so other hosts are judged
126// by the upstream model name alone.
127fn is_openai_target(base_url: &str, model: &str) -> bool {
128    let host = reqwest::Url::parse(base_url)
129        .ok()
130        .and_then(|url| url.host_str().map(str::to_ascii_lowercase));
131    if host.is_some_and(|host| host == "api.openai.com" || host.ends_with(".openai.azure.com")) {
132        return true;
133    }
134    let name = model
135        .rsplit('/')
136        .next()
137        .unwrap_or(model)
138        .to_ascii_lowercase();
139    let is_o_series = name
140        .strip_prefix('o')
141        .is_some_and(|rest| rest.starts_with(|c: char| c.is_ascii_digit()));
142    (name.starts_with("gpt-") && !name.starts_with("gpt-oss"))
143        || name.starts_with("codex-")
144        || is_o_series
145}
146
147#[cfg(test)]
148mod tests {
149    use super::is_openai_target;
150
151    #[test]
152    fn openai_targets_come_from_host_or_model_name() {
153        let hub = "https://inference-api.nvidia.com/v1";
154        let proxy = "http://inference-proxy.internal:8080/v1";
155        for (base_url, model, expected) in [
156            ("https://api.openai.com/v1", "my-alias", true),
157            (
158                "https://acme.openai.azure.com/openai/v1",
159                "deployment",
160                true,
161            ),
162            (hub, "openai/openai/gpt-5.5", true),
163            (proxy, "azure/openai/gpt-5.6-sol", true),
164            (hub, "openai/o4-mini", true),
165            (proxy, "codex-mini-latest", true),
166            (hub, "nvidia/zai-org/glm-5.3-flash", false),
167            (proxy, "nvidia/zai-org/glm-5.3-flash", false),
168            (hub, "openai/gpt-oss-120b", false),
169            (proxy, "switchyard", false),
170            ("not a url", "deepseek-v4-pro", false),
171        ] {
172            assert_eq!(
173                is_openai_target(base_url, model),
174                expected,
175                "{base_url} {model}"
176            );
177        }
178    }
179}
switchyard_llm_client/responses_reasoning.rs

switchyard_llm_client/
responses_reasoning.rs
