switchyard_libsy/algorithms/util/
prompts.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Adding text to a request on its way to the model it was routed to.
5//!
6//! Two shapes, both target-agnostic — any algorithm routing between named
7//! targets can use them, and neither writes anything back into the caller's
8//! conversation:
9//!
10//! * [`append_note`] — a one-off note in the conversation itself, for telling
11//!   the model something about *this* turn.
12//! * [`SystemPromptProcessor`] — standing instructions per target, applied on
13//!   every turn that target serves.
14//!
15//! Which text, and when, is the caller's policy; this module only knows how to
16//! place it so the provider accepts it and the prompt cache survives.
17//!
18//! **Anything added here must call [`drop_exact_replay`].** Both shapes above
19//! mutate the normalized request, and a codec asked to encode for the format the
20//! request arrived in replays the body captured at decode instead of reading that
21//! request — so an addition that leaves exact replay in place never reaches the
22//! model. This is not enforced: a future processor that mutates the request and
23//! forgets the call reintroduces SWITCH-1224, silently and without a failing
24//! test.
25
26use std::collections::BTreeMap;
27
28use async_trait::async_trait;
29use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role};
30
31use crate::Result;
32use crate::core::processor::{Event, Processor};
33
34/// Appends `note` to the request as conversation text.
35///
36/// It joins the trailing user message when there is one, rather than opening a
37/// turn of its own: Anthropic rejects two consecutive user messages, and a
38/// `tool_result` must stay first within its message. A text block after it
39/// satisfies both and keeps the addition a cache-safe suffix. Any other trailing
40/// role — an empty conversation, or one ending on an assistant turn — takes a
41/// fresh user message.
42pub fn append_note(request: &mut Request, note: &str) {
43    match request.llm_request.messages.last_mut() {
44        Some(last) if last.role == Role::User => last.content.push(ContentBlock::Text {
45            text: note.to_string(),
46        }),
47        _ => request
48            .llm_request
49            .messages
50            .push(Message::text(Role::User, note)),
51    }
52    drop_exact_replay(request);
53}
54
55/// Gives up exact same-format replay for this turn.
56///
57/// A codec replays the preserved inbound body verbatim when the target format
58/// matches the source, which is what keeps a same-format hop lossless. That body
59/// predates anything added here, so leaving it in place would encode the request
60/// as it arrived and silently drop the addition. Dropping it sends the codec down
61/// its normal path, which encodes from the request itself.
62///
63/// Every stored body goes, not just the one for the inbound format: preservation
64/// also carries bodies embedded by earlier hops, and the addition is missing from
65/// all of them equally.
66///
67/// Call this from any new code that mutates the request. Nothing checks that you
68/// have.
69fn drop_exact_replay(request: &mut Request) {
70    request.llm_request.preservation.requests.clear();
71}
72
73/// System prompts keyed by routing target. A target left unset is routed
74/// untouched.
75#[derive(Clone, Debug, Default)]
76pub struct TargetPrompts {
77    by_target: BTreeMap<String, String>,
78}
79
80impl TargetPrompts {
81    /// Hand `target` this prompt on every turn it serves.
82    pub fn with(mut self, target: impl Into<String>, prompt: impl Into<String>) -> Self {
83        self.by_target.insert(target.into(), prompt.into());
84        self
85    }
86
87    /// The prompt configured for `target`, if any.
88    pub fn get(&self, target: &str) -> Option<&str> {
89        self.by_target.get(target).map(String::as_str)
90    }
91
92    /// Whether any target has a prompt, so a caller can skip wiring the
93    /// processor when none does.
94    pub fn is_empty(&self) -> bool {
95        self.by_target.is_empty()
96    }
97}
98
99/// Prepends the routed target's system prompt to the outbound request.
100pub struct SystemPromptProcessor {
101    prompts: TargetPrompts,
102}
103
104impl SystemPromptProcessor {
105    /// Hand each target the prompt configured for it.
106    pub fn new(prompts: TargetPrompts) -> Self {
107        Self { prompts }
108    }
109}
110
111#[async_trait]
112impl<S: Send> Processor<S> for SystemPromptProcessor {
113    async fn process(&self, _state: &mut S, event: Event<'_>) -> Result<()> {
114        // The decision event carries both the routing outcome and the outbound request,
115        // so the target is read straight off it — whichever classifier picked it, and
116        // with nothing kept between turns.
117        let Event::Decision { request, decision } = event else {
118            return Ok(());
119        };
120        let Some(prompt) = self.prompts.get(decision.selected_model()) else {
121            return Ok(());
122        };
123        // Ahead of the client's own instructions, so this framing is what the
124        // model reads first.
125        request.llm_request.instructions.insert(
126            0,
127            InstructionBlock {
128                role: Role::System,
129                content: vec![ContentBlock::Text {
130                    text: prompt.to_string(),
131                }],
132            },
133        );
134        drop_exact_replay(request);
135        Ok(())
136    }
137}
138
139#[cfg(test)]
140mod tests {
141    use super::*;
142    use switchyard_protocol::{LlmRequest, ToolResult, text_request};
143
144    const NOTE: &str = "recovering from an error";
145    const STRONG_PROMPT: &str = "diagnose before you edit";
146    const WEAK_PROMPT: &str = "follow the settled plan";
147
148    /// Every test request carries the exact inbound body a codec keeps for
149    /// same-format replay, so each assertion below also says what happens to it.
150    fn request_with(messages: Vec<Message>) -> Request {
151        Request {
152            llm_request: LlmRequest {
153                messages,
154                preservation: preserved_body(),
155                ..LlmRequest::default()
156            },
157            raw_request: None,
158            metadata: None,
159        }
160    }
161
162    fn preserved_body() -> switchyard_protocol::PreservationMetadata {
163        let mut preservation = switchyard_protocol::PreservationMetadata::default();
164        preservation.requests.insert(
165            "openai_chat".into(),
166            serde_json::json!({"model": "weak", "messages": [{"role": "user", "content": "hi"}]}),
167        );
168        preservation
169    }
170
171    fn replays_exactly(request: &Request) -> bool {
172        !request.llm_request.preservation.requests.is_empty()
173    }
174
175    #[test]
176    fn a_note_joins_a_trailing_user_turn_after_its_tool_result() {
177        // The shape a coding-agent turn actually arrives in: the tool result
178        // leads the trailing user message, so the note has to follow it.
179        let tool_result = ContentBlock::ToolResult(ToolResult {
180            tool_call_id: "call_1".to_string(),
181            content: vec![ContentBlock::Text {
182                text: "exit 1".to_string(),
183            }],
184            is_error: Some(true),
185        });
186        let mut request = request_with(vec![Message {
187            role: Role::User,
188            content: vec![tool_result.clone()],
189        }]);
190
191        append_note(&mut request, NOTE);
192
193        let messages = &request.llm_request.messages;
194        assert_eq!(messages.len(), 1, "no second consecutive user turn");
195        assert_eq!(
196            messages[0].content,
197            vec![
198                tool_result,
199                ContentBlock::Text {
200                    text: NOTE.to_string()
201                }
202            ]
203        );
204    }
205
206    #[test]
207    fn a_note_opens_a_user_turn_after_an_assistant_turn() {
208        let mut request = request_with(vec![Message::text(Role::Assistant, "done")]);
209
210        append_note(&mut request, NOTE);
211
212        let messages = &request.llm_request.messages;
213        assert_eq!(messages.len(), 2);
214        assert_eq!(messages[1].role, Role::User);
215        assert_eq!(messages[1].text_content(""), Some(NOTE.to_string()));
216    }
217
218    #[test]
219    fn a_note_leaves_the_rest_of_the_conversation_untouched() {
220        let mut request = Request {
221            llm_request: LlmRequest {
222                preservation: preserved_body(),
223                ..text_request(Some("auto".to_string()), "fix the build")
224            },
225            raw_request: None,
226            metadata: None,
227        };
228
229        append_note(&mut request, NOTE);
230
231        let trail: Vec<String> = request
232            .llm_request
233            .messages
234            .iter()
235            .filter_map(|message| message.text_content("|"))
236            .collect();
237        assert_eq!(trail, vec![format!("fix the build|{NOTE}")]);
238        assert!(
239            !replays_exactly(&request),
240            "a same-format hop would replay the body captured before the note"
241        );
242    }
243
244    /// A decision routed to `target`.
245    struct RoutedTo(&'static str);
246    impl switchyard_protocol::Decision for RoutedTo {
247        fn selected_model(&self) -> &str {
248            self.0
249        }
250        fn reasoning(&self) -> Option<&str> {
251            None
252        }
253        fn as_any(&self) -> &dyn std::any::Any {
254            self
255        }
256    }
257
258    /// The instruction text the request carries.
259    fn instructions(request: &Request) -> Vec<String> {
260        request
261            .llm_request
262            .instructions
263            .iter()
264            .filter_map(|block| {
265                block.content.iter().find_map(|content| match content {
266                    ContentBlock::Text { text } => Some(text.clone()),
267                    _ => None,
268                })
269            })
270            .collect()
271    }
272
273    /// Runs one outbound request routed to `target` through `processor`.
274    async fn run(processor: &SystemPromptProcessor, target: &'static str) -> Result<Request> {
275        let mut request = Request {
276            llm_request: LlmRequest {
277                preservation: preserved_body(),
278                ..LlmRequest::default()
279            },
280            ..Request::default()
281        };
282        processor
283            .process(
284                &mut (),
285                Event::Decision {
286                    request: &mut request,
287                    decision: &RoutedTo(target),
288                },
289            )
290            .await?;
291        Ok(request)
292    }
293
294    fn prompts() -> TargetPrompts {
295        TargetPrompts::default()
296            .with("strong", STRONG_PROMPT)
297            .with("weak", WEAK_PROMPT)
298    }
299
300    #[tokio::test]
301    async fn each_target_gets_its_own_prompt() -> Result<()> {
302        let processor = SystemPromptProcessor::new(prompts());
303        for (target, expected) in [("strong", STRONG_PROMPT), ("weak", WEAK_PROMPT)] {
304            let request = run(&processor, target).await?;
305            assert_eq!(instructions(&request), vec![expected]);
306            assert!(
307                !replays_exactly(&request),
308                "{target}: a same-format hop would replay the body captured before the prompt"
309            );
310        }
311        Ok(())
312    }
313
314    #[tokio::test]
315    async fn an_unconfigured_target_is_left_untouched() -> Result<()> {
316        // One target's prompt must not leak onto another, whatever ran before.
317        let processor =
318            SystemPromptProcessor::new(TargetPrompts::default().with("strong", STRONG_PROMPT));
319        assert_eq!(
320            instructions(&run(&processor, "strong").await?),
321            vec![STRONG_PROMPT]
322        );
323        let untouched = run(&processor, "weak").await?;
324        assert!(instructions(&untouched).is_empty());
325        assert!(
326            replays_exactly(&untouched),
327            "an untouched request must keep its lossless same-format replay"
328        );
329        Ok(())
330    }
331
332    #[tokio::test]
333    async fn the_prompt_leads_the_client_instructions() -> Result<()> {
334        let processor = SystemPromptProcessor::new(prompts());
335        let mut request = Request::default();
336        request.llm_request.instructions.push(InstructionBlock {
337            role: Role::System,
338            content: vec![ContentBlock::Text {
339                text: "you are a coding agent".to_string(),
340            }],
341        });
342
343        processor
344            .process(
345                &mut (),
346                Event::Decision {
347                    request: &mut request,
348                    decision: &RoutedTo("strong"),
349                },
350            )
351            .await?;
352
353        assert_eq!(
354            instructions(&request),
355            vec![STRONG_PROMPT, "you are a coding agent"]
356        );
357        Ok(())
358    }
359
360    #[tokio::test]
361    async fn the_inbound_request_is_left_alone() -> Result<()> {
362        // The inbound hook runs before the cascade has picked anything.
363        let processor = SystemPromptProcessor::new(prompts());
364        let mut request = Request::default();
365        processor
366            .process(&mut (), Event::Request(&mut request))
367            .await?;
368        assert!(instructions(&request).is_empty());
369        Ok(())
370    }
371}
switchyard_libsy/algorithms/util/prompts.rs

switchyard_libsy/algorithms/util/
prompts.rs
