switchyard_libsy/algorithms/util/
prompts.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Adding text to a request on its way to the model it was routed to.
5//!
6//! Two shapes, both target-agnostic — any algorithm routing between named
7//! targets can use them, and neither writes anything back into the caller's
8//! conversation:
9//!
10//! * [`append_note`] — a one-off note in the conversation itself, for telling
11//!   the model something about *this* turn.
12//! * [`SystemPromptProcessor`] — standing instructions per target, applied on
13//!   every turn that target serves.
14//!
15//! Which text, and when, is the caller's policy; this module only knows how to
16//! place it so the provider accepts it and the prompt cache survives.
17
18use std::collections::BTreeMap;
19
20use async_trait::async_trait;
21use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role};
22
23use crate::Result;
24use crate::core::processor::{Event, Processor};
25
26/// Appends `note` to the request as conversation text.
27///
28/// It joins the trailing user message when there is one, rather than opening a
29/// turn of its own: Anthropic rejects two consecutive user messages, and a
30/// `tool_result` must stay first within its message. A text block after it
31/// satisfies both and keeps the addition a cache-safe suffix. Any other trailing
32/// role — an empty conversation, or one ending on an assistant turn — takes a
33/// fresh user message.
34pub fn append_note(request: &mut Request, note: &str) {
35    match request.llm_request.messages.last_mut() {
36        Some(last) if last.role == Role::User => last.content.push(ContentBlock::Text {
37            text: note.to_string(),
38        }),
39        _ => request
40            .llm_request
41            .messages
42            .push(Message::text(Role::User, note)),
43    }
44}
45
46/// System prompts keyed by routing target. A target left unset is routed
47/// untouched.
48#[derive(Clone, Debug, Default)]
49pub struct TargetPrompts {
50    by_target: BTreeMap<String, String>,
51}
52
53impl TargetPrompts {
54    /// Hand `target` this prompt on every turn it serves.
55    pub fn with(mut self, target: impl Into<String>, prompt: impl Into<String>) -> Self {
56        self.by_target.insert(target.into(), prompt.into());
57        self
58    }
59
60    /// The prompt configured for `target`, if any.
61    pub fn get(&self, target: &str) -> Option<&str> {
62        self.by_target.get(target).map(String::as_str)
63    }
64
65    /// Whether any target has a prompt, so a caller can skip wiring the
66    /// processor when none does.
67    pub fn is_empty(&self) -> bool {
68        self.by_target.is_empty()
69    }
70}
71
72/// Prepends the routed target's system prompt to the outbound request.
73pub struct SystemPromptProcessor {
74    prompts: TargetPrompts,
75}
76
77impl SystemPromptProcessor {
78    /// Hand each target the prompt configured for it.
79    pub fn new(prompts: TargetPrompts) -> Self {
80        Self { prompts }
81    }
82}
83
84#[async_trait]
85impl<S: Send> Processor<S> for SystemPromptProcessor {
86    async fn process(&self, _state: &mut S, event: Event<'_>) -> Result<()> {
87        // The decision event carries both the routing outcome and the outbound request,
88        // so the target is read straight off it — whichever classifier picked it, and
89        // with nothing kept between turns.
90        let Event::Decision { request, decision } = event else {
91            return Ok(());
92        };
93        let Some(prompt) = self.prompts.get(decision.selected_model()) else {
94            return Ok(());
95        };
96        // Ahead of the client's own instructions, so this framing is what the
97        // model reads first.
98        request.llm_request.instructions.insert(
99            0,
100            InstructionBlock {
101                role: Role::System,
102                content: vec![ContentBlock::Text {
103                    text: prompt.to_string(),
104                }],
105            },
106        );
107        Ok(())
108    }
109}
110
111#[cfg(test)]
112mod tests {
113    use super::*;
114    use crate::text::text_request;
115    use switchyard_protocol::{LlmRequest, ToolResult};
116
117    const NOTE: &str = "recovering from an error";
118    const STRONG_PROMPT: &str = "diagnose before you edit";
119    const WEAK_PROMPT: &str = "follow the settled plan";
120
121    fn request_with(messages: Vec<Message>) -> Request {
122        Request {
123            llm_request: LlmRequest {
124                messages,
125                ..LlmRequest::default()
126            },
127            raw_request: None,
128            metadata: None,
129        }
130    }
131
132    #[test]
133    fn a_note_joins_a_trailing_user_turn_after_its_tool_result() {
134        // The shape a coding-agent turn actually arrives in: the tool result
135        // leads the trailing user message, so the note has to follow it.
136        let tool_result = ContentBlock::ToolResult(ToolResult {
137            tool_call_id: "call_1".to_string(),
138            content: vec![ContentBlock::Text {
139                text: "exit 1".to_string(),
140            }],
141            is_error: Some(true),
142        });
143        let mut request = request_with(vec![Message {
144            role: Role::User,
145            content: vec![tool_result.clone()],
146        }]);
147
148        append_note(&mut request, NOTE);
149
150        let messages = &request.llm_request.messages;
151        assert_eq!(messages.len(), 1, "no second consecutive user turn");
152        assert_eq!(
153            messages[0].content,
154            vec![
155                tool_result,
156                ContentBlock::Text {
157                    text: NOTE.to_string()
158                }
159            ]
160        );
161    }
162
163    #[test]
164    fn a_note_opens_a_user_turn_after_an_assistant_turn() {
165        let mut request = request_with(vec![Message::text(Role::Assistant, "done")]);
166
167        append_note(&mut request, NOTE);
168
169        let messages = &request.llm_request.messages;
170        assert_eq!(messages.len(), 2);
171        assert_eq!(messages[1].role, Role::User);
172        assert_eq!(messages[1].text_content(""), Some(NOTE.to_string()));
173    }
174
175    #[test]
176    fn a_note_leaves_the_rest_of_the_conversation_untouched() {
177        let mut request = Request {
178            llm_request: text_request(Some("auto".to_string()), "fix the build"),
179            raw_request: None,
180            metadata: None,
181        };
182
183        append_note(&mut request, NOTE);
184
185        let trail: Vec<String> = request
186            .llm_request
187            .messages
188            .iter()
189            .filter_map(|message| message.text_content("|"))
190            .collect();
191        assert_eq!(trail, vec![format!("fix the build|{NOTE}")]);
192    }
193
194    /// A decision routed to `target`.
195    struct RoutedTo(&'static str);
196    impl switchyard_protocol::Decision for RoutedTo {
197        fn selected_model(&self) -> &str {
198            self.0
199        }
200        fn reasoning(&self) -> Option<&str> {
201            None
202        }
203        fn as_any(&self) -> &dyn std::any::Any {
204            self
205        }
206    }
207
208    /// The instruction text the request carries.
209    fn instructions(request: &Request) -> Vec<String> {
210        request
211            .llm_request
212            .instructions
213            .iter()
214            .filter_map(|block| {
215                block.content.iter().find_map(|content| match content {
216                    ContentBlock::Text { text } => Some(text.clone()),
217                    _ => None,
218                })
219            })
220            .collect()
221    }
222
223    /// Runs one outbound request routed to `target` through `processor`.
224    async fn run(processor: &SystemPromptProcessor, target: &'static str) -> Result<Request> {
225        let mut request = Request::default();
226        processor
227            .process(
228                &mut (),
229                Event::Decision {
230                    request: &mut request,
231                    decision: &RoutedTo(target),
232                },
233            )
234            .await?;
235        Ok(request)
236    }
237
238    fn prompts() -> TargetPrompts {
239        TargetPrompts::default()
240            .with("strong", STRONG_PROMPT)
241            .with("weak", WEAK_PROMPT)
242    }
243
244    #[tokio::test]
245    async fn each_target_gets_its_own_prompt() -> Result<()> {
246        let processor = SystemPromptProcessor::new(prompts());
247        for (target, expected) in [("strong", STRONG_PROMPT), ("weak", WEAK_PROMPT)] {
248            assert_eq!(
249                instructions(&run(&processor, target).await?),
250                vec![expected]
251            );
252        }
253        Ok(())
254    }
255
256    #[tokio::test]
257    async fn an_unconfigured_target_is_left_untouched() -> Result<()> {
258        // One target's prompt must not leak onto another, whatever ran before.
259        let processor =
260            SystemPromptProcessor::new(TargetPrompts::default().with("strong", STRONG_PROMPT));
261        assert_eq!(
262            instructions(&run(&processor, "strong").await?),
263            vec![STRONG_PROMPT]
264        );
265        assert!(instructions(&run(&processor, "weak").await?).is_empty());
266        Ok(())
267    }
268
269    #[tokio::test]
270    async fn the_prompt_leads_the_client_instructions() -> Result<()> {
271        let processor = SystemPromptProcessor::new(prompts());
272        let mut request = Request::default();
273        request.llm_request.instructions.push(InstructionBlock {
274            role: Role::System,
275            content: vec![ContentBlock::Text {
276                text: "you are a coding agent".to_string(),
277            }],
278        });
279
280        processor
281            .process(
282                &mut (),
283                Event::Decision {
284                    request: &mut request,
285                    decision: &RoutedTo("strong"),
286                },
287            )
288            .await?;
289
290        assert_eq!(
291            instructions(&request),
292            vec![STRONG_PROMPT, "you are a coding agent"]
293        );
294        Ok(())
295    }
296
297    #[tokio::test]
298    async fn the_inbound_request_is_left_alone() -> Result<()> {
299        // The inbound hook runs before the cascade has picked anything.
300        let processor = SystemPromptProcessor::new(prompts());
301        let mut request = Request::default();
302        processor
303            .process(&mut (), Event::Request(&mut request))
304            .await?;
305        assert!(instructions(&request).is_empty());
306        Ok(())
307    }
308}
switchyard_libsy/algorithms/util/prompts.rs

switchyard_libsy/algorithms/util/
prompts.rs
