switchyard_libsy/algorithms/util/
escalation.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and
5//! the transcript condenser they read.
6//!
7//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts
8//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
9
10use serde::Deserialize;
11use serde_json::Value;
12use switchyard_protocol::{ContentBlock, Message, ModelId, Role};
13
14use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};
15use super::llm_judge::{
16    ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder,
17    StructuredJudge,
18};
19use crate::core::classifier::{Classification, Score};
20use crate::core::state::State;
21use crate::{LibsyError, Result};
22use switchyard_protocol::Request;
23
24const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md");
25const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json");
26
27/// Separator marking where [`truncate_middle`] dropped a message's interior.
28const TRIM_MARKER: &str = " ...[trimmed] ";
29
30/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`].
31const TRUNCATION_SUFFIX: &str = "...<truncated>";
32
33/// Per-message cap for system and developer anchors, which carry no trajectory signal but
34/// which coding-agent harnesses make very large.
35const SYSTEM_CHARS: usize = 1_000;
36
37/// Cap for the first user message — the task statement, so it gets the widest anchor budget.
38const FIRST_USER_CHARS: usize = 2_000;
39
40/// Backstop on the assembled transcript; the per-message caps normally bind first.
41const MAX_REQUEST_CHARS: usize = 18_000;
42
43/// The tuning surface for the trajectory judge.
44///
45/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant
46/// (the constants above).
47#[derive(Clone, Debug, Deserialize)]
48#[serde(default, deny_unknown_fields)]
49pub struct EscalationJudgeConfig {
50    /// Consecutive escalate verdicts required before a turn moves to the capable tier, which
51    /// is also the turn that latches the session. Any decline clears the streak.
52    /// `1` escalates on the first verdict; the router's main cost dial.
53    /// `2` or higher needs a session id, since the streak is retained per session.
54    pub confirmations: u32,
55    /// Trailing messages shown on top of the anchors. A loop longer than this is invisible.
56    pub recent_turn_window: usize,
57    /// Per-message cap inside the trailing window.
58    pub window_message_chars: usize,
59}
60
61impl EscalationJudgeConfig {
62    /// Rejects settings that would leave the judge with nothing useful to read.
63    fn validate(&self) -> Result<()> {
64        let reject = |message: String| Err(LibsyError::AlgorithmError { message });
65        if self.confirmations == 0 {
66            return reject("confirmations must be at least 1".to_string());
67        }
68        if self.recent_turn_window == 0 {
69            return reject("recent_turn_window must be at least 1".to_string());
70        }
71        if self.window_message_chars < 50 {
72            return reject(format!(
73                "window_message_chars must be at least 50, got {}",
74                self.window_message_chars
75            ));
76        }
77        Ok(())
78    }
79}
80
81impl Default for EscalationJudgeConfig {
82    fn default() -> Self {
83        Self {
84            confirmations: 2,
85            recent_turn_window: 28,
86            window_message_chars: 500,
87        }
88    }
89}
90
91/// The judge's verdict. The schema also requires a `reason`, which makes the judge state its
92/// case and measurably sharpens the verdict — routing reads only the boolean, so it is
93/// deserialized away rather than carried.
94#[derive(Deserialize)]
95pub(crate) struct EscalationVerdict {
96    escalate: bool,
97}
98
99/// Builds the condensed trajectory presented to the escalation judge.
100pub(crate) struct EscalationInput {
101    config: EscalationJudgeConfig,
102}
103
104impl ClassifierInput for EscalationInput {
105    fn build_messages(&self, _state: &State, request: &Request) -> Vec<Message> {
106        let messages = &request.llm_request.messages;
107        let summary = summarize_for_judge(messages, conversation_turn(request), &self.config);
108        vec![Message::text(Role::User, summary)]
109    }
110}
111
112/// Structured trajectory judge with a typed escalation verdict.
113pub(crate) type EscalationJudge = StructuredJudge<EscalationInput, SerdeDecoder<EscalationVerdict>>;
114
115/// Maps the judge's verdict to a classification. A verdict names the tier to serve — capable
116/// on escalate, efficient on decline — so the caller reads it straight off the winning score.
117/// [`Classification::Ambiguous`] carries the unavailable case, which names no tier: both a
118/// decline and an outage stay efficient, but only a decline is evidence, so only a decline
119/// clears the streak.
120pub(crate) struct EscalationPolicy {
121    capable: ModelId,
122    efficient: ModelId,
123}
124
125impl JudgePolicy for EscalationPolicy {
126    type Verdict = EscalationVerdict;
127
128    fn to_classification(&self, verdict: Option<&EscalationVerdict>) -> Classification {
129        match verdict {
130            Some(verdict) if verdict.escalate => Classification::Scores(vec![Score {
131                target: self.capable.clone(),
132                confidence: 1.0,
133            }]),
134            Some(_) => Classification::Scores(vec![Score {
135                target: self.efficient.clone(),
136                confidence: 1.0,
137            }]),
138            None => Classification::Ambiguous(Vec::new()),
139        }
140    }
141}
142
143/// Maps present verdicts to stable `escalate` or `continue` values; absent verdicts add nothing.
144fn escalation_evidence(
145    _policy: &EscalationPolicy,
146    verdict: Option<&EscalationVerdict>,
147) -> Option<Value> {
148    verdict.map(|verdict| {
149        serde_json::json!({
150            "source": "escalation",
151            "verdict": if verdict.escalate { "escalate" } else { "continue" },
152        })
153    })
154}
155
156/// Builds the trajectory judge over `judge_target`, scoring `capable` when it escalates.
157///
158/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
159/// fails here rather than on the first request.
160pub(crate) fn build_judge(
161    judge_target: ModelId,
162    capable: ModelId,
163    efficient: ModelId,
164    contract_config: &ClassifierContractConfig,
165    config: EscalationJudgeConfig,
166    max_output_tokens: u64,
167) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
168    config.validate()?;
169    let contract =
170        ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
171    Ok(JudgeClassifier::new(
172        StructuredJudge::new(
173            EscalationInput { config },
174            contract,
175            SerdeDecoder::new(),
176            JudgeRuntimeConfig::new(max_output_tokens)?,
177        ),
178        judge_target,
179        EscalationPolicy { capable, efficient },
180    )
181    .with_evidence(escalation_evidence))
182}
183
184/// The 1-indexed model invocation the transcript ends on: one per assistant reply.
185///
186/// The judge reads the turn *including* the reply it is judging, so the newest assistant
187/// message is this turn — no `+ 1`. Counting the caller's request instead would report the
188/// turn after the one under judgement.
189///
190/// Messages are already normalized by `switchyard-protocol`, so this needs no
191/// per-format branching.
192pub(crate) fn conversation_turn(request: &Request) -> usize {
193    request
194        .llm_request
195        .messages
196        .iter()
197        .filter(|message| message.role == Role::Assistant)
198        .count()
199}
200
201/// Flattens a message to plain text, tool calls and tool results included.
202///
203/// [`Message::text_content`] is deliberately not used here: it keeps only text and refusal
204/// blocks, which would erase exactly the repeated-command signal the judge's loop detection
205/// relies on.
206fn message_text(message: &Message) -> String {
207    let mut parts = Vec::new();
208    collect_text(&message.content, &mut parts);
209    parts.join(" ")
210}
211
212/// Appends the judge-relevant text of each block, descending into tool results.
213fn collect_text(content: &[ContentBlock], parts: &mut Vec<String>) {
214    for block in content {
215        match block {
216            ContentBlock::Text { text } | ContentBlock::Refusal { text } => {
217                parts.push(text.clone());
218            }
219            ContentBlock::ToolCall(call) => {
220                parts.push(format!("tool_call {}({})", call.name, call.arguments));
221            }
222            ContentBlock::ToolResult(result) => collect_text(&result.content, parts),
223            _ => {}
224        }
225    }
226}
227
228/// Keeps the head and tail of `text` within `limit` characters.
229///
230/// The head gets two thirds of the surviving budget: for a trajectory judge the command or
231/// error signature that opens a message carries more signal than its trailing output.
232fn truncate_middle(text: &str, limit: usize) -> String {
233    let chars: Vec<char> = text.chars().collect();
234    if chars.len() <= limit {
235        return text.to_string();
236    }
237    let keep = limit
238        .saturating_sub(TRIM_MARKER.chars().count())
239        .max(20)
240        .min(chars.len());
241    let head = keep * 2 / 3;
242    let tail = keep - head;
243    let mut out: String = chars[..head].iter().collect();
244    out.push_str(TRIM_MARKER);
245    out.extend(chars[chars.len() - tail..].iter());
246    out
247}
248
249/// Renders a compact role-labelled transcript for the judge.
250///
251/// The framing anchors — system/developer messages and the first user message, where agent
252/// harnesses put the task statement — are kept unconditionally and capped individually. The
253/// trailing window carries recent activity. A coverage header states how much history is not
254/// shown, so the judge can reason about pace rather than assuming it sees everything.
255///
256/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go
257/// first: for a trajectory judge the newest evidence is strictly the most valuable.
258fn summarize_for_judge(
259    messages: &[Message],
260    turn: usize,
261    config: &EscalationJudgeConfig,
262) -> String {
263    let mut anchors: Vec<String> = Vec::new();
264    let mut window: Vec<String> = Vec::new();
265    let mut first_user_seen = false;
266
267    for message in messages {
268        let text = message_text(message);
269        match message.role {
270            Role::System | Role::Developer => anchors.push(format!(
271                "[{}] {}",
272                role_label(message.role),
273                truncate_middle(&text, SYSTEM_CHARS)
274            )),
275            Role::User if !first_user_seen => {
276                first_user_seen = true;
277                anchors.push(format!(
278                    "[user (task)] {}",
279                    truncate_middle(&text, FIRST_USER_CHARS)
280                ));
281            }
282            role => window.push(format!(
283                "[{}] {}",
284                role_label(role),
285                truncate_middle(&text, config.window_message_chars)
286            )),
287        }
288    }
289
290    if window.len() > config.recent_turn_window {
291        window.drain(..window.len() - config.recent_turn_window);
292    }
293
294    let assemble = |window: &[String]| {
295        let header = format!(
296            "Conversation turn {turn}; showing the last {} of {} messages after the task framing.",
297            window.len(),
298            messages.len(),
299        );
300        std::iter::once(header)
301            .chain(anchors.iter().cloned())
302            .chain(window.iter().cloned())
303            .collect::<Vec<_>>()
304            .join("\n")
305    };
306
307    let mut text = assemble(&window);
308    while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() {
309        window.remove(0);
310        text = assemble(&window);
311    }
312    if text.chars().count() > MAX_REQUEST_CHARS {
313        let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1);
314        text = text.chars().take(keep).collect::<String>() + TRUNCATION_SUFFIX;
315    }
316    text
317}
318
319/// The transcript label for a role.
320fn role_label(role: Role) -> &'static str {
321    match role {
322        Role::System => "system",
323        Role::Developer => "developer",
324        Role::User => "user",
325        Role::Assistant => "assistant",
326        Role::Tool => "tool",
327    }
328}
329
330/// A request whose conversation sits at `turn`: `turn - 1` prior assistant replies, each
331/// answered by a further user message.
332///
333/// Shared with the assembled router's tests, which drive the same conversation shape.
334#[cfg(test)]
335pub(crate) fn request_at_turn(session_id: Option<&str>, turn: usize) -> Request {
336    use switchyard_protocol::{LlmRequest, Metadata};
337
338    let mut messages = vec![Message::text(Role::User, "What is 2+2?")];
339    for attempt in 1..turn {
340        messages.push(Message::text(Role::Assistant, format!("attempt {attempt}")));
341        messages.push(Message::text(Role::User, format!("still wrong {attempt}")));
342    }
343    Request {
344        llm_request: LlmRequest {
345            model: Some("auto".to_string()),
346            messages,
347            ..LlmRequest::default()
348        },
349        raw_request: None,
350        metadata: session_id.map(|id| Metadata {
351            session_id: Some(id.to_string()),
352            ..Metadata::default()
353        }),
354    }
355}
356
357#[cfg(test)]
358mod tests {
359    use serde_json::json;
360    use switchyard_protocol::{ContentBlock, Message, Role, ToolCall, ToolResult};
361
362    use super::*;
363    use crate::algorithms::util::llm_judge::Judge;
364
365    fn escalation_judge(max_output_tokens: u64) -> Result<EscalationJudge> {
366        Ok(StructuredJudge::new(
367            EscalationInput {
368                config: EscalationJudgeConfig::default(),
369            },
370            ClassifierContract::from_config(
371                &ClassifierContractConfig::default(),
372                PROMPT_TEMPLATE,
373                SCHEMA_TEMPLATE,
374            )?,
375            SerdeDecoder::new(),
376            JudgeRuntimeConfig::new(max_output_tokens)?,
377        ))
378    }
379
380    #[test]
381    fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> {
382        let judge = escalation_judge(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?;
383
384        // As the classifier calls it: the turn's reply is already on the transcript.
385        let mut judged = request_at_turn(None, 4);
386        judged
387            .llm_request
388            .messages
389            .push(Message::text(Role::Assistant, "this turn's reply"));
390        let built = judge.build_request(&State::default(), &judged);
391
392        // Rubric in instructions, condensed trajectory as the sole user message.
393        assert_eq!(built.llm_request.instructions.len(), 1);
394        assert_eq!(built.llm_request.instructions[0].role, Role::System);
395        assert_eq!(built.llm_request.messages.len(), 1);
396        assert_eq!(built.llm_request.messages[0].role, Role::User);
397        assert!(
398            built.llm_request.messages[0]
399                .text_content("")
400                .is_some_and(|text| text.contains("Conversation turn 4"))
401        );
402        // Bounded output, so a reasoning judge cannot run away mid-verdict.
403        assert_eq!(
404            built.llm_request.output.max_output_tokens,
405            Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
406        );
407        assert!(built.llm_request.output.response_format.is_some());
408        Ok(())
409    }
410
411    #[test]
412    fn judge_request_uses_the_configured_completion_cap() -> Result<()> {
413        let judge = escalation_judge(512)?;
414
415        let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
416
417        assert_eq!(built.llm_request.output.max_output_tokens, Some(512));
418        Ok(())
419    }
420
421    #[test]
422    fn conversation_turn_counts_assistant_replies() {
423        // The judge is handed the transcript with this turn's reply already appended, which
424        // is the shape asserted here: a request entering turn N plus its reply *is* turn N.
425        for turn in [1, 5] {
426            let mut judged = request_at_turn(None, turn);
427            judged
428                .llm_request
429                .messages
430                .push(Message::text(Role::Assistant, "this turn's reply"));
431            assert_eq!(conversation_turn(&judged), turn);
432        }
433    }
434
435    #[test]
436    fn message_text_keeps_tool_calls_and_results() {
437        let call = Message {
438            role: Role::Assistant,
439            content: vec![
440                ContentBlock::Text {
441                    text: "running it".to_string(),
442                },
443                ContentBlock::ToolCall(ToolCall {
444                    id: "call-1".to_string(),
445                    name: "bash".to_string(),
446                    arguments: json!({"cmd": "ls"}),
447                }),
448            ],
449        };
450        let text = message_text(&call);
451        assert!(text.contains("running it"), "{text}");
452        assert!(text.contains(r#"tool_call bash({"cmd":"ls"})"#), "{text}");
453
454        let result = Message {
455            role: Role::Tool,
456            content: vec![ContentBlock::ToolResult(ToolResult {
457                tool_call_id: "call-1".to_string(),
458                content: vec![ContentBlock::Text {
459                    text: "no such file".to_string(),
460                }],
461                is_error: Some(true),
462            })],
463        };
464        assert_eq!(message_text(&result), "no such file");
465    }
466
467    #[test]
468    fn truncate_middle_keeps_head_and_tail() {
469        let text = "a".repeat(40) + &"z".repeat(40);
470        let trimmed = truncate_middle(&text, 50);
471        assert!(trimmed.chars().count() <= 50, "{trimmed}");
472        assert!(trimmed.starts_with('a'));
473        assert!(trimmed.ends_with('z'));
474        assert!(trimmed.contains("[trimmed]"));
475
476        // Under the limit the text is returned untouched.
477        assert_eq!(truncate_middle("short", 50), "short");
478    }
479
480    #[test]
481    fn summary_keeps_anchors_and_the_recent_window() {
482        let mut messages = vec![
483            Message::text(Role::System, "you are a coding agent"),
484            Message::text(Role::User, "fix the failing test"),
485        ];
486        for i in 0..10 {
487            messages.push(Message::text(Role::Assistant, format!("step {i}")));
488        }
489        let config = EscalationJudgeConfig {
490            recent_turn_window: 3,
491            ..EscalationJudgeConfig::default()
492        };
493
494        let summary = summarize_for_judge(&messages, 11, &config);
495
496        assert!(
497            summary.contains("[system] you are a coding agent"),
498            "{summary}"
499        );
500        assert!(
501            summary.contains("[user (task)] fix the failing test"),
502            "{summary}"
503        );
504        assert!(summary.contains("Conversation turn 11; showing the last 3 of 12 messages"));
505        // Only the newest window entries survive.
506        assert!(summary.contains("step 9"), "{summary}");
507        assert!(summary.contains("step 7"), "{summary}");
508        assert!(!summary.contains("step 6"), "{summary}");
509    }
510
511    #[test]
512    fn summary_drops_oldest_window_lines_under_the_char_cap() {
513        // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps
514        // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually
515        // wide per-message cap. That is the point — it only fires on pathological input.
516        let mut messages = vec![
517            Message::text(Role::System, "framing"),
518            Message::text(Role::User, "task"),
519        ];
520        for i in 0..20 {
521            messages.push(Message::text(
522                Role::Assistant,
523                format!("{i} {}", "x".repeat(2_000)),
524            ));
525        }
526        let config = EscalationJudgeConfig {
527            window_message_chars: 2_000,
528            ..EscalationJudgeConfig::default()
529        };
530
531        let summary = summarize_for_judge(&messages, 21, &config);
532
533        assert!(
534            summary.chars().count() <= MAX_REQUEST_CHARS,
535            "{}",
536            summary.chars().count()
537        );
538        // Anchors are never dropped, and the newest activity outlives the oldest.
539        assert!(summary.contains("[system] framing"), "{summary}");
540        assert!(summary.contains("[user (task)] task"), "{summary}");
541        assert!(summary.contains("19 xxx"), "{summary}");
542        assert!(!summary.contains("0 xxx"), "{summary}");
543    }
544}
switchyard_libsy/algorithms/util/escalation.rs

switchyard_libsy/algorithms/util/
escalation.rs
