switchyard_libsy/algorithms/util/
escalation.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and
5//! the transcript condenser they read.
6//!
7//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts
8//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
9
10use serde::Deserialize;
11use serde_json::Value;
12use switchyard_protocol::{ContentBlock, Message, ModelId, Role};
13
14use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};
15use super::llm_judge::{
16    ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder,
17    StructuredJudge,
18};
19use crate::core::classifier::{Classification, Score};
20use crate::core::state::State;
21use crate::{LibsyError, Result};
22use switchyard_protocol::Request;
23
24const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md");
25const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json");
26
27/// Separator marking where [`truncate_middle`] dropped a message's interior.
28const TRIM_MARKER: &str = " ...[trimmed] ";
29
30/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`].
31const TRUNCATION_SUFFIX: &str = "...<truncated>";
32
33/// Per-message cap for system and developer anchors, which carry no trajectory signal but
34/// which coding-agent harnesses make very large.
35const SYSTEM_CHARS: usize = 1_000;
36
37/// Per-message cap for task-framing user messages — every user message that precedes the first
38/// assistant reply. Coding-agent harnesses often send environment boilerplate as the first user
39/// message and the task itself as the second, so anchoring only the first would pin the
40/// boilerplate and let the task scroll out of the window. Feature specifications run to several
41/// thousand characters, so this gets the widest anchor budget.
42const TASK_CHARS: usize = 4_000;
43
44/// Backstop on the assembled transcript; the per-message caps normally bind first.
45const MAX_REQUEST_CHARS: usize = 18_000;
46
47/// The tuning surface for the trajectory judge.
48///
49/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant
50/// (the constants above).
51#[derive(Clone, Debug, Deserialize)]
52#[serde(default, deny_unknown_fields)]
53pub struct EscalationJudgeConfig {
54    /// Consecutive escalate verdicts required before a turn moves to the capable tier, which
55    /// is also the turn that latches the session. Any decline clears the streak.
56    /// `1` escalates on the first verdict; the router's main cost dial.
57    /// `2` or higher needs a session id, since the streak is retained per session.
58    pub confirmations: u32,
59    /// Trailing messages shown on top of the anchors. A loop longer than this is invisible.
60    pub recent_turn_window: usize,
61    /// Per-message cap inside the trailing window.
62    pub window_message_chars: usize,
63    /// Note appended to the request on every turn the judge has sent to the capable tier: the
64    /// latching turn and each confirmed turn after it. Turns that reach the capable tier by
65    /// fallback (context overflow, transport failure) carry no note, since the judge did not
66    /// speak. The note rides in the forwarded request only, never in the caller's conversation,
67    /// so it cannot accumulate across turns. `None` sends nothing.
68    pub handoff_note: Option<String>,
69}
70
71impl EscalationJudgeConfig {
72    /// Rejects settings that would leave the judge with nothing useful to read.
73    fn validate(&self) -> Result<()> {
74        let reject = |message: String| Err(LibsyError::AlgorithmError { message });
75        if self.confirmations == 0 {
76            return reject("confirmations must be at least 1".to_string());
77        }
78        if self.recent_turn_window == 0 {
79            return reject("recent_turn_window must be at least 1".to_string());
80        }
81        if self.window_message_chars < 50 {
82            return reject(format!(
83                "window_message_chars must be at least 50, got {}",
84                self.window_message_chars
85            ));
86        }
87        if self
88            .handoff_note
89            .as_deref()
90            .is_some_and(|note| note.trim().is_empty())
91        {
92            return reject("handoff_note must not be blank".to_string());
93        }
94        Ok(())
95    }
96}
97
98impl Default for EscalationJudgeConfig {
99    fn default() -> Self {
100        Self {
101            confirmations: 2,
102            recent_turn_window: 28,
103            window_message_chars: 500,
104            handoff_note: None,
105        }
106    }
107}
108
109/// The judge's verdict. The schema also requires a `reason`, which makes the judge state its
110/// case and measurably sharpens the verdict. Routing reads only the boolean; the reason is
111/// kept solely so an operator can see why the judge held or escalated when the
112/// `switchyard_libsy::algorithms::util::escalation` target is enabled at `debug`.
113#[derive(Deserialize)]
114pub(crate) struct EscalationVerdict {
115    escalate: bool,
116    #[serde(default)]
117    reason: String,
118}
119
120/// Builds the condensed trajectory presented to the escalation judge.
121pub(crate) struct EscalationInput {
122    config: EscalationJudgeConfig,
123}
124
125impl ClassifierInput for EscalationInput {
126    fn build_messages(&self, _state: &State, request: &Request) -> Vec<Message> {
127        let messages = &request.llm_request.messages;
128        let summary = summarize_for_judge(messages, conversation_turn(request), &self.config);
129        vec![Message::text(Role::User, summary)]
130    }
131}
132
133/// Structured trajectory judge with a typed escalation verdict.
134pub(crate) type EscalationJudge = StructuredJudge<EscalationInput, SerdeDecoder<EscalationVerdict>>;
135
136/// Maps the judge's verdict to a classification. A verdict names the tier to serve — capable
137/// on escalate, efficient on decline — so the caller reads it straight off the winning score.
138/// [`Classification::Ambiguous`] carries the unavailable case, which names no tier: both a
139/// decline and an outage stay efficient, but only a decline is evidence, so only a decline
140/// clears the streak.
141pub(crate) struct EscalationPolicy {
142    capable: ModelId,
143    efficient: ModelId,
144}
145
146impl JudgePolicy for EscalationPolicy {
147    type Verdict = EscalationVerdict;
148
149    fn to_classification(&self, verdict: Option<&EscalationVerdict>) -> Classification {
150        if let Some(verdict) = verdict {
151            tracing::debug!(
152                escalate = verdict.escalate,
153                reason = %verdict.reason,
154                "escalation judge verdict"
155            );
156        }
157        match verdict {
158            Some(verdict) if verdict.escalate => Classification::Scores(vec![Score {
159                target: self.capable.clone(),
160                confidence: 1.0,
161            }]),
162            Some(_) => Classification::Scores(vec![Score {
163                target: self.efficient.clone(),
164                confidence: 1.0,
165            }]),
166            None => Classification::Ambiguous(Vec::new()),
167        }
168    }
169}
170
171/// Maps present verdicts to stable `escalate` or `continue` values; absent verdicts add nothing.
172fn escalation_evidence(
173    _policy: &EscalationPolicy,
174    verdict: Option<&EscalationVerdict>,
175) -> Option<Value> {
176    verdict.map(|verdict| {
177        serde_json::json!({
178            "source": "escalation",
179            "verdict": if verdict.escalate { "escalate" } else { "continue" },
180        })
181    })
182}
183
184/// Builds the trajectory judge over `judge_target`, scoring `capable` when it escalates.
185///
186/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
187/// fails here rather than on the first request.
188pub(crate) fn build_judge(
189    judge_target: ModelId,
190    capable: ModelId,
191    efficient: ModelId,
192    contract_config: &ClassifierContractConfig,
193    config: EscalationJudgeConfig,
194    max_output_tokens: u64,
195) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
196    config.validate()?;
197    let contract =
198        ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
199    Ok(JudgeClassifier::new(
200        StructuredJudge::new(
201            EscalationInput { config },
202            contract,
203            SerdeDecoder::new(),
204            JudgeRuntimeConfig::new(max_output_tokens)?,
205        ),
206        judge_target,
207        EscalationPolicy { capable, efficient },
208    )
209    .with_evidence(escalation_evidence))
210}
211
212/// The 1-indexed model invocation the transcript ends on: one per assistant reply.
213///
214/// The judge reads the turn *including* the reply it is judging, so the newest assistant
215/// message is this turn — no `+ 1`. Counting the caller's request instead would report the
216/// turn after the one under judgement.
217///
218/// Messages are already normalized by `switchyard-protocol`, so this needs no
219/// per-format branching.
220pub(crate) fn conversation_turn(request: &Request) -> usize {
221    request
222        .llm_request
223        .messages
224        .iter()
225        .filter(|message| message.role == Role::Assistant)
226        .count()
227}
228
229/// Flattens a message to plain text, tool calls and tool results included.
230///
231/// [`Message::text_content`] is deliberately not used here: it keeps only text and refusal
232/// blocks, which would erase exactly the repeated-command signal the judge's loop detection
233/// relies on.
234fn message_text(message: &Message) -> String {
235    let mut parts = Vec::new();
236    collect_text(&message.content, &mut parts);
237    parts.join(" ")
238}
239
240/// Appends the judge-relevant text of each block, descending into tool results.
241fn collect_text(content: &[ContentBlock], parts: &mut Vec<String>) {
242    for block in content {
243        match block {
244            ContentBlock::Text { text } | ContentBlock::Refusal { text } => {
245                parts.push(text.clone());
246            }
247            ContentBlock::ToolCall(call) => {
248                parts.push(format!("tool_call {}({})", call.name, call.arguments));
249            }
250            ContentBlock::ToolResult(result) => collect_text(&result.content, parts),
251            _ => {}
252        }
253    }
254}
255
256/// Keeps the head and tail of `text` within `limit` characters.
257///
258/// The head gets two thirds of the surviving budget: for a trajectory judge the command or
259/// error signature that opens a message carries more signal than its trailing output.
260fn truncate_middle(text: &str, limit: usize) -> String {
261    let chars: Vec<char> = text.chars().collect();
262    if chars.len() <= limit {
263        return text.to_string();
264    }
265    let keep = limit
266        .saturating_sub(TRIM_MARKER.chars().count())
267        .max(20)
268        .min(chars.len());
269    let head = keep * 2 / 3;
270    let tail = keep - head;
271    let mut out: String = chars[..head].iter().collect();
272    out.push_str(TRIM_MARKER);
273    out.extend(chars[chars.len() - tail..].iter());
274    out
275}
276
277/// Renders a compact role-labelled transcript for the judge.
278///
279/// The framing anchors — system/developer messages and the first user message, where agent
280/// harnesses put the task statement — are kept unconditionally and capped individually. The
281/// trailing window carries recent activity. A coverage header states how much history is not
282/// shown, so the judge can reason about pace rather than assuming it sees everything.
283///
284/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go
285/// first: for a trajectory judge the newest evidence is strictly the most valuable.
286fn summarize_for_judge(
287    messages: &[Message],
288    turn: usize,
289    config: &EscalationJudgeConfig,
290) -> String {
291    let mut anchors: Vec<String> = Vec::new();
292    let mut window: Vec<String> = Vec::new();
293    let mut assistant_seen = false;
294
295    for message in messages {
296        let text = message_text(message);
297        match message.role {
298            Role::System | Role::Developer => anchors.push(format!(
299                "[{}] {}",
300                role_label(message.role),
301                truncate_middle(&text, SYSTEM_CHARS)
302            )),
303            // Everything the user said before the agent first replied is task framing.
304            Role::User if !assistant_seen => {
305                anchors.push(format!(
306                    "[user (task)] {}",
307                    truncate_middle(&text, TASK_CHARS)
308                ));
309            }
310            role => {
311                if role == Role::Assistant {
312                    assistant_seen = true;
313                }
314                window.push(format!(
315                    "[{}] {}",
316                    role_label(role),
317                    truncate_middle(&text, config.window_message_chars)
318                ));
319            }
320        }
321    }
322
323    if window.len() > config.recent_turn_window {
324        window.drain(..window.len() - config.recent_turn_window);
325    }
326
327    let assemble = |window: &[String]| {
328        let header = format!(
329            "Conversation turn {turn}; showing the last {} of {} messages after the task framing.",
330            window.len(),
331            messages.len(),
332        );
333        std::iter::once(header)
334            .chain(anchors.iter().cloned())
335            .chain(window.iter().cloned())
336            .collect::<Vec<_>>()
337            .join("\n")
338    };
339
340    let mut text = assemble(&window);
341    while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() {
342        window.remove(0);
343        text = assemble(&window);
344    }
345    if text.chars().count() > MAX_REQUEST_CHARS {
346        let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1);
347        text = text.chars().take(keep).collect::<String>() + TRUNCATION_SUFFIX;
348    }
349    text
350}
351
352/// The transcript label for a role.
353fn role_label(role: Role) -> &'static str {
354    match role {
355        Role::System => "system",
356        Role::Developer => "developer",
357        Role::User => "user",
358        Role::Assistant => "assistant",
359        Role::Tool => "tool",
360    }
361}
362
363/// A request whose conversation sits at `turn`: `turn - 1` prior assistant replies, each
364/// answered by a further user message.
365///
366/// Shared with the assembled router's tests, which drive the same conversation shape.
367#[cfg(test)]
368pub(crate) fn request_at_turn(session_id: Option<&str>, turn: usize) -> Request {
369    use switchyard_protocol::{LlmRequest, Metadata};
370
371    let mut messages = vec![Message::text(Role::User, "What is 2+2?")];
372    for attempt in 1..turn {
373        messages.push(Message::text(Role::Assistant, format!("attempt {attempt}")));
374        messages.push(Message::text(Role::User, format!("still wrong {attempt}")));
375    }
376    Request {
377        llm_request: LlmRequest {
378            model: Some("auto".to_string()),
379            messages,
380            ..LlmRequest::default()
381        },
382        raw_request: None,
383        metadata: session_id.map(|id| Metadata {
384            session_id: Some(id.to_string()),
385            ..Metadata::default()
386        }),
387    }
388}
389
390#[cfg(test)]
391mod tests {
392    use serde_json::json;
393    use switchyard_protocol::{ContentBlock, Message, Role, ToolCall, ToolResult};
394
395    use super::*;
396    use crate::algorithms::util::llm_judge::Judge;
397
398    fn escalation_judge(max_output_tokens: u64) -> Result<EscalationJudge> {
399        Ok(StructuredJudge::new(
400            EscalationInput {
401                config: EscalationJudgeConfig::default(),
402            },
403            ClassifierContract::from_config(
404                &ClassifierContractConfig::default(),
405                PROMPT_TEMPLATE,
406                SCHEMA_TEMPLATE,
407            )?,
408            SerdeDecoder::new(),
409            JudgeRuntimeConfig::new(max_output_tokens)?,
410        ))
411    }
412
413    #[test]
414    fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> {
415        let judge = escalation_judge(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?;
416
417        // As the classifier calls it: the turn's reply is already on the transcript.
418        let mut judged = request_at_turn(None, 4);
419        judged
420            .llm_request
421            .messages
422            .push(Message::text(Role::Assistant, "this turn's reply"));
423        let built = judge.build_request(&State::default(), &judged);
424
425        // Rubric in instructions, condensed trajectory as the sole user message.
426        assert_eq!(built.llm_request.instructions.len(), 1);
427        assert_eq!(built.llm_request.instructions[0].role, Role::System);
428        assert_eq!(built.llm_request.messages.len(), 1);
429        assert_eq!(built.llm_request.messages[0].role, Role::User);
430        assert!(
431            built.llm_request.messages[0]
432                .text_content("")
433                .is_some_and(|text| text.contains("Conversation turn 4"))
434        );
435        // Bounded output, so a reasoning judge cannot run away mid-verdict.
436        assert_eq!(
437            built.llm_request.output.max_output_tokens,
438            Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
439        );
440        assert!(built.llm_request.output.response_format.is_some());
441        Ok(())
442    }
443
444    #[test]
445    fn judge_request_uses_the_configured_completion_cap() -> Result<()> {
446        let judge = escalation_judge(512)?;
447
448        let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
449
450        assert_eq!(built.llm_request.output.max_output_tokens, Some(512));
451        Ok(())
452    }
453
454    #[test]
455    fn conversation_turn_counts_assistant_replies() {
456        // The judge is handed the transcript with this turn's reply already appended, which
457        // is the shape asserted here: a request entering turn N plus its reply *is* turn N.
458        for turn in [1, 5] {
459            let mut judged = request_at_turn(None, turn);
460            judged
461                .llm_request
462                .messages
463                .push(Message::text(Role::Assistant, "this turn's reply"));
464            assert_eq!(conversation_turn(&judged), turn);
465        }
466    }
467
468    #[test]
469    fn message_text_keeps_tool_calls_and_results() {
470        let call = Message {
471            role: Role::Assistant,
472            content: vec![
473                ContentBlock::Text {
474                    text: "running it".to_string(),
475                },
476                ContentBlock::ToolCall(ToolCall {
477                    id: "call-1".to_string(),
478                    name: "bash".to_string(),
479                    arguments: json!({"cmd": "ls"}),
480                }),
481            ],
482        };
483        let text = message_text(&call);
484        assert!(text.contains("running it"), "{text}");
485        assert!(text.contains(r#"tool_call bash({"cmd":"ls"})"#), "{text}");
486
487        let result = Message {
488            role: Role::Tool,
489            content: vec![ContentBlock::ToolResult(ToolResult {
490                tool_call_id: "call-1".to_string(),
491                content: vec![ContentBlock::Text {
492                    text: "no such file".to_string(),
493                }],
494                is_error: Some(true),
495            })],
496        };
497        assert_eq!(message_text(&result), "no such file");
498    }
499
500    #[test]
501    fn truncate_middle_keeps_head_and_tail() {
502        let text = "a".repeat(40) + &"z".repeat(40);
503        let trimmed = truncate_middle(&text, 50);
504        assert!(trimmed.chars().count() <= 50, "{trimmed}");
505        assert!(trimmed.starts_with('a'));
506        assert!(trimmed.ends_with('z'));
507        assert!(trimmed.contains("[trimmed]"));
508
509        // Under the limit the text is returned untouched.
510        assert_eq!(truncate_middle("short", 50), "short");
511    }
512
513    #[test]
514    fn summary_keeps_anchors_and_the_recent_window() {
515        let mut messages = vec![
516            Message::text(Role::System, "you are a coding agent"),
517            Message::text(Role::User, "fix the failing test"),
518        ];
519        for i in 0..10 {
520            messages.push(Message::text(Role::Assistant, format!("step {i}")));
521        }
522        let config = EscalationJudgeConfig {
523            recent_turn_window: 3,
524            ..EscalationJudgeConfig::default()
525        };
526
527        let summary = summarize_for_judge(&messages, 11, &config);
528
529        assert!(
530            summary.contains("[system] you are a coding agent"),
531            "{summary}"
532        );
533        assert!(
534            summary.contains("[user (task)] fix the failing test"),
535            "{summary}"
536        );
537        assert!(summary.contains("Conversation turn 11; showing the last 3 of 12 messages"));
538        // Only the newest window entries survive.
539        assert!(summary.contains("step 9"), "{summary}");
540        assert!(summary.contains("step 7"), "{summary}");
541        assert!(!summary.contains("step 6"), "{summary}");
542    }
543
544    #[test]
545    fn summary_anchors_every_user_message_before_the_first_reply() {
546        // Codex sends environment boilerplate as the first user message and the task as the
547        // second. Both are framing; the task must stay visible after the window has moved on.
548        let mut messages = vec![
549            Message::text(
550                Role::Developer,
551                "<skills_instructions>...</skills_instructions>",
552            ),
553            Message::text(
554                Role::User,
555                "<environment_context><cwd>/app</cwd></environment_context>",
556            ),
557            Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."),
558        ];
559        for i in 0..40 {
560            messages.push(Message::text(Role::Assistant, format!("step {i}")));
561            messages.push(Message::text(Role::User, format!("later user note {i}")));
562        }
563        let config = EscalationJudgeConfig {
564            recent_turn_window: 3,
565            ..EscalationJudgeConfig::default()
566        };
567
568        let summary = summarize_for_judge(&messages, 40, &config);
569
570        assert!(
571            summary.contains("[user (task)] <environment_context>"),
572            "{summary}"
573        );
574        assert!(
575            summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."),
576            "{summary}"
577        );
578        // User messages after the first reply are ordinary window entries, not anchors.
579        assert!(
580            !summary.contains("[user (task)] later user note"),
581            "{summary}"
582        );
583        assert!(summary.contains("[user] later user note 39"), "{summary}");
584        assert!(!summary.contains("later user note 0\n"), "{summary}");
585    }
586
587    #[test]
588    fn summary_drops_oldest_window_lines_under_the_char_cap() {
589        // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps
590        // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually
591        // wide per-message cap. That is the point — it only fires on pathological input.
592        let mut messages = vec![
593            Message::text(Role::System, "framing"),
594            Message::text(Role::User, "task"),
595        ];
596        for i in 0..20 {
597            messages.push(Message::text(
598                Role::Assistant,
599                format!("{i} {}", "x".repeat(2_000)),
600            ));
601        }
602        let config = EscalationJudgeConfig {
603            window_message_chars: 2_000,
604            ..EscalationJudgeConfig::default()
605        };
606
607        let summary = summarize_for_judge(&messages, 21, &config);
608
609        assert!(
610            summary.chars().count() <= MAX_REQUEST_CHARS,
611            "{}",
612            summary.chars().count()
613        );
614        // Anchors are never dropped, and the newest activity outlives the oldest.
615        assert!(summary.contains("[system] framing"), "{summary}");
616        assert!(summary.contains("[user (task)] task"), "{summary}");
617        assert!(summary.contains("19 xxx"), "{summary}");
618        assert!(!summary.contains("0 xxx"), "{summary}");
619    }
620}
switchyard_libsy/algorithms/util/escalation.rs

switchyard_libsy/algorithms/util/
escalation.rs
