switchyard_libsy/algorithms/util/
escalation.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and
5//! the transcript condenser they read.
6//!
7//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts
8//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
9
10use serde::Deserialize;
11use serde_json::Value;
12use switchyard_protocol::{Category, InstructionBlock, Message, Role};
13
14use super::classifier_contract::{ClassifierContract, ClassifierContractConfig, validate_prompt};
15use super::llm_judge::{
16    ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder,
17    StructuredJudge,
18};
19use super::transcript::{collect_text, message_text};
20use crate::core::algorithm::Driver;
21use crate::core::classifier::{Classification, Score};
22use crate::core::state::State;
23use crate::{LibsyError, Result};
24use switchyard_protocol::Request;
25
26const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md");
27const DEESCALATION_PROMPT: &str = include_str!("../../prompts/escalation/deescalation.md");
28const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json");
29
30/// Separator marking where [`truncate_middle`] dropped a message's interior.
31const TRIM_MARKER: &str = " ...[trimmed] ";
32
33/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`].
34const TRUNCATION_SUFFIX: &str = "...<truncated>";
35
36/// Per-message cap for system and developer anchors, which carry no trajectory signal but
37/// which coding-agent harnesses make very large.
38const SYSTEM_CHARS: usize = 1_000;
39
40/// Per-message cap for task-framing user messages — every user message that precedes the first
41/// assistant reply. Coding-agent harnesses often send environment boilerplate as the first user
42/// message and the task itself as the second, so anchoring only the first would pin the
43/// boilerplate and let the task scroll out of the window. Feature specifications run to several
44/// thousand characters, so this gets the widest anchor budget.
45const TASK_CHARS: usize = 4_000;
46
47/// Backstop on the assembled transcript; the per-message caps normally bind first.
48const MAX_REQUEST_CHARS: usize = 18_000;
49
50/// Optional policy for returning an escalated session to the efficient tier.
51#[derive(Clone, Copy, Debug, Deserialize)]
52#[serde(deny_unknown_fields)]
53pub struct DeescalationConfig {
54    /// Minimum number of capable-tier turns before the judge may release the session.
55    pub strong_min_calls: u32,
56    /// Optional hard limit on capable-tier turns before a forced return.
57    #[serde(default)]
58    pub strong_max_calls: Option<u32>,
59    /// Consecutive judge declines required to return to the efficient tier.
60    pub confirmations: u32,
61    /// Efficient calls served without judging after a hard-limit return.
62    #[serde(default)]
63    pub weak_cooldown_calls: u32,
64}
65
66impl DeescalationConfig {
67    fn validate(&self) -> Result<()> {
68        let reject = |message: &str| {
69            Err(LibsyError::AlgorithmError {
70                message: message.to_string(),
71            })
72        };
73        if self.strong_min_calls == 0 {
74            return reject("deescalation.strong_min_calls must be at least 1");
75        }
76        if self.confirmations == 0 {
77            return reject("deescalation.confirmations must be at least 1");
78        }
79        if self
80            .strong_max_calls
81            .is_some_and(|strong_max_calls| strong_max_calls < self.strong_min_calls)
82        {
83            return reject(
84                "deescalation.strong_max_calls must be at least deescalation.strong_min_calls",
85            );
86        }
87        Ok(())
88    }
89}
90
91/// The tuning surface for the trajectory judge.
92///
93/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant
94/// (the constants above).
95#[derive(Clone, Debug, Deserialize)]
96#[serde(default, deny_unknown_fields)]
97pub struct EscalationJudgeConfig {
98    /// Consecutive fresh-evidence verdicts in the same category required before a turn moves to
99    /// the capable tier, which is also the turn that latches the session. Any decline or stale
100    /// evidence clears the streak.
101    /// `1` escalates on the first verdict; the router's main cost dial.
102    /// `2` or higher needs a session id, since the streak is retained per session.
103    pub confirmations: u32,
104    /// Trailing messages shown on top of the anchors. A loop longer than this is invisible.
105    pub recent_turn_window: usize,
106    /// Per-message cap inside the trailing window.
107    pub window_message_chars: usize,
108    /// De-escalation policy. `None` preserves permanent latching to the capable tier.
109    pub deescalation: Option<DeescalationConfig>,
110}
111
112impl EscalationJudgeConfig {
113    /// Rejects settings that would leave the judge with nothing useful to read.
114    fn validate(&self) -> Result<()> {
115        let reject = |message: String| Err(LibsyError::AlgorithmError { message });
116        if self.confirmations == 0 {
117            return reject("confirmations must be at least 1".to_string());
118        }
119        if self.recent_turn_window == 0 {
120            return reject("recent_turn_window must be at least 1".to_string());
121        }
122        if self.window_message_chars < 50 {
123            return reject(format!(
124                "window_message_chars must be at least 50, got {}",
125                self.window_message_chars
126            ));
127        }
128        if let Some(deescalation) = self.deescalation {
129            deescalation.validate()?;
130        }
131        Ok(())
132    }
133}
134
135impl Default for EscalationJudgeConfig {
136    fn default() -> Self {
137        Self {
138            confirmations: 2,
139            recent_turn_window: 28,
140            window_message_chars: 500,
141            deescalation: None,
142        }
143    }
144}
145
146/// Router-controlled phase attached to judge input when de-escalation is enabled.
147#[derive(Clone, Copy, Debug, Eq, PartialEq)]
148pub(crate) enum EvaluationPhase {
149    Efficient,
150    Strong,
151}
152
153impl EvaluationPhase {
154    fn marker(self) -> &'static str {
155        match self {
156            Self::Efficient => "EFFICIENT_EVALUATION",
157            Self::Strong => "STRONG_EVALUATION",
158        }
159    }
160}
161
162/// Bounded trouble pattern used to correlate escalation confirmations across turns.
163#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)]
164#[serde(rename_all = "snake_case")]
165pub(crate) enum EscalationCategory {
166    None,
167    Repetition,
168    FalseProgress,
169    Drift,
170    Desperation,
171    CapabilityGap,
172}
173
174impl EscalationCategory {
175    /// Stable state and telemetry label.
176    pub(crate) const fn label(self) -> &'static str {
177        match self {
178            Self::None => "none",
179            Self::Repetition => "repetition",
180            Self::FalseProgress => "false_progress",
181            Self::Drift => "drift",
182            Self::Desperation => "desperation",
183            Self::CapabilityGap => "capability_gap",
184        }
185    }
186}
187
188/// The judge's typed verdict, including the evidence needed to confirm a stable pattern.
189#[derive(Deserialize)]
190pub(crate) struct EscalationVerdict {
191    pub(crate) escalate: bool,
192    pub(crate) category: EscalationCategory,
193    pub(crate) new_evidence: bool,
194    pub(crate) reason: String,
195}
196
197/// Builds the condensed trajectory presented to the escalation judge.
198pub(crate) struct EscalationInput {
199    config: EscalationJudgeConfig,
200    phase: Option<EvaluationPhase>,
201}
202
203impl ClassifierInput for EscalationInput {
204    fn build_messages(&self, _state: &State, request: &Request) -> Vec<Message> {
205        let conversation = switchyard_translation::conversation(&request.llm_request);
206        let summary = summarize_for_judge(
207            &conversation.instructions,
208            &conversation.messages,
209            conversation_turn(&conversation.messages),
210            self.phase,
211            &self.config,
212        );
213        vec![Message::text(Role::User, summary)]
214    }
215}
216
217/// Structured trajectory judge with a typed escalation verdict.
218pub(crate) type EscalationJudge = StructuredJudge<EscalationInput, SerdeDecoder<EscalationVerdict>>;
219
220/// Maps the judge's verdict to a classification. A verdict names the tier to serve — capable
221/// on escalate, efficient on decline — so the caller reads it straight off the winning score.
222/// [`Classification::Ambiguous`] carries the unavailable case, which names no tier: both a
223/// decline and an outage stay efficient, but only a decline is evidence, so only a decline
224/// clears the streak.
225pub(crate) struct EscalationPolicy {
226    phase: Option<EvaluationPhase>,
227}
228
229impl JudgePolicy for EscalationPolicy {
230    type Verdict = EscalationVerdict;
231
232    fn to_classification(
233        &self,
234        verdict: Option<&EscalationVerdict>,
235        driver: &Driver,
236    ) -> Result<Classification> {
237        if let Some(verdict) = verdict {
238            tracing::debug!(
239                escalate = verdict.escalate,
240                reason = %verdict.reason,
241                "escalation judge verdict"
242            );
243        }
244        match verdict {
245            Some(verdict) if verdict.escalate => Ok(Classification::Scores(vec![Score {
246                target: driver.first_model_for(&Category::Capable)?.clone(),
247                confidence: 1.0,
248                category: Some(Category::Capable),
249            }])),
250            Some(_) => Ok(Classification::Scores(vec![Score {
251                target: driver.first_model_for(&Category::Efficient)?.clone(),
252                confidence: 1.0,
253                category: Some(Category::Efficient),
254            }])),
255            None => Ok(Classification::Ambiguous(Vec::new())),
256        }
257    }
258}
259
260/// Maps present verdicts to phase-aware evidence; absent verdicts add nothing.
261fn escalation_evidence(
262    policy: &EscalationPolicy,
263    verdict: Option<&EscalationVerdict>,
264) -> Option<Value> {
265    verdict.map(|verdict| {
266        let verdict = match (policy.phase, verdict.escalate) {
267            (Some(EvaluationPhase::Strong), true) => "retain",
268            (Some(EvaluationPhase::Strong), false) => "deescalate",
269            (_, true) => "escalate",
270            (_, false) => "continue",
271        };
272        serde_json::json!({
273            "source": "escalation",
274            "verdict": verdict,
275        })
276    })
277}
278
279/// Builds the trajectory judge, scoring the runtime capable category when it escalates.
280///
281/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
282/// fails here rather than on the first request.
283pub(crate) fn build_judge(
284    contract_config: &ClassifierContractConfig,
285    config: EscalationJudgeConfig,
286    phase: Option<EvaluationPhase>,
287    max_output_tokens: u64,
288    judge_deadline_ms: Option<u64>,
289) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
290    config.validate()?;
291    let contract = build_contract(contract_config, phase.is_some())?;
292    Ok(JudgeClassifier::new(
293        StructuredJudge::new(
294            EscalationInput { config, phase },
295            contract,
296            SerdeDecoder::new(),
297            JudgeRuntimeConfig::new(max_output_tokens)?.with_deadline_ms(judge_deadline_ms)?,
298        ),
299        EscalationPolicy { phase },
300    )
301    .with_evidence(escalation_evidence))
302}
303
304fn build_contract(
305    contract_config: &ClassifierContractConfig,
306    phase_aware: bool,
307) -> Result<ClassifierContract> {
308    let prompt = contract_config.prompt().unwrap_or(PROMPT_TEMPLATE);
309    validate_prompt(prompt)?;
310    let phase_aware_config = phase_aware.then(|| {
311        contract_config.clone().with_prompt(format!(
312            "{}\n\n{}",
313            prompt.trim_end(),
314            DEESCALATION_PROMPT.trim()
315        ))
316    });
317    let contract_config = phase_aware_config.as_ref().unwrap_or(contract_config);
318    ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)
319}
320
321/// The 1-indexed model invocation the transcript ends on: one per assistant reply.
322///
323/// The judge reads the turn *including* the reply it is judging, so the newest assistant
324/// message is this turn — no `+ 1`. Counting the caller's request instead would report the
325/// turn after the one under judgement.
326///
327/// Callers pass the conversation view so native Responses items share assistant turns.
328pub(crate) fn conversation_turn(messages: &[Message]) -> usize {
329    messages
330        .iter()
331        .filter(|message| message.role == Role::Assistant)
332        .count()
333}
334
335/// Keeps the head and tail of `text` within `limit` characters.
336///
337/// The head gets two thirds of the surviving budget: for a trajectory judge the command or
338/// error signature that opens a message carries more signal than its trailing output.
339fn truncate_middle(text: &str, limit: usize) -> String {
340    let chars: Vec<char> = text.chars().collect();
341    if chars.len() <= limit {
342        return text.to_string();
343    }
344    let keep = limit
345        .saturating_sub(TRIM_MARKER.chars().count())
346        .max(20)
347        .min(chars.len());
348    let head = keep * 2 / 3;
349    let tail = keep - head;
350    let mut out: String = chars[..head].iter().collect();
351    out.push_str(TRIM_MARKER);
352    out.extend(chars[chars.len() - tail..].iter());
353    out
354}
355
356/// Renders a compact role-labelled transcript for the judge.
357///
358/// Task-framing user messages are capped individually. System/developer instructions share
359/// the remaining budget after reserving space for task framing and the newest window entry.
360/// The trailing window carries recent activity. A coverage header states how much history is
361/// not shown, so the judge can reason about pace rather than assuming it sees everything.
362///
363/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go
364/// first: for a trajectory judge the newest evidence is strictly the most valuable.
365fn summarize_for_judge(
366    instructions: &[InstructionBlock],
367    messages: &[Message],
368    turn: usize,
369    phase: Option<EvaluationPhase>,
370    config: &EscalationJudgeConfig,
371) -> String {
372    let mut instruction_anchors: Vec<String> = Vec::new();
373    let mut anchors: Vec<String> = Vec::new();
374    let mut window: Vec<String> = Vec::new();
375    let mut assistant_seen = false;
376
377    for instruction in instructions {
378        let mut parts = Vec::new();
379        collect_text(&instruction.content, &mut parts, &[]);
380        instruction_anchors.push(format!(
381            "[{}] {}",
382            role_label(instruction.role),
383            truncate_middle(&parts.join(" "), SYSTEM_CHARS)
384        ));
385    }
386
387    for message in messages {
388        let text = message_text(message.role, &message.content);
389        match message.role {
390            Role::System | Role::Developer => instruction_anchors.push(format!(
391                "[{}] {}",
392                role_label(message.role),
393                truncate_middle(&text, SYSTEM_CHARS)
394            )),
395            // Everything the user said before the agent first replied is task framing.
396            Role::User if !assistant_seen => {
397                anchors.push(format!(
398                    "[user (task)] {}",
399                    truncate_middle(&text, TASK_CHARS)
400                ));
401            }
402            role => {
403                if role == Role::Assistant {
404                    assistant_seen = true;
405                }
406                window.push(format!(
407                    "[{}] {}",
408                    role_label(role),
409                    truncate_middle(&text, config.window_message_chars)
410                ));
411            }
412        }
413    }
414
415    if window.len() > config.recent_turn_window {
416        window.drain(..window.len() - config.recent_turn_window);
417    }
418
419    let assemble = |instructions: Option<&str>, window: &[String]| {
420        let header = format!(
421            "Conversation turn {turn}; showing the last {} of {} messages after the task framing.",
422            window.len(),
423            messages.len(),
424        );
425        phase
426            .map(|phase| format!("Routing phase: {}", phase.marker()))
427            .into_iter()
428            .chain(std::iter::once(header))
429            .chain(instructions.map(str::to_owned))
430            .chain(anchors.iter().cloned())
431            .chain(window.iter().cloned())
432            .collect::<Vec<_>>()
433            .join("\n")
434    };
435
436    let reserved = assemble(None, &window[window.len().saturating_sub(1)..])
437        .chars()
438        .count();
439    let instruction_budget = MAX_REQUEST_CHARS.saturating_sub(reserved + 1);
440    // The remaining budget may be smaller than truncate_middle's minimum retained span.
441    let instruction_text = truncate_middle(&instruction_anchors.join("\n"), instruction_budget)
442        .chars()
443        .take(instruction_budget)
444        .collect::<String>();
445    let instructions = (!instruction_text.is_empty()).then_some(instruction_text.as_str());
446    let mut text = assemble(instructions, &window);
447    while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() {
448        window.remove(0);
449        text = assemble(instructions, &window);
450    }
451    if text.chars().count() > MAX_REQUEST_CHARS {
452        let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1);
453        text = text.chars().take(keep).collect::<String>() + TRUNCATION_SUFFIX;
454    }
455    text
456}
457
458/// The transcript label for a role.
459fn role_label(role: Role) -> &'static str {
460    match role {
461        Role::System => "system",
462        Role::Developer => "developer",
463        Role::User => "user",
464        Role::Assistant => "assistant",
465        Role::Tool => "tool",
466    }
467}
468
469/// A request whose conversation sits at `turn`: `turn - 1` prior assistant replies, each
470/// answered by a further user message.
471///
472/// Shared with the assembled router's tests, which drive the same conversation shape.
473#[cfg(test)]
474pub(crate) fn request_at_turn(session_id: Option<&str>, turn: usize) -> Request {
475    use switchyard_protocol::{LlmRequest, Metadata};
476
477    let mut messages = vec![Message::text(Role::User, "What is 2+2?")];
478    for attempt in 1..turn {
479        messages.push(Message::text(Role::Assistant, format!("attempt {attempt}")));
480        messages.push(Message::text(Role::User, format!("still wrong {attempt}")));
481    }
482    Request {
483        llm_request: LlmRequest {
484            model: Some("auto".to_string()),
485            messages,
486            ..LlmRequest::default()
487        },
488        raw_request: None,
489        metadata: session_id.map(|id| Metadata {
490            session_id: Some(id.to_string()),
491            ..Metadata::default()
492        }),
493    }
494}
495
496#[cfg(test)]
497mod tests {
498    use switchyard_protocol::{ContentBlock, Message, Role};
499
500    use super::*;
501    use crate::algorithms::util::llm_judge::Judge;
502
503    fn escalation_judge(
504        max_output_tokens: u64,
505        phase: Option<EvaluationPhase>,
506        contract_config: &ClassifierContractConfig,
507    ) -> Result<EscalationJudge> {
508        Ok(StructuredJudge::new(
509            EscalationInput {
510                config: EscalationJudgeConfig::default(),
511                phase,
512            },
513            build_contract(contract_config, phase.is_some())?,
514            SerdeDecoder::new(),
515            JudgeRuntimeConfig::new(max_output_tokens)?,
516        ))
517    }
518
519    #[test]
520    fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> {
521        let judge = escalation_judge(
522            super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
523            None,
524            &ClassifierContractConfig::default(),
525        )?;
526
527        // As the classifier calls it: the turn's reply is already on the transcript.
528        let mut judged = request_at_turn(None, 4);
529        judged.llm_request.instructions = [
530            (Role::System, "system constraint"),
531            (Role::Developer, "developer constraint"),
532        ]
533        .into_iter()
534        .map(|(role, text)| InstructionBlock {
535            role,
536            content: Message::text(role, text).content,
537        })
538        .collect();
539        judged
540            .llm_request
541            .messages
542            .push(Message::text(Role::Assistant, "this turn's reply"));
543        let built = judge.build_request(&State::default(), &judged);
544
545        // Rubric in instructions, condensed trajectory as the sole user message.
546        assert_eq!(built.llm_request.instructions.len(), 1);
547        assert_eq!(built.llm_request.instructions[0].role, Role::System);
548        assert_eq!(
549            built.llm_request.instructions[0].content.as_slice(),
550            &[ContentBlock::Text {
551                responses: None,
552                text: PROMPT_TEMPLATE.to_string()
553            }]
554        );
555        assert_eq!(built.llm_request.messages.len(), 1);
556        assert_eq!(built.llm_request.messages[0].role, Role::User);
557        let summary = built.llm_request.messages[0]
558            .text_content("")
559            .expect("summary");
560        assert!(summary.contains("Conversation turn 4"));
561        assert!(summary.contains(
562            "[system] system constraint\n[developer] developer constraint\n[user (task)] What is 2+2?"
563        ));
564        assert!(summary.contains("[assistant] this turn's reply"));
565        assert!(!summary.contains("Routing phase:"));
566        // Bounded output, so a reasoning judge cannot run away mid-verdict.
567        assert_eq!(
568            built.llm_request.output.max_output_tokens,
569            Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
570        );
571        assert!(built.llm_request.output.response_format.is_some());
572
573        judged.llm_request.instructions.extend(vec![
574            InstructionBlock {
575                role: Role::Developer,
576                content: Message::text(Role::Developer, "x".repeat(SYSTEM_CHARS)).content,
577            };
578            MAX_REQUEST_CHARS / SYSTEM_CHARS + 1
579        ]);
580        let built = judge.build_request(&State::default(), &judged);
581        let summary = built.llm_request.messages[0]
582            .text_content("")
583            .expect("summary");
584        assert!(summary.chars().count() <= MAX_REQUEST_CHARS);
585        assert!(summary.contains(TRIM_MARKER));
586        assert!(summary.contains("[system] system constraint"));
587        assert!(summary.contains("[developer] developer constraint"));
588        assert!(summary.contains("[user (task)] What is 2+2?"));
589        assert!(summary.contains("[assistant] this turn's reply"));
590        Ok(())
591    }
592
593    #[test]
594    fn judge_request_uses_the_configured_completion_cap() -> Result<()> {
595        let judge = escalation_judge(512, None, &ClassifierContractConfig::default())?;
596
597        let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
598
599        assert_eq!(built.llm_request.output.max_output_tokens, Some(512));
600        Ok(())
601    }
602
603    #[test]
604    fn conversation_turn_counts_assistant_replies() {
605        // The judge is handed the transcript with this turn's reply already appended, which
606        // is the shape asserted here: a request entering turn N plus its reply *is* turn N.
607        for turn in [1, 5] {
608            let mut judged = request_at_turn(None, turn);
609            judged
610                .llm_request
611                .messages
612                .push(Message::text(Role::Assistant, "this turn's reply"));
613            assert_eq!(
614                conversation_turn(
615                    &switchyard_translation::conversation(&judged.llm_request).messages
616                ),
617                turn
618            );
619        }
620    }
621
622    #[test]
623    fn truncate_middle_keeps_head_and_tail() {
624        let text = "a".repeat(40) + &"z".repeat(40);
625        let trimmed = truncate_middle(&text, 50);
626        assert!(trimmed.chars().count() <= 50, "{trimmed}");
627        assert!(trimmed.starts_with('a'));
628        assert!(trimmed.ends_with('z'));
629        assert!(trimmed.contains("[trimmed]"));
630
631        // Under the limit the text is returned untouched.
632        assert_eq!(truncate_middle("short", 50), "short");
633    }
634
635    #[test]
636    fn deescalation_settings_must_be_valid() {
637        let zero = EscalationJudgeConfig {
638            deescalation: Some(DeescalationConfig {
639                strong_min_calls: 0,
640                strong_max_calls: None,
641                confirmations: 2,
642                weak_cooldown_calls: 0,
643            }),
644            ..EscalationJudgeConfig::default()
645        };
646        assert!(
647            zero.validate()
648                .is_err_and(|error| error.to_string().contains("at least 1"))
649        );
650
651        let inverted = EscalationJudgeConfig {
652            deescalation: Some(DeescalationConfig {
653                strong_min_calls: 4,
654                strong_max_calls: Some(3),
655                confirmations: 2,
656                weak_cooldown_calls: 0,
657            }),
658            ..EscalationJudgeConfig::default()
659        };
660        assert!(
661            inverted
662                .validate()
663                .is_err_and(|error| error.to_string().contains("at least deescalation"))
664        );
665    }
666
667    #[test]
668    fn deescalation_contract_marks_both_routing_phases() -> Result<()> {
669        let contract = ClassifierContractConfig::default().with_prompt("Custom trajectory rubric.");
670        for (phase, marker) in [
671            (EvaluationPhase::Efficient, "EFFICIENT_EVALUATION"),
672            (EvaluationPhase::Strong, "STRONG_EVALUATION"),
673        ] {
674            let judge = escalation_judge(512, Some(phase), &contract)?;
675            let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
676            let system_prompt = built.llm_request.instructions[0].content.iter().find_map(
677                |content| match content {
678                    ContentBlock::Text { text, .. } => Some(text.as_str()),
679                    _ => None,
680                },
681            );
682            assert!(system_prompt.is_some_and(|prompt| {
683                prompt.starts_with("Custom trajectory rubric.")
684                    && prompt.contains("EFFICIENT_EVALUATION")
685                    && prompt.contains("STRONG_EVALUATION")
686            }));
687            assert!(
688                built.llm_request.messages[0]
689                    .text_content("")
690                    .is_some_and(|summary| summary.starts_with(&format!(
691                        "Routing phase: {marker}"
692                    )))
693            );
694        }
695        Ok(())
696    }
697
698    #[test]
699    fn summary_keeps_anchors_and_the_recent_window() {
700        let mut messages = vec![
701            Message::text(Role::System, "you are a coding agent"),
702            Message::text(Role::User, "fix the failing test"),
703        ];
704        for i in 0..10 {
705            messages.push(Message::text(Role::Assistant, format!("step {i}")));
706        }
707        let config = EscalationJudgeConfig {
708            recent_turn_window: 3,
709            ..EscalationJudgeConfig::default()
710        };
711
712        let summary = summarize_for_judge(&[], &messages, 11, None, &config);
713
714        assert!(
715            summary.contains("[system] you are a coding agent"),
716            "{summary}"
717        );
718        assert!(
719            summary.contains("[user (task)] fix the failing test"),
720            "{summary}"
721        );
722        assert!(summary.contains("Conversation turn 11; showing the last 3 of 12 messages"));
723        // Only the newest window entries survive.
724        assert!(summary.contains("step 9"), "{summary}");
725        assert!(summary.contains("step 7"), "{summary}");
726        assert!(!summary.contains("step 6"), "{summary}");
727    }
728
729    #[test]
730    fn summary_anchors_every_user_message_before_the_first_reply() {
731        // Codex sends environment boilerplate as the first user message and the task as the
732        // second. Both are framing; the task must stay visible after the window has moved on.
733        let mut messages = vec![
734            Message::text(
735                Role::Developer,
736                "<skills_instructions>...</skills_instructions>",
737            ),
738            Message::text(
739                Role::User,
740                "<environment_context><cwd>/app</cwd></environment_context>",
741            ),
742            Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."),
743        ];
744        for i in 0..40 {
745            messages.push(Message::text(Role::Assistant, format!("step {i}")));
746            messages.push(Message::text(Role::User, format!("later user note {i}")));
747        }
748        let config = EscalationJudgeConfig {
749            recent_turn_window: 3,
750            ..EscalationJudgeConfig::default()
751        };
752
753        let summary = summarize_for_judge(&[], &messages, 40, None, &config);
754
755        assert!(
756            summary.contains("[user (task)] <environment_context>"),
757            "{summary}"
758        );
759        assert!(
760            summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."),
761            "{summary}"
762        );
763        // User messages after the first reply are ordinary window entries, not anchors.
764        assert!(
765            !summary.contains("[user (task)] later user note"),
766            "{summary}"
767        );
768        assert!(summary.contains("[user] later user note 39"), "{summary}");
769        assert!(!summary.contains("later user note 0\n"), "{summary}");
770    }
771
772    #[test]
773    fn summary_drops_oldest_window_lines_under_the_char_cap() {
774        // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps
775        // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually
776        // wide per-message cap. That is the point — it only fires on pathological input.
777        let mut messages = vec![
778            Message::text(Role::System, "framing"),
779            Message::text(Role::User, "task"),
780        ];
781        for i in 0..20 {
782            messages.push(Message::text(
783                Role::Assistant,
784                format!("{i} {}", "x".repeat(2_000)),
785            ));
786        }
787        let config = EscalationJudgeConfig {
788            window_message_chars: 2_000,
789            ..EscalationJudgeConfig::default()
790        };
791
792        let summary = summarize_for_judge(&[], &messages, 21, None, &config);
793
794        assert!(
795            summary.chars().count() <= MAX_REQUEST_CHARS,
796            "{}",
797            summary.chars().count()
798        );
799        // Anchors are never dropped, and the newest activity outlives the oldest.
800        assert!(summary.contains("[system] framing"), "{summary}");
801        assert!(summary.contains("[user (task)] task"), "{summary}");
802        assert!(summary.contains("19 xxx"), "{summary}");
803        assert!(!summary.contains("0 xxx"), "{summary}");
804    }
805}
switchyard_libsy/algorithms/util/escalation.rs

switchyard_libsy/algorithms/util/
escalation.rs
