switchyard_libsy/algorithms/util/
escalation.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and
5//! the transcript condenser they read.
6//!
7//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts
8//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
9
10use serde::Deserialize;
11use serde_json::Value;
12use switchyard_protocol::{Category, InstructionBlock, Message, Role};
13
14use super::classifier_contract::{ClassifierContract, ClassifierContractConfig, validate_prompt};
15use super::llm_judge::{
16    ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder,
17    StructuredJudge,
18};
19use super::transcript::{collect_text, message_text};
20use crate::core::algorithm::Driver;
21use crate::core::classifier::{Classification, Score};
22use crate::core::state::State;
23use crate::{LibsyError, Result};
24use switchyard_protocol::Request;
25
26const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md");
27const DEESCALATION_PROMPT: &str = include_str!("../../prompts/escalation/deescalation.md");
28const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json");
29
30/// Separator marking where [`truncate_middle`] dropped a message's interior.
31const TRIM_MARKER: &str = " ...[trimmed] ";
32
33/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`].
34const TRUNCATION_SUFFIX: &str = "...<truncated>";
35
36/// Per-message cap for system and developer anchors, which carry no trajectory signal but
37/// which coding-agent harnesses make very large.
38const SYSTEM_CHARS: usize = 1_000;
39
40/// Per-message cap for task-framing user messages — every user message that precedes the first
41/// assistant reply. Coding-agent harnesses often send environment boilerplate as the first user
42/// message and the task itself as the second, so anchoring only the first would pin the
43/// boilerplate and let the task scroll out of the window. Feature specifications run to several
44/// thousand characters, so this gets the widest anchor budget.
45const TASK_CHARS: usize = 4_000;
46
47/// Backstop on the assembled transcript; the per-message caps normally bind first.
48const MAX_REQUEST_CHARS: usize = 18_000;
49
50/// Optional policy for returning an escalated session to the efficient tier.
51#[derive(Clone, Copy, Debug, Deserialize)]
52#[serde(deny_unknown_fields)]
53pub struct DeescalationConfig {
54    /// Minimum number of capable-tier turns before the judge may release the session.
55    pub strong_min_calls: u32,
56    /// Optional hard limit on capable-tier turns before a forced return.
57    #[serde(default)]
58    pub strong_max_calls: Option<u32>,
59    /// Consecutive judge declines required to return to the efficient tier.
60    pub confirmations: u32,
61    /// Efficient calls served without judging after a hard-limit return.
62    #[serde(default)]
63    pub weak_cooldown_calls: u32,
64}
65
66impl DeescalationConfig {
67    fn validate(&self) -> Result<()> {
68        let reject = |message: &str| {
69            Err(LibsyError::AlgorithmError {
70                message: message.to_string(),
71            })
72        };
73        if self.strong_min_calls == 0 {
74            return reject("deescalation.strong_min_calls must be at least 1");
75        }
76        if self.confirmations == 0 {
77            return reject("deescalation.confirmations must be at least 1");
78        }
79        if self
80            .strong_max_calls
81            .is_some_and(|strong_max_calls| strong_max_calls < self.strong_min_calls)
82        {
83            return reject(
84                "deescalation.strong_max_calls must be at least deescalation.strong_min_calls",
85            );
86        }
87        Ok(())
88    }
89}
90
91/// The tuning surface for the trajectory judge.
92///
93/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant
94/// (the constants above).
95#[derive(Clone, Debug, Deserialize)]
96#[serde(default, deny_unknown_fields)]
97pub struct EscalationJudgeConfig {
98    /// Consecutive fresh-evidence verdicts in the same category required before a turn moves to
99    /// the capable tier, which is also the turn that latches the session. Any decline or stale
100    /// evidence clears the streak.
101    /// `1` escalates on the first verdict; the router's main cost dial.
102    /// `2` or higher needs a session id, since the streak is retained per session.
103    pub confirmations: u32,
104    /// Trailing messages shown on top of the anchors. A loop longer than this is invisible.
105    pub recent_turn_window: usize,
106    /// Per-message cap inside the trailing window.
107    pub window_message_chars: usize,
108    /// De-escalation policy. `None` preserves permanent latching to the capable tier.
109    pub deescalation: Option<DeescalationConfig>,
110}
111
112impl EscalationJudgeConfig {
113    /// Rejects settings that would leave the judge with nothing useful to read.
114    fn validate(&self) -> Result<()> {
115        let reject = |message: String| Err(LibsyError::AlgorithmError { message });
116        if self.confirmations == 0 {
117            return reject("confirmations must be at least 1".to_string());
118        }
119        if self.recent_turn_window == 0 {
120            return reject("recent_turn_window must be at least 1".to_string());
121        }
122        if self.window_message_chars < 50 {
123            return reject(format!(
124                "window_message_chars must be at least 50, got {}",
125                self.window_message_chars
126            ));
127        }
128        if let Some(deescalation) = self.deescalation {
129            deescalation.validate()?;
130        }
131        Ok(())
132    }
133}
134
135impl Default for EscalationJudgeConfig {
136    fn default() -> Self {
137        Self {
138            confirmations: 2,
139            recent_turn_window: 28,
140            window_message_chars: 500,
141            deescalation: None,
142        }
143    }
144}
145
146/// Router-controlled phase attached to judge input when de-escalation is enabled.
147#[derive(Clone, Copy, Debug, Eq, PartialEq)]
148pub(crate) enum EvaluationPhase {
149    Efficient,
150    Strong,
151}
152
153impl EvaluationPhase {
154    fn marker(self) -> &'static str {
155        match self {
156            Self::Efficient => "EFFICIENT_EVALUATION",
157            Self::Strong => "STRONG_EVALUATION",
158        }
159    }
160}
161
162/// Bounded trouble pattern used to correlate escalation confirmations across turns.
163#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)]
164#[serde(rename_all = "snake_case")]
165pub(crate) enum EscalationCategory {
166    None,
167    Repetition,
168    FalseProgress,
169    Drift,
170    Desperation,
171    CapabilityGap,
172}
173
174impl EscalationCategory {
175    /// Stable state and telemetry label.
176    pub(crate) const fn label(self) -> &'static str {
177        match self {
178            Self::None => "none",
179            Self::Repetition => "repetition",
180            Self::FalseProgress => "false_progress",
181            Self::Drift => "drift",
182            Self::Desperation => "desperation",
183            Self::CapabilityGap => "capability_gap",
184        }
185    }
186}
187
188/// The judge's typed verdict, including the evidence needed to confirm a stable pattern.
189#[derive(Deserialize)]
190pub(crate) struct EscalationVerdict {
191    pub(crate) escalate: bool,
192    pub(crate) category: EscalationCategory,
193    pub(crate) new_evidence: bool,
194    #[serde(rename = "reason")]
195    _reason: String,
196}
197
198/// Builds the condensed trajectory presented to the escalation judge.
199pub(crate) struct EscalationInput {
200    config: EscalationJudgeConfig,
201    phase: Option<EvaluationPhase>,
202}
203
204impl ClassifierInput for EscalationInput {
205    fn build_messages(&self, _state: &State, request: &Request) -> Vec<Message> {
206        let summary = summarize_for_judge(
207            &request.llm_request.instructions,
208            &request.llm_request.messages,
209            conversation_turn(request),
210            self.phase,
211            &self.config,
212        );
213        vec![Message::text(Role::User, summary)]
214    }
215}
216
217/// Structured trajectory judge with a typed escalation verdict.
218pub(crate) type EscalationJudge = StructuredJudge<EscalationInput, SerdeDecoder<EscalationVerdict>>;
219
220/// Maps the judge's verdict to a classification. A verdict names the tier to serve — capable
221/// on escalate, efficient on decline — so the caller reads it straight off the winning score.
222/// [`Classification::Ambiguous`] carries the unavailable case, which names no tier: both a
223/// decline and an outage stay efficient, but only a decline is evidence, so only a decline
224/// clears the streak.
225pub(crate) struct EscalationPolicy {
226    phase: Option<EvaluationPhase>,
227}
228
229impl JudgePolicy for EscalationPolicy {
230    type Verdict = EscalationVerdict;
231
232    fn to_classification(
233        &self,
234        verdict: Option<&EscalationVerdict>,
235        driver: &Driver,
236    ) -> Result<Classification> {
237        if let Some(verdict) = verdict {
238            tracing::debug!(
239                escalate = verdict.escalate,
240                category = verdict.category.label(),
241                new_evidence = verdict.new_evidence,
242                "escalation judge verdict"
243            );
244        }
245        match verdict {
246            Some(verdict) if verdict.escalate => Ok(Classification::Scores(vec![Score {
247                target: driver.first_model_for(&Category::Capable)?.clone(),
248                confidence: 1.0,
249                category: Some(Category::Capable),
250            }])),
251            Some(_) => Ok(Classification::Scores(vec![Score {
252                target: driver.first_model_for(&Category::Efficient)?.clone(),
253                confidence: 1.0,
254                category: Some(Category::Efficient),
255            }])),
256            None => Ok(Classification::Ambiguous(Vec::new())),
257        }
258    }
259}
260
261/// Maps present verdicts to phase-aware evidence; absent verdicts add nothing.
262fn escalation_evidence(
263    policy: &EscalationPolicy,
264    verdict: Option<&EscalationVerdict>,
265) -> Option<Value> {
266    verdict.map(|verdict| {
267        let verdict = match (policy.phase, verdict.escalate) {
268            (Some(EvaluationPhase::Strong), true) => "retain",
269            (Some(EvaluationPhase::Strong), false) => "deescalate",
270            (_, true) => "escalate",
271            (_, false) => "continue",
272        };
273        serde_json::json!({
274            "source": "escalation",
275            "verdict": verdict,
276        })
277    })
278}
279
280/// Builds the trajectory judge, scoring the runtime capable category when it escalates.
281///
282/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
283/// fails here rather than on the first request.
284pub(crate) fn build_judge(
285    contract_config: &ClassifierContractConfig,
286    config: EscalationJudgeConfig,
287    phase: Option<EvaluationPhase>,
288    max_output_tokens: u64,
289    judge_deadline_ms: Option<u64>,
290) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
291    config.validate()?;
292    let contract = build_contract(contract_config, phase.is_some())?;
293    Ok(JudgeClassifier::new(
294        StructuredJudge::new(
295            EscalationInput { config, phase },
296            contract,
297            SerdeDecoder::new(),
298            JudgeRuntimeConfig::new(max_output_tokens)?.with_deadline_ms(judge_deadline_ms)?,
299        ),
300        EscalationPolicy { phase },
301    )
302    .with_evidence(escalation_evidence))
303}
304
305fn build_contract(
306    contract_config: &ClassifierContractConfig,
307    phase_aware: bool,
308) -> Result<ClassifierContract> {
309    let prompt = contract_config.prompt().unwrap_or(PROMPT_TEMPLATE);
310    validate_prompt(prompt)?;
311    let phase_aware_config = phase_aware.then(|| {
312        contract_config.clone().with_prompt(format!(
313            "{}\n\n{}",
314            prompt.trim_end(),
315            DEESCALATION_PROMPT.trim()
316        ))
317    });
318    let contract_config = phase_aware_config.as_ref().unwrap_or(contract_config);
319    ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)
320}
321
322/// The 1-indexed model invocation the transcript ends on: one per assistant reply.
323///
324/// The judge reads the turn *including* the reply it is judging, so the newest assistant
325/// message is this turn — no `+ 1`. Counting the caller's request instead would report the
326/// turn after the one under judgement.
327///
328/// Messages are already normalized by `switchyard-protocol`, so this needs no
329/// per-format branching.
330pub(crate) fn conversation_turn(request: &Request) -> usize {
331    request
332        .llm_request
333        .messages
334        .iter()
335        .filter(|message| message.role == Role::Assistant)
336        .count()
337}
338
339/// Keeps the head and tail of `text` within `limit` characters.
340///
341/// The head gets two thirds of the surviving budget: for a trajectory judge the command or
342/// error signature that opens a message carries more signal than its trailing output.
343fn truncate_middle(text: &str, limit: usize) -> String {
344    let chars: Vec<char> = text.chars().collect();
345    if chars.len() <= limit {
346        return text.to_string();
347    }
348    let keep = limit
349        .saturating_sub(TRIM_MARKER.chars().count())
350        .max(20)
351        .min(chars.len());
352    let head = keep * 2 / 3;
353    let tail = keep - head;
354    let mut out: String = chars[..head].iter().collect();
355    out.push_str(TRIM_MARKER);
356    out.extend(chars[chars.len() - tail..].iter());
357    out
358}
359
360/// Renders a compact role-labelled transcript for the judge.
361///
362/// Task-framing user messages are capped individually. System/developer instructions share
363/// the remaining budget after reserving space for task framing and the newest window entry.
364/// The trailing window carries recent activity. A coverage header states how much history is
365/// not shown, so the judge can reason about pace rather than assuming it sees everything.
366///
367/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go
368/// first: for a trajectory judge the newest evidence is strictly the most valuable.
369fn summarize_for_judge(
370    instructions: &[InstructionBlock],
371    messages: &[Message],
372    turn: usize,
373    phase: Option<EvaluationPhase>,
374    config: &EscalationJudgeConfig,
375) -> String {
376    let mut instruction_anchors: Vec<String> = Vec::new();
377    let mut anchors: Vec<String> = Vec::new();
378    let mut window: Vec<String> = Vec::new();
379    let mut assistant_seen = false;
380
381    for instruction in instructions {
382        let mut parts = Vec::new();
383        collect_text(&instruction.content, &mut parts, &[]);
384        instruction_anchors.push(format!(
385            "[{}] {}",
386            role_label(instruction.role),
387            truncate_middle(&parts.join(" "), SYSTEM_CHARS)
388        ));
389    }
390
391    for message in messages {
392        let text = message_text(message);
393        match message.role {
394            Role::System | Role::Developer => instruction_anchors.push(format!(
395                "[{}] {}",
396                role_label(message.role),
397                truncate_middle(&text, SYSTEM_CHARS)
398            )),
399            // Everything the user said before the agent first replied is task framing.
400            Role::User if !assistant_seen => {
401                anchors.push(format!(
402                    "[user (task)] {}",
403                    truncate_middle(&text, TASK_CHARS)
404                ));
405            }
406            role => {
407                if role == Role::Assistant {
408                    assistant_seen = true;
409                }
410                window.push(format!(
411                    "[{}] {}",
412                    role_label(role),
413                    truncate_middle(&text, config.window_message_chars)
414                ));
415            }
416        }
417    }
418
419    if window.len() > config.recent_turn_window {
420        window.drain(..window.len() - config.recent_turn_window);
421    }
422
423    let assemble = |instructions: Option<&str>, window: &[String]| {
424        let header = format!(
425            "Conversation turn {turn}; showing the last {} of {} messages after the task framing.",
426            window.len(),
427            messages.len(),
428        );
429        phase
430            .map(|phase| format!("Routing phase: {}", phase.marker()))
431            .into_iter()
432            .chain(std::iter::once(header))
433            .chain(instructions.map(str::to_owned))
434            .chain(anchors.iter().cloned())
435            .chain(window.iter().cloned())
436            .collect::<Vec<_>>()
437            .join("\n")
438    };
439
440    let reserved = assemble(None, &window[window.len().saturating_sub(1)..])
441        .chars()
442        .count();
443    let instruction_budget = MAX_REQUEST_CHARS.saturating_sub(reserved + 1);
444    // The remaining budget may be smaller than truncate_middle's minimum retained span.
445    let instruction_text = truncate_middle(&instruction_anchors.join("\n"), instruction_budget)
446        .chars()
447        .take(instruction_budget)
448        .collect::<String>();
449    let instructions = (!instruction_text.is_empty()).then_some(instruction_text.as_str());
450    let mut text = assemble(instructions, &window);
451    while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() {
452        window.remove(0);
453        text = assemble(instructions, &window);
454    }
455    if text.chars().count() > MAX_REQUEST_CHARS {
456        let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1);
457        text = text.chars().take(keep).collect::<String>() + TRUNCATION_SUFFIX;
458    }
459    text
460}
461
462/// The transcript label for a role.
463fn role_label(role: Role) -> &'static str {
464    match role {
465        Role::System => "system",
466        Role::Developer => "developer",
467        Role::User => "user",
468        Role::Assistant => "assistant",
469        Role::Tool => "tool",
470    }
471}
472
473/// A request whose conversation sits at `turn`: `turn - 1` prior assistant replies, each
474/// answered by a further user message.
475///
476/// Shared with the assembled router's tests, which drive the same conversation shape.
477#[cfg(test)]
478pub(crate) fn request_at_turn(session_id: Option<&str>, turn: usize) -> Request {
479    use switchyard_protocol::{LlmRequest, Metadata};
480
481    let mut messages = vec![Message::text(Role::User, "What is 2+2?")];
482    for attempt in 1..turn {
483        messages.push(Message::text(Role::Assistant, format!("attempt {attempt}")));
484        messages.push(Message::text(Role::User, format!("still wrong {attempt}")));
485    }
486    Request {
487        llm_request: LlmRequest {
488            model: Some("auto".to_string()),
489            messages,
490            ..LlmRequest::default()
491        },
492        raw_request: None,
493        metadata: session_id.map(|id| Metadata {
494            session_id: Some(id.to_string()),
495            ..Metadata::default()
496        }),
497    }
498}
499
500#[cfg(test)]
501mod tests {
502    use switchyard_protocol::{ContentBlock, Message, Role};
503
504    use super::*;
505    use crate::algorithms::util::llm_judge::Judge;
506
507    fn escalation_judge(
508        max_output_tokens: u64,
509        phase: Option<EvaluationPhase>,
510        contract_config: &ClassifierContractConfig,
511    ) -> Result<EscalationJudge> {
512        Ok(StructuredJudge::new(
513            EscalationInput {
514                config: EscalationJudgeConfig::default(),
515                phase,
516            },
517            build_contract(contract_config, phase.is_some())?,
518            SerdeDecoder::new(),
519            JudgeRuntimeConfig::new(max_output_tokens)?,
520        ))
521    }
522
523    #[test]
524    fn verdict_logs_keep_bounded_fields_without_the_free_form_reason() -> Result<()> {
525        use crate::core::algorithm::RuntimeModels;
526        use std::sync::{Arc, Mutex};
527
528        #[derive(Clone)]
529        struct LogCapture(Arc<Mutex<Vec<u8>>>);
530        impl std::io::Write for LogCapture {
531            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
532                self.0.lock().unwrap().extend_from_slice(buf);
533                Ok(buf.len())
534            }
535            fn flush(&mut self) -> std::io::Result<()> {
536                Ok(())
537            }
538        }
539        const REASON: &str = "PRIVATE_SYNTHETIC_JUDGE_REASON";
540        let mut invalid =
541            serde_json::json!({"escalate": false, "category": "none", "new_evidence": false});
542        assert!(serde_json::from_value::<EscalationVerdict>(invalid.clone()).is_err());
543        invalid["reason"] = serde_json::json!(123);
544        assert!(serde_json::from_value::<EscalationVerdict>(invalid).is_err());
545        for phase in [
546            None,
547            Some(EvaluationPhase::Efficient),
548            Some(EvaluationPhase::Strong),
549        ] {
550            for escalate in [false, true] {
551                let capture = Arc::new(Mutex::new(Vec::<u8>::new()));
552                let writer = LogCapture(capture.clone());
553                let subscriber = tracing_subscriber::fmt()
554                    .without_time()
555                    .with_ansi(false)
556                    .with_max_level(tracing::Level::DEBUG)
557                    .with_writer(move || writer.clone())
558                    .finish();
559                let models = RuntimeModels::new(
560                    [
561                        (Category::Efficient, vec!["efficient".into()]),
562                        (Category::Capable, vec!["capable".into()]),
563                    ]
564                    .into(),
565                );
566                let (driver, _receiver) = Driver::new("escalation-log-probe", Arc::new(models));
567                let verdict = serde_json::from_value(serde_json::json!({
568                    "escalate": escalate, "category": "repetition",
569                    "new_evidence": true, "reason": REASON,
570                }))
571                .expect("valid synthetic verdict");
572                let classification = tracing::subscriber::with_default(subscriber, || {
573                    EscalationPolicy { phase }.to_classification(Some(&verdict), &driver)
574                })?;
575                let Classification::Scores(scores) = classification else {
576                    panic!("parsed verdict must yield scores");
577                };
578                assert_eq!(
579                    scores[0].target.as_str(),
580                    if escalate { "capable" } else { "efficient" }
581                );
582                let logs = String::from_utf8(capture.lock().unwrap().clone()).unwrap();
583                assert!(logs.contains("escalation judge verdict"), "{logs}");
584                assert!(!logs.contains(REASON), "{logs}");
585                assert!(!logs.contains("reason="), "{logs}");
586                assert!(logs.contains(&format!("escalate={escalate}")), "{logs}");
587                assert!(logs.contains("category=\"repetition\""), "{logs}");
588                assert!(logs.contains("new_evidence=true"), "{logs}");
589            }
590        }
591        Ok(())
592    }
593
594    #[test]
595    fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> {
596        let judge = escalation_judge(
597            super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
598            None,
599            &ClassifierContractConfig::default(),
600        )?;
601
602        // As the classifier calls it: the turn's reply is already on the transcript.
603        let mut judged = request_at_turn(None, 4);
604        judged.llm_request.instructions = [
605            (Role::System, "system constraint"),
606            (Role::Developer, "developer constraint"),
607        ]
608        .into_iter()
609        .map(|(role, text)| InstructionBlock {
610            role,
611            content: Message::text(role, text).content,
612        })
613        .collect();
614        judged
615            .llm_request
616            .messages
617            .push(Message::text(Role::Assistant, "this turn's reply"));
618        let built = judge.build_request(&State::default(), &judged);
619
620        // Rubric in instructions, condensed trajectory as the sole user message.
621        assert_eq!(built.llm_request.instructions.len(), 1);
622        assert_eq!(built.llm_request.instructions[0].role, Role::System);
623        assert_eq!(
624            built.llm_request.instructions[0].content.as_slice(),
625            &[ContentBlock::Text {
626                text: PROMPT_TEMPLATE.to_string()
627            }]
628        );
629        assert_eq!(built.llm_request.messages.len(), 1);
630        assert_eq!(built.llm_request.messages[0].role, Role::User);
631        let summary = built.llm_request.messages[0]
632            .text_content("")
633            .expect("summary");
634        assert!(summary.contains("Conversation turn 4"));
635        assert!(summary.contains(
636            "[system] system constraint\n[developer] developer constraint\n[user (task)] What is 2+2?"
637        ));
638        assert!(summary.contains("[assistant] this turn's reply"));
639        assert!(!summary.contains("Routing phase:"));
640        // Bounded output, so a reasoning judge cannot run away mid-verdict.
641        assert_eq!(
642            built.llm_request.output.max_output_tokens,
643            Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
644        );
645        assert!(built.llm_request.output.response_format.is_some());
646
647        judged.llm_request.instructions.extend(vec![
648            InstructionBlock {
649                role: Role::Developer,
650                content: Message::text(Role::Developer, "x".repeat(SYSTEM_CHARS)).content,
651            };
652            MAX_REQUEST_CHARS / SYSTEM_CHARS + 1
653        ]);
654        let built = judge.build_request(&State::default(), &judged);
655        let summary = built.llm_request.messages[0]
656            .text_content("")
657            .expect("summary");
658        assert!(summary.chars().count() <= MAX_REQUEST_CHARS);
659        assert!(summary.contains(TRIM_MARKER));
660        assert!(summary.contains("[system] system constraint"));
661        assert!(summary.contains("[developer] developer constraint"));
662        assert!(summary.contains("[user (task)] What is 2+2?"));
663        assert!(summary.contains("[assistant] this turn's reply"));
664        Ok(())
665    }
666
667    #[test]
668    fn judge_request_uses_the_configured_completion_cap() -> Result<()> {
669        let judge = escalation_judge(512, None, &ClassifierContractConfig::default())?;
670
671        let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
672
673        assert_eq!(built.llm_request.output.max_output_tokens, Some(512));
674        Ok(())
675    }
676
677    #[test]
678    fn conversation_turn_counts_assistant_replies() {
679        // The judge is handed the transcript with this turn's reply already appended, which
680        // is the shape asserted here: a request entering turn N plus its reply *is* turn N.
681        for turn in [1, 5] {
682            let mut judged = request_at_turn(None, turn);
683            judged
684                .llm_request
685                .messages
686                .push(Message::text(Role::Assistant, "this turn's reply"));
687            assert_eq!(conversation_turn(&judged), turn);
688        }
689    }
690
691    #[test]
692    fn truncate_middle_keeps_head_and_tail() {
693        let text = "a".repeat(40) + &"z".repeat(40);
694        let trimmed = truncate_middle(&text, 50);
695        assert!(trimmed.chars().count() <= 50, "{trimmed}");
696        assert!(trimmed.starts_with('a'));
697        assert!(trimmed.ends_with('z'));
698        assert!(trimmed.contains("[trimmed]"));
699
700        // Under the limit the text is returned untouched.
701        assert_eq!(truncate_middle("short", 50), "short");
702    }
703
704    #[test]
705    fn deescalation_settings_must_be_valid() {
706        let zero = EscalationJudgeConfig {
707            deescalation: Some(DeescalationConfig {
708                strong_min_calls: 0,
709                strong_max_calls: None,
710                confirmations: 2,
711                weak_cooldown_calls: 0,
712            }),
713            ..EscalationJudgeConfig::default()
714        };
715        assert!(
716            zero.validate()
717                .is_err_and(|error| error.to_string().contains("at least 1"))
718        );
719
720        let inverted = EscalationJudgeConfig {
721            deescalation: Some(DeescalationConfig {
722                strong_min_calls: 4,
723                strong_max_calls: Some(3),
724                confirmations: 2,
725                weak_cooldown_calls: 0,
726            }),
727            ..EscalationJudgeConfig::default()
728        };
729        assert!(
730            inverted
731                .validate()
732                .is_err_and(|error| error.to_string().contains("at least deescalation"))
733        );
734    }
735
736    #[test]
737    fn deescalation_contract_marks_both_routing_phases() -> Result<()> {
738        let contract = ClassifierContractConfig::default().with_prompt("Custom trajectory rubric.");
739        for (phase, marker) in [
740            (EvaluationPhase::Efficient, "EFFICIENT_EVALUATION"),
741            (EvaluationPhase::Strong, "STRONG_EVALUATION"),
742        ] {
743            let judge = escalation_judge(512, Some(phase), &contract)?;
744            let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
745            let system_prompt = built.llm_request.instructions[0].content.iter().find_map(
746                |content| match content {
747                    ContentBlock::Text { text } => Some(text.as_str()),
748                    _ => None,
749                },
750            );
751            assert!(system_prompt.is_some_and(|prompt| {
752                prompt.starts_with("Custom trajectory rubric.")
753                    && prompt.contains("EFFICIENT_EVALUATION")
754                    && prompt.contains("STRONG_EVALUATION")
755            }));
756            assert!(
757                built.llm_request.messages[0]
758                    .text_content("")
759                    .is_some_and(|summary| summary.starts_with(&format!(
760                        "Routing phase: {marker}"
761                    )))
762            );
763        }
764        Ok(())
765    }
766
767    #[test]
768    fn summary_keeps_anchors_and_the_recent_window() {
769        let mut messages = vec![
770            Message::text(Role::System, "you are a coding agent"),
771            Message::text(Role::User, "fix the failing test"),
772        ];
773        for i in 0..10 {
774            messages.push(Message::text(Role::Assistant, format!("step {i}")));
775        }
776        let config = EscalationJudgeConfig {
777            recent_turn_window: 3,
778            ..EscalationJudgeConfig::default()
779        };
780
781        let summary = summarize_for_judge(&[], &messages, 11, None, &config);
782
783        assert!(
784            summary.contains("[system] you are a coding agent"),
785            "{summary}"
786        );
787        assert!(
788            summary.contains("[user (task)] fix the failing test"),
789            "{summary}"
790        );
791        assert!(summary.contains("Conversation turn 11; showing the last 3 of 12 messages"));
792        // Only the newest window entries survive.
793        assert!(summary.contains("step 9"), "{summary}");
794        assert!(summary.contains("step 7"), "{summary}");
795        assert!(!summary.contains("step 6"), "{summary}");
796    }
797
798    #[test]
799    fn summary_anchors_every_user_message_before_the_first_reply() {
800        // Codex sends environment boilerplate as the first user message and the task as the
801        // second. Both are framing; the task must stay visible after the window has moved on.
802        let mut messages = vec![
803            Message::text(
804                Role::Developer,
805                "<skills_instructions>...</skills_instructions>",
806            ),
807            Message::text(
808                Role::User,
809                "<environment_context><cwd>/app</cwd></environment_context>",
810            ),
811            Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."),
812        ];
813        for i in 0..40 {
814            messages.push(Message::text(Role::Assistant, format!("step {i}")));
815            messages.push(Message::text(Role::User, format!("later user note {i}")));
816        }
817        let config = EscalationJudgeConfig {
818            recent_turn_window: 3,
819            ..EscalationJudgeConfig::default()
820        };
821
822        let summary = summarize_for_judge(&[], &messages, 40, None, &config);
823
824        assert!(
825            summary.contains("[user (task)] <environment_context>"),
826            "{summary}"
827        );
828        assert!(
829            summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."),
830            "{summary}"
831        );
832        // User messages after the first reply are ordinary window entries, not anchors.
833        assert!(
834            !summary.contains("[user (task)] later user note"),
835            "{summary}"
836        );
837        assert!(summary.contains("[user] later user note 39"), "{summary}");
838        assert!(!summary.contains("later user note 0\n"), "{summary}");
839    }
840
841    #[test]
842    fn summary_drops_oldest_window_lines_under_the_char_cap() {
843        // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps
844        // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually
845        // wide per-message cap. That is the point — it only fires on pathological input.
846        let mut messages = vec![
847            Message::text(Role::System, "framing"),
848            Message::text(Role::User, "task"),
849        ];
850        for i in 0..20 {
851            messages.push(Message::text(
852                Role::Assistant,
853                format!("{i} {}", "x".repeat(2_000)),
854            ));
855        }
856        let config = EscalationJudgeConfig {
857            window_message_chars: 2_000,
858            ..EscalationJudgeConfig::default()
859        };
860
861        let summary = summarize_for_judge(&[], &messages, 21, None, &config);
862
863        assert!(
864            summary.chars().count() <= MAX_REQUEST_CHARS,
865            "{}",
866            summary.chars().count()
867        );
868        // Anchors are never dropped, and the newest activity outlives the oldest.
869        assert!(summary.contains("[system] framing"), "{summary}");
870        assert!(summary.contains("[user (task)] task"), "{summary}");
871        assert!(summary.contains("19 xxx"), "{summary}");
872        assert!(!summary.contains("0 xxx"), "{summary}");
873    }
874}
switchyard_libsy/algorithms/util/escalation.rs

switchyard_libsy/algorithms/util/
escalation.rs
