1use serde::Deserialize;
11use serde_json::Value;
12use switchyard_protocol::{Category, ContentBlock, InstructionBlock, Message, Role};
13
14use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};
15use super::llm_judge::{
16 ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder,
17 StructuredJudge,
18};
19use crate::core::algorithm::Driver;
20use crate::core::classifier::{Classification, Score};
21use crate::core::state::State;
22use crate::{LibsyError, Result};
23use switchyard_protocol::Request;
24
25const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md");
26const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json");
27
28const TRIM_MARKER: &str = " ...[trimmed] ";
30
31const TRUNCATION_SUFFIX: &str = "...<truncated>";
33
34const SYSTEM_CHARS: usize = 1_000;
37
38const TASK_CHARS: usize = 4_000;
44
45const MAX_REQUEST_CHARS: usize = 18_000;
47
48#[derive(Clone, Debug, Deserialize)]
53#[serde(default, deny_unknown_fields)]
54pub struct EscalationJudgeConfig {
55 pub confirmations: u32,
60 pub recent_turn_window: usize,
62 pub window_message_chars: usize,
64}
65
66impl EscalationJudgeConfig {
67 fn validate(&self) -> Result<()> {
69 let reject = |message: String| Err(LibsyError::AlgorithmError { message });
70 if self.confirmations == 0 {
71 return reject("confirmations must be at least 1".to_string());
72 }
73 if self.recent_turn_window == 0 {
74 return reject("recent_turn_window must be at least 1".to_string());
75 }
76 if self.window_message_chars < 50 {
77 return reject(format!(
78 "window_message_chars must be at least 50, got {}",
79 self.window_message_chars
80 ));
81 }
82 Ok(())
83 }
84}
85
86impl Default for EscalationJudgeConfig {
87 fn default() -> Self {
88 Self {
89 confirmations: 2,
90 recent_turn_window: 28,
91 window_message_chars: 500,
92 }
93 }
94}
95
96#[derive(Deserialize)]
101pub(crate) struct EscalationVerdict {
102 escalate: bool,
103 #[serde(default)]
104 reason: String,
105}
106
107pub(crate) struct EscalationInput {
109 config: EscalationJudgeConfig,
110}
111
112impl ClassifierInput for EscalationInput {
113 fn build_messages(&self, _state: &State, request: &Request) -> Vec<Message> {
114 let summary = summarize_for_judge(
115 &request.llm_request.instructions,
116 &request.llm_request.messages,
117 conversation_turn(request),
118 &self.config,
119 );
120 vec![Message::text(Role::User, summary)]
121 }
122}
123
124pub(crate) type EscalationJudge = StructuredJudge<EscalationInput, SerdeDecoder<EscalationVerdict>>;
126
127pub(crate) struct EscalationPolicy;
133
134impl JudgePolicy for EscalationPolicy {
135 type Verdict = EscalationVerdict;
136
137 fn to_classification(
138 &self,
139 verdict: Option<&EscalationVerdict>,
140 driver: &Driver,
141 ) -> Result<Classification> {
142 if let Some(verdict) = verdict {
143 tracing::debug!(
144 escalate = verdict.escalate,
145 reason = %verdict.reason,
146 "escalation judge verdict"
147 );
148 }
149 match verdict {
150 Some(verdict) if verdict.escalate => Ok(Classification::Scores(vec![Score {
151 target: driver.first_model_for(&Category::Capable)?.clone(),
152 confidence: 1.0,
153 category: Some(Category::Capable),
154 }])),
155 Some(_) => Ok(Classification::Scores(vec![Score {
156 target: driver.first_model_for(&Category::Efficient)?.clone(),
157 confidence: 1.0,
158 category: Some(Category::Efficient),
159 }])),
160 None => Ok(Classification::Ambiguous(Vec::new())),
161 }
162 }
163}
164
165fn escalation_evidence(
167 _policy: &EscalationPolicy,
168 verdict: Option<&EscalationVerdict>,
169) -> Option<Value> {
170 verdict.map(|verdict| {
171 serde_json::json!({
172 "source": "escalation",
173 "verdict": if verdict.escalate { "escalate" } else { "continue" },
174 })
175 })
176}
177
178pub(crate) fn build_judge(
183 contract_config: &ClassifierContractConfig,
184 config: EscalationJudgeConfig,
185 max_output_tokens: u64,
186) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
187 config.validate()?;
188 let contract =
189 ClassifierContract::from_config(contract_config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
190 Ok(JudgeClassifier::new(
191 StructuredJudge::new(
192 EscalationInput { config },
193 contract,
194 SerdeDecoder::new(),
195 JudgeRuntimeConfig::new(max_output_tokens)?,
196 ),
197 EscalationPolicy,
198 )
199 .with_evidence(escalation_evidence))
200}
201
202pub(crate) fn conversation_turn(request: &Request) -> usize {
211 request
212 .llm_request
213 .messages
214 .iter()
215 .filter(|message| message.role == Role::Assistant)
216 .count()
217}
218
219fn message_text(message: &Message) -> String {
225 let mut parts = Vec::new();
226 collect_text(&message.content, &mut parts);
227 parts.join(" ")
228}
229
230fn collect_text(content: &[ContentBlock], parts: &mut Vec<String>) {
232 for block in content {
233 match block {
234 ContentBlock::Text { text } | ContentBlock::Refusal { text } => {
235 parts.push(text.clone());
236 }
237 ContentBlock::ToolCall(call) => {
238 parts.push(format!("tool_call {}({})", call.name, call.arguments));
239 }
240 ContentBlock::ToolResult(result) => collect_text(&result.content, parts),
241 _ => {}
242 }
243 }
244}
245
246fn truncate_middle(text: &str, limit: usize) -> String {
251 let chars: Vec<char> = text.chars().collect();
252 if chars.len() <= limit {
253 return text.to_string();
254 }
255 let keep = limit
256 .saturating_sub(TRIM_MARKER.chars().count())
257 .max(20)
258 .min(chars.len());
259 let head = keep * 2 / 3;
260 let tail = keep - head;
261 let mut out: String = chars[..head].iter().collect();
262 out.push_str(TRIM_MARKER);
263 out.extend(chars[chars.len() - tail..].iter());
264 out
265}
266
267fn summarize_for_judge(
277 instructions: &[InstructionBlock],
278 messages: &[Message],
279 turn: usize,
280 config: &EscalationJudgeConfig,
281) -> String {
282 let mut instruction_anchors: Vec<String> = Vec::new();
283 let mut anchors: Vec<String> = Vec::new();
284 let mut window: Vec<String> = Vec::new();
285 let mut assistant_seen = false;
286
287 for instruction in instructions {
288 let mut parts = Vec::new();
289 collect_text(&instruction.content, &mut parts);
290 instruction_anchors.push(format!(
291 "[{}] {}",
292 role_label(instruction.role),
293 truncate_middle(&parts.join(" "), SYSTEM_CHARS)
294 ));
295 }
296
297 for message in messages {
298 let text = message_text(message);
299 match message.role {
300 Role::System | Role::Developer => instruction_anchors.push(format!(
301 "[{}] {}",
302 role_label(message.role),
303 truncate_middle(&text, SYSTEM_CHARS)
304 )),
305 Role::User if !assistant_seen => {
307 anchors.push(format!(
308 "[user (task)] {}",
309 truncate_middle(&text, TASK_CHARS)
310 ));
311 }
312 role => {
313 if role == Role::Assistant {
314 assistant_seen = true;
315 }
316 window.push(format!(
317 "[{}] {}",
318 role_label(role),
319 truncate_middle(&text, config.window_message_chars)
320 ));
321 }
322 }
323 }
324
325 if window.len() > config.recent_turn_window {
326 window.drain(..window.len() - config.recent_turn_window);
327 }
328
329 let assemble = |instructions: Option<&str>, window: &[String]| {
330 let header = format!(
331 "Conversation turn {turn}; showing the last {} of {} messages after the task framing.",
332 window.len(),
333 messages.len(),
334 );
335 std::iter::once(header)
336 .chain(instructions.map(str::to_owned))
337 .chain(anchors.iter().cloned())
338 .chain(window.iter().cloned())
339 .collect::<Vec<_>>()
340 .join("\n")
341 };
342
343 let reserved = assemble(None, &window[window.len().saturating_sub(1)..])
344 .chars()
345 .count();
346 let instruction_budget = MAX_REQUEST_CHARS.saturating_sub(reserved + 1);
347 let instruction_text = truncate_middle(&instruction_anchors.join("\n"), instruction_budget)
349 .chars()
350 .take(instruction_budget)
351 .collect::<String>();
352 let instructions = (!instruction_text.is_empty()).then_some(instruction_text.as_str());
353 let mut text = assemble(instructions, &window);
354 while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() {
355 window.remove(0);
356 text = assemble(instructions, &window);
357 }
358 if text.chars().count() > MAX_REQUEST_CHARS {
359 let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1);
360 text = text.chars().take(keep).collect::<String>() + TRUNCATION_SUFFIX;
361 }
362 text
363}
364
365fn role_label(role: Role) -> &'static str {
367 match role {
368 Role::System => "system",
369 Role::Developer => "developer",
370 Role::User => "user",
371 Role::Assistant => "assistant",
372 Role::Tool => "tool",
373 }
374}
375
376#[cfg(test)]
381pub(crate) fn request_at_turn(session_id: Option<&str>, turn: usize) -> Request {
382 use switchyard_protocol::{LlmRequest, Metadata};
383
384 let mut messages = vec![Message::text(Role::User, "What is 2+2?")];
385 for attempt in 1..turn {
386 messages.push(Message::text(Role::Assistant, format!("attempt {attempt}")));
387 messages.push(Message::text(Role::User, format!("still wrong {attempt}")));
388 }
389 Request {
390 llm_request: LlmRequest {
391 model: Some("auto".to_string()),
392 messages,
393 ..LlmRequest::default()
394 },
395 raw_request: None,
396 metadata: session_id.map(|id| Metadata {
397 session_id: Some(id.to_string()),
398 ..Metadata::default()
399 }),
400 }
401}
402
403#[cfg(test)]
404mod tests {
405 use serde_json::json;
406 use switchyard_protocol::{ContentBlock, Message, Role, ToolCall, ToolResult};
407
408 use super::*;
409 use crate::algorithms::util::llm_judge::Judge;
410
411 fn escalation_judge(max_output_tokens: u64) -> Result<EscalationJudge> {
412 Ok(StructuredJudge::new(
413 EscalationInput {
414 config: EscalationJudgeConfig::default(),
415 },
416 ClassifierContract::from_config(
417 &ClassifierContractConfig::default(),
418 PROMPT_TEMPLATE,
419 SCHEMA_TEMPLATE,
420 )?,
421 SerdeDecoder::new(),
422 JudgeRuntimeConfig::new(max_output_tokens)?,
423 ))
424 }
425
426 #[test]
427 fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> {
428 let judge = escalation_judge(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?;
429
430 let mut judged = request_at_turn(None, 4);
432 judged.llm_request.instructions = [
433 (Role::System, "system constraint"),
434 (Role::Developer, "developer constraint"),
435 ]
436 .into_iter()
437 .map(|(role, text)| InstructionBlock {
438 role,
439 content: Message::text(role, text).content,
440 })
441 .collect();
442 judged
443 .llm_request
444 .messages
445 .push(Message::text(Role::Assistant, "this turn's reply"));
446 let built = judge.build_request(&State::default(), &judged);
447
448 assert_eq!(built.llm_request.instructions.len(), 1);
450 assert_eq!(built.llm_request.instructions[0].role, Role::System);
451 assert_eq!(built.llm_request.messages.len(), 1);
452 assert_eq!(built.llm_request.messages[0].role, Role::User);
453 let summary = built.llm_request.messages[0]
454 .text_content("")
455 .expect("summary");
456 assert!(summary.contains("Conversation turn 4"));
457 assert!(summary.contains(
458 "[system] system constraint\n[developer] developer constraint\n[user (task)] What is 2+2?"
459 ));
460 assert!(summary.contains("[assistant] this turn's reply"));
461 assert_eq!(
463 built.llm_request.output.max_output_tokens,
464 Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
465 );
466 assert!(built.llm_request.output.response_format.is_some());
467
468 judged.llm_request.instructions.extend(vec![
469 InstructionBlock {
470 role: Role::Developer,
471 content: Message::text(Role::Developer, "x".repeat(SYSTEM_CHARS)).content,
472 };
473 MAX_REQUEST_CHARS / SYSTEM_CHARS + 1
474 ]);
475 let built = judge.build_request(&State::default(), &judged);
476 let summary = built.llm_request.messages[0]
477 .text_content("")
478 .expect("summary");
479 assert!(summary.chars().count() <= MAX_REQUEST_CHARS);
480 assert!(summary.contains(TRIM_MARKER));
481 assert!(summary.contains("[system] system constraint"));
482 assert!(summary.contains("[developer] developer constraint"));
483 assert!(summary.contains("[user (task)] What is 2+2?"));
484 assert!(summary.contains("[assistant] this turn's reply"));
485 Ok(())
486 }
487
488 #[test]
489 fn judge_request_uses_the_configured_completion_cap() -> Result<()> {
490 let judge = escalation_judge(512)?;
491
492 let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
493
494 assert_eq!(built.llm_request.output.max_output_tokens, Some(512));
495 Ok(())
496 }
497
498 #[test]
499 fn conversation_turn_counts_assistant_replies() {
500 for turn in [1, 5] {
503 let mut judged = request_at_turn(None, turn);
504 judged
505 .llm_request
506 .messages
507 .push(Message::text(Role::Assistant, "this turn's reply"));
508 assert_eq!(conversation_turn(&judged), turn);
509 }
510 }
511
512 #[test]
513 fn message_text_keeps_tool_calls_and_results() {
514 let call = Message {
515 role: Role::Assistant,
516 content: vec![
517 ContentBlock::Text {
518 text: "running it".to_string(),
519 },
520 ContentBlock::ToolCall(ToolCall {
521 id: "call-1".to_string(),
522 name: "bash".to_string(),
523 arguments: json!({"cmd": "ls"}),
524 }),
525 ],
526 };
527 let text = message_text(&call);
528 assert!(text.contains("running it"), "{text}");
529 assert!(text.contains(r#"tool_call bash({"cmd":"ls"})"#), "{text}");
530
531 let result = Message {
532 role: Role::Tool,
533 content: vec![ContentBlock::ToolResult(ToolResult {
534 tool_call_id: "call-1".to_string(),
535 content: vec![ContentBlock::Text {
536 text: "no such file".to_string(),
537 }],
538 is_error: Some(true),
539 })],
540 };
541 assert_eq!(message_text(&result), "no such file");
542 }
543
544 #[test]
545 fn truncate_middle_keeps_head_and_tail() {
546 let text = "a".repeat(40) + &"z".repeat(40);
547 let trimmed = truncate_middle(&text, 50);
548 assert!(trimmed.chars().count() <= 50, "{trimmed}");
549 assert!(trimmed.starts_with('a'));
550 assert!(trimmed.ends_with('z'));
551 assert!(trimmed.contains("[trimmed]"));
552
553 assert_eq!(truncate_middle("short", 50), "short");
555 }
556
557 #[test]
558 fn summary_keeps_anchors_and_the_recent_window() {
559 let mut messages = vec![
560 Message::text(Role::System, "you are a coding agent"),
561 Message::text(Role::User, "fix the failing test"),
562 ];
563 for i in 0..10 {
564 messages.push(Message::text(Role::Assistant, format!("step {i}")));
565 }
566 let config = EscalationJudgeConfig {
567 recent_turn_window: 3,
568 ..EscalationJudgeConfig::default()
569 };
570
571 let summary = summarize_for_judge(&[], &messages, 11, &config);
572
573 assert!(
574 summary.contains("[system] you are a coding agent"),
575 "{summary}"
576 );
577 assert!(
578 summary.contains("[user (task)] fix the failing test"),
579 "{summary}"
580 );
581 assert!(summary.contains("Conversation turn 11; showing the last 3 of 12 messages"));
582 assert!(summary.contains("step 9"), "{summary}");
584 assert!(summary.contains("step 7"), "{summary}");
585 assert!(!summary.contains("step 6"), "{summary}");
586 }
587
588 #[test]
589 fn summary_anchors_every_user_message_before_the_first_reply() {
590 let mut messages = vec![
593 Message::text(
594 Role::Developer,
595 "<skills_instructions>...</skills_instructions>",
596 ),
597 Message::text(
598 Role::User,
599 "<environment_context><cwd>/app</cwd></environment_context>",
600 ),
601 Message::text(Role::User, "Implement RFC 5545 timezone interop in rrule."),
602 ];
603 for i in 0..40 {
604 messages.push(Message::text(Role::Assistant, format!("step {i}")));
605 messages.push(Message::text(Role::User, format!("later user note {i}")));
606 }
607 let config = EscalationJudgeConfig {
608 recent_turn_window: 3,
609 ..EscalationJudgeConfig::default()
610 };
611
612 let summary = summarize_for_judge(&[], &messages, 40, &config);
613
614 assert!(
615 summary.contains("[user (task)] <environment_context>"),
616 "{summary}"
617 );
618 assert!(
619 summary.contains("[user (task)] Implement RFC 5545 timezone interop in rrule."),
620 "{summary}"
621 );
622 assert!(
624 !summary.contains("[user (task)] later user note"),
625 "{summary}"
626 );
627 assert!(summary.contains("[user] later user note 39"), "{summary}");
628 assert!(!summary.contains("later user note 0\n"), "{summary}");
629 }
630
631 #[test]
632 fn summary_drops_oldest_window_lines_under_the_char_cap() {
633 let mut messages = vec![
637 Message::text(Role::System, "framing"),
638 Message::text(Role::User, "task"),
639 ];
640 for i in 0..20 {
641 messages.push(Message::text(
642 Role::Assistant,
643 format!("{i} {}", "x".repeat(2_000)),
644 ));
645 }
646 let config = EscalationJudgeConfig {
647 window_message_chars: 2_000,
648 ..EscalationJudgeConfig::default()
649 };
650
651 let summary = summarize_for_judge(&[], &messages, 21, &config);
652
653 assert!(
654 summary.chars().count() <= MAX_REQUEST_CHARS,
655 "{}",
656 summary.chars().count()
657 );
658 assert!(summary.contains("[system] framing"), "{summary}");
660 assert!(summary.contains("[user (task)] task"), "{summary}");
661 assert!(summary.contains("19 xxx"), "{summary}");
662 assert!(!summary.contains("0 xxx"), "{summary}");
663 }
664}