1#![allow(dead_code)]
14
15use std::collections::HashMap;
16use std::path::Path;
17use std::sync::LazyLock;
18
19use async_trait::async_trait;
20use regex::Regex;
21use serde::Deserialize;
22use serde_json::Value;
23use switchyard_protocol::codex_namespaces::{split_qualified_name, tool_namespaces};
24use switchyard_protocol::{ContentBlock, Request, Role, WireFormat};
25
26use super::affinity::is_user_turn;
27use crate::{LibsyError, Result};
28
29use crate::core::processor::{Event, Processor};
30use crate::core::state::State;
31
32const SOFT: f32 = 0.3;
35const HARD: f32 = 0.7;
36const CRITICAL: f32 = 1.0;
37
38static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[
42 (
43 "oom",
44 CRITICAL,
45 &["out of memory", "memoryerror", "cannot allocate memory"],
46 ),
47 (
48 "connection_refused",
49 HARD,
50 &[
51 "connection refused",
52 "connectionrefusederror",
53 "econnrefused",
54 ],
55 ),
56 ("traceback", HARD, &["traceback (most recent call last)"]),
57 (
58 "import_error",
59 HARD,
60 &["modulenotfounderror:", "importerror:", "no module named "],
61 ),
62 (
63 "cmd_not_found",
64 HARD,
65 &["command not found", "/usr/bin/env: "],
66 ),
67 ("assertion", HARD, &["assertionerror"]),
68 ("value_error", HARD, &["valueerror:"]),
69 ("syntax_error", HARD, &["syntaxerror:"]),
70 (
71 "timeout",
72 HARD,
73 &[
74 "timed out",
75 "timeouterror",
76 "timeout expired",
77 "deadline exceeded",
78 ],
79 ),
80 (
81 "no_such_file",
82 HARD,
83 &[
84 "filenotfounderror:",
85 "no such file or directory",
86 "file does not exist",
91 ],
92 ),
93 ("exit_nonzero", SOFT, &["returned non-zero"]),
95];
96
97static NONZERO_EXIT_PHRASES: &[&str] = &[
98 "exit code",
99 "exit status",
100 "exited with code",
101 "exited with status",
102];
103
104static EDIT_TOOL_NAMES: &[&str] = &[
105 "edit",
106 "multiedit",
107 "notebookedit",
108 "str_replace",
109 "str_replace_based_edit_tool",
110 "apply_patch", "text_editor",
112 "patch", ];
114
115static EDITOR_TOOL_NAMES: &[&str] = &["str_replace_based_edit_tool", "text_editor"];
117
118static WRITE_TOOL_NAMES: &[&str] = &["write", "create_file", "new_file", "write_file"];
119
120static BASH_WRITE_PATTERNS: &[&str] = &[
124 "cat >",
125 "cat >>",
126 "echo >",
127 "echo >>",
128 "tee ",
129 "printf >",
130 "printf >>",
131 "> /",
132 ">> /",
133 "<< 'eof'",
134 "<<eof",
135 "<<'eof'",
136 "<< eof",
137];
138
139static PYTHON_WRITE_PATTERNS: &[&str] = &["write_text(", "writelines(", ".write("];
142
143static JAVASCRIPT_WRITE_PATTERNS: &[&str] = &[
144 "writefilesync(",
145 "writefile(",
146 "appendfilesync(",
147 "appendfile(",
148];
149
150static BASH_EDIT_PATTERNS: &[&str] = &[
151 "sed -i",
152 "sed --in-place",
153 "awk -i inplace",
154 "awk 'inplace=1'",
155 "patch ",
156 "patch -p",
157 "perl -i",
158 "perl -p -i",
159 "perl -pi",
160];
161
162static BASH_READ_PATTERNS: &[&str] = &[
165 "cat /", "cat ./", "cat ../", "grep ", "ls ", "ls -", "find ", "head ", "tail ", "wc ",
166 "diff ", "which ", "ps ", "df ", "du ", "stat ", "file ", "less ", "more ",
167];
168
169static BASH_READ_COMMANDS: &[&str] = &[
172 "cat", "rg", "nl", "jq", "pwd", "tree", "sed", "grep", "ls", "find", "head", "tail", "wc",
173 "diff", "which", "ps", "df", "du", "stat", "file", "less", "more", "readlink", "realpath",
174 "basename", "dirname", "printenv",
175];
176
177static GIT_READ_SUBCOMMANDS: &[&str] = &[
178 "status",
179 "diff",
180 "log",
181 "show",
182 "show-ref",
183 "rev-parse",
184 "ls-files",
185 "ls-remote",
186 "ls-tree",
187 "grep",
188 "blame",
189 "merge-base",
190 "check-ignore",
191 "tag",
192];
193
194static READ_TOOL_NAMES: &[&str] = &[
195 "read",
196 "view",
197 "read_file",
198 "search_files",
199 "glob",
200 "grep",
201 "find",
202 "ls",
203];
204
205static PLAN_TOOL_NAMES: &[&str] = &[
208 "todowrite",
209 "todo_write",
210 "todo",
211 "update_plan",
212 "todo_list",
213];
214
215static BASH_TOOL_NAMES: &[&str] = &[
220 "bash",
221 "shell_command",
222 "shell",
223 "local_shell_call",
224 "terminal",
225 "exec_command", "exec", "powershell", ];
229
230static TEST_PASS_PHRASES: &[&str] = &[
233 " passed",
234 "passed in",
235 "tests passed",
236 "all tests passed",
237 "test ok",
238 "test result: ok",
239 "passed.\n",
240 "tests pass",
241 "\nok ", "✓ ",
243];
244
245static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "error:"];
250
251static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"];
255
256pub const DEFAULT_RECENT_WINDOW: usize = 3;
263
264#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
270#[serde(default, deny_unknown_fields)]
271pub struct ToolSemantics {
272 pub observe: Vec<String>,
274 pub mutate: Vec<String>,
276 pub plan: Vec<String>,
278 pub new: Vec<String>,
280}
281
282impl ToolSemantics {
283 pub fn validate(&self) -> Result<()> {
285 let mut seen: Vec<(String, &'static str)> = Vec::new();
286 for (category, names) in [
287 ("observe", &self.observe),
288 ("mutate", &self.mutate),
289 ("plan", &self.plan),
290 ("new", &self.new),
291 ] {
292 for name in names {
293 if name.trim().is_empty() {
294 return Err(tool_semantics_error(format!(
295 "tool_semantics.{category} contains an empty tool name"
296 )));
297 }
298 let normalized = name.to_ascii_lowercase();
299 if is_builtin_tool_name(&name.to_lowercase()) {
300 return Err(tool_semantics_error(format!(
301 "tool {name:?} already has built-in semantics and cannot be reclassified"
302 )));
303 }
304 if let Some((_, previous)) = seen.iter().find(|(seen, _)| seen == &normalized) {
305 return Err(tool_semantics_error(format!(
306 "tool {name:?} appears in both tool_semantics.{previous} and tool_semantics.{category}"
307 )));
308 }
309 seen.push((normalized, category));
310 }
311 }
312 Ok(())
313 }
314
315 fn classify(&self, name: &str) -> Option<ToolSemantic> {
316 if contains_name(&self.observe, name) {
317 Some(ToolSemantic::Observe)
318 } else if contains_name(&self.mutate, name) {
319 Some(ToolSemantic::Mutate(MutationKind::Write))
322 } else if contains_name(&self.plan, name) {
323 Some(ToolSemantic::Plan)
324 } else if contains_name(&self.new, name) {
325 Some(ToolSemantic::New)
326 } else {
327 None
328 }
329 }
330}
331
332fn contains_name(names: &[String], candidate: &str) -> bool {
333 names
334 .iter()
335 .any(|name| name.eq_ignore_ascii_case(candidate))
336}
337
338fn tool_semantics_error(message: String) -> LibsyError {
339 LibsyError::AlgorithmError { message }
340}
341
342#[derive(Clone, Debug, Default)]
351pub struct ToolSignals {
352 pub severity: f32,
357 pub repeated_failure: bool,
360 pub no_error_streak: u32,
362 pub edit_count: u32,
364 pub write_count: u32,
366 pub read_count: u32,
368 pub todowrite_count: u32,
371 pub recent_edit_count: u32,
373 pub recent_write_count: u32,
375 pub recent_read_count: u32,
377 pub recent_todowrite_count: u32,
379 pub new_count: u32,
381 pub recent_new_count: u32,
383 pub pure_bash_streak: u32,
386 pub tests_passed: bool,
388 pub tool_result_count: u32,
391 pub assistant_turn_count: u32,
394 pub turn_depth: u32,
398 pub compacted: bool,
404}
405
406impl ToolSignals {
407 pub fn from_request(request: &Request, window_size: Option<usize>) -> Self {
412 Self::from_request_with_semantics(request, window_size, &ToolSemantics::default())
413 }
414
415 pub fn from_request_with_semantics(
417 request: &Request,
418 window_size: Option<usize>,
419 semantics: &ToolSemantics,
420 ) -> Self {
421 extract_tool_signals_with_window_and_semantics(
422 request,
423 window_size.unwrap_or(DEFAULT_RECENT_WINDOW),
424 semantics,
425 false,
426 )
427 }
428}
429
430#[derive(Debug, Clone)]
434struct ObservedToolCall<'a> {
435 name: String,
436 bare_name: Option<&'a str>,
437 command: Option<String>,
438 is_retrieval: bool,
439 code_semantic: Option<ToolSemantic>,
440}
441
442#[derive(Clone, Copy, Default)]
443struct ToolResultContext {
444 is_retrieval: bool,
445 search_no_match: bool,
446 codex_session: bool,
447}
448
449#[derive(Debug, Clone, Copy, PartialEq, Eq)]
450enum MutationKind {
451 Write,
452 Edit,
453}
454
455#[derive(Debug, Clone, Copy, PartialEq, Eq)]
457enum ToolSemantic {
458 Mutate(MutationKind),
459 Observe,
460 Plan,
461 New,
462 Unknown,
463}
464
465#[derive(Debug, Clone)]
468pub struct ToolSignalProcessor {
469 pub recent_window: usize,
472 pub tool_semantics: ToolSemantics,
474}
475
476impl Default for ToolSignalProcessor {
477 fn default() -> Self {
478 Self {
479 recent_window: DEFAULT_RECENT_WINDOW,
480 tool_semantics: ToolSemantics::default(),
481 }
482 }
483}
484
485impl ToolSignalProcessor {
486 pub(crate) fn for_user_turn(&self, request: &Request) -> ToolSignals {
487 extract_tool_signals_with_window_and_semantics(
488 request,
489 self.recent_window,
490 &self.tool_semantics,
491 true,
492 )
493 }
494}
495
496#[async_trait]
497impl Processor for ToolSignalProcessor {
498 async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
499 if let Event::Request { request: req, .. } = event {
500 let tool_signal = ToolSignals::from_request_with_semantics(
501 req,
502 Some(self.recent_window),
503 &self.tool_semantics,
504 );
505 state.tool_signals = Some(tool_signal);
506 }
507 Ok(())
508 }
509}
510
511fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic {
512 classify_tool_call_with_semantics(name, command, &ToolSemantics::default())
513}
514
515fn code_mode_has_patch(mut input: &str) -> bool {
518 static STATEMENT: LazyLock<Regex> = LazyLock::new(|| {
519 let identifier = r"[a-zA-Z_$][a-zA-Z0-9_$]*";
520 let literal = r#"(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|`(?:[^`\\$]|\\[^\r\n])*`)"#;
521 let argument = format!(r"(?:{literal}|{identifier})");
522 let patch = format!(r"await\s+tools\s*\.\s*apply_patch\s*\(\s*{argument}\s*\)");
523 Regex::new(&format!(
525 r"(?x)\A(?://[^\r\n]*|(?s:/\*.*?\*/)|(?:
526 (?:const|let)\s+(?P<binding>{identifier})\s*=\s*{literal}|
527 (?P<patch>{patch}|text\s*\(\s*{patch}\s*\))|
528 text\s*\(\s*{argument}\s*\)
529 )[\t\x20]*(?:;|\r?\n|\z))"
530 ))
531 .expect("constant code-mode statement pattern")
532 });
533 let mut has_patch = false;
534 while !input.trim_start().is_empty() {
535 input = input.trim_start();
536 let Some(statement) = STATEMENT.captures(input) else {
537 return false;
538 };
539 if statement
540 .name("binding")
541 .is_some_and(|binding| matches!(binding.as_str(), "tools" | "text"))
542 {
543 return false;
544 }
545 has_patch |= statement.name("patch").is_some();
546 input = &input[statement[0].len()..];
547 }
548 has_patch
549}
550
551fn classify_tool_call_with_semantics(
552 name: &str,
553 command: Option<&str>,
554 semantics: &ToolSemantics,
555) -> ToolSemantic {
556 let lower = name.to_lowercase();
558 if WRITE_TOOL_NAMES.contains(&lower.as_str()) {
559 return ToolSemantic::Mutate(MutationKind::Write);
560 }
561 if EDITOR_TOOL_NAMES.contains(&lower.as_str()) && command == Some("view") {
562 return ToolSemantic::Observe;
563 }
564 if EDIT_TOOL_NAMES.contains(&lower.as_str()) {
565 return ToolSemantic::Mutate(MutationKind::Edit);
566 }
567 if READ_TOOL_NAMES.contains(&lower.as_str()) {
568 return ToolSemantic::Observe;
569 }
570 if PLAN_TOOL_NAMES.contains(&lower.as_str()) {
571 return ToolSemantic::Plan;
572 }
573 if BASH_TOOL_NAMES.contains(&lower.as_str())
574 && let Some(cmd) = command
575 {
576 if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_write(cmd) {
578 return ToolSemantic::Mutate(MutationKind::Write);
579 }
580 if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) {
581 return ToolSemantic::Mutate(MutationKind::Write);
582 }
583 if shell_invokes_program(cmd, "node")
584 && JAVASCRIPT_WRITE_PATTERNS
585 .iter()
586 .any(|pattern| cmd.contains(pattern))
587 {
588 return ToolSemantic::Mutate(MutationKind::Write);
589 }
590 if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_edit(cmd) {
591 return ToolSemantic::Mutate(MutationKind::Edit);
592 }
593 if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_read(cmd) {
594 return ToolSemantic::Observe;
595 }
596 }
597 semantics.classify(name).unwrap_or(ToolSemantic::Unknown)
598}
599
600fn is_retrieval_tool(name: &str, command: Option<&Value>) -> bool {
602 let lower = name.to_lowercase();
603 READ_TOOL_NAMES.contains(&lower.as_str())
604 || (EDITOR_TOOL_NAMES.contains(&lower.as_str())
605 && command
606 .and_then(Value::as_str)
607 .is_some_and(|command| command.eq_ignore_ascii_case("view")))
608 || (lower != "powershell"
610 && BASH_TOOL_NAMES.contains(&lower.as_str())
611 && command.is_some_and(command_is_retrieval))
612}
613
614fn command_is_retrieval(command: &Value) -> bool {
616 match command {
617 Value::String(line) => shell_is_retrieval(line, 0),
618 Value::Array(argv) => {
619 argv.iter()
622 .try_fold(0, |size, word| {
623 let size = size + word.as_str()?.len() + 1;
624 (size <= MAX_RETRIEVAL_COMMAND_BYTES).then_some(size)
625 })
626 .is_some()
627 && retrieval_command(
628 &argv.iter().filter_map(Value::as_str).collect::<Vec<_>>(),
629 0,
630 )
631 }
632 _ => false,
633 }
634}
635
636fn command_ends_with_search(command: &Value) -> bool {
639 match command {
640 Value::String(command) => {
641 if quoted_chars(command).any(|(_, c, quote)| quote.is_none() && c == '&') {
642 return false;
643 }
644 shell_segments(command)
645 .last()
646 .and_then(shlex::split)
647 .is_some_and(|words| {
648 search_command(&words.iter().map(String::as_str).collect::<Vec<_>>(), 0)
649 })
650 }
651 Value::Array(words) => search_command(
652 &words.iter().filter_map(Value::as_str).collect::<Vec<_>>(),
653 0,
654 ),
655 _ => false,
656 }
657}
658
659fn search_command(words: &[&str], depth: usize) -> bool {
660 if depth > MAX_COMMAND_DEPTH {
661 return false;
662 }
663 let words = skip_shell_assignments(words);
664 match words.split_first() {
665 Some((program, args)) => match program_name(program) {
666 "env" => search_command(
667 skip_shell_options(args, &["-u", "--unset", "-C", "--chdir"]),
668 depth + 1,
669 ),
670 "rg" | "grep" => true,
671 _ => false,
672 },
673 None => false,
674 }
675}
676
677fn codex_session_id(header: &str) -> Option<&str> {
678 header.lines().find_map(|line| {
679 line.strip_prefix("Process running with session ID ")
680 .filter(|id| !id.is_empty() && id.bytes().all(|c| c.is_ascii_digit()))
681 })
682}
683
684const MAX_RETRIEVAL_COMMAND_BYTES: usize = 16 * 1024;
687
688fn shell_is_retrieval(command: &str, depth: usize) -> bool {
692 if command.len() > MAX_RETRIEVAL_COMMAND_BYTES {
693 return false;
694 }
695 if quoted_chars(command).any(|(_, c, quote)| {
696 (quote != Some('\'') && matches!(c, '$' | '`'))
697 || (quote.is_none() && matches!(c, '<' | '>' | '(' | ')' | '{' | '}' | '#'))
698 }) {
699 return false;
700 }
701 let mut segments = shell_segments(command).peekable();
702 segments.peek().is_some()
703 && segments.all(|segment| {
704 shlex::split(segment).is_some_and(|words| {
706 retrieval_command(&words.iter().map(String::as_str).collect::<Vec<_>>(), depth)
707 })
708 })
709}
710
711fn skip_shell_options<'a>(mut args: &'a [&'a str], takes_value: &[&str]) -> &'a [&'a str] {
712 while let Some((option, rest)) = args.split_first() {
713 if !option.starts_with('-') || *option == "-" {
714 break;
715 }
716 args = rest;
717 if *option == "--" {
718 break;
719 }
720 if takes_value.contains(option) {
721 args = args.get(1..).unwrap_or_default();
722 }
723 }
724 args
725}
726
727fn has_short_flag(word: &str, flags: &[char]) -> bool {
729 word.starts_with('-') && !word.starts_with("--") && word.contains(flags)
730}
731
732const MAX_COMMAND_DEPTH: usize = 20;
734
735fn skip_shell_assignments<'a>(mut words: &'a [&'a str]) -> &'a [&'a str] {
736 while words.first().is_some_and(|word| {
737 word.split_once('=').is_some_and(|(name, _)| {
738 !name.is_empty()
739 && name.bytes().enumerate().all(|(i, c)| {
740 c == b'_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())
741 })
742 })
743 }) {
744 words = &words[1..];
745 }
746 words
747}
748
749fn retrieval_command(words: &[&str], depth: usize) -> bool {
750 if depth > MAX_COMMAND_DEPTH {
751 return false;
752 }
753 let words = skip_shell_assignments(words);
754 let Some((&program, args)) = words.split_first() else {
755 return false;
756 };
757 let program = [
760 "/bin/",
761 "/sbin/",
762 "/usr/bin/",
763 "/usr/sbin/",
764 "/usr/local/bin/",
765 "/opt/homebrew/bin/",
766 ]
767 .iter()
768 .find_map(|dir| program.strip_prefix(dir))
769 .unwrap_or(program);
770 let wrapper_options: Option<&[&str]> = match program {
772 "sudo" => Some(&[
773 "-u", "-g", "-h", "-p", "-C", "-T", "--user", "--group", "--host",
774 ]),
775 "env" => Some(&["-u", "--unset", "-C", "--chdir"]),
776 "command" | "nohup" => Some(&[]),
777 "exec" => Some(&["-a"]),
778 "time" => Some(&["-f", "--format", "-o", "--output"]),
779 "timeout" => Some(&["-s", "--signal", "-k", "--kill-after"]),
780 "xargs" => Some(&[
781 "-n",
782 "--max-args",
783 "-P",
784 "--max-procs",
785 "-I",
786 "--replace",
787 "-d",
788 "--delimiter",
789 "-L",
790 "--max-lines",
791 "-s",
792 "--max-chars",
793 ]),
794 _ => None,
795 };
796 if let Some(options) = wrapper_options {
797 if program == "env"
799 && args
800 .iter()
801 .any(|arg| arg.starts_with("--split-string") || has_short_flag(arg, &['S']))
802 {
803 return false;
804 }
805 let mut rest = skip_shell_options(args, options);
806 if program == "timeout" {
807 rest = rest.get(1..).unwrap_or_default();
808 }
809 return (program == "env" && rest.is_empty()) || retrieval_command(rest, depth + 1);
810 }
811 if matches!(program, "bash" | "sh" | "dash" | "zsh" | "ksh") {
812 return args
813 .first()
814 .is_some_and(|option| has_short_flag(option, &['c']))
815 && args
816 .get(1)
817 .is_some_and(|command| shell_is_retrieval(command, depth + 1));
818 }
819 if program == "git" {
820 return git_is_retrieval(args);
821 }
822 if matches!(args, ["--help" | "--version"]) {
823 return true;
824 }
825 match program {
826 "find" => !args.iter().any(|arg| {
827 matches!(
828 *arg,
829 "-delete"
830 | "-exec"
831 | "-execdir"
832 | "-ok"
833 | "-okdir"
834 | "-fprint"
835 | "-fprint0"
836 | "-fprintf"
837 )
838 }),
839 "fd" => !args
841 .iter()
842 .any(|arg| arg.starts_with("--exec") || has_short_flag(arg, &['x', 'X'])),
843 "sed" => {
845 args.len() >= 2
846 && args[0] == "-n"
847 && args[2..].iter().all(|arg| !arg.starts_with('-'))
848 && args[1].strip_suffix('p').is_some_and(|range| {
849 !range.is_empty()
850 && range
851 .bytes()
852 .all(|c| c.is_ascii_digit() || matches!(c, b',' | b'$'))
853 })
854 }
855 "sort" => !args.iter().any(|arg| {
856 arg.starts_with("--compress-program")
857 || arg.starts_with("--output")
858 || arg.starts_with("-o")
859 }),
860 "rg" => !args.iter().any(|arg| arg.starts_with("--pre")),
861 "xxd" => !args.contains(&"-r") && !args.contains(&"-revert"),
862 "go" => {
863 matches!(args.first(), Some(&"list" | &"doc" | &"env" | &"version"))
864 && !args.iter().any(|arg| matches!(*arg, "-w" | "-u"))
865 }
866 "docker" | "podman" => matches!(args.first(), Some(&"ps" | &"version")),
867 "cat" | "grep" | "ls" | "nl" | "head" | "tail" | "wc" | "pwd" | "stat" | "file" | "du"
868 | "df" | "which" | "type" | "diff" | "cmp" | "jq" | "uniq" | "cut" | "readlink"
869 | "realpath" | "tree" | "basename" | "dirname" | "printenv" | "echo" | "printf"
870 | "less" | "more" | "test" | "[" | "ps" | "pgrep" | "pkg-config" | "strings" | "uname"
871 | "od" | "true" | ":" | "cd" | "pstree" | "sha256sum" | "sha1sum" | "md5sum" | "lsof"
872 | "tr" | "free" | "id" | "namei" | "whoami" | "paste" | "ss" | "getent" => true,
873 _ => false,
874 }
875}
876
877fn git_is_retrieval(args: &[&str]) -> bool {
878 let args = skip_shell_options(
879 args,
880 &["-C", "-c", "--git-dir", "--work-tree", "--namespace"],
881 );
882 let Some((subcommand, rest)) = args.split_first() else {
883 return false;
884 };
885 match *subcommand {
886 "diff" => !rest.contains(&"--check"),
887 "branch" => {
888 rest.is_empty()
889 || rest.iter().all(|arg| {
890 matches!(
891 *arg,
892 "-a" | "-r"
893 | "--all"
894 | "--list"
895 | "--show-current"
896 | "-v"
897 | "-vv"
898 | "--verbose"
899 )
900 })
901 }
902 "remote" => {
903 rest.is_empty()
904 || matches!(rest, ["-v" | "--verbose"])
905 || rest.first() == Some(&"get-url")
906 }
907 "tag" => rest.is_empty() || matches!(rest[0], "-l" | "--list"),
908 "config" => matches!(
909 rest.first(),
910 Some(&"--get" | &"--get-all" | &"--list" | &"-l")
911 ),
912 "worktree" => rest.first() == Some(&"list"),
913 "submodule" => rest.first() == Some(&"status"),
914 "status" | "log" | "show" | "blame" | "ls-files" | "ls-remote" | "rev-parse"
915 | "merge-base" | "grep" | "describe" | "show-ref" | "check-ignore" | "rev-list"
916 | "ls-tree" | "diff-tree" | "version" => true,
917 _ => false,
918 }
919}
920
921fn is_builtin_tool_name(lower: &str) -> bool {
922 WRITE_TOOL_NAMES.contains(&lower)
923 || EDIT_TOOL_NAMES.contains(&lower)
924 || READ_TOOL_NAMES.contains(&lower)
925 || PLAN_TOOL_NAMES.contains(&lower)
926 || BASH_TOOL_NAMES.contains(&lower)
927}
928
929fn shell_segments(command: &str) -> impl Iterator<Item = &str> {
933 let mut start = 0;
934 quoted_chars(command)
935 .filter(|&(_, c, quote)| quote.is_none() && matches!(c, '\n' | ';' | '|' | '&'))
936 .map(|(index, _, _)| index)
937 .chain([command.len()])
938 .filter_map(move |end| {
939 let segment = command[start..end].trim();
940 start = end + 1;
942 (!segment.is_empty()).then_some(segment)
943 })
944}
945
946fn quoted_chars(command: &str) -> impl Iterator<Item = (usize, char, Option<char>)> {
948 let mut quote = None;
949 let mut escaped = false;
950 command.char_indices().filter_map(move |(index, c)| {
951 if escaped {
952 escaped = false;
953 return None;
954 }
955 if c == '\\' && quote != Some('\'') {
956 escaped = true;
957 return None;
958 }
959 let around = quote;
960 if quote == Some(c) {
961 quote = None;
962 } else if quote.is_none() && matches!(c, '\'' | '"') {
963 quote = Some(c);
964 }
965 Some((index, c, around))
966 })
967}
968
969fn shell_words(segment: &str) -> std::iter::Peekable<std::str::SplitAsciiWhitespace<'_>> {
970 let mut words = segment.split_ascii_whitespace().peekable();
971
972 if words.peek().copied() == Some("env") {
973 words.next();
974 while words.peek().is_some_and(|word| word.starts_with('-')) {
975 words.next();
976 }
977 }
978 while words
979 .peek()
980 .is_some_and(|word| word.contains('=') && !word.starts_with('='))
981 {
982 words.next();
983 }
984
985 words
986}
987
988fn program_name(word: &str) -> &str {
989 Path::new(word)
990 .file_name()
991 .and_then(|name| name.to_str())
992 .unwrap_or(word)
993}
994
995fn shell_invokes_program(command: &str, expected: &str) -> bool {
996 shell_segments(command).any(|segment| {
997 shell_words(segment)
998 .next()
999 .is_some_and(|word| program_name(word) == expected)
1000 })
1001}
1002
1003fn shell_command_is_write(command: &str) -> bool {
1004 shell_segments(command).any(|segment| {
1005 let mut words = shell_words(segment);
1006 let Some(program) = words.next().map(program_name) else {
1007 return false;
1008 };
1009 if matches!(program, "cp" | "mkdir" | "touch" | "install") {
1010 return true;
1011 }
1012
1013 let redirects_output = words.any(|word| matches!(word, ">" | ">>"));
1014 redirects_output
1015 && (matches!(program, "echo" | "printf" | "git")
1016 || BASH_READ_COMMANDS.contains(&program))
1017 })
1018}
1019
1020fn shell_command_is_edit(command: &str) -> bool {
1021 shell_segments(command).any(|segment| {
1022 let mut words = shell_words(segment);
1023 let Some(program) = words.next().map(program_name) else {
1024 return false;
1025 };
1026 let has_arg = |arg: &str| words.clone().any(|word| word == arg);
1027
1028 match program {
1029 "mv" | "rm" => true,
1030 "perl" => words
1031 .take_while(|word| word.starts_with('-'))
1032 .any(|option| {
1033 option
1034 .trim_start_matches('-')
1035 .chars()
1036 .any(|flag| flag == 'i')
1037 }),
1038 "git" => words
1039 .next()
1040 .is_some_and(|subcommand| matches!(subcommand, "apply" | "am" | "restore")),
1041 "gofmt" => has_arg("-w"),
1042 "cargo" => words.clone().next() == Some("fmt") && !has_arg("--check"),
1043 "ruff" => {
1044 let subcommand = words.clone().next();
1045 (subcommand == Some("format") && !has_arg("--check"))
1046 || (subcommand == Some("check") && has_arg("--fix"))
1047 }
1048 "prettier" => has_arg("--write"),
1049 "black" => !has_arg("--check"),
1050 _ => {
1051 (words.clone().any(|word| program_name(word) == "prettier") && has_arg("--write"))
1052 || (words.clone().any(|word| program_name(word) == "ruff")
1053 && ((has_arg("format") && !has_arg("--check"))
1054 || (has_arg("check") && has_arg("--fix"))))
1055 }
1056 }
1057 })
1058}
1059
1060fn shell_command_is_read(command: &str) -> bool {
1061 shell_segments(command).any(|segment| {
1062 if segment == "env" {
1063 return true;
1064 }
1065 let mut words = shell_words(segment);
1066 let Some(program) = words.next().map(program_name) else {
1067 return false;
1068 };
1069
1070 if BASH_READ_COMMANDS.contains(&program) {
1071 return true;
1072 }
1073 if program == "command" && words.next() == Some("-v") {
1074 return true;
1075 }
1076 if program == "type" {
1077 return true;
1078 }
1079 if program != "git" {
1080 return false;
1081 }
1082
1083 match words.next() {
1084 Some("branch") => words.next().is_none_or(|arg| arg.starts_with('-')),
1085 Some("remote") => words
1086 .next()
1087 .is_none_or(|arg| arg.starts_with('-') || arg == "get-url"),
1088 Some("config") => words
1089 .next()
1090 .is_some_and(|arg| matches!(arg, "--get" | "--get-all" | "--list" | "-l")),
1091 Some(subcommand) => GIT_READ_SUBCOMMANDS.contains(&subcommand),
1092 None => false,
1093 }
1094 })
1095}
1096
1097fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals {
1104 extract_tool_signals_with_window_and_semantics(
1105 request,
1106 recent_window,
1107 &ToolSemantics::default(),
1108 false,
1109 )
1110}
1111
1112fn extract_tool_signals_with_window_and_semantics(
1113 request: &Request,
1114 recent_window: usize,
1115 semantics: &ToolSemantics,
1116 current_user_turn: bool,
1117) -> ToolSignals {
1118 let messages = &request.llm_request.messages;
1120 let boundary = if current_user_turn {
1121 messages.iter().rposition(is_user_turn)
1122 } else {
1123 None
1124 };
1125 let start = boundary.map_or(0, |index| index + 1);
1126 let namespaces = tool_namespaces(&request.llm_request.extensions);
1127 let mut tool_texts: Vec<(String, bool)> = Vec::new();
1128 let mut tool_calls: Vec<ObservedToolCall> = Vec::new();
1129 let mut result_contexts: HashMap<&str, ToolResultContext> = HashMap::new();
1130 let mut sessions: HashMap<String, ToolResultContext> = HashMap::new();
1131 let mut tool_result_count = 0usize;
1132 let mut assistant_turn_count = 0usize;
1133
1134 for (index, message) in messages.iter().enumerate() {
1135 if index >= start && message.role == Role::Assistant {
1136 assistant_turn_count += 1;
1137 }
1138 for block in &message.content {
1139 match block {
1140 ContentBlock::ToolCall(call) => {
1141 let bare_name = namespaces
1143 .and_then(|namespaces| split_qualified_name(namespaces, &call.name))
1144 .map(|(tool, _)| tool)
1145 .or_else(|| mcp_tool_name(&call.name));
1146 let decoded = call
1148 .arguments
1149 .as_str()
1150 .and_then(|raw| serde_json::from_str::<Value>(raw).ok());
1151 let arguments = decoded.as_ref().unwrap_or(&call.arguments);
1152 let command_field = command_of(arguments);
1153 let mut code_semantic = (bare_name.unwrap_or(&call.name) == "exec"
1156 && arguments.get("command").is_none()
1157 && arguments.get("cmd").is_none())
1158 .then(|| arguments.get("input").and_then(Value::as_str))
1159 .flatten()
1160 .map(|input| {
1161 if code_mode_has_patch(input) {
1162 ToolSemantic::Mutate(MutationKind::Edit)
1163 } else {
1164 ToolSemantic::Unknown
1165 }
1166 });
1167 let command = command_field.and_then(command_text);
1168 let full = classify_tool_call_with_semantics(
1172 &call.name,
1173 command.as_deref(),
1174 semantics,
1175 );
1176 let name = match (full, bare_name) {
1177 (ToolSemantic::Unknown | ToolSemantic::Observe, Some(bare_name)) => {
1178 bare_name
1179 }
1180 _ => call.name.as_str(),
1181 };
1182 let mut is_retrieval =
1183 code_semantic.is_none() && is_retrieval_tool(name, command_field);
1184 let mut context = ToolResultContext {
1185 is_retrieval,
1186 search_no_match: is_retrieval
1187 && command_field.is_some_and(command_ends_with_search),
1188 codex_session: name == "exec_command",
1189 };
1190 if name == "write_stdin" {
1191 let session_id = arguments.get("session_id").map(|id| {
1192 id.as_str()
1193 .map(str::to_owned)
1194 .unwrap_or_else(|| id.to_string())
1195 });
1196 if let Some(id) = session_id {
1197 if arguments
1198 .get("chars")
1199 .and_then(Value::as_str)
1200 .unwrap_or("")
1201 .is_empty()
1202 {
1203 context = sessions.get(&id).copied().unwrap_or_default();
1204 is_retrieval = context.is_retrieval;
1205 code_semantic = Some(if is_retrieval {
1208 ToolSemantic::Observe
1209 } else {
1210 ToolSemantic::Unknown
1211 });
1212 } else {
1213 sessions.remove(&id);
1215 }
1216 }
1217 }
1218 if !call.id.is_empty() {
1219 result_contexts.insert(call.id.as_str(), context);
1221 }
1222 if index < start {
1223 continue;
1224 }
1225 tool_calls.push(ObservedToolCall {
1226 name: call.name.clone(),
1227 bare_name,
1228 command,
1229 is_retrieval,
1230 code_semantic,
1231 });
1232 }
1233 ContentBlock::ToolResult(result) => {
1234 let is_error = result.is_error == Some(true);
1235 let text = result
1236 .content
1237 .iter()
1238 .filter_map(text_of)
1239 .collect::<Vec<_>>()
1240 .join("\n");
1241 let context = result_contexts
1242 .get(result.tool_call_id.as_str())
1243 .copied()
1244 .unwrap_or_default();
1245 let status = text
1247 .split_once("\nOutput:\n")
1248 .map_or(text.as_str(), |(header, _)| header);
1249 if context.codex_session
1250 && !is_error
1251 && let Some(id) = codex_session_id(status)
1252 {
1253 sessions.insert(id.to_owned(), context);
1254 }
1255 if index < start {
1258 continue;
1259 }
1260 tool_result_count += 1;
1261 if context.is_retrieval {
1262 let shell_result = serde_json::Deserializer::from_str(&text)
1265 .into_iter::<Value>()
1266 .next()
1267 .and_then(|result| result.ok())
1268 .filter(|value| {
1269 value["output"].is_string() && value.get("exit_code").is_some()
1270 });
1271 let shell_failed = match &shell_result {
1275 Some(value) => {
1276 value["exit_code"].as_i64().is_some_and(|code| code != 0)
1277 || value["success"].as_bool() == Some(false)
1278 || value["error"]
1279 .as_str()
1280 .is_some_and(|error| !error.trim().is_empty())
1281 }
1282 None => has_nonzero_exit_status(&status.to_lowercase()),
1283 };
1284 let exit_code = shell_result
1285 .as_ref()
1286 .and_then(|value| value["exit_code"].as_i64())
1287 .or_else(|| {
1288 status.lines().find_map(|line| {
1289 line.strip_prefix("Process exited with code ")?
1290 .parse::<i64>()
1291 .ok()
1292 })
1293 });
1294 let no_match = context.search_no_match
1295 && exit_code == Some(1)
1296 && classify_text(&text).0 <= SOFT;
1297 if is_missing_file(&text) || (!is_error && (!shell_failed || no_match)) {
1298 if !text.is_empty() {
1300 tool_texts.push((String::new(), false));
1301 }
1302 continue;
1303 }
1304 }
1305 if !text.is_empty() || is_error {
1307 tool_texts.push((text, is_error));
1308 }
1309 }
1310 ContentBlock::Unknown { provider, raw }
1312 if index >= start
1313 && provider.as_str() == WireFormat::OpenAiResponses.as_str()
1314 && raw.get("type").and_then(Value::as_str)
1315 == Some("apply_patch_call_output") =>
1316 {
1317 tool_result_count += 1;
1318 let text = raw
1319 .get("output")
1320 .and_then(Value::as_str)
1321 .unwrap_or_default();
1322 let is_error = raw.get("status").and_then(Value::as_str) == Some("failed");
1323 tool_texts.push((text.to_owned(), is_error));
1324 }
1325 ContentBlock::Unknown { provider, raw }
1326 if index >= start
1327 && provider.as_str() == WireFormat::OpenAiResponses.as_str()
1328 && raw.get("type").and_then(Value::as_str) == Some("shell_call_output") =>
1329 {
1330 tool_result_count += 1;
1331 let mut texts = Vec::new();
1332 let mut is_error = false;
1333 if let Some(outputs) = raw.get("output").and_then(Value::as_array) {
1334 for output in outputs {
1335 for field in ["stdout", "stderr"] {
1336 if let Some(text) = output.get(field).and_then(Value::as_str)
1337 && !text.is_empty()
1338 {
1339 texts.push(text);
1340 }
1341 }
1342 if let Some(outcome) = output.get("outcome") {
1343 is_error |= match outcome.get("type").and_then(Value::as_str) {
1344 Some("timeout") => true,
1345 Some("exit")
1346 if outcome
1347 .get("exit_code")
1348 .and_then(Value::as_i64)
1349 .is_some_and(|code| code != 0) =>
1350 {
1351 texts.push("returned non-zero");
1353 output
1354 .get("stderr")
1355 .and_then(Value::as_str)
1356 .is_some_and(|text| !text.trim().is_empty())
1357 }
1358 _ => false,
1359 };
1360 }
1361 }
1362 }
1363 let text = texts.join("\n");
1364 tool_texts.push((text, is_error));
1365 }
1366 _ => {}
1367 }
1368 }
1369 }
1370
1371 let mut signal = build_signal(
1372 tool_texts,
1373 tool_calls,
1374 (messages.len() - boundary.unwrap_or(0)) as u32,
1375 recent_window,
1376 semantics,
1377 );
1378 signal.compacted = request.llm_request.has_compaction;
1379 signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);
1380 signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);
1381 signal
1382}
1383
1384fn mcp_tool_name(name: &str) -> Option<&str> {
1388 let (_server, tool) = name.strip_prefix("mcp__")?.split_once("__")?;
1389 (!tool.is_empty()).then_some(tool)
1390}
1391
1392fn command_of(arguments: &Value) -> Option<&Value> {
1396 ["command", "cmd", "input"]
1397 .iter()
1398 .find_map(|key| arguments.get(*key))
1399}
1400
1401fn command_text(value: &Value) -> Option<String> {
1403 match value {
1404 Value::String(text) => Some(text.to_lowercase()),
1405 Value::Array(parts) => {
1406 let joined = parts
1407 .iter()
1408 .filter_map(Value::as_str)
1409 .collect::<Vec<_>>()
1410 .join(" ");
1411 (!joined.is_empty()).then(|| joined.to_lowercase())
1412 }
1413 _ => None,
1414 }
1415}
1416
1417fn text_of(block: &ContentBlock) -> Option<&str> {
1419 match block {
1420 ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()),
1421 _ => None,
1422 }
1423}
1424
1425fn is_missing_file(text: &str) -> bool {
1428 let lower = text.to_lowercase();
1429 lower.contains("no such file or directory") || lower.contains("file does not exist")
1431}
1432
1433fn build_signal(
1434 tool_texts: Vec<(String, bool)>,
1435 tool_calls: Vec<ObservedToolCall>,
1436 turn_depth: u32,
1437 recent_window: usize,
1438 semantics: &ToolSemantics,
1439) -> ToolSignals {
1440 let sev_start = tool_texts.len().saturating_sub(recent_window.max(1));
1446 let mut severity = 0.0f32;
1447 let mut failure_fingerprints = Vec::new();
1448 let mut repeated_failure = false;
1449 for (text, is_error) in &tool_texts[sev_start..] {
1450 let (sev, _patterns) = classify_text(text);
1451 let sev = if *is_error { sev.max(HARD) } else { sev };
1453 if sev > severity {
1454 severity = sev;
1455 }
1456 if let Some(fingerprint) = failure_fingerprint(text, *is_error) {
1457 repeated_failure |= failure_fingerprints.contains(&fingerprint);
1458 failure_fingerprints.push(fingerprint);
1459 }
1460 }
1461
1462 let no_error_streak = compute_no_error_streak(&tool_texts);
1463
1464 let recent_start = tool_calls.len().saturating_sub(recent_window);
1468 let mut write_count = 0u32;
1469 let mut edit_count = 0u32;
1470 let mut read_count = 0u32;
1471 let mut todowrite_count = 0u32;
1472 let mut recent_write_count = 0u32;
1473 let mut recent_edit_count = 0u32;
1474 let mut recent_read_count = 0u32;
1475 let mut recent_todowrite_count = 0u32;
1476 let mut new_count = 0u32;
1477 let mut recent_new_count = 0u32;
1478 let mut pure_bash_streak = 0u32;
1479 let mut streak_open = true;
1480 for (i, tc) in tool_calls.iter().enumerate().rev() {
1481 let mut cat = if let Some(semantic) = tc.code_semantic {
1485 semantic
1486 } else if tc.is_retrieval {
1487 ToolSemantic::Observe
1488 } else {
1489 classify_tool_call_with_semantics(&tc.name, tc.command.as_deref(), semantics)
1490 };
1491 if matches!(cat, ToolSemantic::Unknown)
1492 && tc.code_semantic.is_none()
1493 && let Some(bare_name) = tc.bare_name
1494 {
1495 cat = classify_tool_call_with_semantics(bare_name, tc.command.as_deref(), semantics);
1496 }
1497 if streak_open {
1498 if matches!(cat, ToolSemantic::Unknown) {
1499 pure_bash_streak += 1;
1500 } else {
1501 streak_open = false;
1502 }
1503 }
1504 match cat {
1505 ToolSemantic::Mutate(MutationKind::Write) => {
1506 write_count += 1;
1507 if i >= recent_start {
1508 recent_write_count += 1;
1509 }
1510 }
1511 ToolSemantic::Mutate(MutationKind::Edit) => {
1512 edit_count += 1;
1513 if i >= recent_start {
1514 recent_edit_count += 1;
1515 }
1516 }
1517 ToolSemantic::Observe => {
1518 read_count += 1;
1519 if i >= recent_start {
1520 recent_read_count += 1;
1521 }
1522 }
1523 ToolSemantic::Plan => {
1524 todowrite_count += 1;
1525 if i >= recent_start {
1526 recent_todowrite_count += 1;
1527 }
1528 }
1529 ToolSemantic::New => {
1530 new_count += 1;
1531 if i >= recent_start {
1532 recent_new_count += 1;
1533 }
1534 }
1535 ToolSemantic::Unknown => {}
1536 }
1537 }
1538
1539 let tests_passed = detect_tests_passed(&tool_texts, recent_window);
1540
1541 ToolSignals {
1542 severity,
1543 repeated_failure,
1544 no_error_streak,
1545 edit_count,
1546 write_count,
1547 read_count,
1548 todowrite_count,
1549 recent_edit_count,
1550 recent_write_count,
1551 recent_read_count,
1552 recent_todowrite_count,
1553 new_count,
1554 recent_new_count,
1555 pure_bash_streak,
1556 tests_passed,
1557 turn_depth,
1558 tool_result_count: 0,
1562 assistant_turn_count: 0,
1563 compacted: false,
1564 }
1565}
1566
1567fn content_to_text(content: Option<&Value>) -> Option<String> {
1571 match content? {
1572 Value::String(s) => Some(s.clone()),
1573 Value::Array(blocks) => {
1574 let parts: Vec<&str> = blocks
1575 .iter()
1576 .filter_map(|b| {
1577 b.as_object()
1578 .filter(|o| o.get("type").and_then(Value::as_str) == Some("text"))
1579 .and_then(|o| o.get("text"))
1580 .and_then(Value::as_str)
1581 })
1582 .collect();
1583 if parts.is_empty() {
1584 None
1585 } else {
1586 Some(parts.join("\n"))
1587 }
1588 }
1589 _ => None,
1590 }
1591}
1592
1593pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {
1597 let lower = text.to_lowercase();
1598 let mut patterns = Vec::new();
1599 let mut severity: f32 = 0.0;
1600 for (name, sev, substrings) in ERROR_PATTERNS {
1601 if substrings.iter().any(|sub| lower.contains(sub)) {
1602 patterns.push(name.to_string());
1603 severity = severity.max(*sev);
1604 }
1605 }
1606 let result = serde_json::Deserializer::from_str(text)
1608 .into_iter::<Value>()
1609 .next()
1610 .and_then(|result| result.ok())
1611 .unwrap_or_default();
1612 let nonzero_exit = result["exit_code"].as_i64().is_some_and(|code| code != 0);
1613 let tool_error = result["success"].as_bool() == Some(false)
1614 || result["error"]
1615 .as_str()
1616 .is_some_and(|error| !error.trim().is_empty());
1617 if (nonzero_exit || has_nonzero_exit_status(&lower))
1618 && !patterns.iter().any(|p| p == "exit_nonzero")
1619 {
1620 patterns.push("exit_nonzero".to_string());
1621 severity = severity.max(SOFT);
1622 }
1623 for (name, matched) in [
1624 ("tool_error", tool_error),
1625 ("compile_error", has_compiler_diagnostic(&lower)),
1626 ("runtime_exception", has_runtime_exception(&lower)),
1627 ("runtime_panic", has_runtime_panic(&lower)),
1628 ("patch_error", has_patch_failure(&lower)),
1629 ("cmd_not_found", ends_with_not_found(&lower)),
1630 ] {
1631 if matched && !patterns.iter().any(|pattern| pattern == name) {
1632 patterns.push(name.to_string());
1633 severity = severity.max(HARD);
1634 }
1635 }
1636 (severity, patterns)
1637}
1638
1639fn failure_fingerprint(text: &str, is_error: bool) -> Option<String> {
1642 let (severity, patterns) = classify_text(text);
1643 if severity < HARD && !is_error {
1644 return None;
1645 }
1646
1647 let lower = text.to_lowercase();
1648 let diagnostic = lower
1649 .lines()
1650 .find(|line| is_failure_diagnostic(line))
1651 .or_else(|| lower.lines().find(|line| !line.trim().is_empty()))
1652 .unwrap_or_default();
1653 let normalized = normalize_failure_text(diagnostic);
1654 Some(format!("{}|{normalized}", patterns.join(",")))
1655}
1656
1657fn is_failure_diagnostic(line: &str) -> bool {
1658 let line = line.trim();
1659 [
1660 "error",
1661 "exception",
1662 "panic",
1663 "failed",
1664 "timed out",
1665 "timeout",
1666 "connection refused",
1667 "cannot allocate memory",
1668 "out of memory",
1669 "not found",
1670 ]
1671 .iter()
1672 .any(|marker| line.contains(marker))
1673}
1674
1675fn normalize_failure_text(text: &str) -> String {
1678 let mut normalized = String::new();
1679 for word in text.split_whitespace() {
1680 if !normalized.is_empty() {
1681 normalized.push(' ');
1682 }
1683 let mut in_digits = false;
1684 if word.starts_with('/') || word.contains("/src/") || word.contains("/tmp/") {
1685 normalized.push_str("<path>");
1686 continue;
1687 }
1688 for character in word.chars() {
1689 if character.is_ascii_digit() {
1690 if !in_digits {
1691 normalized.push('#');
1692 in_digits = true;
1693 }
1694 } else {
1695 normalized.push(character);
1696 in_digits = false;
1697 }
1698 }
1699 }
1700 normalized.chars().take(240).collect()
1701}
1702
1703fn has_compiler_diagnostic(lower: &str) -> bool {
1704 lower.lines().any(|line| {
1705 let line = line.trim_start();
1706 if matches!(
1707 line,
1708 "compilation failed" | "error: compilation failed" | "error: could not compile"
1709 ) || line.starts_with("error: could not compile ")
1710 {
1711 return true;
1712 }
1713
1714 let Some(rest) = line.strip_prefix("error[e") else {
1715 return false;
1716 };
1717 let Some((code, _)) = rest.split_once("]:") else {
1718 return false;
1719 };
1720 !code.is_empty() && code.chars().all(|character| character.is_ascii_digit())
1721 })
1722}
1723
1724fn has_runtime_exception(lower: &str) -> bool {
1725 let has_exception_line = lower.lines().any(|line| {
1726 let line = line.trim_start();
1727 [
1728 "typeerror:",
1729 "referenceerror:",
1730 "rangeerror:",
1731 "runtimeerror:",
1732 "keyerror:",
1733 "attributeerror:",
1734 ]
1735 .iter()
1736 .any(|prefix| line.starts_with(prefix))
1737 });
1738 has_exception_line && (lower.contains("\n at ") || lower.contains("\n at "))
1739}
1740
1741fn has_runtime_panic(lower: &str) -> bool {
1742 lower
1743 .lines()
1744 .any(|line| line.trim_start().starts_with("panic: runtime error:"))
1745 && (lower.contains("\ngoroutine ") || lower.contains("[signal sig"))
1746}
1747
1748fn has_patch_failure(lower: &str) -> bool {
1749 lower.lines().any(|line| {
1750 let line = line.trim_start();
1751 line.starts_with("error: patch failed:")
1752 || line.starts_with("patch failed:")
1753 || line.contains(": patch does not apply")
1754 || line.starts_with("invalid context")
1755 })
1756}
1757
1758fn ends_with_not_found(lower: &str) -> bool {
1767 lower
1768 .lines()
1769 .any(|line| line.trim_end().ends_with(": not found"))
1770}
1771
1772fn has_nonzero_exit_status(lower: &str) -> bool {
1778 NONZERO_EXIT_PHRASES
1779 .iter()
1780 .any(|phrase| phrase_followed_by_nonzero_integer(lower, phrase))
1781}
1782
1783fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool {
1785 let mut cursor = 0usize;
1786 while let Some(rel) = lower[cursor..].find(phrase) {
1787 let value_start = cursor + rel + phrase.len();
1788 let rest = lower[value_start..].trim_start_matches(|c: char| {
1789 c.is_ascii_whitespace() || matches!(c, ':' | '=' | '\'' | '"' | '`')
1790 });
1791 let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
1792 if !digits.is_empty() && digits.chars().any(|d| d != '0') {
1793 return true;
1794 }
1795 cursor = value_start;
1796 }
1797 false
1798}
1799
1800fn compute_no_error_streak(tool_texts: &[(String, bool)]) -> u32 {
1801 let mut streak = 0u32;
1802 for (text, is_error) in tool_texts.iter().rev() {
1803 let (sev, _) = classify_text(text);
1804 if *is_error || sev > 0.0 {
1805 break;
1806 }
1807 streak += 1;
1808 }
1809 streak
1810}
1811
1812fn detect_tests_passed(tool_texts: &[(String, bool)], recent_window: usize) -> bool {
1813 let start = tool_texts.len().saturating_sub(recent_window.max(1));
1814 let recent = &tool_texts[start..];
1815 let after_latest_failure = recent
1816 .iter()
1817 .rposition(|(text, is_error)| *is_error || classify_text(text).0 > 0.0)
1818 .map_or(recent, |index| &recent[index + 1..]);
1819 after_latest_failure.iter().any(|(text, _)| {
1820 let lower = text.to_lowercase();
1821 TEST_PASS_PHRASES.iter().any(|p| lower.contains(p))
1822 && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p))
1823 && !has_nonzero_failure_count(&lower)
1824 })
1825}
1826
1827fn has_nonzero_failure_count(lower: &str) -> bool {
1833 for kw in NUMERIC_FAILURE_KEYWORDS {
1834 let mut cursor = 0usize;
1835 while let Some(rel) = lower[cursor..].find(kw) {
1836 let kw_start = cursor + rel;
1837 let kw_end = kw_start + kw.len();
1838 let boundary_after = lower[kw_end..]
1841 .chars()
1842 .next()
1843 .is_none_or(|c| !c.is_ascii_alphanumeric());
1844 if boundary_after {
1845 let prefix = &lower[..kw_start];
1846 let trimmed = prefix.trim_end_matches(|c: char| c.is_whitespace());
1847 let digits_rev: String = trimmed
1848 .chars()
1849 .rev()
1850 .take_while(|c| c.is_ascii_digit())
1851 .collect();
1852 if !digits_rev.is_empty() && digits_rev.chars().any(|d| d != '0') {
1853 return true;
1854 }
1855 }
1856 cursor = kw_start + kw.len();
1857 }
1858 }
1859 false
1860}
1861
1862#[cfg(test)]
1865mod tests {
1866 use super::*;
1867 use crate::algorithms::util::stage::score_signal;
1868 use serde_json::json;
1869 use switchyard_protocol::codex_namespaces::TOOL_NAMESPACES_KEY;
1870 use switchyard_protocol::{
1871 ContentBlock, LlmRequest, Message, Metadata, Role, ToolCall, ToolResult,
1872 };
1873
1874 fn with_messages(messages: Vec<Message>) -> Request {
1875 let mut request = Request {
1876 llm_request: LlmRequest {
1877 messages,
1878 ..LlmRequest::default()
1879 },
1880 raw_request: None,
1881 metadata: None,
1882 };
1883 switchyard_translation::normalize_compaction(&mut request.llm_request, None);
1884 request
1885 }
1886
1887 fn tc(name: &str) -> Message {
1889 Message::new(
1890 Role::Assistant,
1891 vec![ContentBlock::ToolCall(ToolCall {
1892 id: String::new(),
1893 name: name.to_string(),
1894 arguments: json!({}),
1895 })],
1896 )
1897 }
1898
1899 fn bash(command: &str) -> Message {
1901 Message::new(
1902 Role::Assistant,
1903 vec![ContentBlock::ToolCall(ToolCall {
1904 id: String::new(),
1905 name: "Bash".to_string(),
1906 arguments: json!({"command": command}),
1907 })],
1908 )
1909 }
1910
1911 fn tr(text: &str) -> Message {
1913 Message::new(
1914 Role::User,
1915 vec![ContentBlock::ToolResult(ToolResult {
1916 tool_call_id: String::new(),
1917 content: vec![ContentBlock::Text {
1918 text: text.to_string(),
1919 }],
1920 is_error: None,
1921 })],
1922 )
1923 }
1924
1925 #[test]
1926 fn user_turn_signals_exclude_old_errors_and_keep_user_role_tool_results() {
1927 let mut user = Message::text(Role::User, "next");
1928 user.content.extend(tr("out of memory").content);
1929 let request = with_messages(vec![tr("out of memory"), user, tr("ok")]);
1930 let signals = ToolSignalProcessor::default().for_user_turn(&request);
1931 assert_eq!(signals.tool_result_count, 1);
1932 assert_eq!(signals.severity, 0.0);
1933 assert_eq!(signals.turn_depth, 2);
1934 }
1935
1936 #[test]
1937 fn clean_text_has_zero_severity() {
1938 let (sev, patterns) = classify_text("everything went fine");
1939 assert_eq!(sev, 0.0);
1940 assert!(patterns.is_empty());
1941 }
1942
1943 #[test]
1944 fn structured_tool_failures_affect_recovery_signals() {
1945 for (text, severity) in [
1946 (r#"{"output":"","exit_code":7,"error":null}"#, SOFT),
1947 (r#"{"output":"","exit_code":-1,"error":null}"#, SOFT),
1948 (r#"{"success":false,"error":"No matching text"}"#, HARD),
1949 (r#"{"success":false,"error":null}"#, HARD),
1950 (
1951 r#"{"error":"Overwrite refused","stale_write_blocked":true}"#,
1952 HARD,
1953 ),
1954 (r#"{"output":"out of memory","exit_code":1}"#, CRITICAL),
1955 (r#"{"output":"done","exit_code":0,"error":null}"#, 0.0),
1956 (r#"{"success":true,"error":" "}"#, 0.0),
1957 (r#"{"output":"running","exit_code":null,"error":null}"#, 0.0),
1958 (r#"{"output":{"success":false},"exit_code":0}"#, 0.0),
1959 ] {
1960 let warned = format!("{text}\n\n[Tool loop warning: repeated identical call]");
1961 let request = with_messages(vec![tr("5 passed in 0.12s"), tr(text), tr(&warned)]);
1962 let signal = ToolSignals::from_request(&request, None);
1963 let clean = severity == 0.0;
1964 assert_eq!(signal.severity, severity, "{text}");
1965 assert_eq!(signal.no_error_streak, if clean { 3 } else { 0 }, "{text}");
1966 assert_eq!(signal.repeated_failure, severity >= HARD, "{text}");
1967 assert_eq!(signal.tests_passed, clean, "{text}");
1968 }
1969 }
1970
1971 #[test]
1972 fn traceback_is_hard() {
1973 let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError");
1974 assert_eq!(sev, HARD);
1975 assert!(patterns.contains(&"traceback".to_string()));
1976 }
1977
1978 #[test]
1979 fn oom_is_critical() {
1980 let (sev, _) = classify_text("Out of memory: kill process 1234");
1981 assert_eq!(sev, CRITICAL);
1982 }
1983
1984 #[test]
1985 fn connection_refused_is_hard() {
1986 let (severity, _) = classify_text("Connection refused on port 8000");
1987 assert_eq!(severity, HARD);
1988 }
1989
1990 #[test]
1991 fn repeated_failure_ignores_volatile_paths_and_numbers() {
1992 let request = with_messages(vec![
1993 tr("error[E0308]: mismatched types at /tmp/a/src/lib.rs:12"),
1994 tr("error[E0308]: mismatched types at /tmp/b/src/lib.rs:47"),
1995 ]);
1996 assert!(ToolSignals::from_request(&request, None).repeated_failure);
1997 }
1998
1999 #[test]
2000 fn different_failures_are_not_repeated() {
2001 let request = with_messages(vec![
2002 tr("error[E0308]: mismatched types"),
2003 tr("error[E0509]: cannot move out"),
2004 ]);
2005 assert!(!ToolSignals::from_request(&request, None).repeated_failure);
2006 }
2007
2008 #[test]
2009 fn one_material_failure_is_not_repeated() {
2010 let request = with_messages(vec![tr("Connection refused on port 8000")]);
2011 assert!(!ToolSignals::from_request(&request, None).repeated_failure);
2012 }
2013
2014 #[test]
2016 fn structured_tool_failures_feed_error_and_recovery_signals() {
2017 for text in ["Dependency unavailable", "", "5 passed in 0.12s"] {
2018 let mut failed = tr(text);
2019 let ContentBlock::ToolResult(result) = &mut failed.content[0] else {
2020 panic!("expected tool result");
2021 };
2022 result.is_error = Some(true);
2023 let mut request = with_messages(vec![tr("5 passed in 0.12s"), failed.clone()]);
2024 let signals = ToolSignals::from_request(&request, Some(3));
2025 assert_eq!(signals.severity, HARD);
2026 assert!(!signals.repeated_failure);
2027 assert_eq!(signals.no_error_streak, 0);
2028 assert!(!signals.tests_passed);
2029
2030 request.llm_request.messages.push(failed);
2031 assert!(ToolSignals::from_request(&request, Some(3)).repeated_failure);
2032 request.llm_request.messages.push(tr("5 passed in 0.12s"));
2033 let recovered = ToolSignals::from_request(&request, Some(1));
2034 assert_eq!(recovered.severity, 0.0);
2035 assert!(!recovered.repeated_failure);
2036 assert_eq!(recovered.no_error_streak, 1);
2037 assert!(recovered.tests_passed);
2038 }
2039 }
2040
2041 #[test]
2042 fn severity_is_max_across_patterns() {
2043 let (sev, _) = classify_text("exit code 1\nTraceback (most recent call last):");
2045 assert_eq!(sev, HARD);
2046 }
2047
2048 #[test]
2049 fn codex_process_exit_zero_stays_clean() {
2050 let (sev, patterns) =
2051 classify_text("Chunk ID: abc\nProcess exited with code 0\nOutput:\nok");
2052 assert_eq!(sev, 0.0);
2053 assert!(!patterns.contains(&"exit_nonzero".to_string()));
2054 }
2055
2056 #[test]
2057 fn nonzero_exit_codes_are_soft_errors() {
2058 let cases = [
2059 "Process exited with code 1",
2060 "Process exited with code 127",
2061 "exit code: 2",
2062 "exit status 3",
2063 "exited with status 9",
2064 ];
2065 for case in cases {
2066 let (sev, patterns) = classify_text(case);
2067 assert_eq!(sev, SOFT, "expected soft severity for {case}");
2068 assert!(patterns.contains(&"exit_nonzero".to_string()));
2069 }
2070 }
2071
2072 #[test]
2073 fn partial_process_failures_are_hard_errors() {
2074 let cases = [
2075 (
2076 "Process running with session ID 12\nOutput:\nerror[E0509]: cannot move out",
2077 "compile_error",
2078 ),
2079 (
2080 "Process exited with code 0\nOutput:\nTypeError: value is undefined\n at main.js:1:2",
2081 "runtime_exception",
2082 ),
2083 (
2084 "Process running with session ID 13\nOutput:\npanic: runtime error: index out of range\n\ngoroutine 6 [running]:",
2085 "runtime_panic",
2086 ),
2087 (
2088 "Process exited with code 0\nOutput:\nerror: patch failed: src/lib.rs:4\nerror: src/lib.rs: patch does not apply",
2089 "patch_error",
2090 ),
2091 ];
2092 for (text, expected_pattern) in cases {
2093 let (severity, patterns) = classify_text(text);
2094 assert_eq!(severity, HARD, "expected hard severity for {text}");
2095 assert!(patterns.iter().any(|pattern| pattern == expected_pattern));
2096 }
2097 }
2098
2099 #[test]
2102 fn trimmed_command_not_found_is_still_hard() {
2103 let cases = [
2104 "git: not found",
2105 "sh: 1: git: not found",
2106 "/bin/sh: 1: git: not found",
2107 ];
2108 for text in cases {
2109 let (sev, patterns) = classify_text(text);
2110 assert_eq!(sev, HARD, "expected hard severity for {text}");
2111 assert!(patterns.contains(&"cmd_not_found".to_string()));
2112 }
2113 }
2114
2115 #[test]
2118 fn not_found_mid_sentence_stays_clean() {
2119 let (sev, _) =
2120 classify_text("Looking for config key, not found in environment, using default.");
2121 assert_eq!(sev, 0.0);
2122 }
2123
2124 #[test]
2127 fn not_found_without_shell_separator_stays_clean() {
2128 for text in [
2129 "HTTP/1.1 404 Not Found",
2130 "lookup finished, key not found",
2131 "GET /healthz -> 404 Not Found",
2132 ] {
2133 let (sev, patterns) = classify_text(text);
2134 assert_eq!(sev, 0.0, "expected clean classification for {text}");
2135 assert!(!patterns.contains(&"cmd_not_found".to_string()));
2136 }
2137 }
2138
2139 #[test]
2140 fn source_text_that_names_exceptions_stays_clean() {
2141 let text =
2142 "pub enum TypeError: this is documentation\nlet sample = 'panic: runtime error:';";
2143 assert_eq!(classify_text(text).0, 0.0);
2144 }
2145
2146 #[test]
2147 fn file_does_not_exist_is_hard() {
2148 let (sev, patterns) =
2150 classify_text("Error: File does not exist. Note: current working directory is /app.");
2151 assert_eq!(sev, HARD);
2152 assert!(patterns.contains(&"no_such_file".to_string()));
2153 }
2154
2155 #[test]
2156 fn bare_does_not_exist_stays_clean() {
2157 let (sev, _) = classify_text("The directory does not exist yet, creating it now.");
2160 assert_eq!(sev, 0.0);
2161 }
2162
2163 #[test]
2164 fn no_error_streak_all_clean() {
2165 let texts = vec![("ok".to_string(), false), ("all good".to_string(), false)];
2166 assert_eq!(compute_no_error_streak(&texts), 2);
2167 }
2168
2169 #[test]
2170 fn no_error_streak_stops_at_error() {
2171 let texts = vec![
2172 ("Traceback (most recent call last):".to_string(), false),
2173 ("ok".to_string(), false),
2174 ("ok".to_string(), false),
2175 ];
2176 assert_eq!(compute_no_error_streak(&texts), 2);
2177 }
2178
2179 #[test]
2180 fn tests_passed_detects_pytest_output() {
2181 assert!(detect_tests_passed(
2182 &[("====== 5 passed in 0.12s ======".to_string(), false)],
2183 DEFAULT_RECENT_WINDOW
2184 ));
2185 }
2186
2187 #[test]
2188 fn tests_passed_ignores_partial_failures() {
2189 assert!(!detect_tests_passed(
2190 &[("2 failed, 5 passed in 0.56s".to_string(), false)],
2191 DEFAULT_RECENT_WINDOW
2192 ));
2193 }
2194
2195 #[test]
2196 fn tests_passed_must_follow_the_latest_failure() {
2197 assert!(!detect_tests_passed(
2198 &[
2199 ("5 passed in 0.12s".to_string(), false),
2200 (
2201 "Traceback (most recent call last):\nValueError".to_string(),
2202 false
2203 ),
2204 ("edit applied".to_string(), false),
2205 ],
2206 DEFAULT_RECENT_WINDOW
2207 ));
2208 assert!(detect_tests_passed(
2209 &[
2210 (
2211 "Traceback (most recent call last):\nValueError".to_string(),
2212 false
2213 ),
2214 ("5 passed in 0.12s".to_string(), false),
2215 ],
2216 DEFAULT_RECENT_WINDOW
2217 ));
2218 }
2219
2220 #[test]
2221 fn retrieved_file_contents_are_ignored() {
2222 let call = |id: &str, name: &str, arguments: Value| {
2223 Message::new(
2224 Role::Assistant,
2225 vec![ContentBlock::ToolCall(ToolCall {
2226 id: id.to_string(),
2227 name: name.to_string(),
2228 arguments,
2229 })],
2230 )
2231 };
2232 let result = |id: &str, text: &str| {
2233 Message::new(
2234 Role::User,
2235 vec![ContentBlock::ToolResult(ToolResult {
2236 tool_call_id: id.to_string(),
2237 content: vec![ContentBlock::Text {
2238 text: text.to_string(),
2239 }],
2240 is_error: None,
2241 })],
2242 )
2243 };
2244 let signal = extract_tool_signals_with_window(
2245 &with_messages(vec![
2246 call("a", "Bash", json!({"command": "pytest"})),
2247 result("a", "Traceback (most recent call last):\nValueError"),
2248 call("b", "Read", json!({"file_path": "notes.md"})),
2249 result(
2250 "b",
2251 r#"{"success":false,"error":"out of memory","exit_code":7}"#,
2252 ),
2253 call("c", "Grep", json!({"pattern": "passed"})),
2254 result("c", "CHANGELOG.md: all tests passed"),
2255 ]),
2256 DEFAULT_RECENT_WINDOW,
2257 );
2258 assert_eq!(signal.severity, HARD);
2260 assert!(!signal.tests_passed);
2261 assert_eq!(signal.tool_result_count, 3);
2262
2263 for (command, text, severity) in [
2264 ("cat config.json", r#"{"error":"fixture data"}"#, 0.0),
2265 (
2266 "cat missing.json",
2267 r#"{"output":"","exit_code":7,"error":null}"#,
2268 SOFT,
2269 ),
2270 (
2271 "cat config.json && python check.py",
2272 r#"{"error":"check rejected"}"#,
2273 HARD,
2274 ),
2275 ] {
2276 let request = with_messages(vec![
2277 call("shell", "Bash", json!({"command": command})),
2278 result("shell", text),
2279 ]);
2280 assert_eq!(
2281 ToolSignals::from_request(&request, None).severity,
2282 severity,
2283 "{text}"
2284 );
2285 }
2286 }
2287
2288 fn call_with_result(mut call: Message, mut result: Message) -> Request {
2290 if let ContentBlock::ToolCall(call) = &mut call.content[0] {
2291 call.id = "call-1".into();
2292 }
2293 if let ContentBlock::ToolResult(result) = &mut result.content[0] {
2294 result.tool_call_id = "call-1".into();
2295 }
2296 with_messages(vec![call, result])
2297 }
2298
2299 #[test]
2300 fn shell_reads_are_ignored_but_keep_their_window_slot() {
2301 for command in [
2302 "/usr/bin/cat logfile.txt",
2303 "cd /repo && tail -n 100 logfile.txt | grep MemoryError",
2304 "env MODE=debug timeout 5s rg 'MemoryError' .",
2305 ] {
2306 let mut request = call_with_result(bash(command), tr("MemoryError\nall tests passed"));
2307 request.llm_request.messages.splice(
2308 0..0,
2309 [bash("pytest"), tr("Traceback (most recent call last):")],
2310 );
2311 let signal = ToolSignals::from_request(&request, Some(2));
2313 assert_eq!(signal.severity, HARD, "{command}");
2314 assert!(!signal.tests_passed, "{command}");
2315 let signal = ToolSignals::from_request(&request, Some(1));
2317 assert_eq!(signal.severity, 0.0, "{command}");
2318 }
2319 }
2320
2321 #[test]
2322 fn shell_commands_that_run_code_keep_signals() {
2323 let oversized = format!("cat {}", "a".repeat(MAX_RETRIEVAL_COMMAND_BYTES));
2324 for command in [
2325 "cat logfile.txt; pytest",
2326 "echo $(pytest)",
2327 "find . -name '*.py' -exec pytest ';'",
2328 oversized.as_str(),
2329 ] {
2330 let request = call_with_result(bash(command), tr("MemoryError"));
2331 let signal = ToolSignals::from_request(&request, None);
2332 assert_eq!(signal.severity, CRITICAL, "{command}");
2333 }
2334 }
2335
2336 #[test]
2337 fn failed_reads_signal_unless_the_file_is_missing() {
2338 let failed = |text: &str| {
2339 let mut result = tr(text);
2340 if let ContentBlock::ToolResult(result) = &mut result.content[0] {
2341 result.is_error = Some(true);
2342 }
2343 result
2344 };
2345 let missing = "missing.txt: No such file or directory";
2346 let severities = [
2347 call_with_result(bash("cat missing.txt"), failed(missing)),
2348 call_with_result(tc("Read"), failed("File does not exist.")),
2349 call_with_result(bash("cat logfile.txt"), failed("MemoryError")),
2351 call_with_result(bash("python missing.txt"), failed(missing)),
2353 ]
2354 .map(|request| ToolSignals::from_request(&request, None).severity);
2355 assert_eq!(severities, [0.0, 0.0, CRITICAL, HARD]);
2356
2357 let request = call_with_result(
2358 bash("cat missing.json"),
2359 tr(r#"{"output":"No such file or directory","exit_code":1}"#),
2360 );
2361 assert_eq!(ToolSignals::from_request(&request, None).severity, 0.0);
2362 }
2363
2364 #[test]
2365 fn codex_read_status_comes_from_the_header() {
2366 let read = |header: &str| {
2367 let output = format!(
2368 "{header}\nOutput:\nTraceback (most recent call last):\nMemoryError\nProcess exited with code 1\n"
2369 );
2370 let call = exec_command(json!({"cmd": "cat run.log"}));
2371 ToolSignals::from_request(&call_with_result(call, tr(&output)), None).severity
2372 };
2373 assert_eq!(read("Chunk ID: abc\nProcess exited with code 0"), 0.0);
2375 assert_eq!(read("Chunk ID: abc\nProcess exited with code 1"), CRITICAL);
2377 }
2378
2379 #[test]
2380 fn search_no_match_is_an_observation_not_a_failure() {
2381 for command in [
2382 json!("rg missing ."),
2383 json!("grep missing file.txt"),
2384 json!("LC_ALL=C rg missing ."),
2385 json!("env LC_ALL=C rg missing ."),
2386 json!(["env", "LC_ALL=C", "rg", "missing", "."]),
2387 json!(["rg", "needle; cat file", "."]),
2388 json!("rg version poetry.lock; rg limits poetry.lock | sed -n '1,80p'; rg headers ."),
2389 ] {
2390 for output in [
2391 "Process exited with code 1\nOutput:\n",
2392 r#"{"output":"","exit_code":1,"error":null}"#,
2393 ] {
2394 let request = call_with_result(exec_command(json!({"cmd": command})), tr(output));
2395 let signal = ToolSignals::from_request(&request, None);
2396 assert_eq!(signal.severity, 0.0, "{command}: {output}");
2397 assert_eq!(signal.read_count, 1);
2398 }
2399 }
2400 }
2401
2402 #[test]
2403 fn search_detection_preserves_argv_boundaries() {
2404 let request = call_with_result(
2405 exec_command(json!({"cmd": ["cat", "log; rg needle ."]})),
2406 tr("Process exited with code 1\nOutput:\nPermission denied"),
2407 );
2408 assert_eq!(ToolSignals::from_request(&request, None).severity, SOFT);
2409 }
2410
2411 #[test]
2412 fn search_errors_and_non_search_exit_one_remain_failures() {
2413 for (command, output) in [
2414 (
2415 "rg missing .",
2416 "Process exited with code 2\nOutput:\nrg: invalid regex",
2417 ),
2418 (
2419 "rg missing .",
2420 "Process exited with code 1\nOutput:\nMemoryError",
2421 ),
2422 (
2423 "rg missing .",
2424 r#"{"output":"","exit_code":1,"error":"execution failed"}"#,
2425 ),
2426 (
2427 "cat file.txt",
2428 "Process exited with code 1\nOutput:\npermission denied",
2429 ),
2430 (
2431 "pytest; rg missing .",
2432 "Process exited with code 1\nOutput:\n",
2433 ),
2434 (
2435 "false && rg missing .",
2436 "Process exited with code 1\nOutput:\n",
2437 ),
2438 ] {
2439 let request = call_with_result(exec_command(json!({"cmd": command})), tr(output));
2440 assert!(
2441 ToolSignals::from_request(&request, None).severity > 0.0,
2442 "{command}: {output}"
2443 );
2444 }
2445 let mut request = call_with_result(
2446 exec_command(json!({"cmd": "rg missing ."})),
2447 tr("Process exited with code 1\nOutput:\n"),
2448 );
2449 if let ContentBlock::ToolResult(result) = &mut request.llm_request.messages[1].content[0] {
2450 result.is_error = Some(true);
2451 }
2452 assert_eq!(ToolSignals::from_request(&request, None).severity, HARD);
2453 }
2454
2455 fn polled_command(command: &str, chars: &str, output: &str) -> Request {
2456 let mut request = call_with_result(
2457 exec_command(json!({"cmd": command})),
2458 tr("Process running with session ID 123\nOutput:\n"),
2459 );
2460 let mut poll = tc("write_stdin");
2461 if let ContentBlock::ToolCall(call) = &mut poll.content[0] {
2462 call.id = "poll-1".into();
2463 call.arguments = json!({"session_id":123, "chars":chars});
2464 }
2465 let mut result = tr(output);
2466 if let ContentBlock::ToolResult(result) = &mut result.content[0] {
2467 result.tool_call_id = "poll-1".into();
2468 }
2469 request.llm_request.messages.extend([poll, result]);
2470 request
2471 }
2472
2473 #[test]
2474 fn codex_poll_preserves_read_context_without_hiding_execution_errors() {
2475 let output = "Process exited with code 0\nOutput:\nMemoryError\n5 passed in 0.12s";
2476 let request = polled_command("cat run.log", "", output);
2477 let signal = ToolSignals::from_request(&request, Some(1));
2478 assert_eq!(signal.severity, 0.0);
2479 assert_eq!(signal.recent_read_count, 1);
2480 assert!(!signal.tests_passed);
2481 for (command, chars) in [("python script.py", ""), ("cat run.log", "x")] {
2482 let signal =
2483 ToolSignals::from_request(&polled_command(command, chars, output), Some(1));
2484 assert_eq!(signal.severity, CRITICAL);
2485 }
2486 let failed = polled_command(
2487 "cat run.log",
2488 "",
2489 "Process exited with code 2\nOutput:\nMemoryError",
2490 );
2491 assert_eq!(
2492 ToolSignals::from_request(&failed, Some(1)).severity,
2493 CRITICAL
2494 );
2495 let no_match = polled_command("rg missing .", "", "Process exited with code 1\nOutput:\n");
2496 assert_eq!(ToolSignals::from_request(&no_match, Some(1)).severity, 0.0);
2497 let tests = polled_command(
2498 "yarn test",
2499 "",
2500 "Process exited with code 0\nOutput:\n5 passed in 0.12s",
2501 );
2502 assert!(ToolSignals::from_request(&tests, Some(1)).tests_passed);
2503 let write = polled_command(
2504 "echo done > result.txt",
2505 "",
2506 "Process exited with code 0\nOutput:\n",
2507 );
2508 assert_eq!(
2509 ToolSignals::from_request(&write, Some(1)).recent_write_count,
2510 0
2511 );
2512 }
2513
2514 #[test]
2515 fn codex_poll_keeps_context_across_user_turns_without_old_activity() {
2516 let mut request = polled_command(
2517 "cat run.log",
2518 "",
2519 "Process exited with code 0\nOutput:\nMemoryError",
2520 );
2521 request
2522 .llm_request
2523 .messages
2524 .insert(2, Message::text(Role::User, "Check the command"));
2525 let signal = ToolSignalProcessor::default().for_user_turn(&request);
2526 assert_eq!(signal.severity, 0.0);
2527 assert_eq!(signal.read_count, 1);
2528 assert_eq!(signal.tool_result_count, 1);
2529 }
2530
2531 #[test]
2532 fn hermes_read_status_comes_from_exit_code() {
2533 let read = |exit_code: i64| {
2534 let output = json!({
2535 "output": "Traceback (most recent call last):\nMemoryError\nProcess exited with code 1\n",
2536 "exit_code": exit_code,
2537 "error": null,
2538 });
2539 let call = Message::new(
2540 Role::Assistant,
2541 vec![ContentBlock::ToolCall(ToolCall {
2542 id: String::new(),
2543 name: "terminal".to_string(),
2544 arguments: json!({"command": "cat run.log"}),
2545 })],
2546 );
2547 ToolSignals::from_request(&call_with_result(call, tr(&output.to_string())), None)
2548 .severity
2549 };
2550 assert_eq!(read(0), 0.0);
2552 assert_eq!(read(1), CRITICAL);
2554 }
2555
2556 #[test]
2557 fn argv_commands_keep_word_boundaries() {
2558 let signal = |argv: Value| {
2559 let call = exec_command(json!({"cmd": argv}));
2560 ToolSignals::from_request(&call_with_result(call, tr("MemoryError")), None)
2561 };
2562 let mut many_args = vec![""; MAX_RETRIEVAL_COMMAND_BYTES];
2563 many_args[0] = "cat";
2564 let severities = [
2565 json!(["bash", "-lc", "cd repo && rg foo"]),
2566 json!(["bash", "-lc", "cat log; pytest"]),
2567 json!(["cat", "log; pytest"]),
2569 json!(["cat", "a".repeat(MAX_RETRIEVAL_COMMAND_BYTES)]),
2571 json!(many_args),
2572 ]
2573 .map(|argv| signal(argv).severity);
2574 assert_eq!(severities, [0.0, CRITICAL, 0.0, CRITICAL, CRITICAL]);
2575 let read = signal(json!(["bash", "-lc", "sed -n 1,200p x"]));
2577 assert_eq!(read.read_count, 1);
2578 let write = signal(json!(["bash", "-lc", "cd repo && mkdir x"]));
2580 assert_eq!(write.write_count, 1);
2581 }
2582
2583 #[test]
2584 fn severity_is_windowed_over_recent_results() {
2585 let request = with_messages(vec![
2587 tr("Traceback (most recent call last):\n ValueError"),
2588 tr("ok"),
2589 tr("ok"),
2590 ]);
2591 assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD);
2593 assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0);
2595 }
2596
2597 #[test]
2598 fn extract_openai_chat_tool_results() {
2599 let request = with_messages(vec![
2600 Message::text(Role::User, "do something"),
2601 tc("Edit"),
2602 tr("Traceback (most recent call last):\n ValueError"),
2603 ]);
2604 let sig = ToolSignals::from_request(&request, None);
2605 assert_eq!(sig.severity, HARD);
2606 assert_eq!(sig.edit_count, 1);
2607 assert_eq!(sig.turn_depth, 3);
2608 }
2609
2610 #[test]
2611 fn extract_anthropic_tool_results() {
2612 let request = with_messages(vec![tr("Traceback (most recent call last):\n ValueError")]);
2613 let sig = ToolSignals::from_request(&request, None);
2614 assert_eq!(sig.severity, HARD);
2615 }
2616
2617 #[test]
2618 fn extract_responses_api_tool_results() {
2619 let request = with_messages(vec![tc("Write"), tr("file written successfully")]);
2620 let sig = ToolSignals::from_request(&request, None);
2621 assert_eq!(sig.severity, 0.0);
2622 assert_eq!(sig.write_count, 1);
2623 }
2624
2625 #[test]
2626 fn responses_builtin_tool_failures_escalate() {
2627 use crate::algorithms::util::stage::{PickOutcome, PickerMode, Tier, pick_tier};
2628
2629 let mut cases = Vec::new();
2630 for (status, output) in [
2631 (
2632 "failed",
2633 "Synthetic dependency unavailable; retry with the recovery path.",
2634 ),
2635 (
2636 "completed",
2637 "Synthetic dependency unavailable; retry with the recovery path.",
2638 ),
2639 ("failed", ""),
2640 ] {
2641 cases.push((
2642 json!({
2643 "type": "apply_patch_call_output",
2644 "status": status,
2645 "output": output,
2646 }),
2647 if status == "failed" { HARD } else { 0.0 },
2648 ));
2649 }
2650 for (outcome, stdout, stderr, severity) in [
2651 (json!({"type": "exit", "exit_code": 1}), "", "", SOFT),
2652 (json!({"type": "exit", "exit_code": 1}), "", " \n", SOFT),
2653 (
2654 json!({"type": "exit", "exit_code": 1}),
2655 "",
2656 "command failed",
2657 HARD,
2658 ),
2659 (json!({"type": "timeout"}), "", "", HARD),
2660 (json!({"type": "exit", "exit_code": 0}), "done", "", 0.0),
2661 (
2662 json!({"type": "exit", "exit_code": 0}),
2663 "Traceback (most recent call last):",
2664 "",
2665 HARD,
2666 ),
2667 (
2668 json!({"type": "exit", "exit_code": 0}),
2669 "",
2670 "Traceback (most recent call last):",
2671 HARD,
2672 ),
2673 ] {
2674 cases.push((
2675 json!({
2676 "type": "shell_call_output",
2677 "output": [
2678 {"stdout": stdout, "stderr": stderr, "outcome": outcome},
2679 {"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}
2680 ],
2681 }),
2682 severity,
2683 ));
2684 }
2685 for (raw, severity) in cases {
2686 let is_error = severity >= HARD;
2687 let mut request = with_messages(
2688 ["call_1", "call_2"]
2689 .into_iter()
2690 .map(|call_id| {
2691 let mut raw = raw.clone();
2692 raw["call_id"] = json!(call_id);
2693 Message::new(
2694 Role::User,
2695 vec![ContentBlock::Unknown {
2696 provider: WireFormat::OpenAiResponses.into(),
2697 raw,
2698 }],
2699 )
2700 })
2701 .collect(),
2702 );
2703 let signal = ToolSignals::from_request(&request, Some(3));
2704 assert_eq!(signal.severity, severity, "{raw}");
2705 assert_eq!(signal.repeated_failure, is_error, "{raw}");
2706 assert_eq!(signal.tool_result_count, 2);
2707 assert_eq!(
2708 matches!(
2709 pick_tier(&signal, PickerMode::EfficientFirst, 0.5),
2710 PickOutcome::Resolved {
2711 tier: Tier::Capable,
2712 ..
2713 }
2714 ),
2715 is_error,
2716 "{raw}"
2717 );
2718
2719 let mut success = match raw["type"].as_str() {
2720 Some("apply_patch_call_output") => json!({
2721 "type": "apply_patch_call_output", "status": "completed", "output": ""
2722 }),
2723 Some("shell_call_output") => json!({
2724 "type": "shell_call_output",
2725 "output": [{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}]
2726 }),
2727 _ => unreachable!(),
2728 };
2729 for index in 0..3 {
2730 success["call_id"] = json!(format!("success_{index}"));
2731 request.llm_request.messages.push(Message::new(
2732 Role::User,
2733 vec![ContentBlock::Unknown {
2734 provider: WireFormat::OpenAiResponses.into(),
2735 raw: success.clone(),
2736 }],
2737 ));
2738 }
2739 let recovered = ToolSignals::from_request(&request, Some(3));
2740 assert_eq!(recovered.severity, 0.0, "{raw}");
2741 assert!(!recovered.repeated_failure, "{raw}");
2742 assert!(recovered.no_error_streak >= 3, "{raw}");
2743 assert_eq!(recovered.tool_result_count, 5);
2744 }
2745 }
2746
2747 #[test]
2748 fn conversation_counts_are_per_block_and_role_aware() {
2749 let result = |content: Vec<ContentBlock>| {
2753 ContentBlock::ToolResult(ToolResult {
2754 tool_call_id: String::new(),
2755 content,
2756 is_error: None,
2757 })
2758 };
2759 let request = with_messages(vec![
2760 Message::text(Role::User, "do something"),
2761 Message::text(Role::Assistant, "working"),
2762 Message::new(
2763 Role::User,
2764 vec![
2765 result(vec![ContentBlock::Text {
2766 text: "ok".to_string(),
2767 }]),
2768 result(Vec::new()),
2769 ],
2770 ),
2771 tc("Bash"),
2772 ]);
2773 let sig = ToolSignals::from_request(&request, None);
2774 assert_eq!(sig.tool_result_count, 2);
2775 assert_eq!(sig.assistant_turn_count, 2);
2776 assert_eq!(sig.turn_depth, 4);
2777 }
2778
2779 #[test]
2780 fn recent_window_counts_only_last_default_window_tool_calls() {
2781 let request = with_messages(vec![
2784 tc("Write"),
2785 tr("ok"),
2786 tc("Write"),
2787 tr("ok"),
2788 tc("Write"),
2789 tr("ok"),
2790 tc("Write"),
2791 tr("ok"),
2792 tc("Write"),
2793 tr("ok"),
2794 tc("Edit"),
2795 tr("ok"),
2796 ]);
2797 let sig = ToolSignals::from_request(&request, None);
2798 assert_eq!(sig.write_count, 5);
2799 assert_eq!(sig.edit_count, 1);
2800 assert_eq!(sig.recent_write_count, 2);
2801 assert_eq!(sig.recent_edit_count, 1);
2802 }
2803
2804 #[test]
2805 fn codex_apply_patch_counts_as_an_edit() {
2806 let request = with_messages(vec![tc("apply_patch"), tr("Success. Updated the file")]);
2807 let sig = ToolSignals::from_request(&request, None);
2808 assert_eq!(sig.edit_count, 1);
2809 assert_eq!(sig.recent_edit_count, 1);
2810 }
2811
2812 fn exec_command(cmd: Value) -> Message {
2813 Message::new(
2814 Role::Assistant,
2815 vec![ContentBlock::ToolCall(ToolCall {
2816 id: String::new(),
2817 name: "exec_command".to_string(),
2818 arguments: cmd,
2819 })],
2820 )
2821 }
2822
2823 #[test]
2824 fn codex_exec_command_is_classified() {
2825 let args = json!(r#"{"cmd":"sed -i s/a/b/ src/lib.rs","workdir":"/x"}"#);
2827 let request = with_messages(vec![exec_command(args), tr("ok")]);
2828 assert_eq!(
2829 ToolSignals::from_request(&request, None).recent_edit_count,
2830 1
2831 );
2832 }
2833
2834 #[test]
2835 fn python_write_expressions_need_a_python_command() {
2836 let write = with_messages(vec![
2837 exec_command(json!({"cmd": "python3 - <<'PY'\np.write_text(s)\nPY"})),
2838 tr("ok"),
2839 ]);
2840 assert_eq!(
2841 ToolSignals::from_request(&write, None).recent_write_count,
2842 1
2843 );
2844
2845 let search = with_messages(vec![
2846 exec_command(json!({"cmd": "grep -R '.write(' src"})),
2847 tr("ok"),
2848 ]);
2849 assert_eq!(
2850 ToolSignals::from_request(&search, None).recent_write_count,
2851 0
2852 );
2853 }
2854
2855 #[test]
2856 fn recent_window_size_is_caller_overridable() {
2857 let request = with_messages(vec![
2861 tc("Write"),
2862 tr("ok"),
2863 tc("Write"),
2864 tr("ok"),
2865 tc("Write"),
2866 tr("ok"),
2867 tc("Write"),
2868 tr("ok"),
2869 tc("Write"),
2870 tr("ok"),
2871 tc("Edit"),
2872 tr("ok"),
2873 ]);
2874 let narrow = extract_tool_signals_with_window(&request, 3);
2875 assert_eq!(narrow.recent_write_count, 2);
2876 assert_eq!(narrow.recent_edit_count, 1);
2877
2878 let wide = extract_tool_signals_with_window(&request, 6);
2879 assert_eq!(wide.recent_write_count, 5);
2880 assert_eq!(wide.recent_edit_count, 1);
2881 }
2882
2883 #[test]
2884 fn compaction_marker_sets_compacted() {
2885 for summary in [
2886 "This session is being continued from a previous conversation that ran out of context.",
2887 "Another language model started to solve this problem and produced a summary of its thinking process.",
2888 ] {
2889 let request = with_messages(vec![
2890 Message::text(Role::User, summary),
2891 Message::text(Role::User, "Continue with the next task"),
2892 bash("ls"),
2893 ]);
2894 assert!(ToolSignals::from_request(&request, None).compacted);
2895 assert!(
2896 ToolSignalProcessor::default()
2897 .for_user_turn(&request)
2898 .compacted
2899 );
2900 }
2901 }
2902
2903 #[test]
2904 fn hermes_compaction_headers_set_compacted() {
2905 for header in [
2906 "[CONTEXT COMPACTION \u{2014} REFERENCE ONLY] Earlier turns were compacted into the summary below.",
2907 "[STILL IN PROGRESS \u{2014} this is the active request, restated after the compaction boundary because it was not finished yet. Continue it\u{3b} do not start over.]",
2908 ] {
2909 for role in [Role::User, Role::Assistant] {
2910 let request = with_messages(vec![
2911 Message::text(role, format!("{header}\nContinue the task.")),
2912 bash("ls"),
2913 ]);
2914 assert!(ToolSignals::from_request(&request, None).compacted);
2915 }
2916 let request = with_messages(vec![Message::text(
2917 Role::User,
2918 format!("Prior context.\n\n {header}\nContinue the task."),
2919 )]);
2920 assert!(ToolSignals::from_request(&request, None).compacted);
2921 let request = with_messages(vec![tr(header)]);
2922 assert!(!ToolSignals::from_request(&request, None).compacted);
2923 }
2924 }
2925
2926 #[test]
2927 fn ordinary_compaction_text_stays_uncompacted() {
2928 for text in [
2929 "The task is still in progress after the compaction boundary.",
2930 "Explain [CONTEXT COMPACTION \u{2014} REFERENCE ONLY] in the docs.",
2931 "[STILL IN PROGRESS] Continue the task.",
2932 ] {
2933 let request = with_messages(vec![Message::text(Role::User, text)]);
2934 assert!(!ToolSignals::from_request(&request, None).compacted);
2935 }
2936 }
2937
2938 #[test]
2939 fn codex_compaction_metadata_stays_on_parent_route() {
2940 let mut request = with_messages(vec![bash("ls")]);
2941 request.metadata = Some(Metadata {
2942 is_subagent: true,
2943 agent_kind: Some("compact".to_string()),
2944 ..Default::default()
2945 });
2946 assert!(!ToolSignals::from_request(&request, None).compacted);
2947 }
2948
2949 #[test]
2950 fn no_compaction_marker_stays_uncompacted() {
2951 let request = with_messages(vec![
2952 Message::text(Role::User, "Write a script that parses the log file."),
2953 bash("ls"),
2954 ]);
2955 assert!(!ToolSignals::from_request(&request, None).compacted);
2956 }
2957
2958 #[test]
2959 fn bash_heredoc_counts_as_write() {
2960 let request = with_messages(vec![bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF")]);
2962 let sig = ToolSignals::from_request(&request, None);
2963 assert_eq!(
2964 sig.write_count, 1,
2965 "Bash heredoc should bucket into write_count"
2966 );
2967 assert_eq!(sig.edit_count, 0);
2968 }
2969
2970 #[test]
2971 fn bash_sed_inplace_counts_as_edit() {
2972 let request = with_messages(vec![bash("sed -i 's/foo/bar/g' /app/file.py")]);
2973 let sig = ToolSignals::from_request(&request, None);
2974 assert_eq!(
2975 sig.edit_count, 1,
2976 "Bash sed -i should bucket into edit_count"
2977 );
2978 assert_eq!(sig.write_count, 0);
2979 }
2980
2981 #[test]
2982 fn bash_non_mutating_does_not_count() {
2983 let request = with_messages(vec![bash("ls -la /app"), bash("cat /app/main.py")]);
2985 let sig = ToolSignals::from_request(&request, None);
2986 assert_eq!(sig.write_count, 0);
2987 assert_eq!(sig.edit_count, 0);
2988 }
2989
2990 #[test]
2991 fn tests_passed_detects_pytest_with_failure_block() {
2992 assert!(!detect_tests_passed(
2994 &[("2 failed, 5 passed in 0.56s".to_string(), false)],
2995 DEFAULT_RECENT_WINDOW
2996 ));
2997 }
2998
2999 #[test]
3000 fn tests_passed_accepts_cargo_clean_summary() {
3001 assert!(detect_tests_passed(
3004 &[(
3005 "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string(),
3006 false
3007 )],
3008 DEFAULT_RECENT_WINDOW
3009 ));
3010 }
3011
3012 #[test]
3013 fn tests_passed_rejects_cargo_real_failure() {
3014 assert!(!detect_tests_passed(
3016 &[(
3017 "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string(),
3018 false
3019 )],
3020 DEFAULT_RECENT_WINDOW
3021 ));
3022 }
3023
3024 #[test]
3025 fn tests_passed_accepts_go_clean_summary() {
3026 assert!(detect_tests_passed(
3028 &[(
3029 "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string(),
3030 false
3031 )],
3032 DEFAULT_RECENT_WINDOW
3033 ));
3034 }
3035
3036 #[test]
3037 fn tests_passed_accepts_pytest_zero_errors() {
3038 assert!(detect_tests_passed(
3040 &[("5 passed, 0 errors in 0.30s".to_string(), false)],
3041 DEFAULT_RECENT_WINDOW
3042 ));
3043 }
3044
3045 #[test]
3046 fn tests_passed_detects_diy_checkmark() {
3047 assert!(detect_tests_passed(
3048 &[("✓ all checks passed".to_string(), false)],
3049 DEFAULT_RECENT_WINDOW
3050 ));
3051 }
3052
3053 #[test]
3054 fn anthropic_bash_heredoc_extracts_command() {
3055 let request = with_messages(vec![bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF")]);
3057 let sig = ToolSignals::from_request(&request, None);
3058 assert_eq!(
3059 sig.write_count, 1,
3060 "Anthropic Bash heredoc must also be detected"
3061 );
3062 }
3063
3064 #[test]
3065 fn recent_window_falls_back_to_full_history_when_short() {
3066 let request = with_messages(vec![tc("Write")]);
3067 let sig = ToolSignals::from_request(&request, None);
3068 assert_eq!(sig.recent_write_count, 1);
3069 assert_eq!(sig.recent_edit_count, 0);
3070 }
3071
3072 #[test]
3073 fn clean_tool_result_has_zero_severity_and_non_empty_streak() {
3074 let request = with_messages(vec![tr("output ok"), tr("another ok")]);
3075 let sig = ToolSignals::from_request(&request, None);
3076 assert_eq!(sig.severity, 0.0);
3077 assert_eq!(sig.no_error_streak, 2);
3078 }
3079
3080 #[test]
3083 fn todowrite_classifies_as_plan() {
3084 assert_eq!(classify_tool_call("TodoWrite", None), ToolSemantic::Plan);
3085 assert_eq!(classify_tool_call("todo_write", None), ToolSemantic::Plan);
3086 }
3087
3088 #[test]
3089 fn codex_update_plan_classifies_as_plan() {
3090 assert_eq!(classify_tool_call("update_plan", None), ToolSemantic::Plan);
3091 }
3092
3093 #[test]
3094 fn codex_shell_command_runs_bash_pattern_match() {
3095 assert_eq!(
3097 classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")),
3098 ToolSemantic::Mutate(MutationKind::Write),
3099 );
3100 assert_eq!(
3102 classify_tool_call("shell_command", Some("ls /app")),
3103 ToolSemantic::Observe,
3104 );
3105 assert_eq!(
3107 classify_tool_call("shell_command", Some("./run_tests.sh")),
3108 ToolSemantic::Unknown,
3109 );
3110 }
3111
3112 #[test]
3113 fn text_editor_view_is_a_read() {
3114 for name in ["str_replace_based_edit_tool", "text_editor"] {
3115 assert_eq!(
3116 classify_tool_call(name, Some("view")),
3117 ToolSemantic::Observe
3118 );
3119 for command in [
3120 Some("create"),
3121 Some("insert"),
3122 Some("str_replace"),
3123 Some("undo_edit"),
3124 None,
3125 ] {
3126 assert_eq!(
3127 classify_tool_call(name, command),
3128 ToolSemantic::Mutate(MutationKind::Edit),
3129 );
3130 }
3131 }
3132
3133 let arguments = [
3134 json!({"command": "view", "path": "/app/main.py"}),
3135 json!(r#"{"command":"view","path":"/app/main.py"}"#),
3137 ];
3138 for arguments in arguments {
3139 let call = Message::new(
3140 Role::Assistant,
3141 vec![ContentBlock::ToolCall(ToolCall {
3142 id: String::new(),
3143 name: "str_replace_based_edit_tool".to_string(),
3144 arguments,
3145 })],
3146 );
3147 let request = with_messages(vec![call, tr("print('hi')")]);
3148 let sig = ToolSignals::from_request(&request, None);
3149 assert_eq!(sig.read_count, 1);
3150 assert_eq!(sig.recent_read_count, 1);
3151 assert_eq!(sig.edit_count, 0);
3152 }
3153 }
3154
3155 #[test]
3156 fn read_tool_classifies_as_read() {
3157 assert_eq!(classify_tool_call("Read", None), ToolSemantic::Observe);
3158 assert_eq!(classify_tool_call("View", None), ToolSemantic::Observe);
3159 }
3160
3161 #[test]
3162 fn hermes_tool_names_classify() {
3163 assert_eq!(
3165 classify_tool_call("write_file", None),
3166 ToolSemantic::Mutate(MutationKind::Write)
3167 );
3168 assert_eq!(
3169 classify_tool_call("patch", None),
3170 ToolSemantic::Mutate(MutationKind::Edit)
3171 );
3172 assert_eq!(classify_tool_call("read_file", None), ToolSemantic::Observe);
3173 assert_eq!(
3174 classify_tool_call("search_files", None),
3175 ToolSemantic::Observe
3176 );
3177 assert_eq!(
3180 classify_tool_call("terminal", Some("sed -i 's/a/b/' /app/x.py")),
3181 ToolSemantic::Mutate(MutationKind::Edit),
3182 );
3183 assert_eq!(
3184 classify_tool_call("terminal", Some("grep foo /app")),
3185 ToolSemantic::Observe,
3186 );
3187 assert_eq!(
3188 classify_tool_call("terminal", Some("./run_tests.sh")),
3189 ToolSemantic::Unknown,
3190 );
3191 }
3192
3193 #[test]
3194 fn bash_read_patterns_classify_as_read() {
3195 let cases = [
3196 "cat /etc/passwd",
3197 "grep foo bar.txt",
3198 "ls /app",
3199 "find . -name '*.py'",
3200 ];
3201 for cmd in cases {
3202 assert_eq!(
3203 classify_tool_call("Bash", Some(cmd)),
3204 ToolSemantic::Observe,
3205 "expected Read for {cmd}"
3206 );
3207 }
3208 }
3209
3210 #[test]
3211 fn codex_inspection_commands_classify_as_read() {
3212 let cases = [
3213 "sed -n '1,80p' src/lib.rs",
3214 "rg -n 'needle' src",
3215 "nl -ba src/lib.rs",
3216 "cat package.json",
3217 "jq '.scripts' package.json",
3218 "git status --short",
3219 "git log --oneline -5",
3220 "git show HEAD:src/lib.rs",
3221 "git branch --show-current",
3222 "git remote -v",
3223 "git config --get remote.origin.url",
3224 ];
3225 for command in cases {
3226 assert_eq!(
3227 classify_tool_call("exec_command", Some(command)),
3228 ToolSemantic::Observe,
3229 "expected Read for {command}"
3230 );
3231 }
3232 }
3233
3234 #[test]
3235 fn quoted_shell_separators_do_not_create_commands() {
3236 for command in ["rg 'foo|rm obsolete.rs'", "rg \"foo; rm obsolete.rs\""] {
3237 assert_eq!(
3238 classify_tool_call("exec_command", Some(command)),
3239 ToolSemantic::Observe,
3240 "quoted text must not be parsed as a command: {command}"
3241 );
3242 }
3243 }
3244
3245 #[test]
3246 fn codex_shell_mutations_classify_as_production() {
3247 let writes = [
3248 "cp source.rs destination.rs",
3249 "mkdir -p src/generated",
3250 "touch src/generated/mod.rs",
3251 "git show HEAD:file.rs > file.rs",
3252 "node <<'node'\nfs.writefilesync('file.js', text)\nnode",
3253 ];
3254 for command in writes {
3255 assert_eq!(
3256 classify_tool_call("exec_command", Some(command)),
3257 ToolSemantic::Mutate(MutationKind::Write),
3258 "expected Write for {command}"
3259 );
3260 }
3261
3262 let edits = [
3263 "mv old.rs new.rs",
3264 "rm obsolete.rs",
3265 "gofmt -w main.go",
3266 "cargo fmt",
3267 "ruff check --fix src",
3268 "perl -0pi -e 's/old/new/' src/lib.rs",
3269 "npx prettier --write src/lib.ts",
3270 "uv run ruff format src",
3271 "git apply fix.patch",
3272 ];
3273 for command in edits {
3274 assert_eq!(
3275 classify_tool_call("exec_command", Some(command)),
3276 ToolSemantic::Mutate(MutationKind::Edit),
3277 "expected Edit for {command}"
3278 );
3279 }
3280 }
3281
3282 #[test]
3283 fn formatter_checks_are_not_edits() {
3284 for command in [
3285 "cargo fmt --check",
3286 "ruff format --check src",
3287 "black --check src",
3288 ] {
3289 assert_ne!(
3290 classify_tool_call("exec_command", Some(command)),
3291 ToolSemantic::Mutate(MutationKind::Edit),
3292 "read-only formatter check must not be Edit: {command}"
3293 );
3294 }
3295 }
3296
3297 #[test]
3298 fn embedded_comparison_is_not_a_shell_write() {
3299 let command = "node <<'node'\nif (index > 0) console.log(index)\nnode";
3300 assert_eq!(
3301 classify_tool_call("exec_command", Some(command)),
3302 ToolSemantic::Unknown
3303 );
3304 }
3305
3306 #[test]
3307 fn bash_write_precedence_over_read() {
3308 assert_eq!(
3311 classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")),
3312 ToolSemantic::Mutate(MutationKind::Write),
3313 );
3314 }
3315
3316 #[test]
3317 fn pure_bash_streak_counts_trailing_other() {
3318 let request = with_messages(vec![
3320 bash("make"),
3321 tr("ok"),
3322 bash("./configure"),
3323 tr("ok"),
3324 bash("make install"),
3325 tr("ok"),
3326 bash("./run.sh"),
3327 tr("ok"),
3328 bash("./test"),
3329 tr("ok"),
3330 ]);
3331 let sig = ToolSignals::from_request(&request, None);
3332 assert_eq!(sig.pure_bash_streak, 5);
3333 assert_eq!(sig.write_count, 0);
3334 assert_eq!(sig.read_count, 0);
3335 }
3336
3337 #[test]
3338 fn pure_bash_streak_resets_on_write() {
3339 let request = with_messages(vec![bash("make"), tr("ok"), tc("Write"), tr("ok")]);
3340 let sig = ToolSignals::from_request(&request, None);
3341 assert_eq!(sig.pure_bash_streak, 0);
3342 assert_eq!(sig.write_count, 1);
3343 }
3344
3345 #[test]
3346 fn recent_window_tracks_todowrite_and_read() {
3347 let request = with_messages(vec![
3349 bash("make"),
3350 tr("ok"),
3351 tc("TodoWrite"),
3352 tr("ok"),
3353 tc("Read"),
3354 tr("ok"),
3355 tc("TodoWrite"),
3356 tr("ok"),
3357 ]);
3358 let sig = ToolSignals::from_request(&request, None);
3359 assert_eq!(sig.todowrite_count, 2);
3360 assert_eq!(sig.recent_todowrite_count, 2);
3361 assert_eq!(sig.read_count, 1);
3362 assert_eq!(sig.recent_read_count, 1);
3363 }
3364
3365 #[test]
3366 fn configured_tool_semantics_extend_the_builtin_vocabulary() {
3367 let semantics = ToolSemantics {
3368 observe: vec!["KB_search".to_string()],
3369 mutate: vec!["send_payment_request".to_string()],
3370 plan: vec!["create_research_plan".to_string()],
3371 new: vec!["send_message_to_user".to_string()],
3372 };
3373 semantics.validate().expect("valid additive semantics");
3374 let request = with_messages(vec![
3375 tc("Read"),
3376 tc("Write"),
3377 tc("TodoWrite"),
3378 tc("kb_SEARCH"),
3379 tc("send_payment_request"),
3380 tc("create_research_plan"),
3381 tc("send_message_to_user"),
3382 tc("unlisted_tool"),
3383 ]);
3384
3385 let signal = ToolSignals::from_request_with_semantics(&request, None, &semantics);
3386
3387 assert_eq!(signal.read_count, 2);
3388 assert_eq!(signal.write_count, 2);
3389 assert_eq!(signal.todowrite_count, 2);
3390 assert_eq!(signal.new_count, 1);
3391 assert_eq!(signal.recent_new_count, 1);
3392 assert_eq!(signal.pure_bash_streak, 1);
3393 }
3394
3395 #[test]
3396 fn configured_tool_semantics_match_namespaced_and_mcp_tools() {
3397 let mut request = with_messages(vec![tc("mcp__billing__send_payment_request")]);
3399 request.llm_request.extensions.fields.insert(
3400 TOOL_NAMESPACES_KEY.to_string(),
3401 json!({"mcp__billing__send_payment_request": "mcp__billing"}),
3402 );
3403
3404 let claude_request = with_messages(vec![tc("mcp__billing__send_payment_request")]);
3406
3407 for request in [&request, &claude_request] {
3408 for name in ["send_payment_request", "mcp__billing__send_payment_request"] {
3409 let semantics = ToolSemantics {
3410 mutate: vec![name.to_string()],
3411 ..Default::default()
3412 };
3413 let signal = ToolSignals::from_request_with_semantics(request, None, &semantics);
3414 assert_eq!(signal.write_count, 1, "{name}");
3415 }
3416 }
3417 }
3418
3419 #[test]
3420 fn configured_tool_semantics_only_fold_ascii_case() {
3421 let semantics = ToolSemantics {
3422 observe: vec!["kb_search".to_string()],
3423 ..Default::default()
3424 };
3425
3426 assert_eq!(
3427 classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
3428 ToolSemantic::Observe
3429 );
3430 assert_eq!(
3433 classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
3434 ToolSemantic::Unknown
3435 );
3436 }
3437
3438 #[test]
3439 fn custom_semantics_preserve_builtin_unicode_lowercasing() {
3440 let semantics = ToolSemantics {
3441 observe: vec!["lookup_customer".to_string()],
3442 ..Default::default()
3443 };
3444
3445 assert_eq!(
3447 classify_tool_call_with_semantics("notebooKedit", None, &semantics),
3448 ToolSemantic::Mutate(MutationKind::Edit)
3449 );
3450 }
3451
3452 #[test]
3453 fn configured_semantics_never_replace_builtin_classifications() {
3454 let semantics = ToolSemantics {
3455 observe: vec!["lookup_customer".to_string()],
3456 mutate: vec!["send_payment".to_string()],
3457 plan: vec!["create_workflow".to_string()],
3458 new: vec!["send_message".to_string()],
3459 };
3460
3461 for name in WRITE_TOOL_NAMES {
3462 assert_eq!(
3463 classify_tool_call_with_semantics(name, None, &semantics),
3464 ToolSemantic::Mutate(MutationKind::Write),
3465 "write tool {name:?} changed classification"
3466 );
3467 }
3468 for name in EDIT_TOOL_NAMES {
3469 assert_eq!(
3470 classify_tool_call_with_semantics(name, None, &semantics),
3471 ToolSemantic::Mutate(MutationKind::Edit),
3472 "edit tool {name:?} changed classification"
3473 );
3474 }
3475 for name in READ_TOOL_NAMES {
3476 assert_eq!(
3477 classify_tool_call_with_semantics(name, None, &semantics),
3478 ToolSemantic::Observe,
3479 "read tool {name:?} changed classification"
3480 );
3481 }
3482 for name in PLAN_TOOL_NAMES {
3483 assert_eq!(
3484 classify_tool_call_with_semantics(name, None, &semantics),
3485 ToolSemantic::Plan,
3486 "plan tool {name:?} changed classification"
3487 );
3488 }
3489
3490 for (command, expected) in [
3491 ("cat /tmp/input", ToolSemantic::Observe),
3492 (
3493 "cat /tmp/input > /tmp/output",
3494 ToolSemantic::Mutate(MutationKind::Write),
3495 ),
3496 (
3497 "sed -i 's/a/b/' /tmp/file",
3498 ToolSemantic::Mutate(MutationKind::Edit),
3499 ),
3500 ("./run_tests.sh", ToolSemantic::Unknown),
3501 ] {
3502 assert_eq!(
3503 classify_tool_call_with_semantics("BASH", Some(command), &semantics),
3504 expected,
3505 "bash command {command:?} changed classification"
3506 );
3507 }
3508 }
3509
3510 #[test]
3511 fn configured_semantics_score_like_their_builtin_equivalents() {
3512 let semantics = ToolSemantics {
3513 observe: vec!["lookup_customer".to_string()],
3514 mutate: vec!["send_payment".to_string()],
3515 plan: vec!["create_workflow".to_string()],
3516 ..Default::default()
3517 };
3518
3519 for (builtin, configured) in [
3520 ("Read", "lookup_customer"),
3521 ("Write", "send_payment"),
3522 ("TodoWrite", "create_workflow"),
3523 ] {
3524 let messages_before_tool = || {
3525 vec![
3526 Message::text(Role::User, "start"),
3527 Message::text(Role::Assistant, "working"),
3528 Message::text(Role::User, "continue"),
3529 Message::text(Role::Assistant, "working"),
3530 Message::text(Role::User, "continue"),
3531 Message::text(Role::Assistant, "working"),
3532 Message::text(Role::User, "continue"),
3533 ]
3534 };
3535 let mut builtin_messages = messages_before_tool();
3536 builtin_messages.push(tc(builtin));
3537 let mut configured_messages = messages_before_tool();
3538 configured_messages.push(tc(configured));
3539
3540 let builtin_score = score_signal(&ToolSignals::from_request(
3541 &with_messages(builtin_messages),
3542 None,
3543 ));
3544 let configured_score = score_signal(&ToolSignals::from_request_with_semantics(
3545 &with_messages(configured_messages),
3546 None,
3547 &semantics,
3548 ));
3549
3550 assert_ne!(
3551 builtin_score.score, 0.0,
3552 "the {builtin:?} control must exercise a scoring dimension"
3553 );
3554 assert_eq!(
3555 configured_score, builtin_score,
3556 "configured tool {configured:?} must score exactly like {builtin:?}"
3557 );
3558 }
3559 }
3560
3561 #[test]
3562 fn tool_semantics_reject_duplicates_and_builtin_reclassification() {
3563 let duplicate = ToolSemantics {
3564 observe: vec!["lookup".to_string()],
3565 mutate: vec!["LOOKUP".to_string()],
3566 ..Default::default()
3567 };
3568 assert!(
3569 duplicate
3570 .validate()
3571 .expect_err("duplicate should fail")
3572 .to_string()
3573 .contains("appears in both")
3574 );
3575
3576 let builtin = ToolSemantics {
3577 new: vec!["write_file".to_string()],
3578 ..Default::default()
3579 };
3580 assert!(
3581 builtin
3582 .validate()
3583 .expect_err("built-in should fail")
3584 .to_string()
3585 .contains("built-in semantics")
3586 );
3587
3588 let empty = ToolSemantics {
3589 observe: vec![" \t".to_string()],
3590 ..Default::default()
3591 };
3592 assert!(
3593 empty
3594 .validate()
3595 .expect_err("empty name should fail")
3596 .to_string()
3597 .contains("empty tool name")
3598 );
3599 }
3600}