1use std::collections::HashSet;
14use std::path::Path;
15use std::sync::LazyLock;
16
17use async_trait::async_trait;
18use regex::Regex;
19use serde::Deserialize;
20use serde_json::Value;
21use switchyard_protocol::codex_namespaces::{split_qualified_name, tool_namespaces};
22use switchyard_protocol::{ContentBlock, Request, Role, WireFormat};
23
24use super::affinity::is_user_turn;
25use crate::{LibsyError, Result};
26
27use crate::core::processor::{Event, Processor};
28use crate::core::state::State;
29
30const SOFT: f32 = 0.3;
33const HARD: f32 = 0.7;
34const CRITICAL: f32 = 1.0;
35
36static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[
40 (
41 "oom",
42 CRITICAL,
43 &["out of memory", "memoryerror", "cannot allocate memory"],
44 ),
45 (
46 "connection_refused",
47 HARD,
48 &[
49 "connection refused",
50 "connectionrefusederror",
51 "econnrefused",
52 ],
53 ),
54 ("traceback", HARD, &["traceback (most recent call last)"]),
55 (
56 "import_error",
57 HARD,
58 &["modulenotfounderror:", "importerror:", "no module named "],
59 ),
60 (
61 "cmd_not_found",
62 HARD,
63 &["command not found", "/usr/bin/env: "],
64 ),
65 ("assertion", HARD, &["assertionerror"]),
66 ("value_error", HARD, &["valueerror:"]),
67 ("syntax_error", HARD, &["syntaxerror:"]),
68 (
69 "timeout",
70 HARD,
71 &[
72 "timed out",
73 "timeouterror",
74 "timeout expired",
75 "deadline exceeded",
76 ],
77 ),
78 (
79 "no_such_file",
80 HARD,
81 &[
82 "filenotfounderror:",
83 "no such file or directory",
84 "file does not exist",
89 ],
90 ),
91 ("exit_nonzero", SOFT, &["returned non-zero"]),
93];
94
95static NONZERO_EXIT_PHRASES: &[&str] = &[
96 "exit code",
97 "exit status",
98 "exited with code",
99 "exited with status",
100];
101
102static EDIT_TOOL_NAMES: &[&str] = &[
103 "edit",
104 "multiedit",
105 "notebookedit",
106 "str_replace",
107 "str_replace_based_edit_tool",
108 "apply_patch", "text_editor",
110 "patch", ];
112
113static EDITOR_TOOL_NAMES: &[&str] = &["str_replace_based_edit_tool", "text_editor"];
115
116static WRITE_TOOL_NAMES: &[&str] = &["write", "create_file", "new_file", "write_file"];
117
118static BASH_WRITE_PATTERNS: &[&str] = &[
122 "cat >",
123 "cat >>",
124 "echo >",
125 "echo >>",
126 "tee ",
127 "printf >",
128 "printf >>",
129 "> /",
130 ">> /",
131 "<< 'eof'",
132 "<<eof",
133 "<<'eof'",
134 "<< eof",
135];
136
137static PYTHON_WRITE_PATTERNS: &[&str] = &["write_text(", "writelines(", ".write("];
140
141static JAVASCRIPT_WRITE_PATTERNS: &[&str] = &[
142 "writefilesync(",
143 "writefile(",
144 "appendfilesync(",
145 "appendfile(",
146];
147
148static BASH_EDIT_PATTERNS: &[&str] = &[
149 "sed -i",
150 "sed --in-place",
151 "awk -i inplace",
152 "awk 'inplace=1'",
153 "perl -i",
154 "perl -p -i",
155 "perl -pi",
156];
157
158static BASH_READ_PATTERNS: &[&str] = &[
161 "cat /", "cat ./", "cat ../", "grep ", "ls ", "ls -", "find ", "head ", "tail ", "wc ",
162 "diff ", "which ", "ps ", "df ", "du ", "stat ", "file ", "less ", "more ",
163];
164
165static BASH_READ_COMMANDS: &[&str] = &[
168 "cat", "rg", "nl", "jq", "pwd", "tree", "sed", "grep", "ls", "find", "head", "tail", "wc",
169 "diff", "which", "ps", "df", "du", "stat", "file", "less", "more", "readlink", "realpath",
170 "basename", "dirname", "printenv",
171];
172
173static GIT_READ_SUBCOMMANDS: &[&str] = &[
174 "status",
175 "diff",
176 "log",
177 "show",
178 "show-ref",
179 "rev-parse",
180 "ls-files",
181 "ls-remote",
182 "ls-tree",
183 "grep",
184 "blame",
185 "merge-base",
186 "check-ignore",
187 "tag",
188];
189
190static READ_TOOL_NAMES: &[&str] = &[
191 "read",
192 "view",
193 "read_file",
194 "search_files",
195 "glob",
196 "grep",
197 "find",
198 "ls",
199];
200
201static PLAN_TOOL_NAMES: &[&str] = &[
204 "todowrite",
205 "todo_write",
206 "todo",
207 "update_plan",
208 "todo_list",
209];
210
211static BASH_TOOL_NAMES: &[&str] = &[
216 "bash",
217 "shell_command",
218 "shell",
219 "local_shell_call",
220 "terminal",
221 "exec_command", "exec", "powershell", ];
225
226static TEST_PASS_PHRASES: &[&str] = &[
229 " passed",
230 "passed in",
231 "tests passed",
232 "all tests passed",
233 "test ok",
234 "test result: ok",
235 "passed.\n",
236 "tests pass",
237 "\nok ", "✓ ",
239];
240
241static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "error:"];
246
247static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"];
251
252pub const DEFAULT_RECENT_WINDOW: usize = 3;
259
260#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
266#[serde(default, deny_unknown_fields)]
267pub struct ToolSemantics {
268 pub observe: Vec<String>,
270 pub mutate: Vec<String>,
272 pub plan: Vec<String>,
274 pub new: Vec<String>,
276}
277
278impl ToolSemantics {
279 pub fn validate(&self) -> Result<()> {
281 let mut seen: Vec<(String, &'static str)> = Vec::new();
282 for (category, names) in [
283 ("observe", &self.observe),
284 ("mutate", &self.mutate),
285 ("plan", &self.plan),
286 ("new", &self.new),
287 ] {
288 for name in names {
289 if name.trim().is_empty() {
290 return Err(tool_semantics_error(format!(
291 "tool_semantics.{category} contains an empty tool name"
292 )));
293 }
294 let normalized = name.to_ascii_lowercase();
295 if is_builtin_tool_name(&name.to_lowercase()) {
296 return Err(tool_semantics_error(format!(
297 "tool {name:?} already has built-in semantics and cannot be reclassified"
298 )));
299 }
300 if let Some((_, previous)) = seen.iter().find(|(seen, _)| seen == &normalized) {
301 return Err(tool_semantics_error(format!(
302 "tool {name:?} appears in both tool_semantics.{previous} and tool_semantics.{category}"
303 )));
304 }
305 seen.push((normalized, category));
306 }
307 }
308 Ok(())
309 }
310
311 fn classify(&self, name: &str) -> Option<ToolSemantic> {
312 if contains_name(&self.observe, name) {
313 Some(ToolSemantic::Observe)
314 } else if contains_name(&self.mutate, name) {
315 Some(ToolSemantic::Mutate(MutationKind::Write))
318 } else if contains_name(&self.plan, name) {
319 Some(ToolSemantic::Plan)
320 } else if contains_name(&self.new, name) {
321 Some(ToolSemantic::New)
322 } else {
323 None
324 }
325 }
326}
327
328fn contains_name(names: &[String], candidate: &str) -> bool {
329 names
330 .iter()
331 .any(|name| name.eq_ignore_ascii_case(candidate))
332}
333
334fn tool_semantics_error(message: String) -> LibsyError {
335 LibsyError::AlgorithmError { message }
336}
337
338#[derive(Clone, Debug, Default)]
347pub struct ToolSignals {
348 pub severity: f32,
353 pub repeated_failure: bool,
356 pub no_error_streak: u32,
358 pub edit_count: u32,
360 pub write_count: u32,
362 pub read_count: u32,
364 pub todowrite_count: u32,
367 pub recent_edit_count: u32,
369 pub recent_write_count: u32,
371 pub recent_read_count: u32,
373 pub recent_todowrite_count: u32,
375 pub new_count: u32,
377 pub recent_new_count: u32,
379 pub pure_bash_streak: u32,
382 pub tests_passed: bool,
384 pub tool_result_count: u32,
387 pub assistant_turn_count: u32,
390 pub turn_depth: u32,
394 pub compacted: bool,
400}
401
402impl ToolSignals {
403 pub fn from_request(request: &Request, window_size: Option<usize>) -> Self {
408 Self::from_request_with_semantics(request, window_size, &ToolSemantics::default())
409 }
410
411 pub fn from_request_with_semantics(
413 request: &Request,
414 window_size: Option<usize>,
415 semantics: &ToolSemantics,
416 ) -> Self {
417 extract_tool_signals_with_window_and_semantics(
418 request,
419 window_size.unwrap_or(DEFAULT_RECENT_WINDOW),
420 semantics,
421 false,
422 )
423 }
424}
425
426#[derive(Debug, Clone, Copy, PartialEq, Eq)]
427enum MutationKind {
428 Write,
429 Edit,
430}
431
432#[derive(Debug, Clone, Copy, PartialEq, Eq)]
434enum ToolSemantic {
435 Mutate(MutationKind),
436 Observe,
437 Plan,
438 New,
439 Unknown,
440}
441
442#[derive(Debug, Clone)]
445pub struct ToolSignalProcessor {
446 pub recent_window: usize,
449 pub tool_semantics: ToolSemantics,
451}
452
453impl Default for ToolSignalProcessor {
454 fn default() -> Self {
455 Self {
456 recent_window: DEFAULT_RECENT_WINDOW,
457 tool_semantics: ToolSemantics::default(),
458 }
459 }
460}
461
462impl ToolSignalProcessor {
463 pub(crate) fn for_user_turn(&self, request: &Request) -> ToolSignals {
464 extract_tool_signals_with_window_and_semantics(
465 request,
466 self.recent_window,
467 &self.tool_semantics,
468 true,
469 )
470 }
471}
472
473#[async_trait]
474impl Processor for ToolSignalProcessor {
475 async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
476 if let Event::Request { request: req, .. } = event {
477 let tool_signal = ToolSignals::from_request_with_semantics(
478 req,
479 Some(self.recent_window),
480 &self.tool_semantics,
481 );
482 state.tool_signals = Some(tool_signal);
483 }
484 Ok(())
485 }
486}
487
488#[cfg(test)]
489fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic {
490 classify_tool_call_with_semantics(
491 name,
492 command
493 .map(|command| Value::String(command.to_owned()))
494 .as_ref(),
495 &ToolSemantics::default(),
496 )
497}
498
499fn code_mode_has_patch(mut input: &str) -> bool {
502 static STATEMENT: LazyLock<Regex> = LazyLock::new(|| {
503 let identifier = r"[a-zA-Z_$][a-zA-Z0-9_$]*";
504 let literal = r#"(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|`(?:[^`\\$]|\\[^\r\n])*`)"#;
505 let argument = format!(r"(?:{literal}|{identifier})");
506 let patch = format!(r"await\s+tools\s*\.\s*apply_patch\s*\(\s*{argument}\s*\)");
507 Regex::new(&format!(
509 r"(?x)\A(?://[^\r\n]*|(?s:/\*.*?\*/)|(?:
510 (?:const|let)\s+(?P<binding>{identifier})\s*=\s*{literal}|
511 (?P<patch>{patch}|text\s*\(\s*{patch}\s*\))|
512 text\s*\(\s*{argument}\s*\)
513 )[\t\x20]*(?:;|\r?\n|\z))"
514 ))
515 .expect("constant code-mode statement pattern")
516 });
517 let mut has_patch = false;
518 while !input.trim_start().is_empty() {
519 input = input.trim_start();
520 let Some(statement) = STATEMENT.captures(input) else {
521 return false;
522 };
523 if statement
524 .name("binding")
525 .is_some_and(|binding| matches!(binding.as_str(), "tools" | "text"))
526 {
527 return false;
528 }
529 has_patch |= statement.name("patch").is_some();
530 input = &input[statement[0].len()..];
531 }
532 has_patch
533}
534
535fn classify_tool_call_with_semantics(
536 name: &str,
537 command: Option<&Value>,
538 semantics: &ToolSemantics,
539) -> ToolSemantic {
540 let lower = name.to_lowercase();
542 if WRITE_TOOL_NAMES.contains(&lower.as_str()) {
543 return ToolSemantic::Mutate(MutationKind::Write);
544 }
545 if EDITOR_TOOL_NAMES.contains(&lower.as_str())
546 && command
547 .and_then(Value::as_str)
548 .is_some_and(|command| command.eq_ignore_ascii_case("view"))
549 {
550 return ToolSemantic::Observe;
551 }
552 if EDIT_TOOL_NAMES.contains(&lower.as_str()) {
553 return ToolSemantic::Mutate(MutationKind::Edit);
554 }
555 if READ_TOOL_NAMES.contains(&lower.as_str()) {
556 return ToolSemantic::Observe;
557 }
558 if PLAN_TOOL_NAMES.contains(&lower.as_str()) {
559 return ToolSemantic::Plan;
560 }
561 if BASH_TOOL_NAMES.contains(&lower.as_str())
562 && let Some(command) = command
563 && let Some(cmd) = command_text(command)
564 {
565 if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_write(&cmd) {
567 return ToolSemantic::Mutate(MutationKind::Write);
568 }
569 if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) {
570 return ToolSemantic::Mutate(MutationKind::Write);
571 }
572 if shell_invokes_program(&cmd, "node")
573 && JAVASCRIPT_WRITE_PATTERNS
574 .iter()
575 .any(|pattern| cmd.contains(pattern))
576 {
577 return ToolSemantic::Mutate(MutationKind::Write);
578 }
579 if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p))
580 || shell_command_is_edit(&cmd)
581 || command_invokes_patch(command)
582 {
583 return ToolSemantic::Mutate(MutationKind::Edit);
584 }
585 if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_read(&cmd) {
586 return ToolSemantic::Observe;
587 }
588 }
589 semantics.classify(name).unwrap_or(ToolSemantic::Unknown)
590}
591
592fn is_retrieval_tool(name: &str, command: Option<&Value>) -> bool {
594 let lower = name.to_lowercase();
595 READ_TOOL_NAMES.contains(&lower.as_str())
596 || (EDITOR_TOOL_NAMES.contains(&lower.as_str())
597 && command
598 .and_then(Value::as_str)
599 .is_some_and(|command| command.eq_ignore_ascii_case("view")))
600 || (lower != "powershell"
602 && BASH_TOOL_NAMES.contains(&lower.as_str())
603 && command.is_some_and(command_is_retrieval))
604}
605
606fn command_is_retrieval(command: &Value) -> bool {
608 match command {
609 Value::String(line) => shell_is_retrieval(line, 0),
610 Value::Array(argv) => {
611 argv.iter()
614 .try_fold(0, |size, word| {
615 let size = size + word.as_str()?.len() + 1;
616 (size <= MAX_RETRIEVAL_COMMAND_BYTES).then_some(size)
617 })
618 .is_some()
619 && retrieval_command(
620 &argv.iter().filter_map(Value::as_str).collect::<Vec<_>>(),
621 0,
622 )
623 }
624 _ => false,
625 }
626}
627
628const MAX_RETRIEVAL_COMMAND_BYTES: usize = 16 * 1024;
631
632fn shell_is_retrieval(command: &str, depth: usize) -> bool {
636 if command.len() > MAX_RETRIEVAL_COMMAND_BYTES {
637 return false;
638 }
639 if quoted_chars(command).any(|(_, c, quote)| {
640 (quote != Some('\'') && matches!(c, '$' | '`'))
641 || (quote.is_none() && matches!(c, '<' | '>' | '(' | ')' | '{' | '}' | '#'))
642 }) {
643 return false;
644 }
645 let mut segments = shell_segments(command).peekable();
646 segments.peek().is_some()
647 && segments.all(|segment| {
648 shlex::split(segment).is_some_and(|words| {
650 retrieval_command(&words.iter().map(String::as_str).collect::<Vec<_>>(), depth)
651 })
652 })
653}
654
655fn skip_shell_options<'a>(mut args: &'a [&'a str], takes_value: &[&str]) -> &'a [&'a str] {
656 while let Some((option, rest)) = args.split_first() {
657 if !option.starts_with('-') || *option == "-" {
658 break;
659 }
660 args = rest;
661 if *option == "--" {
662 break;
663 }
664 if takes_value.contains(option) {
665 args = args.get(1..).unwrap_or_default();
666 }
667 }
668 args
669}
670
671fn has_short_flag(word: &str, flags: &[char]) -> bool {
673 word.starts_with('-') && !word.starts_with("--") && word.contains(flags)
674}
675
676const MAX_COMMAND_DEPTH: usize = 20;
678
679fn skip_shell_assignments<'a>(mut words: &'a [&'a str]) -> &'a [&'a str] {
681 while words.first().is_some_and(|word| {
682 word.split_once('=').is_some_and(|(name, _)| {
683 name.starts_with(|c: char| c == '_' || c.is_ascii_alphabetic())
684 && name.bytes().all(|c| c == b'_' || c.is_ascii_alphanumeric())
685 })
686 }) {
687 words = &words[1..];
688 }
689 words
690}
691
692fn shell_wrapper_arguments<'a>(program: &str, args: &'a [&'a str]) -> Option<&'a [&'a str]> {
695 let options: &[&str] = match program {
696 "sudo" => &[
697 "-u", "-g", "-h", "-p", "-C", "-T", "--user", "--group", "--host",
698 ],
699 "env" => &["-u", "--unset", "-C", "--chdir"],
700 "command" | "nohup" => &[],
701 "exec" => &["-a"],
702 "time" => &["-f", "--format", "-o", "--output"],
703 "timeout" => &["-s", "--signal", "-k", "--kill-after"],
704 "xargs" => &[
705 "-n",
706 "--max-args",
707 "-P",
708 "--max-procs",
709 "-I",
710 "--replace",
711 "-d",
712 "--delimiter",
713 "-L",
714 "--max-lines",
715 "-s",
716 "--max-chars",
717 ],
718 _ => return None,
719 };
720 if program == "env"
722 && args
723 .iter()
724 .any(|arg| arg.starts_with("--split-string") || has_short_flag(arg, &['S']))
725 {
726 return None;
727 }
728 let mut rest = skip_shell_options(args, options);
729 if program == "timeout" {
730 rest = rest.get(1..).unwrap_or_default();
731 }
732 Some(rest)
733}
734
735fn retrieval_command(words: &[&str], depth: usize) -> bool {
736 if depth > MAX_COMMAND_DEPTH {
737 return false;
738 }
739 let Some((&program, args)) = skip_shell_assignments(words).split_first() else {
740 return false;
741 };
742 let program = [
745 "/bin/",
746 "/sbin/",
747 "/usr/bin/",
748 "/usr/sbin/",
749 "/usr/local/bin/",
750 "/opt/homebrew/bin/",
751 ]
752 .iter()
753 .find_map(|dir| program.strip_prefix(dir))
754 .unwrap_or(program);
755 if let Some(rest) = shell_wrapper_arguments(program, args) {
757 return (program == "env" && rest.is_empty()) || retrieval_command(rest, depth + 1);
758 }
759 if matches!(program, "bash" | "sh" | "dash" | "zsh" | "ksh") {
760 return args
761 .first()
762 .is_some_and(|option| has_short_flag(option, &['c']))
763 && args
764 .get(1)
765 .is_some_and(|command| shell_is_retrieval(command, depth + 1));
766 }
767 if program == "git" {
768 return git_is_retrieval(args);
769 }
770 if matches!(args, ["--help" | "--version"]) {
771 return true;
772 }
773 match program {
774 "find" => !args.iter().any(|arg| {
775 matches!(
776 *arg,
777 "-delete"
778 | "-exec"
779 | "-execdir"
780 | "-ok"
781 | "-okdir"
782 | "-fprint"
783 | "-fprint0"
784 | "-fprintf"
785 )
786 }),
787 "fd" => !args
789 .iter()
790 .any(|arg| arg.starts_with("--exec") || has_short_flag(arg, &['x', 'X'])),
791 "sed" => {
793 args.len() >= 2
794 && args[0] == "-n"
795 && args[2..].iter().all(|arg| !arg.starts_with('-'))
796 && args[1].strip_suffix('p').is_some_and(|range| {
797 !range.is_empty()
798 && range
799 .bytes()
800 .all(|c| c.is_ascii_digit() || matches!(c, b',' | b'$'))
801 })
802 }
803 "sort" => !args.iter().any(|arg| {
804 arg.starts_with("--compress-program")
805 || arg.starts_with("--output")
806 || arg.starts_with("-o")
807 }),
808 "rg" => !args.iter().any(|arg| arg.starts_with("--pre")),
809 "xxd" => !args.contains(&"-r") && !args.contains(&"-revert"),
810 "go" => {
811 matches!(args.first(), Some(&"list" | &"doc" | &"env" | &"version"))
812 && !args.iter().any(|arg| matches!(*arg, "-w" | "-u"))
813 }
814 "docker" | "podman" => matches!(args.first(), Some(&"ps" | &"version")),
815 "cat" | "grep" | "ls" | "nl" | "head" | "tail" | "wc" | "pwd" | "stat" | "file" | "du"
816 | "df" | "which" | "type" | "diff" | "cmp" | "jq" | "uniq" | "cut" | "readlink"
817 | "realpath" | "tree" | "basename" | "dirname" | "printenv" | "echo" | "printf"
818 | "less" | "more" | "test" | "[" | "ps" | "pgrep" | "pkg-config" | "strings" | "uname"
819 | "od" | "true" | ":" | "cd" | "pstree" | "sha256sum" | "sha1sum" | "md5sum" | "lsof"
820 | "tr" | "free" | "id" | "namei" | "whoami" | "paste" | "ss" | "getent" => true,
821 _ => false,
822 }
823}
824
825fn git_is_retrieval(args: &[&str]) -> bool {
826 let args = skip_shell_options(
827 args,
828 &["-C", "-c", "--git-dir", "--work-tree", "--namespace"],
829 );
830 let Some((subcommand, rest)) = args.split_first() else {
831 return false;
832 };
833 match *subcommand {
834 "diff" => !rest.contains(&"--check"),
835 "branch" => {
836 rest.is_empty()
837 || rest.iter().all(|arg| {
838 matches!(
839 *arg,
840 "-a" | "-r"
841 | "--all"
842 | "--list"
843 | "--show-current"
844 | "-v"
845 | "-vv"
846 | "--verbose"
847 )
848 })
849 }
850 "remote" => {
851 rest.is_empty()
852 || matches!(rest, ["-v" | "--verbose"])
853 || rest.first() == Some(&"get-url")
854 }
855 "tag" => rest.is_empty() || matches!(rest[0], "-l" | "--list"),
856 "config" => matches!(
857 rest.first(),
858 Some(&"--get" | &"--get-all" | &"--list" | &"-l")
859 ),
860 "worktree" => rest.first() == Some(&"list"),
861 "submodule" => rest.first() == Some(&"status"),
862 "status" | "log" | "show" | "blame" | "ls-files" | "ls-remote" | "rev-parse"
863 | "merge-base" | "grep" | "describe" | "show-ref" | "check-ignore" | "rev-list"
864 | "ls-tree" | "diff-tree" | "version" => true,
865 _ => false,
866 }
867}
868
869fn is_builtin_tool_name(lower: &str) -> bool {
870 WRITE_TOOL_NAMES.contains(&lower)
871 || EDIT_TOOL_NAMES.contains(&lower)
872 || READ_TOOL_NAMES.contains(&lower)
873 || PLAN_TOOL_NAMES.contains(&lower)
874 || BASH_TOOL_NAMES.contains(&lower)
875}
876
877fn shell_segments(command: &str) -> impl Iterator<Item = &str> {
881 let mut start = 0;
882 quoted_chars(command)
883 .filter(|&(_, c, quote)| quote.is_none() && matches!(c, '\n' | ';' | '|' | '&'))
884 .map(|(index, _, _)| index)
885 .chain([command.len()])
886 .filter_map(move |end| {
887 let segment = command[start..end].trim();
888 start = end + 1;
890 (!segment.is_empty()).then_some(segment)
891 })
892}
893
894fn quoted_chars(command: &str) -> impl Iterator<Item = (usize, char, Option<char>)> {
896 let mut quote = None;
897 let mut escaped = false;
898 command.char_indices().filter_map(move |(index, c)| {
899 if escaped {
900 escaped = false;
901 return None;
902 }
903 if c == '\\' && quote != Some('\'') {
904 escaped = true;
905 return None;
906 }
907 let around = quote;
908 if quote == Some(c) {
909 quote = None;
910 } else if quote.is_none() && matches!(c, '\'' | '"') {
911 quote = Some(c);
912 }
913 Some((index, c, around))
914 })
915}
916
917fn shell_words(segment: &str) -> std::iter::Peekable<std::str::SplitAsciiWhitespace<'_>> {
918 let mut words = segment.split_ascii_whitespace().peekable();
919
920 if words.peek().copied() == Some("env") {
921 words.next();
922 while words.peek().is_some_and(|word| word.starts_with('-')) {
923 words.next();
924 }
925 }
926 while words
927 .peek()
928 .is_some_and(|word| word.contains('=') && !word.starts_with('='))
929 {
930 words.next();
931 }
932
933 words
934}
935
936fn program_name(word: &str) -> &str {
937 Path::new(word)
938 .file_name()
939 .and_then(|name| name.to_str())
940 .unwrap_or(word)
941}
942
943fn shell_invokes_program(command: &str, expected: &str) -> bool {
944 shell_segments(command).any(|segment| {
945 shell_words(segment)
946 .next()
947 .is_some_and(|word| program_name(word) == expected)
948 })
949}
950
951fn command_invokes_patch(command: &Value) -> bool {
954 match command {
955 Value::String(command) => shell_invokes_patch(command, 0),
956 Value::Array(argv) => argv
957 .iter()
958 .map(Value::as_str)
959 .collect::<Option<Vec<_>>>()
960 .is_some_and(|words| patch_command(&words, 0)),
961 _ => false,
962 }
963}
964
965fn shell_invokes_patch(mut command: &str, depth: usize) -> bool {
968 if depth > MAX_COMMAND_DEPTH {
969 return false;
970 }
971 while !command.is_empty() {
972 let marker = quoted_chars(command).find(|&(index, c, quote)| {
973 quote.is_none()
974 && (command[index..].starts_with("<<")
975 || (c == '#'
976 && command[..index].chars().next_back().is_none_or(|previous| {
977 previous.is_whitespace() || ";|&()".contains(previous)
978 })))
979 });
980 let end = marker.map_or(command.len(), |(index, _, _)| index);
981 if shell_segments(&command[..end]).any(|segment| {
982 shlex::split(segment).is_some_and(|words| {
983 patch_command(&words.iter().map(String::as_str).collect::<Vec<_>>(), depth)
984 })
985 }) {
986 return true;
987 }
988 let Some((index, '#', _)) = marker else {
991 return false;
992 };
993 command = command[index..]
994 .split_once('\n')
995 .map_or("", |(_, remaining)| remaining);
996 }
997 false
998}
999
1000fn patch_command(words: &[&str], depth: usize) -> bool {
1003 if depth > MAX_COMMAND_DEPTH {
1004 return false;
1005 }
1006 let Some((&program, args)) = skip_shell_assignments(words).split_first() else {
1007 return false;
1008 };
1009 let program = program_name(program);
1010 if matches!(program, "patch" | "apply_patch") {
1011 return true;
1012 }
1013 if program == "command"
1015 && args
1016 .iter()
1017 .take_while(|arg| arg.starts_with('-') && **arg != "--")
1018 .any(|arg| has_short_flag(arg, &['v', 'V']))
1019 {
1020 return false;
1021 }
1022 if let Some(rest) = shell_wrapper_arguments(program, args) {
1023 let options = &args[..args.len() - rest.len()];
1024 return !options
1025 .iter()
1026 .any(|option| matches!(*option, "--help" | "--version"))
1027 && patch_command(rest, depth + 1);
1028 }
1029 if matches!(program, "bash" | "sh" | "dash" | "zsh" | "ksh") {
1030 let mut options = args.iter();
1033 while let Some(option) = options.next() {
1034 if !option.starts_with('-') || matches!(*option, "-" | "--") {
1035 break;
1036 }
1037 if has_short_flag(option, &['n']) {
1038 return false;
1039 }
1040 if option.starts_with("--")
1041 && !matches!(
1042 *option,
1043 "--login" | "--noprofile" | "--norc" | "--rcfile" | "--init-file"
1044 )
1045 {
1046 return false;
1047 }
1048 if has_short_flag(option, &['o', 'O']) || matches!(*option, "--rcfile" | "--init-file")
1049 {
1050 let Some(value) = options.next() else {
1051 return false;
1052 };
1053 if has_short_flag(option, &['o']) && *value == "noexec" {
1054 return false;
1055 }
1056 }
1057 if has_short_flag(option, &['c']) {
1058 return options
1059 .next()
1060 .is_some_and(|command| shell_invokes_patch(command, depth + 1));
1061 }
1062 }
1063 }
1064 false
1065}
1066
1067fn shell_command_is_write(command: &str) -> bool {
1068 shell_segments(command).any(|segment| {
1069 let mut words = shell_words(segment);
1070 let Some(program) = words.next().map(program_name) else {
1071 return false;
1072 };
1073 if matches!(program, "cp" | "mkdir" | "touch" | "install") {
1074 return true;
1075 }
1076
1077 let redirects_output = words.any(|word| matches!(word, ">" | ">>"));
1078 redirects_output
1079 && (matches!(program, "echo" | "printf" | "git")
1080 || BASH_READ_COMMANDS.contains(&program))
1081 })
1082}
1083
1084fn shell_command_is_edit(command: &str) -> bool {
1085 shell_segments(command).any(|segment| {
1086 let mut words = shell_words(segment);
1087 let Some(program) = words.next().map(program_name) else {
1088 return false;
1089 };
1090 let has_arg = |arg: &str| words.clone().any(|word| word == arg);
1091
1092 match program {
1093 "mv" | "rm" => true,
1094 "perl" => words
1095 .take_while(|word| word.starts_with('-'))
1096 .any(|option| {
1097 option
1098 .trim_start_matches('-')
1099 .chars()
1100 .any(|flag| flag == 'i')
1101 }),
1102 "git" => words
1103 .next()
1104 .is_some_and(|subcommand| matches!(subcommand, "apply" | "am" | "restore")),
1105 "gofmt" => has_arg("-w"),
1106 "cargo" => words.clone().next() == Some("fmt") && !has_arg("--check"),
1107 "ruff" => {
1108 let subcommand = words.clone().next();
1109 (subcommand == Some("format") && !has_arg("--check"))
1110 || (subcommand == Some("check") && has_arg("--fix"))
1111 }
1112 "prettier" => has_arg("--write"),
1113 "black" => !has_arg("--check"),
1114 _ => {
1115 (words.clone().any(|word| program_name(word) == "prettier") && has_arg("--write"))
1116 || (words.clone().any(|word| program_name(word) == "ruff")
1117 && ((has_arg("format") && !has_arg("--check"))
1118 || (has_arg("check") && has_arg("--fix"))))
1119 }
1120 }
1121 })
1122}
1123
1124fn shell_command_is_read(command: &str) -> bool {
1125 shell_segments(command).any(|segment| {
1126 if segment == "env" {
1127 return true;
1128 }
1129 let mut words = shell_words(segment);
1130 let Some(program) = words.next().map(program_name) else {
1131 return false;
1132 };
1133
1134 if BASH_READ_COMMANDS.contains(&program) {
1135 return true;
1136 }
1137 if program == "command" && words.next() == Some("-v") {
1138 return true;
1139 }
1140 if program == "type" {
1141 return true;
1142 }
1143 if program != "git" {
1144 return false;
1145 }
1146
1147 match words.next() {
1148 Some("branch") => words.next().is_none_or(|arg| arg.starts_with('-')),
1149 Some("remote") => words
1150 .next()
1151 .is_none_or(|arg| arg.starts_with('-') || arg == "get-url"),
1152 Some("config") => words
1153 .next()
1154 .is_some_and(|arg| matches!(arg, "--get" | "--get-all" | "--list" | "-l")),
1155 Some(subcommand) => GIT_READ_SUBCOMMANDS.contains(&subcommand),
1156 None => false,
1157 }
1158 })
1159}
1160
1161#[cfg(test)]
1168fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals {
1169 extract_tool_signals_with_window_and_semantics(
1170 request,
1171 recent_window,
1172 &ToolSemantics::default(),
1173 false,
1174 )
1175}
1176
1177fn extract_tool_signals_with_window_and_semantics(
1178 request: &Request,
1179 recent_window: usize,
1180 semantics: &ToolSemantics,
1181 current_user_turn: bool,
1182) -> ToolSignals {
1183 let messages = &request.llm_request.messages;
1185 let boundary = if current_user_turn {
1186 messages.iter().rposition(is_user_turn)
1187 } else {
1188 None
1189 };
1190 let start = boundary.map_or(0, |index| index + 1);
1191 let namespaces = tool_namespaces(&request.llm_request.extensions);
1192 let mut tool_texts: Vec<(String, bool)> = Vec::new();
1193 let mut tool_calls: Vec<ToolSemantic> = Vec::new();
1194 let mut retrieval_calls: HashSet<&str> = HashSet::new();
1196 let mut tool_result_count = 0usize;
1197 let mut assistant_turn_count = 0usize;
1198
1199 for (index, message) in messages.iter().enumerate() {
1200 if index >= start && message.role == Role::Assistant {
1201 assistant_turn_count += 1;
1202 }
1203 for block in &message.content {
1204 match block {
1205 _ if index < start => continue,
1206 ContentBlock::ToolCall(call) => {
1207 let bare_name = namespaces
1209 .and_then(|namespaces| split_qualified_name(namespaces, &call.name))
1210 .map(|(tool, _)| tool)
1211 .or_else(|| mcp_tool_name(&call.name));
1212 let decoded = call
1214 .arguments
1215 .as_str()
1216 .and_then(|raw| serde_json::from_str::<Value>(raw).ok());
1217 let arguments = decoded.as_ref().unwrap_or(&call.arguments);
1218 let command_field = command_of(arguments);
1219 let code_semantic = (bare_name.unwrap_or(&call.name) == "exec"
1222 && arguments.get("command").is_none()
1223 && arguments.get("cmd").is_none())
1224 .then(|| arguments.get("input").and_then(Value::as_str))
1225 .flatten()
1226 .map(|input| {
1227 if code_mode_has_patch(input) {
1228 ToolSemantic::Mutate(MutationKind::Edit)
1229 } else {
1230 ToolSemantic::Unknown
1231 }
1232 });
1233 let full =
1237 classify_tool_call_with_semantics(&call.name, command_field, semantics);
1238 let name = match (full, bare_name) {
1239 (ToolSemantic::Unknown | ToolSemantic::Observe, Some(bare_name)) => {
1240 bare_name
1241 }
1242 _ => call.name.as_str(),
1243 };
1244 let is_retrieval =
1245 code_semantic.is_none() && is_retrieval_tool(name, command_field);
1246 if !call.id.is_empty() {
1247 if is_retrieval {
1249 retrieval_calls.insert(call.id.as_str());
1250 } else {
1251 retrieval_calls.remove(call.id.as_str());
1252 }
1253 }
1254 let semantic = if let Some(semantic) = code_semantic {
1255 semantic
1256 } else if is_retrieval {
1257 ToolSemantic::Observe
1258 } else if full == ToolSemantic::Unknown {
1259 bare_name.map_or(full, |name| {
1260 classify_tool_call_with_semantics(name, command_field, semantics)
1261 })
1262 } else {
1263 full
1264 };
1265 tool_calls.push(semantic);
1266 }
1267 ContentBlock::ToolResult(result) => {
1268 tool_result_count += 1;
1270 let is_error = result.is_error == Some(true);
1271 let text = result
1272 .content
1273 .iter()
1274 .filter_map(text_of)
1275 .collect::<Vec<_>>()
1276 .join("\n");
1277 let is_retrieval_result =
1278 retrieval_calls.contains(result.tool_call_id.as_str());
1279 if is_retrieval_result {
1280 let shell_result = serde_json::Deserializer::from_str(&text)
1283 .into_iter::<Value>()
1284 .next()
1285 .and_then(|result| result.ok())
1286 .filter(|value| {
1287 value["output"].is_string() && value.get("exit_code").is_some()
1288 });
1289 let status = text
1292 .split_once("\nOutput:\n")
1293 .map_or(text.as_str(), |(header, _)| header);
1294 let shell_failed = match &shell_result {
1296 Some(value) => {
1297 value["exit_code"].as_i64().is_some_and(|code| code != 0)
1298 || value["success"].as_bool() == Some(false)
1299 || value["error"]
1300 .as_str()
1301 .is_some_and(|error| !error.trim().is_empty())
1302 }
1303 None => has_nonzero_exit_status(&status.to_lowercase()),
1304 };
1305 if is_missing_file(&text) || (!is_error && !shell_failed) {
1306 if !text.is_empty() {
1308 tool_texts.push((String::new(), false));
1309 }
1310 continue;
1311 }
1312 }
1313 if !text.is_empty() || is_error {
1315 tool_texts.push((text, is_error));
1316 }
1317 }
1318 ContentBlock::Unknown { provider, raw }
1320 if provider.as_str() == WireFormat::OpenAiResponses.as_str()
1321 && raw.get("type").and_then(Value::as_str)
1322 == Some("apply_patch_call_output") =>
1323 {
1324 tool_result_count += 1;
1325 let text = raw
1326 .get("output")
1327 .and_then(Value::as_str)
1328 .unwrap_or_default();
1329 let is_error = raw.get("status").and_then(Value::as_str) == Some("failed");
1330 tool_texts.push((text.to_owned(), is_error));
1331 }
1332 ContentBlock::Unknown { provider, raw }
1333 if provider.as_str() == WireFormat::OpenAiResponses.as_str()
1334 && raw.get("type").and_then(Value::as_str) == Some("shell_call_output") =>
1335 {
1336 tool_result_count += 1;
1337 let mut texts = Vec::new();
1338 let mut is_error = false;
1339 if let Some(outputs) = raw.get("output").and_then(Value::as_array) {
1340 for output in outputs {
1341 for field in ["stdout", "stderr"] {
1342 if let Some(text) = output.get(field).and_then(Value::as_str)
1343 && !text.is_empty()
1344 {
1345 texts.push(text);
1346 }
1347 }
1348 if let Some(outcome) = output.get("outcome") {
1349 is_error |= match outcome.get("type").and_then(Value::as_str) {
1350 Some("timeout") => true,
1351 Some("exit")
1352 if outcome
1353 .get("exit_code")
1354 .and_then(Value::as_i64)
1355 .is_some_and(|code| code != 0) =>
1356 {
1357 texts.push("returned non-zero");
1359 output
1360 .get("stderr")
1361 .and_then(Value::as_str)
1362 .is_some_and(|text| !text.trim().is_empty())
1363 }
1364 _ => false,
1365 };
1366 }
1367 }
1368 }
1369 let text = texts.join("\n");
1370 tool_texts.push((text, is_error));
1371 }
1372 _ => {}
1373 }
1374 }
1375 }
1376
1377 let mut signal = build_signal(
1378 tool_texts,
1379 tool_calls,
1380 (messages.len() - boundary.unwrap_or(0)) as u32,
1381 recent_window,
1382 );
1383 signal.compacted = request.llm_request.has_compaction;
1384 signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);
1385 signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);
1386 signal
1387}
1388
1389fn mcp_tool_name(name: &str) -> Option<&str> {
1393 let (_server, tool) = name.strip_prefix("mcp__")?.split_once("__")?;
1394 (!tool.is_empty()).then_some(tool)
1395}
1396
1397fn command_of(arguments: &Value) -> Option<&Value> {
1401 ["command", "cmd", "input"]
1402 .iter()
1403 .find_map(|key| arguments.get(*key))
1404}
1405
1406fn command_text(value: &Value) -> Option<String> {
1408 match value {
1409 Value::String(text) => Some(text.to_lowercase()),
1410 Value::Array(parts) => {
1411 let joined = parts
1412 .iter()
1413 .filter_map(Value::as_str)
1414 .collect::<Vec<_>>()
1415 .join(" ");
1416 (!joined.is_empty()).then(|| joined.to_lowercase())
1417 }
1418 _ => None,
1419 }
1420}
1421
1422fn text_of(block: &ContentBlock) -> Option<&str> {
1424 match block {
1425 ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()),
1426 _ => None,
1427 }
1428}
1429
1430fn is_missing_file(text: &str) -> bool {
1433 let lower = text.to_lowercase();
1434 lower.contains("no such file or directory") || lower.contains("file does not exist")
1436}
1437
1438fn build_signal(
1439 tool_texts: Vec<(String, bool)>,
1440 tool_calls: Vec<ToolSemantic>,
1441 turn_depth: u32,
1442 recent_window: usize,
1443) -> ToolSignals {
1444 let sev_start = tool_texts.len().saturating_sub(recent_window.max(1));
1450 let mut severity = 0.0f32;
1451 let mut failure_fingerprints = Vec::new();
1452 let mut repeated_failure = false;
1453 for (text, is_error) in &tool_texts[sev_start..] {
1454 let (sev, _patterns) = classify_text(text);
1455 let sev = if *is_error { sev.max(HARD) } else { sev };
1457 if sev > severity {
1458 severity = sev;
1459 }
1460 if let Some(fingerprint) = failure_fingerprint(text, *is_error) {
1461 repeated_failure |= failure_fingerprints.contains(&fingerprint);
1462 failure_fingerprints.push(fingerprint);
1463 }
1464 }
1465
1466 let no_error_streak = compute_no_error_streak(&tool_texts);
1467
1468 let recent_start = tool_calls.len().saturating_sub(recent_window);
1472 let mut write_count = 0u32;
1473 let mut edit_count = 0u32;
1474 let mut read_count = 0u32;
1475 let mut todowrite_count = 0u32;
1476 let mut recent_write_count = 0u32;
1477 let mut recent_edit_count = 0u32;
1478 let mut recent_read_count = 0u32;
1479 let mut recent_todowrite_count = 0u32;
1480 let mut new_count = 0u32;
1481 let mut recent_new_count = 0u32;
1482 let mut pure_bash_streak = 0u32;
1483 let mut streak_open = true;
1484 for (i, cat) in tool_calls.iter().enumerate().rev() {
1485 if streak_open {
1486 if matches!(cat, ToolSemantic::Unknown) {
1487 pure_bash_streak += 1;
1488 } else {
1489 streak_open = false;
1490 }
1491 }
1492 match cat {
1493 ToolSemantic::Mutate(MutationKind::Write) => {
1494 write_count += 1;
1495 if i >= recent_start {
1496 recent_write_count += 1;
1497 }
1498 }
1499 ToolSemantic::Mutate(MutationKind::Edit) => {
1500 edit_count += 1;
1501 if i >= recent_start {
1502 recent_edit_count += 1;
1503 }
1504 }
1505 ToolSemantic::Observe => {
1506 read_count += 1;
1507 if i >= recent_start {
1508 recent_read_count += 1;
1509 }
1510 }
1511 ToolSemantic::Plan => {
1512 todowrite_count += 1;
1513 if i >= recent_start {
1514 recent_todowrite_count += 1;
1515 }
1516 }
1517 ToolSemantic::New => {
1518 new_count += 1;
1519 if i >= recent_start {
1520 recent_new_count += 1;
1521 }
1522 }
1523 ToolSemantic::Unknown => {}
1524 }
1525 }
1526
1527 let tests_passed = detect_tests_passed(&tool_texts, recent_window);
1528
1529 ToolSignals {
1530 severity,
1531 repeated_failure,
1532 no_error_streak,
1533 edit_count,
1534 write_count,
1535 read_count,
1536 todowrite_count,
1537 recent_edit_count,
1538 recent_write_count,
1539 recent_read_count,
1540 recent_todowrite_count,
1541 new_count,
1542 recent_new_count,
1543 pure_bash_streak,
1544 tests_passed,
1545 turn_depth,
1546 tool_result_count: 0,
1550 assistant_turn_count: 0,
1551 compacted: false,
1552 }
1553}
1554
1555pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {
1561 let lower = text.to_lowercase();
1562 let mut patterns = Vec::new();
1563 let mut severity: f32 = 0.0;
1564 for (name, sev, substrings) in ERROR_PATTERNS {
1565 if substrings.iter().any(|sub| lower.contains(sub)) {
1566 patterns.push(name.to_string());
1567 severity = severity.max(*sev);
1568 }
1569 }
1570 let result = serde_json::Deserializer::from_str(text)
1572 .into_iter::<Value>()
1573 .next()
1574 .and_then(|result| result.ok())
1575 .unwrap_or_default();
1576 let nonzero_exit = result["exit_code"].as_i64().is_some_and(|code| code != 0);
1577 let tool_error = result["success"].as_bool() == Some(false)
1578 || result["error"]
1579 .as_str()
1580 .is_some_and(|error| !error.trim().is_empty());
1581 if (nonzero_exit || has_nonzero_exit_status(&lower))
1582 && !patterns.iter().any(|p| p == "exit_nonzero")
1583 {
1584 patterns.push("exit_nonzero".to_string());
1585 severity = severity.max(SOFT);
1586 }
1587 for (name, matched) in [
1588 ("tool_error", tool_error),
1589 ("compile_error", has_compiler_diagnostic(&lower)),
1590 ("runtime_exception", has_runtime_exception(&lower)),
1591 ("runtime_panic", has_runtime_panic(&lower)),
1592 ("patch_error", has_patch_failure(&lower)),
1593 ("cmd_not_found", ends_with_not_found(&lower)),
1594 ] {
1595 if matched && !patterns.iter().any(|pattern| pattern == name) {
1596 patterns.push(name.to_string());
1597 severity = severity.max(HARD);
1598 }
1599 }
1600 (severity, patterns)
1601}
1602
1603fn failure_fingerprint(text: &str, is_error: bool) -> Option<String> {
1606 let (severity, patterns) = classify_text(text);
1607 if severity < HARD && !is_error {
1608 return None;
1609 }
1610
1611 let lower = text.to_lowercase();
1612 let diagnostic = lower
1613 .lines()
1614 .find(|line| is_failure_diagnostic(line))
1615 .or_else(|| lower.lines().find(|line| !line.trim().is_empty()))
1616 .unwrap_or_default();
1617 let normalized = normalize_failure_text(diagnostic);
1618 Some(format!("{}|{normalized}", patterns.join(",")))
1619}
1620
1621fn is_failure_diagnostic(line: &str) -> bool {
1622 let line = line.trim();
1623 [
1624 "error",
1625 "exception",
1626 "panic",
1627 "failed",
1628 "timed out",
1629 "timeout",
1630 "connection refused",
1631 "cannot allocate memory",
1632 "out of memory",
1633 "not found",
1634 ]
1635 .iter()
1636 .any(|marker| line.contains(marker))
1637}
1638
1639fn normalize_failure_text(text: &str) -> String {
1642 let mut normalized = String::new();
1643 for word in text.split_whitespace() {
1644 if !normalized.is_empty() {
1645 normalized.push(' ');
1646 }
1647 let mut in_digits = false;
1648 if word.starts_with('/') || word.contains("/src/") || word.contains("/tmp/") {
1649 normalized.push_str("<path>");
1650 continue;
1651 }
1652 for character in word.chars() {
1653 if character.is_ascii_digit() {
1654 if !in_digits {
1655 normalized.push('#');
1656 in_digits = true;
1657 }
1658 } else {
1659 normalized.push(character);
1660 in_digits = false;
1661 }
1662 }
1663 }
1664 normalized.chars().take(240).collect()
1665}
1666
1667fn has_compiler_diagnostic(lower: &str) -> bool {
1668 lower.lines().any(|line| {
1669 let line = line.trim_start();
1670 if matches!(
1671 line,
1672 "compilation failed" | "error: compilation failed" | "error: could not compile"
1673 ) || line.starts_with("error: could not compile ")
1674 {
1675 return true;
1676 }
1677
1678 let Some(rest) = line.strip_prefix("error[e") else {
1679 return false;
1680 };
1681 let Some((code, _)) = rest.split_once("]:") else {
1682 return false;
1683 };
1684 !code.is_empty() && code.chars().all(|character| character.is_ascii_digit())
1685 })
1686}
1687
1688fn has_runtime_exception(lower: &str) -> bool {
1689 let has_exception_line = lower.lines().any(|line| {
1690 let line = line.trim_start();
1691 [
1692 "typeerror:",
1693 "referenceerror:",
1694 "rangeerror:",
1695 "runtimeerror:",
1696 "keyerror:",
1697 "attributeerror:",
1698 ]
1699 .iter()
1700 .any(|prefix| line.starts_with(prefix))
1701 });
1702 has_exception_line && (lower.contains("\n at ") || lower.contains("\n at "))
1703}
1704
1705fn has_runtime_panic(lower: &str) -> bool {
1706 lower
1707 .lines()
1708 .any(|line| line.trim_start().starts_with("panic: runtime error:"))
1709 && (lower.contains("\ngoroutine ") || lower.contains("[signal sig"))
1710}
1711
1712fn has_patch_failure(lower: &str) -> bool {
1713 lower.lines().any(|line| {
1714 let line = line.trim_start();
1715 line.starts_with("error: patch failed:")
1716 || line.starts_with("patch failed:")
1717 || line.contains(": patch does not apply")
1718 || line.starts_with("invalid context")
1719 })
1720}
1721
1722fn ends_with_not_found(lower: &str) -> bool {
1731 lower
1732 .lines()
1733 .any(|line| line.trim_end().ends_with(": not found"))
1734}
1735
1736fn has_nonzero_exit_status(lower: &str) -> bool {
1742 NONZERO_EXIT_PHRASES
1743 .iter()
1744 .any(|phrase| phrase_followed_by_nonzero_integer(lower, phrase))
1745}
1746
1747fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool {
1749 let mut cursor = 0usize;
1750 while let Some(rel) = lower[cursor..].find(phrase) {
1751 let value_start = cursor + rel + phrase.len();
1752 let rest = lower[value_start..].trim_start_matches(|c: char| {
1753 c.is_ascii_whitespace() || matches!(c, ':' | '=' | '\'' | '"' | '`')
1754 });
1755 let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
1756 if !digits.is_empty() && digits.chars().any(|d| d != '0') {
1757 return true;
1758 }
1759 cursor = value_start;
1760 }
1761 false
1762}
1763
1764fn compute_no_error_streak(tool_texts: &[(String, bool)]) -> u32 {
1765 let mut streak = 0u32;
1766 for (text, is_error) in tool_texts.iter().rev() {
1767 let (sev, _) = classify_text(text);
1768 if *is_error || sev > 0.0 {
1769 break;
1770 }
1771 streak += 1;
1772 }
1773 streak
1774}
1775
1776fn detect_tests_passed(tool_texts: &[(String, bool)], recent_window: usize) -> bool {
1777 let start = tool_texts.len().saturating_sub(recent_window.max(1));
1778 let recent = &tool_texts[start..];
1779 let after_latest_failure = recent
1780 .iter()
1781 .rposition(|(text, is_error)| *is_error || classify_text(text).0 > 0.0)
1782 .map_or(recent, |index| &recent[index + 1..]);
1783 after_latest_failure.iter().any(|(text, _)| {
1784 let lower = text.to_lowercase();
1785 TEST_PASS_PHRASES.iter().any(|p| lower.contains(p))
1786 && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p))
1787 && !has_nonzero_failure_count(&lower)
1788 })
1789}
1790
1791fn has_nonzero_failure_count(lower: &str) -> bool {
1797 for kw in NUMERIC_FAILURE_KEYWORDS {
1798 let mut cursor = 0usize;
1799 while let Some(rel) = lower[cursor..].find(kw) {
1800 let kw_start = cursor + rel;
1801 let kw_end = kw_start + kw.len();
1802 let boundary_after = lower[kw_end..]
1805 .chars()
1806 .next()
1807 .is_none_or(|c| !c.is_ascii_alphanumeric());
1808 if boundary_after {
1809 let prefix = &lower[..kw_start];
1810 let trimmed = prefix.trim_end_matches(|c: char| c.is_whitespace());
1811 let digits_rev: String = trimmed
1812 .chars()
1813 .rev()
1814 .take_while(|c| c.is_ascii_digit())
1815 .collect();
1816 if !digits_rev.is_empty() && digits_rev.chars().any(|d| d != '0') {
1817 return true;
1818 }
1819 }
1820 cursor = kw_start + kw.len();
1821 }
1822 }
1823 false
1824}
1825
1826#[cfg(test)]
1829mod tests {
1830 use super::*;
1831 use crate::algorithms::util::stage::score_signal;
1832 use serde_json::json;
1833 use switchyard_protocol::codex_namespaces::TOOL_NAMESPACES_KEY;
1834 use switchyard_protocol::{
1835 ContentBlock, LlmRequest, Message, Metadata, Role, ToolCall, ToolResult,
1836 };
1837
1838 fn with_messages(messages: Vec<Message>) -> Request {
1839 let mut request = Request {
1840 llm_request: LlmRequest {
1841 messages,
1842 ..LlmRequest::default()
1843 },
1844 raw_request: None,
1845 metadata: None,
1846 };
1847 switchyard_translation::normalize_compaction(&mut request.llm_request, None);
1848 request
1849 }
1850
1851 fn tc(name: &str) -> Message {
1853 Message::new(
1854 Role::Assistant,
1855 vec![ContentBlock::ToolCall(ToolCall {
1856 id: String::new(),
1857 name: name.to_string(),
1858 arguments: json!({}),
1859 })],
1860 )
1861 }
1862
1863 fn bash(command: &str) -> Message {
1865 Message::new(
1866 Role::Assistant,
1867 vec![ContentBlock::ToolCall(ToolCall {
1868 id: String::new(),
1869 name: "Bash".to_string(),
1870 arguments: json!({"command": command}),
1871 })],
1872 )
1873 }
1874
1875 fn tr(text: &str) -> Message {
1877 Message::new(
1878 Role::User,
1879 vec![ContentBlock::ToolResult(ToolResult {
1880 tool_call_id: String::new(),
1881 content: vec![ContentBlock::Text {
1882 text: text.to_string(),
1883 }],
1884 is_error: None,
1885 })],
1886 )
1887 }
1888
1889 #[test]
1890 fn user_turn_signals_exclude_old_errors_and_keep_user_role_tool_results() {
1891 let mut user = Message::text(Role::User, "next");
1892 user.content.extend(tr("out of memory").content);
1893 let request = with_messages(vec![tr("out of memory"), user, tr("ok")]);
1894 let signals = ToolSignalProcessor::default().for_user_turn(&request);
1895 assert_eq!(signals.tool_result_count, 1);
1896 assert_eq!(signals.severity, 0.0);
1897 assert_eq!(signals.turn_depth, 2);
1898 }
1899
1900 #[test]
1901 fn clean_text_has_zero_severity() {
1902 let (sev, patterns) = classify_text("everything went fine");
1903 assert_eq!(sev, 0.0);
1904 assert!(patterns.is_empty());
1905 }
1906
1907 #[test]
1908 fn structured_tool_failures_affect_recovery_signals() {
1909 for (text, severity) in [
1910 (r#"{"output":"","exit_code":7,"error":null}"#, SOFT),
1911 (r#"{"output":"","exit_code":-1,"error":null}"#, SOFT),
1912 (r#"{"success":false,"error":"No matching text"}"#, HARD),
1913 (r#"{"success":false,"error":null}"#, HARD),
1914 (
1915 r#"{"error":"Overwrite refused","stale_write_blocked":true}"#,
1916 HARD,
1917 ),
1918 (r#"{"output":"out of memory","exit_code":1}"#, CRITICAL),
1919 (r#"{"output":"done","exit_code":0,"error":null}"#, 0.0),
1920 (r#"{"success":true,"error":" "}"#, 0.0),
1921 (r#"{"output":"running","exit_code":null,"error":null}"#, 0.0),
1922 (r#"{"output":{"success":false},"exit_code":0}"#, 0.0),
1923 ] {
1924 let warned = format!("{text}\n\n[Tool loop warning: repeated identical call]");
1925 let request = with_messages(vec![tr("5 passed in 0.12s"), tr(text), tr(&warned)]);
1926 let signal = ToolSignals::from_request(&request, None);
1927 let clean = severity == 0.0;
1928 assert_eq!(signal.severity, severity, "{text}");
1929 assert_eq!(signal.no_error_streak, if clean { 3 } else { 0 }, "{text}");
1930 assert_eq!(signal.repeated_failure, severity >= HARD, "{text}");
1931 assert_eq!(signal.tests_passed, clean, "{text}");
1932 }
1933 }
1934
1935 #[test]
1936 fn traceback_is_hard() {
1937 let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError");
1938 assert_eq!(sev, HARD);
1939 assert!(patterns.contains(&"traceback".to_string()));
1940 }
1941
1942 #[test]
1943 fn oom_is_critical() {
1944 let (sev, _) = classify_text("Out of memory: kill process 1234");
1945 assert_eq!(sev, CRITICAL);
1946 }
1947
1948 #[test]
1949 fn connection_refused_is_hard() {
1950 let (severity, _) = classify_text("Connection refused on port 8000");
1951 assert_eq!(severity, HARD);
1952 }
1953
1954 #[test]
1955 fn repeated_failure_ignores_volatile_paths_and_numbers() {
1956 let request = with_messages(vec![
1957 tr("error[E0308]: mismatched types at /tmp/a/src/lib.rs:12"),
1958 tr("error[E0308]: mismatched types at /tmp/b/src/lib.rs:47"),
1959 ]);
1960 assert!(ToolSignals::from_request(&request, None).repeated_failure);
1961 }
1962
1963 #[test]
1964 fn different_failures_are_not_repeated() {
1965 let request = with_messages(vec![
1966 tr("error[E0308]: mismatched types"),
1967 tr("error[E0509]: cannot move out"),
1968 ]);
1969 assert!(!ToolSignals::from_request(&request, None).repeated_failure);
1970 }
1971
1972 #[test]
1973 fn one_material_failure_is_not_repeated() {
1974 let request = with_messages(vec![tr("Connection refused on port 8000")]);
1975 assert!(!ToolSignals::from_request(&request, None).repeated_failure);
1976 }
1977
1978 #[test]
1980 fn structured_tool_failures_feed_error_and_recovery_signals() {
1981 for text in ["Dependency unavailable", "", "5 passed in 0.12s"] {
1982 let mut failed = tr(text);
1983 let ContentBlock::ToolResult(result) = &mut failed.content[0] else {
1984 panic!("expected tool result");
1985 };
1986 result.is_error = Some(true);
1987 let mut request = with_messages(vec![tr("5 passed in 0.12s"), failed.clone()]);
1988 let signals = ToolSignals::from_request(&request, Some(3));
1989 assert_eq!(signals.severity, HARD);
1990 assert!(!signals.repeated_failure);
1991 assert_eq!(signals.no_error_streak, 0);
1992 assert!(!signals.tests_passed);
1993
1994 request.llm_request.messages.push(failed);
1995 assert!(ToolSignals::from_request(&request, Some(3)).repeated_failure);
1996 request.llm_request.messages.push(tr("5 passed in 0.12s"));
1997 let recovered = ToolSignals::from_request(&request, Some(1));
1998 assert_eq!(recovered.severity, 0.0);
1999 assert!(!recovered.repeated_failure);
2000 assert_eq!(recovered.no_error_streak, 1);
2001 assert!(recovered.tests_passed);
2002 }
2003 }
2004
2005 #[test]
2006 fn severity_is_max_across_patterns() {
2007 let (sev, _) = classify_text("exit code 1\nTraceback (most recent call last):");
2009 assert_eq!(sev, HARD);
2010 }
2011
2012 #[test]
2013 fn codex_process_exit_zero_stays_clean() {
2014 let (sev, patterns) =
2015 classify_text("Chunk ID: abc\nProcess exited with code 0\nOutput:\nok");
2016 assert_eq!(sev, 0.0);
2017 assert!(!patterns.contains(&"exit_nonzero".to_string()));
2018 }
2019
2020 #[test]
2021 fn nonzero_exit_codes_are_soft_errors() {
2022 let cases = [
2023 "Process exited with code 1",
2024 "Process exited with code 127",
2025 "exit code: 2",
2026 "exit status 3",
2027 "exited with status 9",
2028 ];
2029 for case in cases {
2030 let (sev, patterns) = classify_text(case);
2031 assert_eq!(sev, SOFT, "expected soft severity for {case}");
2032 assert!(patterns.contains(&"exit_nonzero".to_string()));
2033 }
2034 }
2035
2036 #[test]
2037 fn partial_process_failures_are_hard_errors() {
2038 let cases = [
2039 (
2040 "Process running with session ID 12\nOutput:\nerror[E0509]: cannot move out",
2041 "compile_error",
2042 ),
2043 (
2044 "Process exited with code 0\nOutput:\nTypeError: value is undefined\n at main.js:1:2",
2045 "runtime_exception",
2046 ),
2047 (
2048 "Process running with session ID 13\nOutput:\npanic: runtime error: index out of range\n\ngoroutine 6 [running]:",
2049 "runtime_panic",
2050 ),
2051 (
2052 "Process exited with code 0\nOutput:\nerror: patch failed: src/lib.rs:4\nerror: src/lib.rs: patch does not apply",
2053 "patch_error",
2054 ),
2055 ];
2056 for (text, expected_pattern) in cases {
2057 let (severity, patterns) = classify_text(text);
2058 assert_eq!(severity, HARD, "expected hard severity for {text}");
2059 assert!(patterns.iter().any(|pattern| pattern == expected_pattern));
2060 }
2061 }
2062
2063 #[test]
2066 fn trimmed_command_not_found_is_still_hard() {
2067 let cases = [
2068 "git: not found",
2069 "sh: 1: git: not found",
2070 "/bin/sh: 1: git: not found",
2071 ];
2072 for text in cases {
2073 let (sev, patterns) = classify_text(text);
2074 assert_eq!(sev, HARD, "expected hard severity for {text}");
2075 assert!(patterns.contains(&"cmd_not_found".to_string()));
2076 }
2077 }
2078
2079 #[test]
2082 fn not_found_mid_sentence_stays_clean() {
2083 let (sev, _) =
2084 classify_text("Looking for config key, not found in environment, using default.");
2085 assert_eq!(sev, 0.0);
2086 }
2087
2088 #[test]
2091 fn not_found_without_shell_separator_stays_clean() {
2092 for text in [
2093 "HTTP/1.1 404 Not Found",
2094 "lookup finished, key not found",
2095 "GET /healthz -> 404 Not Found",
2096 ] {
2097 let (sev, patterns) = classify_text(text);
2098 assert_eq!(sev, 0.0, "expected clean classification for {text}");
2099 assert!(!patterns.contains(&"cmd_not_found".to_string()));
2100 }
2101 }
2102
2103 #[test]
2104 fn source_text_that_names_exceptions_stays_clean() {
2105 let text =
2106 "pub enum TypeError: this is documentation\nlet sample = 'panic: runtime error:';";
2107 assert_eq!(classify_text(text).0, 0.0);
2108 }
2109
2110 #[test]
2111 fn file_does_not_exist_is_hard() {
2112 let (sev, patterns) =
2114 classify_text("Error: File does not exist. Note: current working directory is /app.");
2115 assert_eq!(sev, HARD);
2116 assert!(patterns.contains(&"no_such_file".to_string()));
2117 }
2118
2119 #[test]
2120 fn bare_does_not_exist_stays_clean() {
2121 let (sev, _) = classify_text("The directory does not exist yet, creating it now.");
2124 assert_eq!(sev, 0.0);
2125 }
2126
2127 #[test]
2128 fn no_error_streak_all_clean() {
2129 let texts = vec![("ok".to_string(), false), ("all good".to_string(), false)];
2130 assert_eq!(compute_no_error_streak(&texts), 2);
2131 }
2132
2133 #[test]
2134 fn no_error_streak_stops_at_error() {
2135 let texts = vec![
2136 ("Traceback (most recent call last):".to_string(), false),
2137 ("ok".to_string(), false),
2138 ("ok".to_string(), false),
2139 ];
2140 assert_eq!(compute_no_error_streak(&texts), 2);
2141 }
2142
2143 #[test]
2144 fn tests_passed_detects_pytest_output() {
2145 assert!(detect_tests_passed(
2146 &[("====== 5 passed in 0.12s ======".to_string(), false)],
2147 DEFAULT_RECENT_WINDOW
2148 ));
2149 }
2150
2151 #[test]
2152 fn tests_passed_ignores_partial_failures() {
2153 assert!(!detect_tests_passed(
2154 &[("2 failed, 5 passed in 0.56s".to_string(), false)],
2155 DEFAULT_RECENT_WINDOW
2156 ));
2157 }
2158
2159 #[test]
2160 fn tests_passed_must_follow_the_latest_failure() {
2161 assert!(!detect_tests_passed(
2162 &[
2163 ("5 passed in 0.12s".to_string(), false),
2164 (
2165 "Traceback (most recent call last):\nValueError".to_string(),
2166 false
2167 ),
2168 ("edit applied".to_string(), false),
2169 ],
2170 DEFAULT_RECENT_WINDOW
2171 ));
2172 assert!(detect_tests_passed(
2173 &[
2174 (
2175 "Traceback (most recent call last):\nValueError".to_string(),
2176 false
2177 ),
2178 ("5 passed in 0.12s".to_string(), false),
2179 ],
2180 DEFAULT_RECENT_WINDOW
2181 ));
2182 }
2183
2184 #[test]
2185 fn retrieved_file_contents_are_ignored() {
2186 let call = |id: &str, name: &str, arguments: Value| {
2187 Message::new(
2188 Role::Assistant,
2189 vec![ContentBlock::ToolCall(ToolCall {
2190 id: id.to_string(),
2191 name: name.to_string(),
2192 arguments,
2193 })],
2194 )
2195 };
2196 let result = |id: &str, text: &str| {
2197 Message::new(
2198 Role::User,
2199 vec![ContentBlock::ToolResult(ToolResult {
2200 tool_call_id: id.to_string(),
2201 content: vec![ContentBlock::Text {
2202 text: text.to_string(),
2203 }],
2204 is_error: None,
2205 })],
2206 )
2207 };
2208 let signal = extract_tool_signals_with_window(
2209 &with_messages(vec![
2210 call("a", "Bash", json!({"command": "pytest"})),
2211 result("a", "Traceback (most recent call last):\nValueError"),
2212 call("b", "Read", json!({"file_path": "notes.md"})),
2213 result(
2214 "b",
2215 r#"{"success":false,"error":"out of memory","exit_code":7}"#,
2216 ),
2217 call("c", "Grep", json!({"pattern": "passed"})),
2218 result("c", "CHANGELOG.md: all tests passed"),
2219 ]),
2220 DEFAULT_RECENT_WINDOW,
2221 );
2222 assert_eq!(signal.severity, HARD);
2224 assert!(!signal.tests_passed);
2225 assert_eq!(signal.tool_result_count, 3);
2226
2227 for (command, text, severity) in [
2228 ("cat config.json", r#"{"error":"fixture data"}"#, 0.0),
2229 (
2230 "cat missing.json",
2231 r#"{"output":"","exit_code":7,"error":null}"#,
2232 SOFT,
2233 ),
2234 (
2235 "cat config.json && python check.py",
2236 r#"{"error":"check rejected"}"#,
2237 HARD,
2238 ),
2239 ] {
2240 let request = with_messages(vec![
2241 call("shell", "Bash", json!({"command": command})),
2242 result("shell", text),
2243 ]);
2244 assert_eq!(
2245 ToolSignals::from_request(&request, None).severity,
2246 severity,
2247 "{text}"
2248 );
2249 }
2250 }
2251
2252 fn call_with_result(mut call: Message, mut result: Message) -> Request {
2254 if let ContentBlock::ToolCall(call) = &mut call.content[0] {
2255 call.id = "call-1".into();
2256 }
2257 if let ContentBlock::ToolResult(result) = &mut result.content[0] {
2258 result.tool_call_id = "call-1".into();
2259 }
2260 with_messages(vec![call, result])
2261 }
2262
2263 #[test]
2264 fn shell_reads_are_ignored_but_keep_their_window_slot() {
2265 for command in [
2266 "/usr/bin/cat logfile.txt",
2267 "cd /repo && tail -n 100 logfile.txt | grep MemoryError",
2268 "env MODE=debug timeout 5s rg 'MemoryError' .",
2269 ] {
2270 let mut request = call_with_result(bash(command), tr("MemoryError\nall tests passed"));
2271 request.llm_request.messages.splice(
2272 0..0,
2273 [bash("pytest"), tr("Traceback (most recent call last):")],
2274 );
2275 let signal = ToolSignals::from_request(&request, Some(2));
2277 assert_eq!(signal.severity, HARD, "{command}");
2278 assert!(!signal.tests_passed, "{command}");
2279 let signal = ToolSignals::from_request(&request, Some(1));
2281 assert_eq!(signal.severity, 0.0, "{command}");
2282 }
2283 }
2284
2285 #[test]
2286 fn shell_commands_that_run_code_keep_signals() {
2287 let oversized = format!("cat {}", "a".repeat(MAX_RETRIEVAL_COMMAND_BYTES));
2288 for command in [
2289 "cat logfile.txt; pytest",
2290 "echo $(pytest)",
2291 "find . -name '*.py' -exec pytest ';'",
2292 oversized.as_str(),
2293 ] {
2294 let request = call_with_result(bash(command), tr("MemoryError"));
2295 let signal = ToolSignals::from_request(&request, None);
2296 assert_eq!(signal.severity, CRITICAL, "{command}");
2297 }
2298 }
2299
2300 #[test]
2301 fn failed_reads_signal_unless_the_file_is_missing() {
2302 let failed = |text: &str| {
2303 let mut result = tr(text);
2304 if let ContentBlock::ToolResult(result) = &mut result.content[0] {
2305 result.is_error = Some(true);
2306 }
2307 result
2308 };
2309 let missing = "missing.txt: No such file or directory";
2310 let severities = [
2311 call_with_result(bash("cat missing.txt"), failed(missing)),
2312 call_with_result(tc("Read"), failed("File does not exist.")),
2313 call_with_result(bash("cat logfile.txt"), failed("MemoryError")),
2315 call_with_result(bash("python missing.txt"), failed(missing)),
2317 ]
2318 .map(|request| ToolSignals::from_request(&request, None).severity);
2319 assert_eq!(severities, [0.0, 0.0, CRITICAL, HARD]);
2320
2321 let request = call_with_result(
2322 bash("cat missing.json"),
2323 tr(r#"{"output":"No such file or directory","exit_code":1}"#),
2324 );
2325 assert_eq!(ToolSignals::from_request(&request, None).severity, 0.0);
2326 }
2327
2328 #[test]
2329 fn codex_read_status_comes_from_the_header() {
2330 let read = |header: &str| {
2331 let output = format!(
2332 "{header}\nOutput:\nTraceback (most recent call last):\nMemoryError\nProcess exited with code 1\n"
2333 );
2334 let call = exec_command(json!({"cmd": "cat run.log"}));
2335 ToolSignals::from_request(&call_with_result(call, tr(&output)), None).severity
2336 };
2337 assert_eq!(read("Chunk ID: abc\nProcess exited with code 0"), 0.0);
2339 assert_eq!(read("Chunk ID: abc\nProcess exited with code 1"), CRITICAL);
2341 }
2342
2343 #[test]
2344 fn hermes_read_status_comes_from_exit_code() {
2345 let read = |exit_code: i64| {
2346 let output = json!({
2347 "output": "Traceback (most recent call last):\nMemoryError\nProcess exited with code 1\n",
2348 "exit_code": exit_code,
2349 "error": null,
2350 });
2351 let call = Message::new(
2352 Role::Assistant,
2353 vec![ContentBlock::ToolCall(ToolCall {
2354 id: String::new(),
2355 name: "terminal".to_string(),
2356 arguments: json!({"command": "cat run.log"}),
2357 })],
2358 );
2359 ToolSignals::from_request(&call_with_result(call, tr(&output.to_string())), None)
2360 .severity
2361 };
2362 assert_eq!(read(0), 0.0);
2364 assert_eq!(read(1), CRITICAL);
2366 }
2367
2368 #[test]
2369 fn argv_commands_keep_word_boundaries() {
2370 let signal = |argv: Value| {
2371 let call = exec_command(json!({"cmd": argv}));
2372 ToolSignals::from_request(&call_with_result(call, tr("MemoryError")), None)
2373 };
2374 let mut many_args = vec![""; MAX_RETRIEVAL_COMMAND_BYTES];
2375 many_args[0] = "cat";
2376 let severities = [
2377 json!(["bash", "-lc", "cd repo && rg foo"]),
2378 json!(["bash", "-lc", "cat log; pytest"]),
2379 json!(["cat", "log; pytest"]),
2381 json!(["cat", "a".repeat(MAX_RETRIEVAL_COMMAND_BYTES)]),
2383 json!(many_args),
2384 ]
2385 .map(|argv| signal(argv).severity);
2386 assert_eq!(severities, [0.0, CRITICAL, 0.0, CRITICAL, CRITICAL]);
2387 let read = signal(json!(["bash", "-lc", "sed -n 1,200p x"]));
2389 assert_eq!(read.read_count, 1);
2390 let write = signal(json!(["bash", "-lc", "cd repo && mkdir x"]));
2392 assert_eq!(write.write_count, 1);
2393 }
2394
2395 #[test]
2396 fn severity_is_windowed_over_recent_results() {
2397 let request = with_messages(vec![
2399 tr("Traceback (most recent call last):\n ValueError"),
2400 tr("ok"),
2401 tr("ok"),
2402 ]);
2403 assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD);
2405 assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0);
2407 }
2408
2409 #[test]
2410 fn extract_openai_chat_tool_results() {
2411 let request = with_messages(vec![
2412 Message::text(Role::User, "do something"),
2413 tc("Edit"),
2414 tr("Traceback (most recent call last):\n ValueError"),
2415 ]);
2416 let sig = ToolSignals::from_request(&request, None);
2417 assert_eq!(sig.severity, HARD);
2418 assert_eq!(sig.edit_count, 1);
2419 assert_eq!(sig.turn_depth, 3);
2420 }
2421
2422 #[test]
2423 fn extract_anthropic_tool_results() {
2424 let request = with_messages(vec![tr("Traceback (most recent call last):\n ValueError")]);
2425 let sig = ToolSignals::from_request(&request, None);
2426 assert_eq!(sig.severity, HARD);
2427 }
2428
2429 #[test]
2430 fn extract_responses_api_tool_results() {
2431 let request = with_messages(vec![tc("Write"), tr("file written successfully")]);
2432 let sig = ToolSignals::from_request(&request, None);
2433 assert_eq!(sig.severity, 0.0);
2434 assert_eq!(sig.write_count, 1);
2435 }
2436
2437 #[test]
2438 fn responses_builtin_tool_failures_escalate() {
2439 use crate::algorithms::util::stage::{PickOutcome, PickerMode, Tier, pick_tier};
2440
2441 let mut cases = Vec::new();
2442 for (status, output) in [
2443 (
2444 "failed",
2445 "Synthetic dependency unavailable; retry with the recovery path.",
2446 ),
2447 (
2448 "completed",
2449 "Synthetic dependency unavailable; retry with the recovery path.",
2450 ),
2451 ("failed", ""),
2452 ] {
2453 cases.push((
2454 json!({
2455 "type": "apply_patch_call_output",
2456 "status": status,
2457 "output": output,
2458 }),
2459 if status == "failed" { HARD } else { 0.0 },
2460 ));
2461 }
2462 for (outcome, stdout, stderr, severity) in [
2463 (json!({"type": "exit", "exit_code": 1}), "", "", SOFT),
2464 (json!({"type": "exit", "exit_code": 1}), "", " \n", SOFT),
2465 (
2466 json!({"type": "exit", "exit_code": 1}),
2467 "",
2468 "command failed",
2469 HARD,
2470 ),
2471 (json!({"type": "timeout"}), "", "", HARD),
2472 (json!({"type": "exit", "exit_code": 0}), "done", "", 0.0),
2473 (
2474 json!({"type": "exit", "exit_code": 0}),
2475 "Traceback (most recent call last):",
2476 "",
2477 HARD,
2478 ),
2479 (
2480 json!({"type": "exit", "exit_code": 0}),
2481 "",
2482 "Traceback (most recent call last):",
2483 HARD,
2484 ),
2485 ] {
2486 cases.push((
2487 json!({
2488 "type": "shell_call_output",
2489 "output": [
2490 {"stdout": stdout, "stderr": stderr, "outcome": outcome},
2491 {"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}
2492 ],
2493 }),
2494 severity,
2495 ));
2496 }
2497 for (raw, severity) in cases {
2498 let is_error = severity >= HARD;
2499 let mut request = with_messages(
2500 ["call_1", "call_2"]
2501 .into_iter()
2502 .map(|call_id| {
2503 let mut raw = raw.clone();
2504 raw["call_id"] = json!(call_id);
2505 Message::new(
2506 Role::User,
2507 vec![ContentBlock::Unknown {
2508 provider: WireFormat::OpenAiResponses.into(),
2509 raw,
2510 }],
2511 )
2512 })
2513 .collect(),
2514 );
2515 let signal = ToolSignals::from_request(&request, Some(3));
2516 assert_eq!(signal.severity, severity, "{raw}");
2517 assert_eq!(signal.repeated_failure, is_error, "{raw}");
2518 assert_eq!(signal.tool_result_count, 2);
2519 assert_eq!(
2520 matches!(
2521 pick_tier(&signal, PickerMode::EfficientFirst, 0.5),
2522 PickOutcome::Resolved {
2523 tier: Tier::Capable,
2524 ..
2525 }
2526 ),
2527 is_error,
2528 "{raw}"
2529 );
2530
2531 let mut success = match raw["type"].as_str() {
2532 Some("apply_patch_call_output") => json!({
2533 "type": "apply_patch_call_output", "status": "completed", "output": ""
2534 }),
2535 Some("shell_call_output") => json!({
2536 "type": "shell_call_output",
2537 "output": [{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}]
2538 }),
2539 _ => unreachable!(),
2540 };
2541 for index in 0..3 {
2542 success["call_id"] = json!(format!("success_{index}"));
2543 request.llm_request.messages.push(Message::new(
2544 Role::User,
2545 vec![ContentBlock::Unknown {
2546 provider: WireFormat::OpenAiResponses.into(),
2547 raw: success.clone(),
2548 }],
2549 ));
2550 }
2551 let recovered = ToolSignals::from_request(&request, Some(3));
2552 assert_eq!(recovered.severity, 0.0, "{raw}");
2553 assert!(!recovered.repeated_failure, "{raw}");
2554 assert!(recovered.no_error_streak >= 3, "{raw}");
2555 assert_eq!(recovered.tool_result_count, 5);
2556 }
2557 }
2558
2559 #[test]
2560 fn conversation_counts_are_per_block_and_role_aware() {
2561 let result = |content: Vec<ContentBlock>| {
2565 ContentBlock::ToolResult(ToolResult {
2566 tool_call_id: String::new(),
2567 content,
2568 is_error: None,
2569 })
2570 };
2571 let request = with_messages(vec![
2572 Message::text(Role::User, "do something"),
2573 Message::text(Role::Assistant, "working"),
2574 Message::new(
2575 Role::User,
2576 vec![
2577 result(vec![ContentBlock::Text {
2578 text: "ok".to_string(),
2579 }]),
2580 result(Vec::new()),
2581 ],
2582 ),
2583 tc("Bash"),
2584 ]);
2585 let sig = ToolSignals::from_request(&request, None);
2586 assert_eq!(sig.tool_result_count, 2);
2587 assert_eq!(sig.assistant_turn_count, 2);
2588 assert_eq!(sig.turn_depth, 4);
2589 }
2590
2591 #[test]
2592 fn recent_window_counts_only_last_default_window_tool_calls() {
2593 let request = with_messages(vec![
2596 tc("Write"),
2597 tr("ok"),
2598 tc("Write"),
2599 tr("ok"),
2600 tc("Write"),
2601 tr("ok"),
2602 tc("Write"),
2603 tr("ok"),
2604 tc("Write"),
2605 tr("ok"),
2606 tc("Edit"),
2607 tr("ok"),
2608 ]);
2609 let sig = ToolSignals::from_request(&request, None);
2610 assert_eq!(sig.write_count, 5);
2611 assert_eq!(sig.edit_count, 1);
2612 assert_eq!(sig.recent_write_count, 2);
2613 assert_eq!(sig.recent_edit_count, 1);
2614 }
2615
2616 #[test]
2617 fn codex_apply_patch_counts_as_an_edit() {
2618 let request = with_messages(vec![tc("apply_patch"), tr("Success. Updated the file")]);
2619 let sig = ToolSignals::from_request(&request, None);
2620 assert_eq!(sig.edit_count, 1);
2621 assert_eq!(sig.recent_edit_count, 1);
2622 }
2623
2624 fn exec_command(cmd: Value) -> Message {
2625 Message::new(
2626 Role::Assistant,
2627 vec![ContentBlock::ToolCall(ToolCall {
2628 id: String::new(),
2629 name: "exec_command".to_string(),
2630 arguments: cmd,
2631 })],
2632 )
2633 }
2634
2635 #[test]
2636 fn codex_exec_command_is_classified() {
2637 let args = json!(r#"{"cmd":"sed -i s/a/b/ src/lib.rs","workdir":"/x"}"#);
2639 let request = with_messages(vec![exec_command(args), tr("ok")]);
2640 assert_eq!(
2641 ToolSignals::from_request(&request, None).recent_edit_count,
2642 1
2643 );
2644 }
2645
2646 #[test]
2647 fn shell_patch_mentions_do_not_count_as_edits() {
2648 for arguments in [
2649 json!({"cmd": "printf 'worker dispatch locations\\n'; rg worker src 2>/dev/null"}),
2650 json!(r#"{"cmd":"find /usr /opt -name apply_patch -o -name patch 2>/dev/null"}"#),
2651 json!({"command": ["bash", "-lc", "find . -name apply_patch -o -name patch 2>/dev/null"]}),
2652 json!({"input": "printf 'patch utility\\n'; command -v apply_patch 2>/dev/null"}),
2653 json!({"cmd": "rg 'patch -p1|apply_patch fix.diff' src 2>/dev/null"}),
2654 json!({"cmd": "bash -lc \"printf 'patch -p1' 2>/dev/null\""}),
2655 json!({"cmd": ["inspect", "patch -p1; apply_patch fix.diff"]}),
2656 json!({"cmd": "printf 'patch -p1"}),
2657 json!({"cmd": "cat <<'TEXT'\npatch -p1\nTEXT"}),
2658 json!({"cmd": "printf x # ; patch -p1"}),
2659 json!({"cmd": ["bash", "script.sh", "-c", "patch -p1"]}),
2660 json!({"cmd": "env --help patch"}),
2661 json!({"cmd": "bash -nc 'patch -p1'"}),
2662 json!({"cmd": "bash -o noexec -c 'patch -p1'"}),
2663 json!({"cmd": "bash --norc -n -c 'patch -p1'"}),
2664 json!({"cmd": "bash --help -c 'patch -p1'"}),
2665 json!({"cmd": "bash --version -c 'patch -p1'"}),
2666 json!({"cmd": "bash --dump-strings -c 'patch -p1'"}),
2667 json!({"cmd": "bash --dump-po-strings -c 'patch -p1'"}),
2668 json!({"cmd": "zsh --noexec -c 'patch -p1'"}),
2669 json!({"cmd": "ksh --no-exec -c 'patch -p1'"}),
2670 json!({"cmd": "zsh --NO_EXEC -c 'patch -p1'"}),
2671 json!({"command": ["bash", "--rcfile", "-c", "script.sh", "patch -p1"]}),
2672 json!({"cmd": "bash -- -c 'patch -p1'"}),
2673 json!({"cmd": "bash -o pipefail -c \"printf 'patch -p1' 2>/dev/null\""}),
2674 ] {
2675 let request = call_with_result(exec_command(arguments.clone()), tr("MemoryError"));
2676 let signal = ToolSignals::from_request(&request, None);
2677 assert_eq!(signal.recent_edit_count, 0, "{arguments}");
2678 assert_eq!(signal.recent_write_count, 0, "{arguments}");
2679 assert_eq!(signal.severity, CRITICAL, "{arguments}");
2680 }
2681 }
2682
2683 #[test]
2684 fn shell_patch_invocations_count_as_edits() {
2685 for arguments in [
2686 json!({"cmd": "patch -p1 < fix.diff"}),
2687 json!({"cmd": "apply_patch <<'PATCH'\n*** Begin Patch\n*** End Patch\nPATCH"}),
2688 json!({"cmd": "env -u MODE MODE=debug /usr/bin/patch -p1 < fix.diff"}),
2689 json!({"cmd": "cat fix.diff | command patch -p1"}),
2690 json!({"cmd": "command patch -v -p1 < fix.diff"}),
2691 json!({"cmd": "sudo timeout 5 /usr/local/bin/apply_patch fix.diff"}),
2692 json!({"cmd": "bash -lc 'patch -p1 < fix.diff'"}),
2693 json!({"cmd": "bash -l -c 'patch -p1 < fix.diff'"}),
2694 json!({"cmd": "bash -o pipefail -c 'patch -p1 < fix.diff'"}),
2695 json!({"cmd": "bash -oc pipefail 'patch -p1 < fix.diff'"}),
2696 json!({"cmd": "bash -O extglob -c 'patch -p1 < fix.diff'"}),
2697 json!({"cmd": "bash --login -c 'patch -p1 < fix.diff'"}),
2698 json!({"cmd": "bash --noprofile --norc -c 'apply_patch fix.diff'"}),
2699 json!({"command": ["bash", "--rcfile", "/dev/null", "-c", "apply_patch fix.diff"]}),
2700 json!({"cmd": ["bash", "--init-file", "/dev/null", "-c", "apply_patch fix.diff"]}),
2701 json!(r#"{"cmd":["bash","-o","pipefail","-c","apply_patch fix.diff"]}"#),
2702 json!({"cmd": "printf x # note\npatch -p1 < fix.diff"}),
2703 json!({"cmd": "printf '%s' name#suffix; patch -p1 < fix.diff"}),
2704 json!({"command": ["bash", "-lc", "apply_patch fix.diff"]}),
2705 json!(r#"{"cmd":["env","MODE=debug","patch","-p1"]}"#),
2706 ] {
2707 let request = with_messages(vec![exec_command(arguments.clone()), tr("ok")]);
2708 let signal = ToolSignals::from_request(&request, None);
2709 assert_eq!(signal.recent_edit_count, 1, "{arguments}");
2710 assert_eq!(signal.recent_write_count, 0, "{arguments}");
2711 }
2712 }
2713
2714 #[test]
2715 fn python_write_expressions_need_a_python_command() {
2716 let write = with_messages(vec![
2717 exec_command(json!({"cmd": "python3 - <<'PY'\np.write_text(s)\nPY"})),
2718 tr("ok"),
2719 ]);
2720 assert_eq!(
2721 ToolSignals::from_request(&write, None).recent_write_count,
2722 1
2723 );
2724
2725 let search = with_messages(vec![
2726 exec_command(json!({"cmd": "grep -R '.write(' src"})),
2727 tr("ok"),
2728 ]);
2729 assert_eq!(
2730 ToolSignals::from_request(&search, None).recent_write_count,
2731 0
2732 );
2733 }
2734
2735 #[test]
2736 fn recent_window_size_is_caller_overridable() {
2737 let request = with_messages(vec![
2741 tc("Write"),
2742 tr("ok"),
2743 tc("Write"),
2744 tr("ok"),
2745 tc("Write"),
2746 tr("ok"),
2747 tc("Write"),
2748 tr("ok"),
2749 tc("Write"),
2750 tr("ok"),
2751 tc("Edit"),
2752 tr("ok"),
2753 ]);
2754 let narrow = extract_tool_signals_with_window(&request, 3);
2755 assert_eq!(narrow.recent_write_count, 2);
2756 assert_eq!(narrow.recent_edit_count, 1);
2757
2758 let wide = extract_tool_signals_with_window(&request, 6);
2759 assert_eq!(wide.recent_write_count, 5);
2760 assert_eq!(wide.recent_edit_count, 1);
2761 }
2762
2763 #[test]
2764 fn compaction_marker_sets_compacted() {
2765 for summary in [
2766 "This session is being continued from a previous conversation that ran out of context.",
2767 "Another language model started to solve this problem and produced a summary of its thinking process.",
2768 ] {
2769 let request = with_messages(vec![
2770 Message::text(Role::User, summary),
2771 Message::text(Role::User, "Continue with the next task"),
2772 bash("ls"),
2773 ]);
2774 assert!(ToolSignals::from_request(&request, None).compacted);
2775 assert!(
2776 ToolSignalProcessor::default()
2777 .for_user_turn(&request)
2778 .compacted
2779 );
2780 }
2781 }
2782
2783 #[test]
2784 fn hermes_compaction_headers_set_compacted() {
2785 for header in [
2786 "[CONTEXT COMPACTION \u{2014} REFERENCE ONLY] Earlier turns were compacted into the summary below.",
2787 "[STILL IN PROGRESS \u{2014} this is the active request, restated after the compaction boundary because it was not finished yet. Continue it\u{3b} do not start over.]",
2788 ] {
2789 for role in [Role::User, Role::Assistant] {
2790 let request = with_messages(vec![
2791 Message::text(role, format!("{header}\nContinue the task.")),
2792 bash("ls"),
2793 ]);
2794 assert!(ToolSignals::from_request(&request, None).compacted);
2795 }
2796 let request = with_messages(vec![Message::text(
2797 Role::User,
2798 format!("Prior context.\n\n {header}\nContinue the task."),
2799 )]);
2800 assert!(ToolSignals::from_request(&request, None).compacted);
2801 let request = with_messages(vec![tr(header)]);
2802 assert!(!ToolSignals::from_request(&request, None).compacted);
2803 }
2804 }
2805
2806 #[test]
2807 fn ordinary_compaction_text_stays_uncompacted() {
2808 for text in [
2809 "The task is still in progress after the compaction boundary.",
2810 "Explain [CONTEXT COMPACTION \u{2014} REFERENCE ONLY] in the docs.",
2811 "[STILL IN PROGRESS] Continue the task.",
2812 ] {
2813 let request = with_messages(vec![Message::text(Role::User, text)]);
2814 assert!(!ToolSignals::from_request(&request, None).compacted);
2815 }
2816 }
2817
2818 #[test]
2819 fn codex_compaction_metadata_stays_on_parent_route() {
2820 let mut request = with_messages(vec![bash("ls")]);
2821 request.metadata = Some(Metadata {
2822 is_subagent: true,
2823 agent_kind: Some("compact".to_string()),
2824 ..Default::default()
2825 });
2826 assert!(!ToolSignals::from_request(&request, None).compacted);
2827 }
2828
2829 #[test]
2830 fn no_compaction_marker_stays_uncompacted() {
2831 let request = with_messages(vec![
2832 Message::text(Role::User, "Write a script that parses the log file."),
2833 bash("ls"),
2834 ]);
2835 assert!(!ToolSignals::from_request(&request, None).compacted);
2836 }
2837
2838 #[test]
2839 fn bash_heredoc_counts_as_write() {
2840 let request = with_messages(vec![bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF")]);
2842 let sig = ToolSignals::from_request(&request, None);
2843 assert_eq!(
2844 sig.write_count, 1,
2845 "Bash heredoc should bucket into write_count"
2846 );
2847 assert_eq!(sig.edit_count, 0);
2848 }
2849
2850 #[test]
2851 fn bash_sed_inplace_counts_as_edit() {
2852 let request = with_messages(vec![bash("sed -i 's/foo/bar/g' /app/file.py")]);
2853 let sig = ToolSignals::from_request(&request, None);
2854 assert_eq!(
2855 sig.edit_count, 1,
2856 "Bash sed -i should bucket into edit_count"
2857 );
2858 assert_eq!(sig.write_count, 0);
2859 }
2860
2861 #[test]
2862 fn bash_non_mutating_does_not_count() {
2863 let request = with_messages(vec![bash("ls -la /app"), bash("cat /app/main.py")]);
2865 let sig = ToolSignals::from_request(&request, None);
2866 assert_eq!(sig.write_count, 0);
2867 assert_eq!(sig.edit_count, 0);
2868 }
2869
2870 #[test]
2871 fn tests_passed_detects_pytest_with_failure_block() {
2872 assert!(!detect_tests_passed(
2874 &[("2 failed, 5 passed in 0.56s".to_string(), false)],
2875 DEFAULT_RECENT_WINDOW
2876 ));
2877 }
2878
2879 #[test]
2880 fn tests_passed_accepts_cargo_clean_summary() {
2881 assert!(detect_tests_passed(
2884 &[(
2885 "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string(),
2886 false
2887 )],
2888 DEFAULT_RECENT_WINDOW
2889 ));
2890 }
2891
2892 #[test]
2893 fn tests_passed_rejects_cargo_real_failure() {
2894 assert!(!detect_tests_passed(
2896 &[(
2897 "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string(),
2898 false
2899 )],
2900 DEFAULT_RECENT_WINDOW
2901 ));
2902 }
2903
2904 #[test]
2905 fn tests_passed_accepts_go_clean_summary() {
2906 assert!(detect_tests_passed(
2908 &[(
2909 "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string(),
2910 false
2911 )],
2912 DEFAULT_RECENT_WINDOW
2913 ));
2914 }
2915
2916 #[test]
2917 fn tests_passed_accepts_pytest_zero_errors() {
2918 assert!(detect_tests_passed(
2920 &[("5 passed, 0 errors in 0.30s".to_string(), false)],
2921 DEFAULT_RECENT_WINDOW
2922 ));
2923 }
2924
2925 #[test]
2926 fn tests_passed_detects_diy_checkmark() {
2927 assert!(detect_tests_passed(
2928 &[("✓ all checks passed".to_string(), false)],
2929 DEFAULT_RECENT_WINDOW
2930 ));
2931 }
2932
2933 #[test]
2934 fn anthropic_bash_heredoc_extracts_command() {
2935 let request = with_messages(vec![bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF")]);
2937 let sig = ToolSignals::from_request(&request, None);
2938 assert_eq!(
2939 sig.write_count, 1,
2940 "Anthropic Bash heredoc must also be detected"
2941 );
2942 }
2943
2944 #[test]
2945 fn recent_window_falls_back_to_full_history_when_short() {
2946 let request = with_messages(vec![tc("Write")]);
2947 let sig = ToolSignals::from_request(&request, None);
2948 assert_eq!(sig.recent_write_count, 1);
2949 assert_eq!(sig.recent_edit_count, 0);
2950 }
2951
2952 #[test]
2953 fn clean_tool_result_has_zero_severity_and_non_empty_streak() {
2954 let request = with_messages(vec![tr("output ok"), tr("another ok")]);
2955 let sig = ToolSignals::from_request(&request, None);
2956 assert_eq!(sig.severity, 0.0);
2957 assert_eq!(sig.no_error_streak, 2);
2958 }
2959
2960 #[test]
2963 fn todowrite_classifies_as_plan() {
2964 assert_eq!(classify_tool_call("TodoWrite", None), ToolSemantic::Plan);
2965 assert_eq!(classify_tool_call("todo_write", None), ToolSemantic::Plan);
2966 }
2967
2968 #[test]
2969 fn codex_update_plan_classifies_as_plan() {
2970 assert_eq!(classify_tool_call("update_plan", None), ToolSemantic::Plan);
2971 }
2972
2973 #[test]
2974 fn codex_shell_command_runs_bash_pattern_match() {
2975 assert_eq!(
2977 classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")),
2978 ToolSemantic::Mutate(MutationKind::Write),
2979 );
2980 assert_eq!(
2982 classify_tool_call("shell_command", Some("ls /app")),
2983 ToolSemantic::Observe,
2984 );
2985 assert_eq!(
2987 classify_tool_call("shell_command", Some("./run_tests.sh")),
2988 ToolSemantic::Unknown,
2989 );
2990 }
2991
2992 #[test]
2993 fn text_editor_view_is_a_read() {
2994 for name in ["str_replace_based_edit_tool", "text_editor"] {
2995 assert_eq!(
2996 classify_tool_call(name, Some("view")),
2997 ToolSemantic::Observe
2998 );
2999 for command in [
3000 Some("create"),
3001 Some("insert"),
3002 Some("str_replace"),
3003 Some("undo_edit"),
3004 None,
3005 ] {
3006 assert_eq!(
3007 classify_tool_call(name, command),
3008 ToolSemantic::Mutate(MutationKind::Edit),
3009 );
3010 }
3011 }
3012
3013 let arguments = [
3014 json!({"command": "view", "path": "/app/main.py"}),
3015 json!(r#"{"command":"view","path":"/app/main.py"}"#),
3017 ];
3018 for arguments in arguments {
3019 let call = Message::new(
3020 Role::Assistant,
3021 vec![ContentBlock::ToolCall(ToolCall {
3022 id: String::new(),
3023 name: "str_replace_based_edit_tool".to_string(),
3024 arguments,
3025 })],
3026 );
3027 let request = with_messages(vec![call, tr("print('hi')")]);
3028 let sig = ToolSignals::from_request(&request, None);
3029 assert_eq!(sig.read_count, 1);
3030 assert_eq!(sig.recent_read_count, 1);
3031 assert_eq!(sig.edit_count, 0);
3032 }
3033 }
3034
3035 #[test]
3036 fn read_tool_classifies_as_read() {
3037 assert_eq!(classify_tool_call("Read", None), ToolSemantic::Observe);
3038 assert_eq!(classify_tool_call("View", None), ToolSemantic::Observe);
3039 }
3040
3041 #[test]
3042 fn hermes_tool_names_classify() {
3043 assert_eq!(
3045 classify_tool_call("write_file", None),
3046 ToolSemantic::Mutate(MutationKind::Write)
3047 );
3048 assert_eq!(
3049 classify_tool_call("patch", None),
3050 ToolSemantic::Mutate(MutationKind::Edit)
3051 );
3052 assert_eq!(classify_tool_call("read_file", None), ToolSemantic::Observe);
3053 assert_eq!(
3054 classify_tool_call("search_files", None),
3055 ToolSemantic::Observe
3056 );
3057 assert_eq!(
3060 classify_tool_call("terminal", Some("sed -i 's/a/b/' /app/x.py")),
3061 ToolSemantic::Mutate(MutationKind::Edit),
3062 );
3063 assert_eq!(
3064 classify_tool_call("terminal", Some("grep foo /app")),
3065 ToolSemantic::Observe,
3066 );
3067 assert_eq!(
3068 classify_tool_call("terminal", Some("./run_tests.sh")),
3069 ToolSemantic::Unknown,
3070 );
3071 }
3072
3073 #[test]
3074 fn bash_read_patterns_classify_as_read() {
3075 let cases = [
3076 "cat /etc/passwd",
3077 "grep foo bar.txt",
3078 "ls /app",
3079 "find . -name '*.py'",
3080 ];
3081 for cmd in cases {
3082 assert_eq!(
3083 classify_tool_call("Bash", Some(cmd)),
3084 ToolSemantic::Observe,
3085 "expected Read for {cmd}"
3086 );
3087 }
3088 }
3089
3090 #[test]
3091 fn codex_inspection_commands_classify_as_read() {
3092 let cases = [
3093 "sed -n '1,80p' src/lib.rs",
3094 "rg -n 'needle' src",
3095 "nl -ba src/lib.rs",
3096 "cat package.json",
3097 "jq '.scripts' package.json",
3098 "git status --short",
3099 "git log --oneline -5",
3100 "git show HEAD:src/lib.rs",
3101 "git branch --show-current",
3102 "git remote -v",
3103 "git config --get remote.origin.url",
3104 ];
3105 for command in cases {
3106 assert_eq!(
3107 classify_tool_call("exec_command", Some(command)),
3108 ToolSemantic::Observe,
3109 "expected Read for {command}"
3110 );
3111 }
3112 }
3113
3114 #[test]
3115 fn quoted_shell_separators_do_not_create_commands() {
3116 for command in ["rg 'foo|rm obsolete.rs'", "rg \"foo; rm obsolete.rs\""] {
3117 assert_eq!(
3118 classify_tool_call("exec_command", Some(command)),
3119 ToolSemantic::Observe,
3120 "quoted text must not be parsed as a command: {command}"
3121 );
3122 }
3123 }
3124
3125 #[test]
3126 fn codex_shell_mutations_classify_as_production() {
3127 let writes = [
3128 "cp source.rs destination.rs",
3129 "mkdir -p src/generated",
3130 "touch src/generated/mod.rs",
3131 "git show HEAD:file.rs > file.rs",
3132 "node <<'node'\nfs.writefilesync('file.js', text)\nnode",
3133 ];
3134 for command in writes {
3135 assert_eq!(
3136 classify_tool_call("exec_command", Some(command)),
3137 ToolSemantic::Mutate(MutationKind::Write),
3138 "expected Write for {command}"
3139 );
3140 }
3141
3142 let edits = [
3143 "mv old.rs new.rs",
3144 "rm obsolete.rs",
3145 "gofmt -w main.go",
3146 "cargo fmt",
3147 "ruff check --fix src",
3148 "perl -0pi -e 's/old/new/' src/lib.rs",
3149 "npx prettier --write src/lib.ts",
3150 "uv run ruff format src",
3151 "git apply fix.patch",
3152 ];
3153 for command in edits {
3154 assert_eq!(
3155 classify_tool_call("exec_command", Some(command)),
3156 ToolSemantic::Mutate(MutationKind::Edit),
3157 "expected Edit for {command}"
3158 );
3159 }
3160 }
3161
3162 #[test]
3163 fn formatter_checks_are_not_edits() {
3164 for command in [
3165 "cargo fmt --check",
3166 "ruff format --check src",
3167 "black --check src",
3168 ] {
3169 assert_ne!(
3170 classify_tool_call("exec_command", Some(command)),
3171 ToolSemantic::Mutate(MutationKind::Edit),
3172 "read-only formatter check must not be Edit: {command}"
3173 );
3174 }
3175 }
3176
3177 #[test]
3178 fn embedded_comparison_is_not_a_shell_write() {
3179 let command = "node <<'node'\nif (index > 0) console.log(index)\nnode";
3180 assert_eq!(
3181 classify_tool_call("exec_command", Some(command)),
3182 ToolSemantic::Unknown
3183 );
3184 }
3185
3186 #[test]
3187 fn bash_write_precedence_over_read() {
3188 assert_eq!(
3191 classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")),
3192 ToolSemantic::Mutate(MutationKind::Write),
3193 );
3194 }
3195
3196 #[test]
3197 fn pure_bash_streak_counts_trailing_other() {
3198 let request = with_messages(vec![
3200 bash("make"),
3201 tr("ok"),
3202 bash("./configure"),
3203 tr("ok"),
3204 bash("make install"),
3205 tr("ok"),
3206 bash("./run.sh"),
3207 tr("ok"),
3208 bash("./test"),
3209 tr("ok"),
3210 ]);
3211 let sig = ToolSignals::from_request(&request, None);
3212 assert_eq!(sig.pure_bash_streak, 5);
3213 assert_eq!(sig.write_count, 0);
3214 assert_eq!(sig.read_count, 0);
3215 }
3216
3217 #[test]
3218 fn pure_bash_streak_resets_on_write() {
3219 let request = with_messages(vec![bash("make"), tr("ok"), tc("Write"), tr("ok")]);
3220 let sig = ToolSignals::from_request(&request, None);
3221 assert_eq!(sig.pure_bash_streak, 0);
3222 assert_eq!(sig.write_count, 1);
3223 }
3224
3225 #[test]
3226 fn recent_window_tracks_todowrite_and_read() {
3227 let request = with_messages(vec![
3229 bash("make"),
3230 tr("ok"),
3231 tc("TodoWrite"),
3232 tr("ok"),
3233 tc("Read"),
3234 tr("ok"),
3235 tc("TodoWrite"),
3236 tr("ok"),
3237 ]);
3238 let sig = ToolSignals::from_request(&request, None);
3239 assert_eq!(sig.todowrite_count, 2);
3240 assert_eq!(sig.recent_todowrite_count, 2);
3241 assert_eq!(sig.read_count, 1);
3242 assert_eq!(sig.recent_read_count, 1);
3243 }
3244
3245 #[test]
3246 fn configured_tool_semantics_extend_the_builtin_vocabulary() {
3247 let semantics = ToolSemantics {
3248 observe: vec!["KB_search".to_string()],
3249 mutate: vec!["send_payment_request".to_string()],
3250 plan: vec!["create_research_plan".to_string()],
3251 new: vec!["send_message_to_user".to_string()],
3252 };
3253 semantics.validate().expect("valid additive semantics");
3254 let request = with_messages(vec![
3255 tc("Read"),
3256 tc("Write"),
3257 tc("TodoWrite"),
3258 tc("kb_SEARCH"),
3259 tc("send_payment_request"),
3260 tc("create_research_plan"),
3261 tc("send_message_to_user"),
3262 tc("unlisted_tool"),
3263 ]);
3264
3265 let signal = ToolSignals::from_request_with_semantics(&request, None, &semantics);
3266
3267 assert_eq!(signal.read_count, 2);
3268 assert_eq!(signal.write_count, 2);
3269 assert_eq!(signal.todowrite_count, 2);
3270 assert_eq!(signal.new_count, 1);
3271 assert_eq!(signal.recent_new_count, 1);
3272 assert_eq!(signal.pure_bash_streak, 1);
3273 }
3274
3275 #[test]
3276 fn configured_tool_semantics_match_namespaced_and_mcp_tools() {
3277 let mut request = with_messages(vec![tc("mcp__billing__send_payment_request")]);
3279 request.llm_request.extensions.fields.insert(
3280 TOOL_NAMESPACES_KEY.to_string(),
3281 json!({"mcp__billing__send_payment_request": "mcp__billing"}),
3282 );
3283
3284 let claude_request = with_messages(vec![tc("mcp__billing__send_payment_request")]);
3286
3287 for request in [&request, &claude_request] {
3288 for name in ["send_payment_request", "mcp__billing__send_payment_request"] {
3289 let semantics = ToolSemantics {
3290 mutate: vec![name.to_string()],
3291 ..Default::default()
3292 };
3293 let signal = ToolSignals::from_request_with_semantics(request, None, &semantics);
3294 assert_eq!(signal.write_count, 1, "{name}");
3295 }
3296 }
3297 }
3298
3299 #[test]
3300 fn configured_tool_semantics_only_fold_ascii_case() {
3301 let semantics = ToolSemantics {
3302 observe: vec!["kb_search".to_string()],
3303 ..Default::default()
3304 };
3305
3306 assert_eq!(
3307 classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
3308 ToolSemantic::Observe
3309 );
3310 assert_eq!(
3313 classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
3314 ToolSemantic::Unknown
3315 );
3316 }
3317
3318 #[test]
3319 fn custom_semantics_preserve_builtin_unicode_lowercasing() {
3320 let semantics = ToolSemantics {
3321 observe: vec!["lookup_customer".to_string()],
3322 ..Default::default()
3323 };
3324
3325 assert_eq!(
3327 classify_tool_call_with_semantics("notebooKedit", None, &semantics),
3328 ToolSemantic::Mutate(MutationKind::Edit)
3329 );
3330 }
3331
3332 #[test]
3333 fn configured_semantics_never_replace_builtin_classifications() {
3334 let semantics = ToolSemantics {
3335 observe: vec!["lookup_customer".to_string()],
3336 mutate: vec!["send_payment".to_string()],
3337 plan: vec!["create_workflow".to_string()],
3338 new: vec!["send_message".to_string()],
3339 };
3340
3341 for name in WRITE_TOOL_NAMES {
3342 assert_eq!(
3343 classify_tool_call_with_semantics(name, None, &semantics),
3344 ToolSemantic::Mutate(MutationKind::Write),
3345 "write tool {name:?} changed classification"
3346 );
3347 }
3348 for name in EDIT_TOOL_NAMES {
3349 assert_eq!(
3350 classify_tool_call_with_semantics(name, None, &semantics),
3351 ToolSemantic::Mutate(MutationKind::Edit),
3352 "edit tool {name:?} changed classification"
3353 );
3354 }
3355 for name in READ_TOOL_NAMES {
3356 assert_eq!(
3357 classify_tool_call_with_semantics(name, None, &semantics),
3358 ToolSemantic::Observe,
3359 "read tool {name:?} changed classification"
3360 );
3361 }
3362 for name in PLAN_TOOL_NAMES {
3363 assert_eq!(
3364 classify_tool_call_with_semantics(name, None, &semantics),
3365 ToolSemantic::Plan,
3366 "plan tool {name:?} changed classification"
3367 );
3368 }
3369
3370 for (command, expected) in [
3371 ("cat /tmp/input", ToolSemantic::Observe),
3372 (
3373 "cat /tmp/input > /tmp/output",
3374 ToolSemantic::Mutate(MutationKind::Write),
3375 ),
3376 (
3377 "sed -i 's/a/b/' /tmp/file",
3378 ToolSemantic::Mutate(MutationKind::Edit),
3379 ),
3380 ("./run_tests.sh", ToolSemantic::Unknown),
3381 ] {
3382 assert_eq!(
3383 classify_tool_call_with_semantics("BASH", Some(&json!(command)), &semantics),
3384 expected,
3385 "bash command {command:?} changed classification"
3386 );
3387 }
3388 }
3389
3390 #[test]
3391 fn configured_semantics_score_like_their_builtin_equivalents() {
3392 let semantics = ToolSemantics {
3393 observe: vec!["lookup_customer".to_string()],
3394 mutate: vec!["send_payment".to_string()],
3395 plan: vec!["create_workflow".to_string()],
3396 ..Default::default()
3397 };
3398
3399 for (builtin, configured) in [
3400 ("Read", "lookup_customer"),
3401 ("Write", "send_payment"),
3402 ("TodoWrite", "create_workflow"),
3403 ] {
3404 let messages_before_tool = || {
3405 vec![
3406 Message::text(Role::User, "start"),
3407 Message::text(Role::Assistant, "working"),
3408 Message::text(Role::User, "continue"),
3409 Message::text(Role::Assistant, "working"),
3410 Message::text(Role::User, "continue"),
3411 Message::text(Role::Assistant, "working"),
3412 Message::text(Role::User, "continue"),
3413 ]
3414 };
3415 let mut builtin_messages = messages_before_tool();
3416 builtin_messages.push(tc(builtin));
3417 let mut configured_messages = messages_before_tool();
3418 configured_messages.push(tc(configured));
3419
3420 let builtin_score = score_signal(&ToolSignals::from_request(
3421 &with_messages(builtin_messages),
3422 None,
3423 ));
3424 let configured_score = score_signal(&ToolSignals::from_request_with_semantics(
3425 &with_messages(configured_messages),
3426 None,
3427 &semantics,
3428 ));
3429
3430 assert_ne!(
3431 builtin_score.score, 0.0,
3432 "the {builtin:?} control must exercise a scoring dimension"
3433 );
3434 assert_eq!(
3435 configured_score, builtin_score,
3436 "configured tool {configured:?} must score exactly like {builtin:?}"
3437 );
3438 }
3439 }
3440
3441 #[test]
3442 fn tool_semantics_reject_duplicates_and_builtin_reclassification() {
3443 let duplicate = ToolSemantics {
3444 observe: vec!["lookup".to_string()],
3445 mutate: vec!["LOOKUP".to_string()],
3446 ..Default::default()
3447 };
3448 assert!(
3449 duplicate
3450 .validate()
3451 .expect_err("duplicate should fail")
3452 .to_string()
3453 .contains("appears in both")
3454 );
3455
3456 let builtin = ToolSemantics {
3457 new: vec!["write_file".to_string()],
3458 ..Default::default()
3459 };
3460 assert!(
3461 builtin
3462 .validate()
3463 .expect_err("built-in should fail")
3464 .to_string()
3465 .contains("built-in semantics")
3466 );
3467
3468 let empty = ToolSemantics {
3469 observe: vec![" \t".to_string()],
3470 ..Default::default()
3471 };
3472 assert!(
3473 empty
3474 .validate()
3475 .expect_err("empty name should fail")
3476 .to_string()
3477 .contains("empty tool name")
3478 );
3479 }
3480}