1use std::collections::{HashMap, HashSet};
29use std::hash::{DefaultHasher, Hash, Hasher};
30use std::sync::Arc;
31use std::time::Instant;
32
33use parking_lot::Mutex;
34use switchyard_protocol::{
35 ContentBlock, Decision, InstructionBlock, LlmRequest, Message, ModelId, OutputParams, Request,
36 Response, Role, SamplingParams,
37};
38
39use crate::core::algorithm::{Algorithm, Driver};
40use crate::{LibsyError, Result};
41
42mod telemetry;
43#[cfg(test)]
44mod tests;
45mod transcript;
46mod turn;
47
48use telemetry::{
49 ReviewAudit, emit_discarded_audit, emit_review_audit, record_consult_failure, record_discarded,
50 record_review,
51};
52use transcript::{VERDICT_PATTERN, Verdict, advisor_reply_text, parse_verdict, review_transcript};
53use turn::{
54 GatedTurn, assistant_turns, buffer_turn, count_tool_results, has_tool_use, reasoning_text,
55 visible_text,
56};
57
58pub const REVIEWER_SYSTEM_PROMPT: &str =
60 include_str!("../prompts/advisor-gate/reviewer-system-prompt.md");
61
62pub const REDO_FEEDBACK_PREFIX: &str = concat!(
65 include_str!("../prompts/advisor-gate/redo-feedback-prefix.md"),
66 "\n"
67);
68
69const REASONING_TAIL_LABEL: &str =
73 "(the executor produced no visible text this turn; its internal reasoning follows)\n";
74const EMPTY_ECHO_PLACEHOLDER: &str = "(the executor produced no output this turn)";
77const MAX_FAILED_CONSULTS: u32 = 3;
82const MAX_TRACKED_SCOPES: usize = 1_024;
85const BENCH_SESSION_HEADER: &str = "proxy_x_session_id";
89
90#[derive(Clone, Debug, PartialEq)]
92pub enum GateTrigger {
93 NoToolCall,
95 Pattern(String),
99}
100
101#[derive(Clone, Debug)]
103pub struct AdvisorGateConfig {
104 pub reviewer_system_prompt: String,
106 pub redo_feedback_prefix: String,
108 pub gate_trigger: GateTrigger,
110 pub max_reviews: u32,
114 pub gate_stall_turns: u32,
119 pub gate_min_tool_results: u32,
123 pub advisor_max_tokens: u64,
125 pub advisor_temperature: Option<f64>,
127 pub transcript_max_chars: usize,
130 pub fail_open: bool,
133}
134
135impl Default for AdvisorGateConfig {
136 fn default() -> Self {
137 Self {
138 reviewer_system_prompt: REVIEWER_SYSTEM_PROMPT.to_string(),
139 redo_feedback_prefix: REDO_FEEDBACK_PREFIX.to_string(),
140 gate_trigger: GateTrigger::NoToolCall,
141 max_reviews: 1,
142 gate_stall_turns: 0,
143 gate_min_tool_results: 0,
144 advisor_max_tokens: 2048,
145 advisor_temperature: None,
146 transcript_max_chars: 200_000,
147 fail_open: true,
148 }
149 }
150}
151
152enum CompiledTrigger {
154 NoToolCall,
155 Pattern(regex::Regex),
156}
157
158#[derive(Clone, Debug, Hash, PartialEq, Eq)]
162enum ScopeKey {
163 Instance,
164 Client(String),
165 Session(String),
166}
167
168#[derive(Default)]
170struct ScopeState {
171 reviews: u32,
172 failed_consults: u32,
173 exhaustion_logged: bool,
174}
175
176#[derive(Default)]
179struct GateState {
180 scopes: HashMap<ScopeKey, ScopeState>,
181 stall_fired: HashSet<u64>,
182}
183
184pub struct AdvisorGate {
188 executor: ModelId,
189 advisor: ModelId,
190 config: AdvisorGateConfig,
191 trigger: CompiledTrigger,
192 verdict_re: regex::Regex,
193 state: Mutex<GateState>,
194}
195
196impl AdvisorGate {
197 pub fn new(executor: ModelId, advisor: ModelId, config: AdvisorGateConfig) -> Result<Self> {
199 if config.max_reviews < 1 {
200 return Err(algorithm_error("max_reviews must be at least 1"));
201 }
202 if config.advisor_max_tokens < 1 {
203 return Err(algorithm_error("advisor_max_tokens must be at least 1"));
204 }
205 if config.transcript_max_chars < 256 {
206 return Err(algorithm_error("transcript_max_chars must be at least 256"));
207 }
208 let trigger = match &config.gate_trigger {
209 GateTrigger::NoToolCall => CompiledTrigger::NoToolCall,
210 GateTrigger::Pattern(pattern) => {
211 if pattern.is_empty() {
212 return Err(algorithm_error(
213 "gate_trigger 'pattern' requires a non-empty gate_trigger_pattern",
214 ));
215 }
216 CompiledTrigger::Pattern(regex::Regex::new(pattern).map_err(|error| {
217 algorithm_error(format!(
218 "gate_trigger_pattern is not a valid regex: {error}"
219 ))
220 })?)
221 }
222 };
223 let verdict_re = regex::Regex::new(VERDICT_PATTERN).map_err(|error| {
224 algorithm_error(format!("verdict pattern failed to compile: {error}"))
225 })?;
226 Ok(Self {
227 executor,
228 advisor,
229 config,
230 trigger,
231 verdict_re,
232 state: Mutex::new(GateState::default()),
233 })
234 }
235
236 fn executor_decision(&self, reasoning: &str) -> Decision {
239 Decision::new(
240 self.executor.clone(),
241 Some(format!("advisor gate: {reasoning}")),
242 true,
243 )
244 }
245
246 fn check_exhausted(&self, scope: &ScopeKey) -> bool {
250 let mut state = self.state.lock();
251 let Some(entry) = state.scopes.get_mut(scope) else {
252 return false;
253 };
254 let exhausted = entry.reviews >= self.config.max_reviews
255 || entry.failed_consults >= MAX_FAILED_CONSULTS;
256 if exhausted && !entry.exhaustion_logged {
257 entry.exhaustion_logged = true;
258 tracing::info!(
259 target: "libsy",
260 scope = ?scope,
261 "advisor gate: review budget spent; passing through"
262 );
263 }
264 exhausted
265 }
266
267 fn try_reserve(&self, scope: &ScopeKey) -> bool {
271 let mut state = self.state.lock();
272 if state.scopes.len() >= MAX_TRACKED_SCOPES && !state.scopes.contains_key(scope) {
273 let evict = state
274 .scopes
275 .keys()
276 .find(|key| **key != ScopeKey::Instance)
277 .cloned();
278 if let Some(key) = evict {
279 state.scopes.remove(&key);
280 }
281 }
282 let max_reviews = self.config.max_reviews;
283 let entry = state.scopes.entry(scope.clone()).or_default();
284 if entry.reviews >= max_reviews || entry.failed_consults >= MAX_FAILED_CONSULTS {
285 return false;
286 }
287 entry.reviews += 1;
288 true
289 }
290
291 fn refund_failure(&self, scope: &ScopeKey) {
295 let mut state = self.state.lock();
296 let entry = state.scopes.entry(scope.clone()).or_default();
297 entry.reviews = entry.reviews.saturating_sub(1);
298 entry.failed_consults += 1;
299 }
300
301 fn evict_scope(&self, scope: &ScopeKey) {
303 if *scope == ScopeKey::Instance {
304 return;
305 }
306 self.state.lock().scopes.remove(scope);
307 }
308
309 fn stall_already_fired(&self, key: u64) -> bool {
310 self.state.lock().stall_fired.contains(&key)
311 }
312
313 fn mark_stall_fired(&self, key: u64) {
314 let mut state = self.state.lock();
315 if state.stall_fired.len() >= MAX_TRACKED_SCOPES {
316 let drop = state.stall_fired.iter().next().copied();
317 if let Some(key) = drop {
318 state.stall_fired.remove(&key);
319 }
320 }
321 state.stall_fired.insert(key);
322 }
323
324 async fn route_inner(
327 &self,
328 driver: &Driver,
329 request: Request,
330 scope: &ScopeKey,
331 ) -> Result<Response> {
332 if self.check_exhausted(scope) {
337 let decision = self.executor_decision("review budget spent; passthrough");
338 driver.decide(decision.clone()).await?;
339 return driver.call_model(request, decision).await;
340 }
341
342 let decision = self.executor_decision("executor turn");
345 driver.decide(decision.clone()).await?;
346 let response = driver.call_model(request.clone(), decision).await?;
347 let turn = buffer_turn(self.executor.as_str(), response).await?;
348
349 let stall_key = stall_key(&request);
353 let stall = self.config.gate_stall_turns > 0
354 && !self.stall_already_fired(stall_key)
355 && assistant_turns(&request.llm_request.messages) >= self.config.gate_stall_turns;
356 let triggered = match &self.trigger {
357 CompiledTrigger::Pattern(pattern) => {
358 pattern.is_match(visible_text(&turn.agg).as_deref().unwrap_or(""))
359 }
360 CompiledTrigger::NoToolCall => {
361 !has_tool_use(&turn.agg)
362 && count_tool_results(&request.llm_request.messages)
363 >= self.config.gate_min_tool_results
364 }
365 };
366 if !(triggered || stall) {
367 return Ok(turn.into_response());
368 }
369 if stall && !triggered {
372 self.mark_stall_fired(stall_key);
373 }
374 if !self.try_reserve(scope) {
375 return Ok(turn.into_response());
376 }
377
378 let trigger_label = match (&self.trigger, triggered) {
379 (CompiledTrigger::Pattern(_), true) => "pattern",
380 (CompiledTrigger::NoToolCall, true) => "no_tool_call",
381 _ => "stall",
382 };
383 let review_tail = visible_text(&turn.agg).or_else(|| {
384 reasoning_text(&turn.agg).map(|reasoning| format!("{REASONING_TAIL_LABEL}{reasoning}"))
385 });
386 match self
387 .consult(driver, &request, review_tail.as_deref(), trigger_label)
388 .await
389 {
390 Ok(ConsultOutcome::Approve) => Ok(turn.into_response()),
391 Ok(ConsultOutcome::Redo { plan }) => self.redo(driver, request, turn, &plan).await,
392 Ok(ConsultOutcome::Failed) => {
393 self.refund_failure(scope);
394 Ok(turn.into_response())
395 }
396 Err(error) => {
397 self.refund_failure(scope);
398 Err(error)
399 }
400 }
401 }
402
403 async fn redo(
407 &self,
408 driver: &Driver,
409 request: Request,
410 turn: GatedTurn,
411 plan: &str,
412 ) -> Result<Response> {
413 record_discarded(&turn.agg.usage);
414 emit_discarded_audit(self.executor.as_str(), &turn.agg.usage);
415 let echo = visible_text(&turn.agg)
416 .or_else(|| reasoning_text(&turn.agg))
417 .unwrap_or_else(|| EMPTY_ECHO_PLACEHOLDER.to_string());
418 let mut redo = request;
419 redo.llm_request
420 .messages
421 .push(Message::text(Role::Assistant, echo));
422 redo.llm_request.messages.push(Message::text(
423 Role::User,
424 format!("{}{}", self.config.redo_feedback_prefix, plan),
425 ));
426 crate::algorithms::util::prompts::drop_exact_replay(&mut redo);
430 let decision = self.executor_decision("REDO continuation");
431 driver.decide(decision.clone()).await?;
432 driver.call_model(redo, decision).await
433 }
434
435 async fn consult(
439 &self,
440 driver: &Driver,
441 base: &Request,
442 review_tail: Option<&str>,
443 trigger: &'static str,
444 ) -> Result<ConsultOutcome> {
445 let transcript_messages: Vec<Message> = base
450 .llm_request
451 .instructions
452 .iter()
453 .map(|block| Message {
454 role: block.role,
455 content: block.content.clone(),
456 })
457 .chain(base.llm_request.messages.iter().cloned())
458 .collect();
459 let transcript = review_transcript(
460 &transcript_messages,
461 review_tail,
462 self.config.transcript_max_chars,
463 );
464 let consult_request = self.build_consult_request(base, transcript);
465 let decision = Decision::new(
466 self.advisor.clone(),
467 Some("advisor gate: review consult".to_string()),
468 false,
469 );
470 let started = Instant::now();
471 let reply = match driver.call_model(consult_request, decision).await {
472 Ok(response) => response
473 .llm_response
474 .into_agg()
475 .await
476 .map_err(|source| LibsyError::client_call(self.advisor.clone(), source)),
477 Err(error) => Err(error),
478 };
479 let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
480 let agg = match reply {
481 Ok(agg) => agg,
482 Err(error) => {
483 record_consult_failure(crate::algorithms::util::llm_judge::libsy_error_reason(
484 &error,
485 ));
486 if !self.config.fail_open {
487 return Err(algorithm_error(format!(
493 "advisor consult failed (fail_open = false): {error}"
494 )));
495 }
496 tracing::warn!(
497 target: "libsy",
498 error = %error,
499 "advisor gate: consult failed; passing the turn through (fail open)"
500 );
501 emit_review_audit(ReviewAudit {
502 verdict: "APPROVE",
503 error: Some(error.to_string()),
504 latency_ms,
505 reply_head: None,
506 usage: None,
507 });
508 return Ok(ConsultOutcome::Failed);
509 }
510 };
511 let reply_text = advisor_reply_text(&agg);
512 let reply_head: String = reply_text.chars().take(160).collect();
513 match parse_verdict(&self.verdict_re, &reply_text) {
514 Some(Verdict::Approve) => {
515 record_review("approve", trigger);
516 emit_review_audit(ReviewAudit {
517 verdict: "APPROVE",
518 error: None,
519 latency_ms,
520 reply_head: Some(reply_head),
521 usage: Some(&agg.usage),
522 });
523 Ok(ConsultOutcome::Approve)
524 }
525 Some(Verdict::Redo { plan }) => {
526 record_review("redo", trigger);
527 emit_review_audit(ReviewAudit {
528 verdict: "REDO",
529 error: None,
530 latency_ms,
531 reply_head: Some(reply_head),
532 usage: Some(&agg.usage),
533 });
534 Ok(ConsultOutcome::Redo { plan })
535 }
536 None => {
537 record_review("unparseable", trigger);
541 emit_review_audit(ReviewAudit {
542 verdict: "UNPARSEABLE",
543 error: None,
544 latency_ms,
545 reply_head: Some(reply_head),
546 usage: Some(&agg.usage),
547 });
548 Ok(ConsultOutcome::Failed)
549 }
550 }
551 }
552
553 fn build_consult_request(&self, base: &Request, transcript: String) -> Request {
556 Request {
557 llm_request: LlmRequest {
558 model: base.llm_request.model.clone(),
559 instructions: vec![InstructionBlock {
560 role: Role::System,
561 content: vec![ContentBlock::Text {
562 text: self.config.reviewer_system_prompt.clone(),
563 }],
564 }],
565 messages: vec![Message::text(Role::User, transcript)],
566 sampling: SamplingParams {
567 temperature: self.config.advisor_temperature,
568 ..SamplingParams::default()
569 },
570 output: OutputParams {
571 max_output_tokens: Some(self.config.advisor_max_tokens),
572 response_format: None,
573 },
574 ..LlmRequest::default()
575 },
576 raw_request: None,
577 metadata: base.metadata.clone(),
578 }
579 }
580}
581
582#[async_trait::async_trait]
583impl Algorithm for AdvisorGate {
584 fn name(&self) -> &str {
585 "advisor_gate"
586 }
587
588 async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
589 let scope = budget_scope(&request);
590 let session_final = request
591 .metadata
592 .as_ref()
593 .and_then(|metadata| metadata.session_final)
594 == Some(true);
595 let result = self.route_inner(&driver, request, &scope).await;
596 if session_final {
597 self.evict_scope(&scope);
598 }
599 result
600 }
601}
602
603enum ConsultOutcome {
605 Approve,
606 Redo { plan: String },
607 Failed,
608}
609
610fn budget_scope(request: &Request) -> ScopeKey {
616 let metadata = request.metadata.as_ref();
617 if let Some(value) = metadata
618 .and_then(|metadata| metadata.http_headers.as_ref())
619 .and_then(|headers| headers.get(BENCH_SESSION_HEADER))
620 .and_then(|value| value.to_str().ok())
621 && !value.is_empty()
622 {
623 return ScopeKey::Client(value.to_string());
624 }
625 if let Some(id) = metadata.and_then(|metadata| metadata.session_id.as_deref())
626 && !id.is_empty()
627 {
628 return ScopeKey::Session(id.to_string());
629 }
630 ScopeKey::Instance
631}
632
633fn stall_key(request: &Request) -> u64 {
636 let text = request
637 .llm_request
638 .messages
639 .iter()
640 .find(|message| message.role == Role::User)
641 .and_then(|message| message.text_content("\n"))
642 .unwrap_or_default();
643 let mut hasher = DefaultHasher::new();
644 text.hash(&mut hasher);
645 hasher.finish()
646}
647
648fn algorithm_error(message: impl Into<String>) -> LibsyError {
649 LibsyError::AlgorithmError {
650 message: message.into(),
651 }
652}