hanzoai/dev
2
0
Fork 0
mirror of https://github.com/hanzoai/dev.git synced 2026-10-25 03:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(auto-drive): add final observer validation before stop

Browse Source
This commit is contained in:
James Peter
2025-10-01 09:25:15 +10:00
parent 3891143497
commit 94061ee737
2 changed files with 129 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/tui/src/chatwidget/auto_coordinator.rs
+94 -15
View File
@@ -21,7 +21,7 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use uuid::Uuid;
use crate::app_event::{AppEvent, AutoCoordinatorStatus};
use crate::app_event::{AppEvent, AutoCoordinatorStatus, AutoObserverStatus, AutoObserverTelemetry};
use crate::app_event_sender::AppEventSender;
use crate::chatwidget::retry::{retry_with_backoff, RetryDecision, RetryError, RetryOptions};
#[cfg(feature = "dev-faults")]
@@ -31,10 +31,13 @@ use chrono::{DateTime, Local, Utc};
use rand::Rng;
use super::auto_observer::{
build_observer_conversation,
run_observer_once,
start_auto_observer,
AutoObserverCommand,
ObserverOutcome,
ObserverReason,
ObserverTrigger,
summarize_intervention,
};
const RATE_LIMIT_BUFFER: Duration = Duration::from_secs(120);
@@ -154,7 +157,7 @@ fn run_auto_loop(
let model_text_verbosity = config.model_text_verbosity;
let sandbox_policy = config.sandbox_policy.clone();
let config = Arc::new(config);
let client = ModelClient::new(
let client = Arc::new(ModelClient::new(
config.clone(),
Some(auth_mgr),
model_provider,
@@ -166,7 +169,7 @@ fn run_auto_loop(
DebugLogger::new(debug_enabled)
.unwrap_or_else(|_| DebugLogger::new(false).expect("debug logger")),
)),
);
));
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -205,6 +208,7 @@ fn run_auto_loop(
let mut stopped = false;
let mut requests_completed: u64 = 0;
let mut observer_guidance: Vec<String> = Vec::new();
let mut observer_telemetry = AutoObserverTelemetry::default();
let mut observer_handle = if observer_cadence == 0 {
None
} else {
@@ -236,7 +240,7 @@ fn run_auto_loop(
compose_developer_intro(&base_developer_intro, &observer_guidance);
match request_coordinator_decision(
&runtime,
&client,
client.as_ref(),
developer_intro.as_str(),
&primary_goal_message,
&schema,
@@ -246,16 +250,16 @@ fn run_auto_loop(
&cancel_token,
) {
Ok((status, summary, prompt_opt, transcript_items)) => {
let prompt_clone = prompt_opt.clone();
let event = AppEvent::AutoCoordinatorDecision {
status,
summary,
prompt: prompt_opt,
transcript: transcript_items.clone(),
};
app_event_tx.send(event);
if matches!(status, AutoCoordinatorStatus::Continue) {
let prompt_clone = prompt_opt.clone();
let event = AppEvent::AutoCoordinatorDecision {
status,
summary,
prompt: prompt_opt,
transcript: transcript_items.clone(),
};
app_event_tx.send(event);
if let (Some(handle), Some(cadence)) =
(observer_handle.as_ref(), observer_cadence)
{
@@ -268,16 +272,73 @@ fn run_auto_loop(
conversation,
goal_text: goal_text.clone(),
environment_details: environment_details.clone(),
reason: ObserverReason::Cadence,
};
if handle.tx.send(AutoObserverCommand::Trigger(trigger)).is_err() {
tracing::warn!("failed to trigger auto observer");
}
}
}
} else {
stopped = true;
continue;
}
let observer_conversation =
build_observer_conversation(conv_for_observer.clone(), None);
let validation_result = run_final_observer_validation(
&runtime,
client.clone(),
observer_conversation,
&goal_text,
&environment_details,
status,
);
if let Ok((observer_status, replace_message, additional_instructions)) =
&validation_result
{
let telemetry = AutoObserverTelemetry {
trigger_count: observer_telemetry.trigger_count.saturating_add(1),
last_status: *observer_status,
last_intervention: summarize_intervention(
replace_message.as_deref(),
additional_instructions.as_deref(),
),
};
observer_telemetry = telemetry.clone();
let observer_event = AppEvent::AutoObserverReport {
status: *observer_status,
telemetry,
replace_message: replace_message.clone(),
additional_instructions: additional_instructions.clone(),
};
app_event_tx.send(observer_event);
if matches!(observer_status, AutoObserverStatus::Failing) {
if let Some(instr) = additional_instructions
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
if !observer_guidance.iter().any(|existing| existing == instr) {
observer_guidance.push(instr.to_string());
}
}
pending_conversation = Some(conv_for_observer);
continue;
}
} else if let Err(err) = validation_result {
tracing::warn!("final observer validation failed: {err:#}");
}
let event = AppEvent::AutoCoordinatorDecision {
status,
summary,
prompt: prompt_opt,
transcript: transcript_items,
};
app_event_tx.send(event);
stopped = true;
continue;
}
Err(err) => {
if err.downcast_ref::<AutoCoordinatorCancelled>().is_some() {
@@ -320,6 +381,7 @@ fn run_auto_loop(
}
}
observer_telemetry = telemetry.clone();
let event = AppEvent::AutoObserverReport {
status,
telemetry,
@@ -365,6 +427,23 @@ fn should_trigger_observer(requests_completed: u64, cadence: u64) -> bool {
cadence != 0 && requests_completed > 0 && requests_completed % cadence == 0
}
fn run_final_observer_validation(
runtime: &tokio::runtime::Runtime,
client: Arc<ModelClient>,
conversation: Vec<ResponseItem>,
goal_text: &str,
environment_details: &str,
finish_status: AutoCoordinatorStatus,
) -> Result<(AutoObserverStatus, Option<String>, Option<String>)> {
let trigger = ObserverTrigger {
conversation,
goal_text: goal_text.to_string(),
environment_details: environment_details.to_string(),
reason: ObserverReason::FinalCheck { finish_status },
};
run_observer_once(runtime, client, trigger)
}
fn build_developer_message(goal_text: &str, environment_details: &str) -> (String, String) {
let intro = format!(
"You are coordinating prompts sent to a running Code CLI process. You should act like a human maintainer of the project would act. You will see a **Primary Goal** below - this is what you are always working towards.\n\n**JSON Structure**\n- `finish_status`: one of `continue`, `finish_success`, or `finish_failed`.\n * Use `continue` when another prompt is reasonable. Always prefer this option.\n * Use `finish_success` when the goal has been completed in its entirety and absolutely no work remains.\n * Use `finish_failed` when the goal absolutely cannot be satisfied or you are stuck in a loop. This should almost never be used. Try other approaches and gather more information if there is no clear path forward.\n- `summary`: short summary (<= 160 characters) describing what will happen when the CLI performs the next prompt\n- `prompt`: the exact prompt to provide to the Code CLI process. You will receive the response the CLI provides.\n\n**Rules**\n- You set direction, not implementation. Keep the CLI on track, but let it do all the thinking and implementation. You do not have the context the CLI has.\n- When working on an existing code base, start by prompting the CLI to explain the problem and outline plausible approaches. This lets it build context rather than jumping in naively with a solution.\n- Keep every prompt minimal to give the CLI room to make independent decisions.\n- Don't repeat yourself. If something doesn't work, take a different approach. Always push the project forward.\n- Often a simple 'Please continue' or 'Work on feature A next' or 'What do you think is the best approach?' is sufficient. Your job is to keep things running in an appropriate direction. The CLI does all the actual work and thinking. You do not need to know much about the project or codebase, allow the CLI to do all this for you. You are focused on overall direction not implementation details.\n- Only stop when no other options remain. A human is observing your work and will step in if they want to go in a different direction. You should not ask them for assistance - you should use your judgement to move on the most likely path forward. The human may override your message send to the CLI if they choose to go in another direction. This allows you to just guess the best path, knowing an overseer will step in if needed.\n\nUseful command:\n`/review <what to review>` e.g. `/review latest commit` - this spins up a specialist review thread for the CLI which excels at identify issues. This is useful for repeatedly reviewing code changes you make and fixing them.\n\nEnvironment:\n{environment_details}"
codex-rs/tui/src/chatwidget/auto_observer.rs
+35 -10
View File
@@ -10,7 +10,7 @@ use serde::Deserialize;
use serde_json::{self, json, Value};
use tracing::{debug, error, warn};
use crate::app_event::{AutoObserverStatus, AutoObserverTelemetry};
use crate::app_event::{AutoCoordinatorStatus, AutoObserverStatus, AutoObserverTelemetry};
use super::auto_coordinator::{
extract_first_json_object,
@@ -37,11 +37,18 @@ pub(super) enum AutoObserverCommand {
Stop,
}
#[derive(Debug, Clone, Copy)]
pub(super) enum ObserverReason {
Cadence,
FinalCheck { finish_status: AutoCoordinatorStatus },
}
#[derive(Debug, Clone)]
pub(super) struct ObserverTrigger {
pub conversation: Vec<ResponseItem>,
pub goal_text: String,
pub environment_details: String,
pub reason: ObserverReason,
}
#[derive(Debug, Clone)]
@@ -55,7 +62,7 @@ pub(super) struct ObserverOutcome {
const OBSERVER_SCHEMA_NAME: &str = "auto_coordinator_observer";
pub(super) fn start_auto_observer(
client: ModelClient,
client: Arc<ModelClient>,
cadence: u32,
coordinator_tx: Sender<AutoCoordinatorCommand>,
) -> Result<AutoObserverHandle> {
@@ -74,8 +81,16 @@ pub(super) fn start_auto_observer(
})
}
pub(super) fn run_observer_once(
runtime: &tokio::runtime::Runtime,
client: Arc<ModelClient>,
trigger: ObserverTrigger,
) -> Result<(AutoObserverStatus, Option<String>, Option<String>)> {
evaluate_observer(runtime, client, trigger)
}
fn run_observer_loop(
client: ModelClient,
client: Arc<ModelClient>,
rx: Receiver<AutoObserverCommand>,
coordinator_tx: Sender<AutoCoordinatorCommand>,
) -> Result<()> {
@@ -84,7 +99,6 @@ fn run_observer_loop(
.build()
.context("creating runtime for auto observer")?;
let client = Arc::new(client);
let mut telemetry = AutoObserverTelemetry {
trigger_count: 0,
last_status: AutoObserverStatus::Ok,
@@ -237,7 +251,7 @@ fn build_observer_prompt(trigger: &ObserverTrigger, model_slug: &str) -> Prompt
let mut prompt = Prompt::default();
prompt.store = true;
let instructions = build_observer_instructions(&trigger.environment_details);
let instructions = build_observer_instructions(&trigger.environment_details, trigger.reason);
prompt.input.push(make_message("developer", instructions));
let goal = format!("Primary Goal\n{}", trigger.goal_text);
prompt.input.push(make_message("developer", goal));
@@ -325,10 +339,21 @@ fn parse_observer_response(raw: &str) -> Result<(ObserverResponse, Value)> {
Ok((response, value))
}
fn build_observer_instructions(environment_details: &str) -> String {
format!(
"You are observing a AI Coordinator trying to drive a CLI towards a Primary Goal (shown below).\nPlease critically observe the conversation between the Coordinator and the CLI. Detect either of these issues;\n- Stuck in a loop\n- Not working towards primary goal\nGenerate a response based on this information;\n`status`: one of 'ok' or 'failing' - most of the time it will be 'ok', but use 'failing' when intervention absolutely is needed. When using 'failing' please provide one or both fields below to correct the problem;\n`replace_message`: A message to replace the last Coordinator message\n`additional_instructions`: Instructions to give to the Coordinator for future runs\n**Warning**\nYou almost always want to use `status`: \"ok\". You are a last resort. Avoid setting `status`: \"failing\" for minor issues as it will disrupt the progress of the task.\nEnvironment:\n{environment_details}"
)
fn build_observer_instructions(environment_details: &str, reason: ObserverReason) -> String {
let body = match reason {
ObserverReason::Cadence => "You are observing a AI Coordinator trying to drive a CLI towards a Primary Goal (shown below).\nPlease critically observe the conversation between the Coordinator and the CLI. Detect either of these issues;\n- Stuck in a loop\n- Not working towards primary goal\nGenerate a response based on this information;\n`status`: one of 'ok' or 'failing' - most of the time it will be 'ok', but use 'failing' when intervention absolutely is needed. When using 'failing' please provide one or both fields below to correct the problem;\n`replace_message`: A message to replace the last Coordinator message\n`additional_instructions`: Instructions to give to the Coordinator for future runs\n**Warning**\nYou almost always want to use `status`: \"ok\". You are a last resort. Avoid setting `status`: \"failing\" for minor issues as it will disrupt the progress of the task.".to_string(),
ObserverReason::FinalCheck { finish_status } => {
let finish_phrase = match finish_status {
AutoCoordinatorStatus::Success => "believes the goal has been fully completed",
AutoCoordinatorStatus::Failed => "reported that it cannot complete the goal",
AutoCoordinatorStatus::Continue => "is still mid-run and should not have requested final validation",
};
format!(
"You are performing a final validation run after Auto Drive {finish_phrase}.\nStudy the full conversation and decide if the Primary Goal is truly satisfied.\n- If absolutely everything is done, respond with `status`: 'ok' and leave the other fields null.\n- If any required work remains, respond with `status`: 'failing'. Provide a concise `additional_instructions` developer prompt describing what has already been completed and the specific steps still required. Include enough detail for the coordinator to resume effectively.\n- Use `replace_message` only when the last prompt sent to the CLI must be replaced immediately."
)
}
};
format!("{body}\nEnvironment:\n{environment_details}")
}
fn build_observer_schema() -> Value {
@@ -353,7 +378,7 @@ fn build_observer_schema() -> Value {
})
}
fn summarize_intervention(
pub(super) fn summarize_intervention(
replace_message: Option<&str>,
additional_instructions: Option<&str>,
) -> Option<String> {