feat(auto-drive): add observer thread and telemetry

This commit is contained in:
James Peter
2025-09-29 12:09:06 +10:00
parent 22b3c09231
commit 67bf7fb260
8 changed files with 713 additions and 16 deletions
+10
View File
@@ -169,6 +169,9 @@ pub struct Config {
/// and turn completions when not focused.
pub tui_notifications: Notifications,
/// Cadence (in requests) for running the Auto Drive observer thread.
pub auto_drive_observer_cadence: u32,
/// The directory that should be treated as the current working directory
/// for the session. All relative paths inside the business-logic layer are
/// resolved against this path.
@@ -1579,6 +1582,9 @@ pub struct ConfigToml {
/// Collection of settings that are specific to the TUI.
pub tui: Option<Tui>,
#[serde(default)]
pub auto_drive_observer_cadence: Option<u32>,
/// Browser configuration for integrated screenshot capabilities.
pub browser: Option<BrowserConfig>,
@@ -2159,6 +2165,7 @@ impl Config {
.as_ref()
.map(|t| t.notifications.clone())
.unwrap_or_default(),
auto_drive_observer_cadence: cfg.auto_drive_observer_cadence.unwrap_or(5),
};
Ok(config)
}
@@ -2919,6 +2926,7 @@ model_verbosity = "high"
validation: ValidationConfig::default(),
experimental_resume: None,
tui_notifications: Default::default(),
auto_drive_observer_cadence: 5,
};
assert_eq!(expected_gpt3_profile_config, gpt3_profile_config);
@@ -3004,6 +3012,7 @@ model_verbosity = "high"
github: GithubConfig::default(),
experimental_resume: None,
tui_notifications: Default::default(),
auto_drive_observer_cadence: 5,
};
assert_eq!(expected_zdr_profile_config, zdr_profile_config);
@@ -3073,6 +3082,7 @@ model_verbosity = "high"
github: GithubConfig::default(),
experimental_resume: None,
tui_notifications: Default::default(),
auto_drive_observer_cadence: 5,
};
assert_eq!(expected_gpt5_profile_config, gpt5_profile_config);
+15
View File
@@ -1629,6 +1629,21 @@ impl App<'_> {
widget.auto_handle_countdown(countdown_id, seconds_left);
}
}
AppEvent::AutoObserverReport {
status,
telemetry,
replace_message,
additional_instructions,
} => {
if let AppState::Chat { widget } = &mut self.app_state {
widget.auto_handle_observer_report(
status,
telemetry,
replace_message,
additional_instructions,
);
}
}
AppEvent::ShowUndoOptions { index } => {
if let AppState::Chat { widget } = &mut self.app_state {
widget.show_undo_restore_options(index);
+25
View File
@@ -80,6 +80,25 @@ pub(crate) enum AutoCoordinatorStatus {
Failed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AutoObserverStatus {
Ok,
Failing,
}
impl Default for AutoObserverStatus {
fn default() -> Self {
Self::Ok
}
}
#[derive(Debug, Clone, Default)]
pub(crate) struct AutoObserverTelemetry {
pub trigger_count: u64,
pub last_status: AutoObserverStatus,
pub last_intervention: Option<String>,
}
#[allow(clippy::large_enum_variant)]
#[derive(Debug)]
pub(crate) enum AppEvent {
@@ -140,6 +159,12 @@ pub(crate) enum AppEvent {
countdown_id: u64,
seconds_left: u8,
},
AutoObserverReport {
status: AutoObserverStatus,
telemetry: AutoObserverTelemetry,
replace_message: Option<String>,
additional_instructions: Option<String>,
},
/// Dispatch a recognized slash command from the UI (composer) to the app
/// layer so it can be handled centrally. Includes the full command text.
+78 -1
View File
@@ -47,6 +47,7 @@ use codex_protocol::num_format::format_with_separators;
mod auto_coordinator;
mod auto_observer;
mod diff_handlers;
mod agent_install;
mod diff_ui;
@@ -189,6 +190,8 @@ use crate::account_label::{account_display_label, account_mode_priority};
use crate::app_event::{
AppEvent,
AutoCoordinatorStatus,
AutoObserverStatus,
AutoObserverTelemetry,
BackgroundPlacement,
TerminalAfter,
TerminalCommandGate,
@@ -460,6 +463,8 @@ struct AutoCoordinatorUiState {
awaiting_goal_input: bool,
last_broadcast_thought: Option<String>,
last_decision_display: Option<String>,
observer_telemetry: Option<AutoObserverTelemetry>,
observer_status: AutoObserverStatus,
}
impl AutoCoordinatorUiState {
@@ -9608,6 +9613,7 @@ impl ChatWidget<'_> {
conversation,
self.config.clone(),
self.config.debug,
self.config.auto_drive_observer_cadence,
) {
Ok(handle) => {
self.auto_tx = Some(handle.tx);
@@ -9760,6 +9766,56 @@ impl ChatWidget<'_> {
}
}
pub(crate) fn auto_handle_observer_report(
&mut self,
status: AutoObserverStatus,
telemetry: AutoObserverTelemetry,
replace_message: Option<String>,
additional_instructions: Option<String>,
) {
if !self.auto_state.active {
return;
}
self.auto_state.observer_status = status;
self.auto_state.observer_telemetry = Some(telemetry);
if matches!(status, AutoObserverStatus::Failing) {
let guidance = additional_instructions
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty());
let banner = guidance
.map(|text| format!("Observer guidance: {text}"))
.unwrap_or_else(|| "Observer flagged the last Auto Drive step.".to_string());
self.push_background_tail(banner);
if let Some(message) = replace_message
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
self.auto_state.current_prompt = Some(message.to_string());
if self.auto_state.awaiting_submission {
self.auto_state.countdown_id = self.auto_state.countdown_id.wrapping_add(1);
self.auto_state.seconds_remaining = AUTO_COUNTDOWN_SECONDS;
self.auto_start_countdown(self.auto_state.countdown_id);
}
let summary = Self::truncate_with_ellipsis(message, 160);
self.push_background_tail(format!("Observer replaced prompt with: {summary}"));
}
} else if let Some(note) = additional_instructions
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
self.push_background_tail(format!("Observer note: {note}"));
}
self.auto_rebuild_live_ring();
self.request_redraw();
}
pub(crate) fn auto_handle_thinking(&mut self, delta: String, summary_index: Option<u32>) {
if !self.auto_state.active {
return;
@@ -9904,7 +9960,28 @@ impl ChatWidget<'_> {
.clone()
};
let status_lines: Vec<String> = vec![append_thought_ellipsis(&status_text)];
let mut status_lines = vec![append_thought_ellipsis(&status_text)];
if let Some(telemetry) = self.auto_state.observer_telemetry.as_ref() {
let status_label = match self.auto_state.observer_status {
AutoObserverStatus::Ok => "ok",
AutoObserverStatus::Failing => "failing",
};
let mut telemetry_line = format!(
"Observer: {status_label} (triggers: {})",
telemetry.trigger_count
);
if let Some(intervention) = telemetry
.last_intervention
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
let display = Self::truncate_with_ellipsis(intervention, 80);
telemetry_line.push_str(" — ");
telemetry_line.push_str(&display);
}
status_lines.push(telemetry_line);
}
let prompt = self
.auto_state
+150 -13
View File
@@ -6,7 +6,7 @@ use anyhow::{anyhow, Context, Result};
use codex_core::config::Config;
use codex_core::config_types::ReasoningEffort;
use codex_core::debug_logger::DebugLogger;
use codex_core::model_family::{find_family_for_model, derive_default_model_family};
use codex_core::model_family::{derive_default_model_family, find_family_for_model};
use codex_core::protocol::SandboxPolicy;
use codex_core::{AuthManager, ModelClient, Prompt, ResponseEvent, TextFormat};
use codex_protocol::models::{ContentItem, ResponseItem};
@@ -19,7 +19,15 @@ use uuid::Uuid;
use crate::app_event::{AppEvent, AutoCoordinatorStatus};
use crate::app_event_sender::AppEventSender;
const MODEL_SLUG: &str = "gpt-5";
use super::auto_observer::{
build_observer_conversation,
start_auto_observer,
AutoObserverCommand,
ObserverOutcome,
ObserverTrigger,
};
pub(super) const MODEL_SLUG: &str = "gpt-5";
const SCHEMA_NAME: &str = "auto_coordinator_flow";
#[derive(Debug, Clone)]
@@ -30,6 +38,7 @@ pub(super) struct AutoCoordinatorHandle {
#[derive(Debug)]
pub(super) enum AutoCoordinatorCommand {
UpdateConversation(Vec<ResponseItem>),
ObserverResult(ObserverOutcome),
Stop,
}
@@ -47,9 +56,11 @@ pub(super) fn start_auto_coordinator(
conversation: Vec<ResponseItem>,
config: Config,
debug_enabled: bool,
observer_cadence: u32,
) -> Result<AutoCoordinatorHandle> {
let (cmd_tx, cmd_rx) = mpsc::channel();
let thread_tx = cmd_tx.clone();
let loop_tx = cmd_tx.clone();
std::thread::spawn(move || {
if let Err(err) = run_auto_loop(
@@ -58,7 +69,9 @@ pub(super) fn start_auto_coordinator(
conversation,
config,
cmd_rx,
loop_tx,
debug_enabled,
observer_cadence,
) {
tracing::error!("auto coordinator loop error: {err:#}");
}
@@ -73,7 +86,9 @@ fn run_auto_loop(
initial_conversation: Vec<ResponseItem>,
config: Config,
cmd_rx: Receiver<AutoCoordinatorCommand>,
cmd_tx: Sender<AutoCoordinatorCommand>,
debug_enabled: bool,
observer_cadence: u32,
) -> Result<()> {
let preferred_auth = if config.using_chatgpt_auth {
codex_protocol::mcp_protocol::AuthMode::ChatGPT
@@ -106,10 +121,14 @@ fn run_auto_loop(
)),
);
let (developer_intro, primary_goal_message) = build_developer_message(
&goal_text,
matches!(sandbox_policy, SandboxPolicy::DangerFullAccess),
);
let sandbox_label = if matches!(sandbox_policy, SandboxPolicy::DangerFullAccess) {
"full access"
} else {
"limited sandbox"
};
let environment_details = format_environment_details(sandbox_label);
let (base_developer_intro, primary_goal_message) =
build_developer_message(&goal_text, &environment_details);
let schema = build_schema();
let platform = std::env::consts::OS;
debug!("[Auto coordinator] starting: goal={goal_text} platform={platform}");
@@ -121,6 +140,22 @@ fn run_auto_loop(
let mut pending_conversation = Some(initial_conversation);
let mut stopped = false;
let mut requests_completed: u64 = 0;
let mut observer_guidance: Vec<String> = Vec::new();
let mut observer_handle = if observer_cadence == 0 {
None
} else {
match start_auto_observer(client.clone(), observer_cadence, cmd_tx.clone()) {
Ok(handle) => Some(handle),
Err(err) => {
tracing::error!("failed to start auto observer: {err:#}");
None
}
}
};
let observer_cadence = observer_handle
.as_ref()
.map(|handle| handle.cadence() as u64);
loop {
if stopped {
@@ -128,23 +163,46 @@ fn run_auto_loop(
}
if let Some(conv) = pending_conversation.take() {
let conv_for_observer = conv.clone();
let developer_intro = compose_developer_intro(&base_developer_intro, &observer_guidance);
match request_coordinator_decision(
&runtime,
&client,
&developer_intro,
developer_intro.as_str(),
&primary_goal_message,
&schema,
conv,
&app_event_tx,
) {
Ok((status, thoughts, prompt_opt)) => {
let prompt_clone = prompt_opt.clone();
let event = AppEvent::AutoCoordinatorDecision {
status,
thoughts,
prompt: prompt_opt,
};
app_event_tx.send(event);
if !matches!(status, AutoCoordinatorStatus::Continue) {
if matches!(status, AutoCoordinatorStatus::Continue) {
if let (Some(handle), Some(cadence)) =
(observer_handle.as_ref(), observer_cadence)
{
if should_trigger_observer(requests_completed, cadence) {
let conversation = build_observer_conversation(
conv_for_observer,
prompt_clone.as_deref(),
);
let trigger = ObserverTrigger {
conversation,
goal_text: goal_text.clone(),
environment_details: environment_details.clone(),
};
if handle.tx.send(AutoObserverCommand::Trigger(trigger)).is_err() {
tracing::warn!("failed to trigger auto observer");
}
}
}
} else {
stopped = true;
continue;
}
@@ -164,20 +222,73 @@ fn run_auto_loop(
match cmd_rx.recv() {
Ok(AutoCoordinatorCommand::UpdateConversation(conv)) => {
requests_completed = requests_completed.saturating_add(1);
pending_conversation = Some(conv);
}
Ok(AutoCoordinatorCommand::ObserverResult(outcome)) => {
let ObserverOutcome {
status,
replace_message,
additional_instructions,
telemetry,
} = outcome;
if let Some(instr) = additional_instructions
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
if !observer_guidance.iter().any(|existing| existing == instr) {
observer_guidance.push(instr.to_string());
}
}
let event = AppEvent::AutoObserverReport {
status,
telemetry,
replace_message,
additional_instructions,
};
app_event_tx.send(event);
}
Ok(AutoCoordinatorCommand::Stop) | Err(_) => {
stopped = true;
}
}
}
if let Some(handle) = observer_handle.take() {
let _ = handle.tx.send(AutoObserverCommand::Stop);
}
Ok(())
}
fn build_developer_message(goal_text: &str, full_access: bool) -> (String, String) {
let sandbox = if full_access { "full access" } else { "limited sandbox" };
let environment_details = format_environment_details(sandbox);
fn compose_developer_intro(base: &str, guidance: &[String]) -> String {
if guidance.is_empty() {
return base.to_string();
}
let mut intro = String::with_capacity(base.len() + guidance.len() * 64);
intro.push_str(base);
intro.push_str("\n\n**Observer Guidance**\n");
for hint in guidance {
let trimmed = hint.trim();
if trimmed.is_empty() {
continue;
}
intro.push_str("- ");
intro.push_str(trimmed);
intro.push('\n');
}
intro
}
fn should_trigger_observer(requests_completed: u64, cadence: u64) -> bool {
cadence != 0 && requests_completed > 0 && requests_completed % cadence == 0
}
fn build_developer_message(goal_text: &str, environment_details: &str) -> (String, String) {
let intro = format!(
"You are coordinating prompts sent to a running Code CLI process. You should act like a human maintainer of the project would act. You will see a **Primary Goal** below - this is what you are always working towards.\n\n**Rules**\n- `finish_status`: one of `continue`, `finish_success`, or `finish_failed`.\n * Use `continue` when another prompt is reasonable. Always prefer this option.\n * Use `finish_success` when the goal has been completed in it's entirety and absolutely no work remains.\n * Use `finish_failed` when the goal absolutely can not be satisfied or you are stuck in a loop. This should almost never be used. Try other approaches and gather more information if there is no clear path forward.\n- `thoughts`: short status (<= 160 characters) describing you thought process around what the next prompt will do\n- `prompt`: the exact prompt to provide to the Code CLI process. You will receive the response the CLI provides.\n- First plan, then execute. Allow the CLI to plan for you. You should get it to do the thinking for you.\n- Keep the prompt minimal to give the CLI room to make independent decision.\n- Don't repeat yourself. You will see past prompts and outputs showing current progress. Always push the project forward.\n- Often a simple 'Please continue' or 'Work on feature A next' or 'What do you think is the best approach?' is sufficient. Your job is to keep things running in an appropriate direction. The CLI does all the actual work and thinking. You do not need to know much about the project or codebase, allow the CLI to do all this for you. You are focused on overall direction not implementation details.\n- Only stop when no other options remain. A human is observing your work and will step in if they want to go in a different direction. You should not ask them for assistance - you should use your judgement to move on the most likely path forward. The human may override your message send to the CLI if they choose to go in another direction. This allows you to just guess the best path, knowing an overseer will step in if needed.\n\nUseful commands:\n`/review <what to review>` e.g. `/review latest commit` - this spins up a specialist review thread for the CLI which excels at identify issues. This is useful for repeatedly reviewing code changes you make and fixing them.\n`/reasoning <high|medium|low>` e.g. set `/reasoning high` if the CLI makes a poor decision or `/reasoning low` to move faster on simple tasks\n\nEnvironment:\\n{environment_details}"
);
@@ -357,7 +468,7 @@ fn parse_decision(raw: &str) -> Result<(CoordinatorDecision, Value)> {
Ok((decision, value))
}
fn extract_first_json_object(input: &str) -> Option<String> {
pub(super) fn extract_first_json_object(input: &str) -> Option<String> {
let mut depth = 0usize;
let mut in_str = false;
let mut escape = false;
@@ -399,7 +510,7 @@ fn extract_first_json_object(input: &str) -> Option<String> {
None
}
fn make_message(role: &str, text: String) -> ResponseItem {
pub(super) fn make_message(role: &str, text: String) -> ResponseItem {
let content = if role.eq_ignore_ascii_case("assistant") {
ContentItem::OutputText { text }
} else {
@@ -427,3 +538,29 @@ fn strip_role_prefix(input: &str) -> &str {
}
input
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn observer_cadence_triggers_every_n_requests() {
assert!(!should_trigger_observer(0, 5));
assert!(!should_trigger_observer(1, 5));
assert!(should_trigger_observer(5, 5));
assert!(should_trigger_observer(10, 5));
assert!(!should_trigger_observer(11, 5));
assert!(!should_trigger_observer(3, 0));
}
#[test]
fn compose_intro_appends_guidance_section() {
let base = "Intro";
let guidance = vec!["First".to_string(), "Second".to_string()];
let combined = compose_developer_intro(base, &guidance);
assert!(combined.contains("Intro"));
assert!(combined.contains("**Observer Guidance**"));
assert!(combined.contains("- First"));
assert!(combined.contains("- Second"));
}
}
@@ -0,0 +1,375 @@
use std::sync::mpsc::{self, Receiver, Sender};
use std::sync::Arc;
use anyhow::{anyhow, Context, Result};
use codex_core::{ModelClient, Prompt, ResponseEvent, TextFormat};
use codex_core::model_family::{derive_default_model_family, find_family_for_model};
use codex_protocol::models::{ContentItem, ResponseItem};
use futures::StreamExt;
use serde::Deserialize;
use serde_json::{self, json, Value};
use tracing::{debug, error, warn};
use crate::app_event::{AutoObserverStatus, AutoObserverTelemetry};
use super::auto_coordinator::{
extract_first_json_object,
make_message,
AutoCoordinatorCommand,
MODEL_SLUG,
};
#[derive(Debug)]
pub(super) struct AutoObserverHandle {
pub tx: Sender<AutoObserverCommand>,
cadence: u32,
}
impl AutoObserverHandle {
pub fn cadence(&self) -> u32 {
self.cadence.max(1)
}
}
#[derive(Debug)]
pub(super) enum AutoObserverCommand {
Trigger(ObserverTrigger),
Stop,
}
#[derive(Debug, Clone)]
pub(super) struct ObserverTrigger {
pub conversation: Vec<ResponseItem>,
pub goal_text: String,
pub environment_details: String,
}
#[derive(Debug, Clone)]
pub(super) struct ObserverOutcome {
pub status: AutoObserverStatus,
pub replace_message: Option<String>,
pub additional_instructions: Option<String>,
pub telemetry: AutoObserverTelemetry,
}
const OBSERVER_SCHEMA_NAME: &str = "auto_coordinator_observer";
pub(super) fn start_auto_observer(
client: ModelClient,
cadence: u32,
coordinator_tx: Sender<AutoCoordinatorCommand>,
) -> Result<AutoObserverHandle> {
let (tx, rx) = mpsc::channel();
let thread_tx = tx.clone();
std::thread::spawn(move || {
if let Err(err) = run_observer_loop(client, rx, coordinator_tx) {
error!("auto observer loop error: {err:#}");
}
});
Ok(AutoObserverHandle {
tx: thread_tx,
cadence,
})
}
fn run_observer_loop(
client: ModelClient,
rx: Receiver<AutoObserverCommand>,
coordinator_tx: Sender<AutoCoordinatorCommand>,
) -> Result<()> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("creating runtime for auto observer")?;
let client = Arc::new(client);
let mut telemetry = AutoObserverTelemetry {
trigger_count: 0,
last_status: AutoObserverStatus::Ok,
last_intervention: None,
};
while let Ok(cmd) = rx.recv() {
match cmd {
AutoObserverCommand::Trigger(trigger) => {
telemetry.trigger_count += 1;
match evaluate_observer(&runtime, client.clone(), trigger) {
Ok((status, replace_message, additional_instructions)) => {
telemetry.last_status = status;
telemetry.last_intervention = summarize_intervention(
replace_message.as_deref(),
additional_instructions.as_deref(),
);
let outcome = ObserverOutcome {
status,
replace_message,
additional_instructions,
telemetry: telemetry.clone(),
};
if coordinator_tx
.send(AutoCoordinatorCommand::ObserverResult(outcome))
.is_err()
{
break;
}
}
Err(err) => {
warn!("auto observer evaluation error: {err:#}");
telemetry.last_status = AutoObserverStatus::Ok;
telemetry.last_intervention = Some(format!("error: {err}"));
let outcome = ObserverOutcome {
status: AutoObserverStatus::Ok,
replace_message: None,
additional_instructions: None,
telemetry: telemetry.clone(),
};
if coordinator_tx
.send(AutoCoordinatorCommand::ObserverResult(outcome))
.is_err()
{
break;
}
}
}
}
AutoObserverCommand::Stop => break,
}
}
Ok(())
}
fn evaluate_observer(
runtime: &tokio::runtime::Runtime,
client: Arc<ModelClient>,
trigger: ObserverTrigger,
) -> Result<(AutoObserverStatus, Option<String>, Option<String>)> {
let mut prompt = Prompt::default();
prompt.store = true;
let instructions = build_observer_instructions(&trigger.environment_details);
prompt.input.push(make_message("developer", instructions));
let goal = format!("Primary Goal\n{}", trigger.goal_text);
prompt.input.push(make_message("developer", goal));
prompt.input.extend(trigger.conversation);
let schema = build_observer_schema();
prompt.text_format = Some(TextFormat {
r#type: "json_schema".to_string(),
name: Some(OBSERVER_SCHEMA_NAME.to_string()),
strict: Some(true),
schema: Some(schema),
});
prompt.model_override = Some(MODEL_SLUG.to_string());
let family = find_family_for_model(MODEL_SLUG)
.unwrap_or_else(|| derive_default_model_family(MODEL_SLUG));
prompt.model_family_override = Some(family);
let raw = runtime.block_on(async {
request_observer_response(client.clone(), &prompt).await
})?;
let (response, _value) = parse_observer_response(&raw)?;
let status = match response.status.as_str() {
"ok" => AutoObserverStatus::Ok,
"failing" => AutoObserverStatus::Failing,
other => return Err(anyhow!("unexpected status '{other}'")),
};
let replace_message = response
.replace_message
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty());
let additional_instructions = response
.additional_instructions
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty());
if matches!(status, AutoObserverStatus::Failing)
&& replace_message.is_none()
&& additional_instructions.is_none()
{
warn!("observer returned failing status without guidance");
}
debug!(
"[Auto observer] status={status:?} replace={} instructions={}",
replace_message.is_some(),
additional_instructions.is_some()
);
Ok((status, replace_message, additional_instructions))
}
async fn request_observer_response(
client: Arc<ModelClient>,
prompt: &Prompt,
) -> Result<String> {
let mut stream = client.stream(prompt).await?;
let mut out = String::new();
while let Some(ev) = stream.next().await {
match ev {
Ok(ResponseEvent::OutputTextDelta { delta, .. }) => out.push_str(&delta),
Ok(ResponseEvent::OutputItemDone { item, .. }) => {
if let ResponseItem::Message { content, .. } = item {
for c in content {
if let ContentItem::OutputText { text } = c {
out.push_str(&text);
}
}
}
}
Ok(ResponseEvent::Completed { .. }) => break,
Err(err) => return Err(anyhow!("observer stream error: {err}")),
_ => {}
}
}
Ok(out)
}
#[derive(Debug, Deserialize)]
struct ObserverResponse {
status: String,
#[serde(default)]
replace_message: Option<String>,
#[serde(default)]
additional_instructions: Option<String>,
}
fn parse_observer_response(raw: &str) -> Result<(ObserverResponse, Value)> {
let value: Value = match serde_json::from_str(raw) {
Ok(v) => v,
Err(_) => {
let Some(blob) = extract_first_json_object(raw) else {
return Err(anyhow!("observer response was not valid JSON"));
};
serde_json::from_str(&blob).context("parsing JSON from observer output")?
}
};
let response: ObserverResponse = serde_json::from_value(value.clone())
.context("decoding observer response")?;
Ok((response, value))
}
fn build_observer_instructions(environment_details: &str) -> String {
format!(
"You are observing a AI Coordinator trying to drive a CLI towards a Primary Goal (shown below).\nPlease critically observe the conversation between the Coordinator and the CLI. Detect either of these issues;\n- Stuck in a loop\n- Not working towards primary goal\nGenerate a response based on this information;\n`status`: one of 'ok' or 'failing' - most of the time it will be 'ok', but use 'failing' when intervention absolutely is needed. When using 'failing' please provide one or both fields below to correct the problem;\n`replace_message`: A message to replace the last Coordinator message\n`additional_instructions`: Instructions to give to the Coordinator for future runs\n**Warning**\nYou almost always want to use `status`: \"ok\". You are a last resort. Avoid setting `status`: \"failing\" for minor issues as it will disrupt the progress of the task.\nEnvironment:\n{environment_details}"
)
}
fn build_observer_schema() -> Value {
json!({
"type": "object",
"properties": {
"status": {
"type": "string",
"enum": ["ok", "failing"],
},
"replace_message": {
"type": ["string", "null"],
"minLength": 1,
},
"additional_instructions": {
"type": ["string", "null"],
"minLength": 1,
}
},
"required": ["status"],
"additionalProperties": false
})
}
fn summarize_intervention(
replace_message: Option<&str>,
additional_instructions: Option<&str>,
) -> Option<String> {
let source = replace_message.or(additional_instructions)?;
let trimmed = source.trim();
if trimmed.is_empty() {
return None;
}
const MAX_LEN: usize = 160;
if trimmed.len() > MAX_LEN {
let mut out = trimmed.chars().take(MAX_LEN).collect::<String>();
out.push('…');
Some(out)
} else {
Some(trimmed.to_string())
}
}
// Helper so observer can append the coordinator's latest prompt.
pub(super) fn build_observer_conversation(
mut conversation: Vec<ResponseItem>,
coordinator_prompt: Option<&str>,
) -> Vec<ResponseItem> {
if let Some(prompt) = coordinator_prompt.and_then(|p| {
let trimmed = p.trim();
(!trimmed.is_empty()).then_some(trimmed)
}) {
let content = ContentItem::OutputText {
text: format!("Coordinator: {prompt}"),
};
conversation.push(ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![content],
});
}
conversation
}
#[cfg(test)]
mod tests {
use super::*;
use codex_protocol::models::{ContentItem, ResponseItem};
#[test]
fn build_observer_conversation_appends_prompt() {
let base = vec![make_message("assistant", "Coordinator: hello".to_string())];
let conversation = build_observer_conversation(base.clone(), Some("Review the logs"));
assert_eq!(conversation.len(), base.len() + 1);
match conversation.last().expect("appended message") {
ResponseItem::Message { role, content, .. } => {
assert_eq!(role, "assistant");
assert_eq!(content.len(), 1);
match &content[0] {
ContentItem::OutputText { text } => {
assert!(text.contains("Coordinator: Review the logs"));
}
other => panic!("unexpected content: {:?}", other),
}
}
other => panic!("unexpected response item: {:?}", other),
}
}
#[test]
fn parse_observer_response_handles_ok() {
let raw = r#"{"status":"ok"}"#;
let (response, _value) = parse_observer_response(raw).expect("parse ok");
assert_eq!(response.status, "ok");
assert!(response.replace_message.is_none());
assert!(response.additional_instructions.is_none());
}
#[test]
fn parse_observer_response_handles_failing() {
let raw = r#"{"status":"failing","replace_message":"Try a different command","additional_instructions":"Confirm environment variables"}"#;
let (response, _value) = parse_observer_response(raw).expect("parse failing");
assert_eq!(response.status, "failing");
assert_eq!(
response.replace_message.as_deref(),
Some("Try a different command")
);
assert_eq!(
response.additional_instructions.as_deref(),
Some("Confirm environment variables")
);
}
}
+48 -1
View File
@@ -1,7 +1,12 @@
#![allow(clippy::unwrap_used, clippy::expect_used, unnameable_test_items)]
use super::*;
use crate::app_event::{AppEvent, BackgroundPlacement};
use crate::app_event::{
AppEvent,
AutoObserverStatus,
AutoObserverTelemetry,
BackgroundPlacement,
};
use crate::app_event_sender::AppEventSender;
use crate::history::state::{ExecStatus, ExploreEntryStatus, HistoryId, HistoryRecord, HistoryState};
use crate::slash_command::SlashCommand;
@@ -200,6 +205,48 @@ fn background_events_append_in_arrival_order() {
assert_eq!(texts, vec!["first background".to_string(), "second background".to_string()]);
}
#[test]
fn observer_report_replaces_prompt_and_resets_countdown() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual();
chat.auto_state.active = true;
chat.auto_state.awaiting_submission = true;
chat.auto_state.current_prompt = Some("old prompt".to_string());
chat.auto_state.countdown_id = 7;
chat.auto_state.seconds_remaining = 3;
let telemetry = AutoObserverTelemetry {
trigger_count: 5,
last_status: AutoObserverStatus::Failing,
last_intervention: Some("replaced".to_string()),
};
chat.auto_handle_observer_report(
AutoObserverStatus::Failing,
telemetry.clone(),
Some("new prompt".to_string()),
Some("double-check the failing command".to_string()),
);
assert_eq!(chat.auto_state.current_prompt.as_deref(), Some("new prompt"));
assert_eq!(chat.auto_state.observer_status, AutoObserverStatus::Failing);
assert_eq!(
chat
.auto_state
.observer_telemetry
.as_ref()
.unwrap()
.trigger_count,
telemetry.trigger_count
);
assert_eq!(chat.auto_state.countdown_id, 8);
assert_eq!(chat.auto_state.seconds_remaining, AUTO_COUNTDOWN_SECONDS);
let texts = cell_texts(&chat);
assert!(texts.iter().any(|line| line.contains("Observer guidance")));
assert!(texts.iter().any(|line| line.contains("Observer replaced prompt")));
}
#[test]
fn background_event_before_next_output_precedes_later_cells() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual();
+12 -1
View File
@@ -704,7 +704,18 @@ notifications = [ "agent-turn-complete", "approval-requested" ]
> Codex emits desktop notifications using terminal escape codes. Not all terminals support these (notably, macOS Terminal.app and VS Code's terminal do not support custom notifications. iTerm2, Ghostty and WezTerm do support these notifications).
> [!NOTE]
> `tui.notifications` is built‑in and limited to the TUI session. For programmatic or cross‑environment notifications—or to integrate with OS‑specific notifiers—use the top‑level `notify` option to run an external program that receives event JSON. The two settings are independent and can be used together.
> `tui.notifications` is built‑in and limited to the TUI session. For programmatic or cross‑environment notifications—or to integrate with OS‑specific notifiers—use the top-level `notify` option to run an external program that receives event JSON. The two settings are independent and can be used together.
### Auto Drive Observer
Codex keeps long-running Auto Drive sessions in check with a lightweight observer thread. Configure its cadence with the top-level `auto_drive_observer_cadence` key (default `5`). After every *n* completed requests the observer reviews the coordinator/CLI transcript, emits telemetry, and—if necessary—suggests a corrected prompt or follow-up guidance. Setting the value to `0` disables the observer entirely.
```toml
# Run the observer after every third Auto Drive request
auto_drive_observer_cadence = 3
```
When the observer reports `status = "failing"`, the TUI banner highlights the intervention, updates the pending prompt when provided, and records guidance for future coordinator turns.
## Project Hooks