hanzoai/dev
2
0
Fork 0
mirror of https://github.com/hanzoai/dev.git synced 2026-10-25 03:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(auto-drive): keep retrying after errors

Browse Source
This commit is contained in:
James Peter
2025-12-30 14:21:10 +10:00
parent 40eff5cca6
commit 7f6c12e8e3
4 changed files with 30 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code-rs/code-auto-drive-core/src/controller.rs
+12 -42
View File
@@ -257,7 +257,6 @@ pub enum AutoResolvePhase {
AwaitingJudge { review: ReviewOutputEvent },
}
pub const AUTO_RESTART_MAX_ATTEMPTS: u32 = 6;
pub const AUTO_RESTART_BASE_DELAY: Duration = Duration::from_secs(5);
pub const AUTO_RESTART_MAX_DELAY: Duration = Duration::from_secs(120);
pub const AUTO_RESOLVE_MAX_REVIEW_ATTEMPTS: u32 = 3;
@@ -600,7 +599,7 @@ impl AutoDriveController {
pub fn pause_for_transient_failure(
&mut self,
now: Instant,
_now: Instant,
reason: String,
) -> Vec<AutoControllerEffect> {
let pending_attempt = self.transient_restart_attempts.saturating_add(1);
@@ -614,40 +613,6 @@ impl AutoDriveController {
let delay = Self::auto_restart_delay(pending_attempt);
self.apply_phase(AutoRunPhase::TransientRecovery { backoff_ms: delay.as_millis() as u64 });
if pending_attempt > AUTO_RESTART_MAX_ATTEMPTS {
self.transient_restart_attempts = pending_attempt;
let summary = AutoRunSummary {
duration: self
.started_at
.map(|start| now.saturating_duration_since(start))
.unwrap_or_default(),
turns_completed: self.turns_completed,
message: Some(format!(
"Auto Drive stopped after {AUTO_RESTART_MAX_ATTEMPTS} reconnect attempts."
)),
goal: self.goal.clone(),
};
self.reset();
self.last_run_summary = Some(summary.clone());
self.apply_phase(AutoRunPhase::AwaitingGoalEntry);
return vec![
AutoControllerEffect::CancelCoordinator,
AutoControllerEffect::ResetHistory,
AutoControllerEffect::ClearCoordinatorView,
AutoControllerEffect::UpdateTerminalHint { hint: None },
AutoControllerEffect::SetTaskRunning { running: false },
AutoControllerEffect::EnsureInputFocus,
AutoControllerEffect::StopCompleted {
summary,
message: Some(format!(
"Auto Drive stopped after {AUTO_RESTART_MAX_ATTEMPTS} reconnect attempts. Last error: {truncated_reason}"
)),
},
AutoControllerEffect::RefreshUi,
];
}
self.transient_restart_attempts = pending_attempt;
let delay = Self::auto_restart_delay(pending_attempt);
let token = self.restart_token.wrapping_add(1);
@@ -660,7 +625,7 @@ impl AutoDriveController {
let human_delay = format_duration(delay);
self.current_display_line = Some(format!(
"Waiting for connection… retrying in {human_delay} (attempt {pending_attempt}/{AUTO_RESTART_MAX_ATTEMPTS})"
"Waiting for connection… retrying in {human_delay} (attempt {pending_attempt})"
));
self.current_display_is_summary = true;
self.current_status_title = Some(format!("Retrying after error"));
@@ -949,7 +914,7 @@ impl AutoDriveController {
#[cfg(test)]
mod tests {
use super::{AutoControllerEffect, AutoDriveController, AutoRunPhase, AUTO_RESTART_MAX_ATTEMPTS};
use super::{AutoControllerEffect, AutoDriveController, AutoRunPhase};
use std::time::Instant;
#[test]
@@ -996,9 +961,8 @@ mod tests {
}
#[test]
fn restart_exhaustion_emits_cancel_coordinator_effect() {
fn transient_failure_enters_recovery_and_schedules_restart() {
let mut controller = AutoDriveController::default();
controller.transient_restart_attempts = AUTO_RESTART_MAX_ATTEMPTS;
controller.goal = Some("Investigate outage".to_string());
controller.started_at = Some(Instant::now());
@@ -1010,8 +974,14 @@ mod tests {
assert!(effects
.iter()
.any(|effect| matches!(effect, AutoControllerEffect::CancelCoordinator)));
assert!(matches!(controller.current_phase(), AutoRunPhase::AwaitingGoalEntry));
assert!(controller.last_run_summary.is_some());
assert!(effects
.iter()
.any(|effect| matches!(effect, AutoControllerEffect::ScheduleRestart { .. })));
assert!(matches!(
controller.current_phase(),
AutoRunPhase::TransientRecovery { .. }
));
assert!(controller.last_run_summary.is_none());
}
#[test]
code-rs/code-auto-drive-core/src/lib.rs
-1
View File
@@ -40,7 +40,6 @@ pub use controller::{
PhaseTransition,
TransitionEffects,
AUTO_RESTART_BASE_DELAY,
AUTO_RESTART_MAX_ATTEMPTS,
AUTO_RESTART_MAX_DELAY,
AUTO_RESOLVE_MAX_REVIEW_ATTEMPTS,
AUTO_RESOLVE_REVIEW_FOLLOWUP,
code-rs/core/src/codex.rs
+4 -1
View File
@@ -6007,9 +6007,12 @@ async fn run_agent(sess: Arc<Session>, turn_context: Arc<TurnContext>, sub_id: S
// repeated compaction attempts within a single iteration.
if token_limit_reached && !did_proactive_compact_this_iteration {
did_proactive_compact_this_iteration = true;
let attempt_req = sess.current_request_ordinal();
let order = sess.next_background_order(&sub_id, attempt_req, None);
sess
.notify_stream_error(
.notify_background_event_with_order(
&sub_id,
order,
"Token limit reached; running /compact and continuing…".to_string(),
)
.await;
code-rs/tui/src/chatwidget.rs
+14 -15
View File
@@ -198,7 +198,6 @@ use code_auto_drive_core::{
AutoTurnReviewState,
AutoResolveState,
AutoResolvePhase,
AUTO_RESTART_MAX_ATTEMPTS,
AUTO_RESOLVE_REVIEW_FOLLOWUP,
CoordinatorContext,
CoordinatorRouterResponse,
@@ -5799,7 +5798,7 @@ impl ChatWidget<'_> {
self.request_redraw();
}
// Fatal error path: show an error cell and clear running state.
// Error path: show an error cell and clear running state.
self.clear_resume_placeholder();
let key = self.next_internal_key();
let state = history_cell::new_error_event(message.clone());
@@ -5810,18 +5809,18 @@ impl ChatWidget<'_> {
"epilogue",
Some(HistoryDomainRecord::Plain(state)),
);
let should_stop_auto = self.auto_state.is_active();
let should_recover_auto = self.auto_state.is_active();
self.bottom_pane.set_task_running(false);
// Ensure any running exec/tool cells are finalized so spinners don't linger
// after fatal errors.
// after errors.
self.finalize_all_running_as_interrupted();
self.stream.clear_all();
self.stream_state.drop_streaming = false;
self.agents_ready_to_start = false;
self.active_task_ids.clear();
self.maybe_hide_spinner();
if should_stop_auto {
self.auto_stop(Some("Auto Drive stopped after error.".to_string()));
if should_recover_auto {
self.auto_pause_for_transient_failure(message);
}
self.mark_needs_redraw();
}
@@ -18028,7 +18027,7 @@ Have we met every part of this goal and is there no further work to do?"#
"Press Esc again to exit Auto Drive".to_string(),
));
let message = format!(
"Auto Drive will retry automatically in {human_delay} (attempt {attempt}/{AUTO_RESTART_MAX_ATTEMPTS}). Last error: {reason}"
"Auto Drive will retry automatically in {human_delay} (attempt {attempt}). Last error: {reason}"
);
self.auto_card_add_action(message, AutoDriveActionKind::Warning);
self.auto_card_set_status(AutoDriveStatus::Paused);
@@ -18160,12 +18159,10 @@ Have we met every part of this goal and is there no further work to do?"#
self.auto_state.restart_token = token;
let resume_message = if restart.reason.is_empty() {
format!(
"Auto Drive resuming automatically (attempt {attempt}/{AUTO_RESTART_MAX_ATTEMPTS})."
)
format!("Auto Drive resuming automatically (attempt {attempt}).")
} else {
format!(
"Auto Drive resuming automatically (attempt {attempt}/{AUTO_RESTART_MAX_ATTEMPTS}); previous error: {}",
"Auto Drive resuming automatically (attempt {attempt}); previous error: {}",
restart.reason
)
};
@@ -29974,7 +29971,7 @@ use code_core::protocol::OrderMeta;
}
#[test]
fn auto_drive_error_clears_waiting_state() {
fn auto_drive_error_enters_transient_recovery() {
let mut harness = ChatWidgetHarness::new();
harness.with_chat(|chat| {
chat.auto_state.set_phase(AutoRunPhase::Active);
@@ -29992,10 +29989,12 @@ use code_core::protocol::OrderMeta;
order: None,
});
let still_active = harness.with_chat(|chat| chat.auto_state.is_active());
let (still_active, in_recovery) = harness.with_chat(|chat| {
(chat.auto_state.is_active(), chat.auto_state.in_transient_recovery())
});
assert!(
!still_active,
"auto drive should stop after a fatal error event"
still_active && in_recovery,
"auto drive should pause for recovery after an error event"
);
}