mirror of
https://github.com/hanzoai/dev.git
synced 2026-10-25 03:32:32 +00:00
fix(core): auto-retry turns after usage limits
When the provider returns usage_limit_reached, keep the turn alive by waiting until the reset window (or 5 minutes when unknown) and retrying. Apply the same behavior to compaction tasks and show a generic retrying status in the TUI.
This commit is contained in:
+34
-22
@@ -5891,30 +5891,42 @@ async fn run_turn(
|
||||
}
|
||||
Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted),
|
||||
Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)),
|
||||
Err(e @ (CodexErr::UsageLimitReached(_)
|
||||
| CodexErr::UsageNotIncluded
|
||||
| CodexErr::QuotaExceeded)) => {
|
||||
if let CodexErr::UsageLimitReached(limit_err) = &e {
|
||||
if let Some(ctx) = account_usage_context(&sess) {
|
||||
let usage_home = ctx.code_home.clone();
|
||||
let usage_account = ctx.account_id.clone();
|
||||
let usage_plan = ctx.plan.clone();
|
||||
let resets = limit_err.resets_in_seconds;
|
||||
spawn_usage_task(move || {
|
||||
if let Err(err) = account_usage::record_usage_limit_hint(
|
||||
&usage_home,
|
||||
&usage_account,
|
||||
usage_plan.as_deref(),
|
||||
resets,
|
||||
Utc::now(),
|
||||
) {
|
||||
warn!("Failed to persist usage limit hint: {err}");
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(CodexErr::UsageLimitReached(limit_err)) => {
|
||||
if let Some(ctx) = account_usage_context(sess) {
|
||||
let usage_home = ctx.code_home.clone();
|
||||
let usage_account = ctx.account_id.clone();
|
||||
let usage_plan = ctx.plan.clone();
|
||||
let resets = limit_err.resets_in_seconds;
|
||||
spawn_usage_task(move || {
|
||||
if let Err(err) = account_usage::record_usage_limit_hint(
|
||||
&usage_home,
|
||||
&usage_account,
|
||||
usage_plan.as_deref(),
|
||||
resets,
|
||||
Utc::now(),
|
||||
) {
|
||||
warn!("Failed to persist usage limit hint: {err}");
|
||||
}
|
||||
});
|
||||
}
|
||||
return Err(e);
|
||||
|
||||
let now = Utc::now();
|
||||
let retry_after = limit_err
|
||||
.retry_after(now)
|
||||
.unwrap_or_else(|| RetryAfter::from_duration(std::time::Duration::from_secs(5 * 60), now));
|
||||
let eta = format_retry_eta(&retry_after);
|
||||
let mut retry_message = format!("{limit_err} Auto-retrying");
|
||||
if let Some(eta) = eta {
|
||||
retry_message.push_str(&format!(" at {eta}"));
|
||||
}
|
||||
retry_message.push('…');
|
||||
sess.notify_stream_error(&sub_id, retry_message).await;
|
||||
tokio::time::sleep(retry_after.delay).await;
|
||||
retries = 0;
|
||||
continue;
|
||||
}
|
||||
Err(CodexErr::UsageNotIncluded) => return Err(CodexErr::UsageNotIncluded),
|
||||
Err(CodexErr::QuotaExceeded) => return Err(CodexErr::QuotaExceeded),
|
||||
Err(e) => {
|
||||
// Detect context-window overflow and auto-run a compact summarization once
|
||||
if !did_auto_compact {
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::Prompt;
|
||||
use crate::client_common::ResponseEvent;
|
||||
use crate::environment_context::EnvironmentContext;
|
||||
use crate::error::CodexErr;
|
||||
use crate::error::RetryAfter;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::protocol::AgentMessageEvent;
|
||||
use crate::protocol::ErrorEvent;
|
||||
@@ -28,8 +29,10 @@ use code_protocol::protocol::CompactedItem;
|
||||
use code_protocol::protocol::InputMessageKind;
|
||||
use code_protocol::protocol::RolloutItem;
|
||||
use base64::Engine;
|
||||
use chrono::Utc;
|
||||
use futures::prelude::*;
|
||||
use code_app_server_protocol::AuthMode;
|
||||
use std::time::Duration;
|
||||
|
||||
pub const SUMMARIZATION_PROMPT: &str = include_str!("../../templates/compact/prompt.md");
|
||||
pub const COMPACTION_CHECKPOINT_MESSAGE: &str = "History checkpoint: earlier conversation compacted.";
|
||||
@@ -259,6 +262,18 @@ pub(super) async fn perform_compaction(
|
||||
break;
|
||||
}
|
||||
Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted),
|
||||
Err(CodexErr::UsageLimitReached(limit_err)) => {
|
||||
let now = Utc::now();
|
||||
let retry_after = limit_err
|
||||
.retry_after(now)
|
||||
.unwrap_or_else(|| RetryAfter::from_duration(Duration::from_secs(5 * 60), now));
|
||||
let mut message = format!("{limit_err} Auto-retrying");
|
||||
message.push('…');
|
||||
sess.notify_stream_error(&sub_id, message).await;
|
||||
tokio::time::sleep(retry_after.delay).await;
|
||||
retries = 0;
|
||||
continue;
|
||||
}
|
||||
Err(e) if is_context_overflow_error(&e) => {
|
||||
if turn_input.len() > 1 {
|
||||
tracing::warn!(
|
||||
@@ -408,6 +423,18 @@ async fn run_compact_task_inner_inline(
|
||||
break;
|
||||
}
|
||||
Err(CodexErr::Interrupted) => return Vec::new(),
|
||||
Err(CodexErr::UsageLimitReached(limit_err)) => {
|
||||
let now = Utc::now();
|
||||
let retry_after = limit_err
|
||||
.retry_after(now)
|
||||
.unwrap_or_else(|| RetryAfter::from_duration(Duration::from_secs(5 * 60), now));
|
||||
let mut message = format!("{limit_err} Auto-retrying");
|
||||
message.push('…');
|
||||
sess.notify_stream_error(&sub_id, message).await;
|
||||
tokio::time::sleep(retry_after.delay).await;
|
||||
retries = 0;
|
||||
continue;
|
||||
}
|
||||
Err(e) if is_context_overflow_error(&e) => {
|
||||
if turn_input.len() > 1 {
|
||||
tracing::warn!(
|
||||
|
||||
@@ -10,7 +10,9 @@ use super::compact::{
|
||||
use super::Session;
|
||||
use super::TurnContext;
|
||||
use crate::Prompt;
|
||||
use crate::error::CodexErr;
|
||||
use crate::error::Result as CodexResult;
|
||||
use crate::error::RetryAfter;
|
||||
use crate::protocol::AgentMessageEvent;
|
||||
use crate::protocol::ErrorEvent;
|
||||
use crate::protocol::EventMsg;
|
||||
@@ -20,6 +22,7 @@ use code_protocol::models::ResponseItem;
|
||||
use code_protocol::protocol::CompactedItem;
|
||||
use code_protocol::protocol::RolloutItem;
|
||||
use crate::util::backoff;
|
||||
use std::time::Duration;
|
||||
|
||||
pub(super) async fn run_inline_remote_auto_compact_task(
|
||||
sess: Arc<Session>,
|
||||
@@ -123,6 +126,18 @@ async fn run_remote_compact_task_inner(
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
Err(CodexErr::UsageLimitReached(limit_err)) => {
|
||||
let now = chrono::Utc::now();
|
||||
let retry_after = limit_err
|
||||
.retry_after(now)
|
||||
.unwrap_or_else(|| RetryAfter::from_duration(Duration::from_secs(5 * 60), now));
|
||||
let mut message = format!("{limit_err} Auto-retrying");
|
||||
message.push('…');
|
||||
sess.notify_stream_error(sub_id, message).await;
|
||||
tokio::time::sleep(retry_after.delay).await;
|
||||
retries = 0;
|
||||
continue;
|
||||
}
|
||||
Err(err) => {
|
||||
if retries < max_retries {
|
||||
retries += 1;
|
||||
|
||||
@@ -223,6 +223,14 @@ pub struct UsageLimitReachedError {
|
||||
pub resets_in_seconds: Option<u64>,
|
||||
}
|
||||
|
||||
impl UsageLimitReachedError {
|
||||
pub fn retry_after(&self, now: DateTime<Utc>) -> Option<RetryAfter> {
|
||||
let seconds = self.resets_in_seconds?;
|
||||
let delay = Duration::from_secs(seconds).saturating_add(Duration::from_secs(5));
|
||||
Some(RetryAfter::from_duration(delay, now))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for UsageLimitReachedError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
// Base message differs slightly for legacy ChatGPT Plus plan users.
|
||||
|
||||
@@ -5627,14 +5627,11 @@ impl ChatWidget<'_> {
|
||||
fn mark_reconnecting(&mut self, message: String) {
|
||||
// Keep task running and surface a concise status in the input header.
|
||||
self.bottom_pane.set_task_running(true);
|
||||
self.bottom_pane.update_status_text("Reconnecting...".to_string());
|
||||
self.bottom_pane.update_status_text("Retrying...".to_string());
|
||||
|
||||
if !self.reconnect_notice_active {
|
||||
self.reconnect_notice_active = true;
|
||||
self.push_background_tail(format!(
|
||||
"Connection lost. Auto-retrying… ({})",
|
||||
message
|
||||
));
|
||||
self.push_background_tail(format!("Auto-retrying… ({message})"));
|
||||
}
|
||||
|
||||
// Do NOT clear running state or streams; the retry will resume them.
|
||||
@@ -5648,7 +5645,7 @@ impl ChatWidget<'_> {
|
||||
self.reconnect_notice_active = false;
|
||||
self.bottom_pane.update_status_text(String::new());
|
||||
self.bottom_pane
|
||||
.flash_footer_notice_for("Reconnected, resuming".to_string(), Duration::from_secs(2));
|
||||
.flash_footer_notice_for("Resuming".to_string(), Duration::from_secs(2));
|
||||
self.request_redraw();
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user