fix(core): auto-retry turns after usage limits

When the provider returns usage_limit_reached, keep the turn alive by waiting until the reset window (or 5 minutes when unknown) and retrying. Apply the same behavior to compaction tasks and show a generic retrying status in the TUI.
This commit is contained in:
James Peter
2025-12-17 12:22:41 +10:00
parent d2dbf60af9
commit 220414d1b9
5 changed files with 87 additions and 28 deletions
+34 -22
View File
@@ -5891,30 +5891,42 @@ async fn run_turn(
}
Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted),
Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)),
Err(e @ (CodexErr::UsageLimitReached(_)
| CodexErr::UsageNotIncluded
| CodexErr::QuotaExceeded)) => {
if let CodexErr::UsageLimitReached(limit_err) = &e {
if let Some(ctx) = account_usage_context(&sess) {
let usage_home = ctx.code_home.clone();
let usage_account = ctx.account_id.clone();
let usage_plan = ctx.plan.clone();
let resets = limit_err.resets_in_seconds;
spawn_usage_task(move || {
if let Err(err) = account_usage::record_usage_limit_hint(
&usage_home,
&usage_account,
usage_plan.as_deref(),
resets,
Utc::now(),
) {
warn!("Failed to persist usage limit hint: {err}");
}
});
}
Err(CodexErr::UsageLimitReached(limit_err)) => {
if let Some(ctx) = account_usage_context(sess) {
let usage_home = ctx.code_home.clone();
let usage_account = ctx.account_id.clone();
let usage_plan = ctx.plan.clone();
let resets = limit_err.resets_in_seconds;
spawn_usage_task(move || {
if let Err(err) = account_usage::record_usage_limit_hint(
&usage_home,
&usage_account,
usage_plan.as_deref(),
resets,
Utc::now(),
) {
warn!("Failed to persist usage limit hint: {err}");
}
});
}
return Err(e);
let now = Utc::now();
let retry_after = limit_err
.retry_after(now)
.unwrap_or_else(|| RetryAfter::from_duration(std::time::Duration::from_secs(5 * 60), now));
let eta = format_retry_eta(&retry_after);
let mut retry_message = format!("{limit_err} Auto-retrying");
if let Some(eta) = eta {
retry_message.push_str(&format!(" at {eta}"));
}
retry_message.push('…');
sess.notify_stream_error(&sub_id, retry_message).await;
tokio::time::sleep(retry_after.delay).await;
retries = 0;
continue;
}
Err(CodexErr::UsageNotIncluded) => return Err(CodexErr::UsageNotIncluded),
Err(CodexErr::QuotaExceeded) => return Err(CodexErr::QuotaExceeded),
Err(e) => {
// Detect context-window overflow and auto-run a compact summarization once
if !did_auto_compact {
+27
View File
@@ -10,6 +10,7 @@ use crate::Prompt;
use crate::client_common::ResponseEvent;
use crate::environment_context::EnvironmentContext;
use crate::error::CodexErr;
use crate::error::RetryAfter;
use crate::error::Result as CodexResult;
use crate::protocol::AgentMessageEvent;
use crate::protocol::ErrorEvent;
@@ -28,8 +29,10 @@ use code_protocol::protocol::CompactedItem;
use code_protocol::protocol::InputMessageKind;
use code_protocol::protocol::RolloutItem;
use base64::Engine;
use chrono::Utc;
use futures::prelude::*;
use code_app_server_protocol::AuthMode;
use std::time::Duration;
pub const SUMMARIZATION_PROMPT: &str = include_str!("../../templates/compact/prompt.md");
pub const COMPACTION_CHECKPOINT_MESSAGE: &str = "History checkpoint: earlier conversation compacted.";
@@ -259,6 +262,18 @@ pub(super) async fn perform_compaction(
break;
}
Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted),
Err(CodexErr::UsageLimitReached(limit_err)) => {
let now = Utc::now();
let retry_after = limit_err
.retry_after(now)
.unwrap_or_else(|| RetryAfter::from_duration(Duration::from_secs(5 * 60), now));
let mut message = format!("{limit_err} Auto-retrying");
message.push('…');
sess.notify_stream_error(&sub_id, message).await;
tokio::time::sleep(retry_after.delay).await;
retries = 0;
continue;
}
Err(e) if is_context_overflow_error(&e) => {
if turn_input.len() > 1 {
tracing::warn!(
@@ -408,6 +423,18 @@ async fn run_compact_task_inner_inline(
break;
}
Err(CodexErr::Interrupted) => return Vec::new(),
Err(CodexErr::UsageLimitReached(limit_err)) => {
let now = Utc::now();
let retry_after = limit_err
.retry_after(now)
.unwrap_or_else(|| RetryAfter::from_duration(Duration::from_secs(5 * 60), now));
let mut message = format!("{limit_err} Auto-retrying");
message.push('…');
sess.notify_stream_error(&sub_id, message).await;
tokio::time::sleep(retry_after.delay).await;
retries = 0;
continue;
}
Err(e) if is_context_overflow_error(&e) => {
if turn_input.len() > 1 {
tracing::warn!(
+15
View File
@@ -10,7 +10,9 @@ use super::compact::{
use super::Session;
use super::TurnContext;
use crate::Prompt;
use crate::error::CodexErr;
use crate::error::Result as CodexResult;
use crate::error::RetryAfter;
use crate::protocol::AgentMessageEvent;
use crate::protocol::ErrorEvent;
use crate::protocol::EventMsg;
@@ -20,6 +22,7 @@ use code_protocol::models::ResponseItem;
use code_protocol::protocol::CompactedItem;
use code_protocol::protocol::RolloutItem;
use crate::util::backoff;
use std::time::Duration;
pub(super) async fn run_inline_remote_auto_compact_task(
sess: Arc<Session>,
@@ -123,6 +126,18 @@ async fn run_remote_compact_task_inner(
return Err(err);
}
Err(CodexErr::UsageLimitReached(limit_err)) => {
let now = chrono::Utc::now();
let retry_after = limit_err
.retry_after(now)
.unwrap_or_else(|| RetryAfter::from_duration(Duration::from_secs(5 * 60), now));
let mut message = format!("{limit_err} Auto-retrying");
message.push('…');
sess.notify_stream_error(sub_id, message).await;
tokio::time::sleep(retry_after.delay).await;
retries = 0;
continue;
}
Err(err) => {
if retries < max_retries {
retries += 1;
+8
View File
@@ -223,6 +223,14 @@ pub struct UsageLimitReachedError {
pub resets_in_seconds: Option<u64>,
}
impl UsageLimitReachedError {
pub fn retry_after(&self, now: DateTime<Utc>) -> Option<RetryAfter> {
let seconds = self.resets_in_seconds?;
let delay = Duration::from_secs(seconds).saturating_add(Duration::from_secs(5));
Some(RetryAfter::from_duration(delay, now))
}
}
impl std::fmt::Display for UsageLimitReachedError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// Base message differs slightly for legacy ChatGPT Plus plan users.
+3 -6
View File
@@ -5627,14 +5627,11 @@ impl ChatWidget<'_> {
fn mark_reconnecting(&mut self, message: String) {
// Keep task running and surface a concise status in the input header.
self.bottom_pane.set_task_running(true);
self.bottom_pane.update_status_text("Reconnecting...".to_string());
self.bottom_pane.update_status_text("Retrying...".to_string());
if !self.reconnect_notice_active {
self.reconnect_notice_active = true;
self.push_background_tail(format!(
"Connection lost. Auto-retrying… ({})",
message
));
self.push_background_tail(format!("Auto-retrying… ({message})"));
}
// Do NOT clear running state or streams; the retry will resume them.
@@ -5648,7 +5645,7 @@ impl ChatWidget<'_> {
self.reconnect_notice_active = false;
self.bottom_pane.update_status_text(String::new());
self.bottom_pane
.flash_footer_notice_for("Reconnected, resuming".to_string(), Duration::from_secs(2));
.flash_footer_notice_for("Resuming".to_string(), Duration::from_secs(2));
self.request_redraw();
}