fix(auto-review): fallback from spark on usage-limit errors

This commit is contained in:
James Peter
2026-02-15 16:18:42 +10:00
parent a541f404e7
commit 2a4c39fd00
3 changed files with 233 additions and 3 deletions
@@ -111,6 +111,36 @@ fn auto_drive_cli_models_for_auth(
models
}
fn spark_fallback_model(model: &str) -> Option<&'static str> {
if model.eq_ignore_ascii_case("gpt-5.3-codex-spark") {
Some("gpt-5.3-codex")
} else if model.eq_ignore_ascii_case("code-gpt-5.3-codex-spark") {
Some("code-gpt-5.3-codex")
} else {
None
}
}
fn is_usage_limit_stream_error_message(message: &str) -> bool {
let lower = message.to_ascii_lowercase();
lower.contains("usage limit")
|| lower.contains("usage_limit_reached")
|| lower.contains("usage_not_included")
}
fn error_mentions_usage_limit(error: &anyhow::Error) -> bool {
error.chain().any(|cause| {
if let Some(code_err) = cause.downcast_ref::<CodexErr>() {
return match code_err {
CodexErr::UsageLimitReached(_) | CodexErr::UsageNotIncluded => true,
CodexErr::Stream(message, _, _) => is_usage_limit_stream_error_message(message),
_ => false,
};
}
false
})
}
#[derive(Debug, Clone)]
struct AutoTimeBudget {
deadline: Instant,
@@ -1232,6 +1262,30 @@ mod tests {
}
}
#[test]
fn usage_limit_stream_errors_are_detected() {
let err = anyhow!(CodexErr::Stream(
"[transport] Transport error: You've hit your usage limit. Try again in 5 days 47 minutes."
.to_string(),
None,
None,
));
assert!(error_mentions_usage_limit(&err));
}
#[test]
fn spark_models_have_non_spark_fallback() {
assert_eq!(
spark_fallback_model("gpt-5.3-codex-spark"),
Some("gpt-5.3-codex")
);
assert_eq!(
spark_fallback_model("code-gpt-5.3-codex-spark"),
Some("code-gpt-5.3-codex")
);
assert!(spark_fallback_model("gpt-5.3-codex").is_none());
}
#[test]
fn push_unique_guidance_trims_and_dedupes() {
let mut guidance = vec!["Keep CLI prompts short".to_string()];
@@ -2788,10 +2842,37 @@ fn request_decision_with_model(
let tx = event_tx.clone();
let cancel = cancel_token.clone();
let mut rate_limit_switch_state = RateLimitSwitchState::default();
let selected_model = Arc::new(Mutex::new(model_slug.to_string()));
let selected_model_for_retry = Arc::clone(&selected_model);
let mut did_usage_limit_model_fallback = false;
let classify = |error: &anyhow::Error| {
if !did_usage_limit_model_fallback && error_mentions_usage_limit(error) {
let active_model = selected_model_for_retry
.lock()
.ok()
.map(|guard| guard.clone())
.unwrap_or_else(|| model_slug.to_string());
if let Some(fallback_model) = spark_fallback_model(&active_model) {
did_usage_limit_model_fallback = true;
if let Ok(mut guard) = selected_model_for_retry.lock() {
*guard = fallback_model.to_string();
}
event_tx.send(AutoCoordinatorEvent::Action {
message: format!(
"Usage limit reached for {active_model}; retrying with {fallback_model}…"
),
});
return RetryDecision::RateLimited {
wait_until: Instant::now(),
reason: "usage limit reached; switched to non-spark model".to_string(),
};
}
}
classify_model_error_with_auto_switch(client, &mut rate_limit_switch_state, event_tx, error)
};
let options = RetryOptions::with_defaults(retry_max_elapsed(time_budget_deadline));
let selected_model_for_run = Arc::clone(&selected_model);
let result = runtime.block_on(async move {
retry_with_backoff(
@@ -2801,6 +2882,11 @@ fn request_decision_with_model(
let time_budget_message = time_budget_message.clone();
let loop_warning = loop_warning.clone();
let conversation = Arc::clone(&conversation);
let model_slug = selected_model_for_run
.lock()
.ok()
.map(|guard| guard.clone())
.unwrap_or_else(|| model_slug.to_string());
let prompt = build_user_turn_prompt(
&developer_intro,
&primary_goal,
@@ -2809,7 +2895,7 @@ fn request_decision_with_model(
loop_warning.as_deref(),
&schema,
conversation.as_ref(),
model_slug,
&model_slug,
instructions.as_deref(),
);
let tx_inner = tx.clone();
+75 -1
View File
@@ -2644,7 +2644,16 @@ async fn process_sse<S>(
if let Some(error) = error {
match serde_json::from_value::<Error>(error.clone()) {
Ok(error) => {
if is_quota_exceeded_error(&error) {
if error.r#type.as_deref() == Some("usage_limit_reached") {
response_error = Some(CodexErr::UsageLimitReached(
UsageLimitReachedError {
plan_type: error.plan_type,
resets_in_seconds: error.resets_in_seconds,
},
));
} else if error.r#type.as_deref() == Some("usage_not_included") {
response_error = Some(CodexErr::UsageNotIncluded);
} else if is_quota_exceeded_error(&error) {
response_error = Some(CodexErr::QuotaExceeded);
} else if is_server_overloaded_error(&error) {
response_error = Some(CodexErr::ServerOverloaded);
@@ -3707,6 +3716,71 @@ mod tests {
}
}
#[tokio::test]
async fn response_failed_usage_limit_maps_to_typed_error() {
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_limit","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"type":"usage_limit_reached","message":"You've hit your usage limit.","plan_type":"pro","resets_in_seconds":120},"incomplete_details":null}}"#;
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
let provider = ModelProviderInfo {
name: "test".to_string(),
base_url: Some("https://test.com".to_string()),
env_key: Some("TEST_API_KEY".to_string()),
env_key_instructions: None,
experimental_bearer_token: None,
wire_api: WireApi::Responses,
query_params: None,
http_headers: None,
env_http_headers: None,
request_max_retries: Some(0),
stream_max_retries: Some(0),
stream_idle_timeout_ms: Some(1000),
requires_openai_auth: false,
openrouter: None,
};
let events = collect_events(&[sse1.as_bytes()], provider).await;
assert_eq!(events.len(), 1);
match &events[0] {
Err(CodexErr::UsageLimitReached(err)) => {
assert_eq!(err.plan_type.as_deref(), Some("pro"));
assert_eq!(err.resets_in_seconds, Some(120));
}
other => panic!("unexpected usage-limit event: {other:?}"),
}
}
#[tokio::test]
async fn response_failed_usage_not_included_maps_to_typed_error() {
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_not_included","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"type":"usage_not_included","message":"Usage is not included for this model."},"incomplete_details":null}}"#;
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
let provider = ModelProviderInfo {
name: "test".to_string(),
base_url: Some("https://test.com".to_string()),
env_key: Some("TEST_API_KEY".to_string()),
env_key_instructions: None,
experimental_bearer_token: None,
wire_api: WireApi::Responses,
query_params: None,
http_headers: None,
env_http_headers: None,
request_max_retries: Some(0),
stream_max_retries: Some(0),
stream_idle_timeout_ms: Some(1000),
requires_openai_auth: false,
openrouter: None,
};
let events = collect_events(&[sse1.as_bytes()], provider).await;
assert_eq!(events.len(), 1);
match &events[0] {
Err(CodexErr::UsageNotIncluded) => {}
other => panic!("unexpected usage-not-included event: {other:?}"),
}
}
#[tokio::test]
async fn server_overloaded_error_is_typed() {
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_slow_down","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"slow_down","message":"Server is overloaded. Please retry shortly."},"incomplete_details":null}}"#;
+71 -1
View File
@@ -1527,6 +1527,13 @@ fn is_context_overflow_stream_error(message: &str) -> bool {
|| lower.contains("too long")))
}
fn is_usage_limit_stream_error(message: &str) -> bool {
let lower = message.to_ascii_lowercase();
lower.contains("usage limit")
|| lower.contains("usage_limit_reached")
|| lower.contains("usage_not_included")
}
fn spark_fallback_model(model: &str) -> Option<String> {
if model.eq_ignore_ascii_case("gpt-5.3-codex-spark") {
Some("gpt-5.3-codex".to_string())
@@ -2381,9 +2388,60 @@ async fn run_turn(
retries = 0;
continue;
}
Err(CodexErr::UsageNotIncluded) => return Err(CodexErr::UsageNotIncluded),
Err(CodexErr::UsageNotIncluded) => {
if !did_usage_limit_model_fallback {
let active_model = prompt
.model_override
.clone()
.unwrap_or_else(|| tc.client.get_model());
if let Some(fallback_model) = spark_fallback_model(&active_model) {
did_usage_limit_model_fallback = true;
forced_model_override = Some(fallback_model.clone());
retries = 0;
sess.clear_scratchpad();
attempt_input = input.clone();
sess
.notify_stream_error(
&sub_id,
format!(
"Usage limit reached for {active_model}; retrying with {fallback_model}…"
),
)
.await;
continue;
}
}
return Err(CodexErr::UsageNotIncluded);
}
Err(CodexErr::QuotaExceeded) => return Err(CodexErr::QuotaExceeded),
Err(e) => {
if let CodexErr::Stream(msg, _maybe_delay, _req_id) = &e
&& is_usage_limit_stream_error(msg)
&& !did_usage_limit_model_fallback
{
let active_model = prompt
.model_override
.clone()
.unwrap_or_else(|| tc.client.get_model());
if let Some(fallback_model) = spark_fallback_model(&active_model) {
did_usage_limit_model_fallback = true;
forced_model_override = Some(fallback_model.clone());
retries = 0;
sess.clear_scratchpad();
attempt_input = input.clone();
sess
.notify_stream_error(
&sub_id,
format!(
"Usage limit reached for {active_model}; retrying with {fallback_model}…"
),
)
.await;
continue;
}
}
if let CodexErr::Stream(msg, _maybe_delay, _req_id) = &e
&& is_context_overflow_stream_error(msg)
{
@@ -12563,6 +12621,7 @@ mod tests {
choose_larger_context_model_from_candidates,
format_exec_output_with_limit,
is_context_overflow_stream_error,
is_usage_limit_stream_error,
spark_fallback_model,
TRUNCATION_MARKER,
};
@@ -12632,6 +12691,17 @@ mod tests {
assert!(!is_context_overflow_stream_error("temporary network timeout"));
}
#[test]
fn usage_limit_detection_matches_transport_errors() {
assert!(is_usage_limit_stream_error(
"[transport] Transport error: You've hit your usage limit. Try again in 5 days 47 minutes."
));
assert!(is_usage_limit_stream_error(
"response.failed: usage_not_included"
));
assert!(!is_usage_limit_stream_error("temporary network timeout"));
}
#[test]
fn picks_larger_context_model_from_candidates() {
let chosen = choose_larger_context_model_from_candidates(