mirror of
https://github.com/hanzoai/dev.git
synced 2026-10-25 03:32:32 +00:00
fix(auto-review): fallback from spark on usage-limit errors
This commit is contained in:
@@ -111,6 +111,36 @@ fn auto_drive_cli_models_for_auth(
|
||||
models
|
||||
}
|
||||
|
||||
fn spark_fallback_model(model: &str) -> Option<&'static str> {
|
||||
if model.eq_ignore_ascii_case("gpt-5.3-codex-spark") {
|
||||
Some("gpt-5.3-codex")
|
||||
} else if model.eq_ignore_ascii_case("code-gpt-5.3-codex-spark") {
|
||||
Some("code-gpt-5.3-codex")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn is_usage_limit_stream_error_message(message: &str) -> bool {
|
||||
let lower = message.to_ascii_lowercase();
|
||||
lower.contains("usage limit")
|
||||
|| lower.contains("usage_limit_reached")
|
||||
|| lower.contains("usage_not_included")
|
||||
}
|
||||
|
||||
fn error_mentions_usage_limit(error: &anyhow::Error) -> bool {
|
||||
error.chain().any(|cause| {
|
||||
if let Some(code_err) = cause.downcast_ref::<CodexErr>() {
|
||||
return match code_err {
|
||||
CodexErr::UsageLimitReached(_) | CodexErr::UsageNotIncluded => true,
|
||||
CodexErr::Stream(message, _, _) => is_usage_limit_stream_error_message(message),
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
false
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct AutoTimeBudget {
|
||||
deadline: Instant,
|
||||
@@ -1232,6 +1262,30 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_limit_stream_errors_are_detected() {
|
||||
let err = anyhow!(CodexErr::Stream(
|
||||
"[transport] Transport error: You've hit your usage limit. Try again in 5 days 47 minutes."
|
||||
.to_string(),
|
||||
None,
|
||||
None,
|
||||
));
|
||||
assert!(error_mentions_usage_limit(&err));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_models_have_non_spark_fallback() {
|
||||
assert_eq!(
|
||||
spark_fallback_model("gpt-5.3-codex-spark"),
|
||||
Some("gpt-5.3-codex")
|
||||
);
|
||||
assert_eq!(
|
||||
spark_fallback_model("code-gpt-5.3-codex-spark"),
|
||||
Some("code-gpt-5.3-codex")
|
||||
);
|
||||
assert!(spark_fallback_model("gpt-5.3-codex").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_unique_guidance_trims_and_dedupes() {
|
||||
let mut guidance = vec!["Keep CLI prompts short".to_string()];
|
||||
@@ -2788,10 +2842,37 @@ fn request_decision_with_model(
|
||||
let tx = event_tx.clone();
|
||||
let cancel = cancel_token.clone();
|
||||
let mut rate_limit_switch_state = RateLimitSwitchState::default();
|
||||
let selected_model = Arc::new(Mutex::new(model_slug.to_string()));
|
||||
let selected_model_for_retry = Arc::clone(&selected_model);
|
||||
let mut did_usage_limit_model_fallback = false;
|
||||
let classify = |error: &anyhow::Error| {
|
||||
if !did_usage_limit_model_fallback && error_mentions_usage_limit(error) {
|
||||
let active_model = selected_model_for_retry
|
||||
.lock()
|
||||
.ok()
|
||||
.map(|guard| guard.clone())
|
||||
.unwrap_or_else(|| model_slug.to_string());
|
||||
if let Some(fallback_model) = spark_fallback_model(&active_model) {
|
||||
did_usage_limit_model_fallback = true;
|
||||
if let Ok(mut guard) = selected_model_for_retry.lock() {
|
||||
*guard = fallback_model.to_string();
|
||||
}
|
||||
event_tx.send(AutoCoordinatorEvent::Action {
|
||||
message: format!(
|
||||
"Usage limit reached for {active_model}; retrying with {fallback_model}…"
|
||||
),
|
||||
});
|
||||
return RetryDecision::RateLimited {
|
||||
wait_until: Instant::now(),
|
||||
reason: "usage limit reached; switched to non-spark model".to_string(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
classify_model_error_with_auto_switch(client, &mut rate_limit_switch_state, event_tx, error)
|
||||
};
|
||||
let options = RetryOptions::with_defaults(retry_max_elapsed(time_budget_deadline));
|
||||
let selected_model_for_run = Arc::clone(&selected_model);
|
||||
|
||||
let result = runtime.block_on(async move {
|
||||
retry_with_backoff(
|
||||
@@ -2801,6 +2882,11 @@ fn request_decision_with_model(
|
||||
let time_budget_message = time_budget_message.clone();
|
||||
let loop_warning = loop_warning.clone();
|
||||
let conversation = Arc::clone(&conversation);
|
||||
let model_slug = selected_model_for_run
|
||||
.lock()
|
||||
.ok()
|
||||
.map(|guard| guard.clone())
|
||||
.unwrap_or_else(|| model_slug.to_string());
|
||||
let prompt = build_user_turn_prompt(
|
||||
&developer_intro,
|
||||
&primary_goal,
|
||||
@@ -2809,7 +2895,7 @@ fn request_decision_with_model(
|
||||
loop_warning.as_deref(),
|
||||
&schema,
|
||||
conversation.as_ref(),
|
||||
model_slug,
|
||||
&model_slug,
|
||||
instructions.as_deref(),
|
||||
);
|
||||
let tx_inner = tx.clone();
|
||||
|
||||
@@ -2644,7 +2644,16 @@ async fn process_sse<S>(
|
||||
if let Some(error) = error {
|
||||
match serde_json::from_value::<Error>(error.clone()) {
|
||||
Ok(error) => {
|
||||
if is_quota_exceeded_error(&error) {
|
||||
if error.r#type.as_deref() == Some("usage_limit_reached") {
|
||||
response_error = Some(CodexErr::UsageLimitReached(
|
||||
UsageLimitReachedError {
|
||||
plan_type: error.plan_type,
|
||||
resets_in_seconds: error.resets_in_seconds,
|
||||
},
|
||||
));
|
||||
} else if error.r#type.as_deref() == Some("usage_not_included") {
|
||||
response_error = Some(CodexErr::UsageNotIncluded);
|
||||
} else if is_quota_exceeded_error(&error) {
|
||||
response_error = Some(CodexErr::QuotaExceeded);
|
||||
} else if is_server_overloaded_error(&error) {
|
||||
response_error = Some(CodexErr::ServerOverloaded);
|
||||
@@ -3707,6 +3716,71 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_failed_usage_limit_maps_to_typed_error() {
|
||||
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_limit","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"type":"usage_limit_reached","message":"You've hit your usage limit.","plan_type":"pro","resets_in_seconds":120},"incomplete_details":null}}"#;
|
||||
|
||||
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
|
||||
let provider = ModelProviderInfo {
|
||||
name: "test".to_string(),
|
||||
base_url: Some("https://test.com".to_string()),
|
||||
env_key: Some("TEST_API_KEY".to_string()),
|
||||
env_key_instructions: None,
|
||||
experimental_bearer_token: None,
|
||||
wire_api: WireApi::Responses,
|
||||
query_params: None,
|
||||
http_headers: None,
|
||||
env_http_headers: None,
|
||||
request_max_retries: Some(0),
|
||||
stream_max_retries: Some(0),
|
||||
stream_idle_timeout_ms: Some(1000),
|
||||
requires_openai_auth: false,
|
||||
openrouter: None,
|
||||
};
|
||||
|
||||
let events = collect_events(&[sse1.as_bytes()], provider).await;
|
||||
|
||||
assert_eq!(events.len(), 1);
|
||||
match &events[0] {
|
||||
Err(CodexErr::UsageLimitReached(err)) => {
|
||||
assert_eq!(err.plan_type.as_deref(), Some("pro"));
|
||||
assert_eq!(err.resets_in_seconds, Some(120));
|
||||
}
|
||||
other => panic!("unexpected usage-limit event: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_failed_usage_not_included_maps_to_typed_error() {
|
||||
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_not_included","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"type":"usage_not_included","message":"Usage is not included for this model."},"incomplete_details":null}}"#;
|
||||
|
||||
let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n");
|
||||
let provider = ModelProviderInfo {
|
||||
name: "test".to_string(),
|
||||
base_url: Some("https://test.com".to_string()),
|
||||
env_key: Some("TEST_API_KEY".to_string()),
|
||||
env_key_instructions: None,
|
||||
experimental_bearer_token: None,
|
||||
wire_api: WireApi::Responses,
|
||||
query_params: None,
|
||||
http_headers: None,
|
||||
env_http_headers: None,
|
||||
request_max_retries: Some(0),
|
||||
stream_max_retries: Some(0),
|
||||
stream_idle_timeout_ms: Some(1000),
|
||||
requires_openai_auth: false,
|
||||
openrouter: None,
|
||||
};
|
||||
|
||||
let events = collect_events(&[sse1.as_bytes()], provider).await;
|
||||
|
||||
assert_eq!(events.len(), 1);
|
||||
match &events[0] {
|
||||
Err(CodexErr::UsageNotIncluded) => {}
|
||||
other => panic!("unexpected usage-not-included event: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_overloaded_error_is_typed() {
|
||||
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_slow_down","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"slow_down","message":"Server is overloaded. Please retry shortly."},"incomplete_details":null}}"#;
|
||||
|
||||
@@ -1527,6 +1527,13 @@ fn is_context_overflow_stream_error(message: &str) -> bool {
|
||||
|| lower.contains("too long")))
|
||||
}
|
||||
|
||||
fn is_usage_limit_stream_error(message: &str) -> bool {
|
||||
let lower = message.to_ascii_lowercase();
|
||||
lower.contains("usage limit")
|
||||
|| lower.contains("usage_limit_reached")
|
||||
|| lower.contains("usage_not_included")
|
||||
}
|
||||
|
||||
fn spark_fallback_model(model: &str) -> Option<String> {
|
||||
if model.eq_ignore_ascii_case("gpt-5.3-codex-spark") {
|
||||
Some("gpt-5.3-codex".to_string())
|
||||
@@ -2381,9 +2388,60 @@ async fn run_turn(
|
||||
retries = 0;
|
||||
continue;
|
||||
}
|
||||
Err(CodexErr::UsageNotIncluded) => return Err(CodexErr::UsageNotIncluded),
|
||||
Err(CodexErr::UsageNotIncluded) => {
|
||||
if !did_usage_limit_model_fallback {
|
||||
let active_model = prompt
|
||||
.model_override
|
||||
.clone()
|
||||
.unwrap_or_else(|| tc.client.get_model());
|
||||
if let Some(fallback_model) = spark_fallback_model(&active_model) {
|
||||
did_usage_limit_model_fallback = true;
|
||||
forced_model_override = Some(fallback_model.clone());
|
||||
retries = 0;
|
||||
sess.clear_scratchpad();
|
||||
attempt_input = input.clone();
|
||||
sess
|
||||
.notify_stream_error(
|
||||
&sub_id,
|
||||
format!(
|
||||
"Usage limit reached for {active_model}; retrying with {fallback_model}…"
|
||||
),
|
||||
)
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return Err(CodexErr::UsageNotIncluded);
|
||||
}
|
||||
Err(CodexErr::QuotaExceeded) => return Err(CodexErr::QuotaExceeded),
|
||||
Err(e) => {
|
||||
if let CodexErr::Stream(msg, _maybe_delay, _req_id) = &e
|
||||
&& is_usage_limit_stream_error(msg)
|
||||
&& !did_usage_limit_model_fallback
|
||||
{
|
||||
let active_model = prompt
|
||||
.model_override
|
||||
.clone()
|
||||
.unwrap_or_else(|| tc.client.get_model());
|
||||
if let Some(fallback_model) = spark_fallback_model(&active_model) {
|
||||
did_usage_limit_model_fallback = true;
|
||||
forced_model_override = Some(fallback_model.clone());
|
||||
retries = 0;
|
||||
sess.clear_scratchpad();
|
||||
attempt_input = input.clone();
|
||||
sess
|
||||
.notify_stream_error(
|
||||
&sub_id,
|
||||
format!(
|
||||
"Usage limit reached for {active_model}; retrying with {fallback_model}…"
|
||||
),
|
||||
)
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if let CodexErr::Stream(msg, _maybe_delay, _req_id) = &e
|
||||
&& is_context_overflow_stream_error(msg)
|
||||
{
|
||||
@@ -12563,6 +12621,7 @@ mod tests {
|
||||
choose_larger_context_model_from_candidates,
|
||||
format_exec_output_with_limit,
|
||||
is_context_overflow_stream_error,
|
||||
is_usage_limit_stream_error,
|
||||
spark_fallback_model,
|
||||
TRUNCATION_MARKER,
|
||||
};
|
||||
@@ -12632,6 +12691,17 @@ mod tests {
|
||||
assert!(!is_context_overflow_stream_error("temporary network timeout"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_limit_detection_matches_transport_errors() {
|
||||
assert!(is_usage_limit_stream_error(
|
||||
"[transport] Transport error: You've hit your usage limit. Try again in 5 days 47 minutes."
|
||||
));
|
||||
assert!(is_usage_limit_stream_error(
|
||||
"response.failed: usage_not_included"
|
||||
));
|
||||
assert!(!is_usage_limit_stream_error("temporary network timeout"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn picks_larger_context_model_from_candidates() {
|
||||
let chosen = choose_larger_context_model_from_candidates(
|
||||
|
||||
Reference in New Issue
Block a user