diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 761c8c0f79e..122cf76883d 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -58,7 +58,12 @@ from agent.model_metadata import ( ) from agent.process_bootstrap import _install_safe_stdio from agent.prompt_caching import apply_anthropic_cache_control -from agent.retry_utils import adaptive_rate_limit_backoff, jittered_backoff +from agent.retry_utils import ( + adaptive_rate_limit_backoff, + is_zai_coding_overload_error, + jittered_backoff, + zai_coding_overload_retry_ceiling, +) from agent.trajectory import has_incomplete_scratchpad from agent.usage_pricing import estimate_usage_cost, normalize_usage from hermes_constants import PARTIAL_STREAM_STUB_ID @@ -3142,6 +3147,21 @@ def run_conversation( FailoverReason.timeout, FailoverReason.overloaded, } + # Z.AI Coding Plan GLM-5.2 signals server overload as HTTP 429 + # code 1305 ("temporarily overloaded"). classify_api_error routes + # that to `overloaded` (not `rate_limit`) so the credential pool + # isn't burned on a valid key — but `overloaded` is excluded from + # `is_rate_limited`, which is exactly what gates the adaptive Z.AI + # backoff below. Detect the overload 429 directly so its adaptive + # long-backoff schedule still runs, and raise the retry ceiling so + # the long tier (30/60/90/120s) is actually reachable: the default + # ceiling equals the short-retry threshold, so the loop otherwise + # gives up after a few quick retries and the long tier is dead code. + _is_zai_coding_overload = is_zai_coding_overload_error( + base_url=str(_base), model=_model, error=api_error + ) + if _is_zai_coding_overload: + max_retries = max(max_retries, zai_coding_overload_retry_ceiling()) _should_fallback = ( is_rate_limited or (_is_transport_failure and retry_count >= 2) @@ -4092,7 +4112,7 @@ def run_conversation( pass wait_time = _retry_after if _retry_after else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0) _backoff_policy = None - if is_rate_limited and not _retry_after: + if (is_rate_limited or _is_zai_coding_overload) and not _retry_after: wait_time, _backoff_policy = adaptive_rate_limit_backoff( retry_count, base_url=str(_base), @@ -4100,13 +4120,14 @@ def run_conversation( error=api_error, default_wait=wait_time, ) - if is_rate_limited: + if is_rate_limited or _is_zai_coding_overload: _policy_note = "" if _backoff_policy == "zai_coding_overload_long": _policy_note = " (Z.AI Coding overload adaptive long backoff)" elif _backoff_policy == "zai_coding_overload_short": _policy_note = " (Z.AI Coding overload short retry)" - _rate_limit_status = f"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}..." + _wait_reason = "Provider overloaded" if _is_zai_coding_overload and not is_rate_limited else "Rate limited" + _rate_limit_status = f"⏱️ {_wait_reason}. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}..." # Normal retries are buffered to avoid noisy transient chatter. Long # Z.AI Coding waits are different: they can last minutes, so surface # progress immediately instead of making the TUI look frozen. diff --git a/agent/retry_utils.py b/agent/retry_utils.py index 2922156847b..3fc5645edb6 100644 --- a/agent/retry_utils.py +++ b/agent/retry_utils.py @@ -127,3 +127,20 @@ def adaptive_rate_limit_backoff( # A smaller jitter ratio keeps long waits readable while still avoiding # synchronized retry storms across concurrent Hermes sessions. return jittered_backoff(1, base_delay=base_delay, max_delay=base_delay, jitter_ratio=0.2), "zai_coding_overload_long" + + +def zai_coding_overload_retry_ceiling(short_attempts: int = 3) -> int: + """Retry-loop ceiling needed for the full Z.AI overload backoff schedule. + + The adaptive policy runs ``short_attempts`` short retries, then walks the + long-backoff table one entry per subsequent attempt. The retry loop gives + up as soon as ``retry_count >= ceiling`` — and that check runs *before* the + attempt's backoff is computed — so the ceiling must sit one past the final + long-backoff entry for every long tier to actually execute. + + With the default ``api_max_retries`` (3) equal to ``short_attempts`` (3), + the loop always gave up before reaching the long tier, leaving the whole + long-backoff schedule as dead code. Callers extend the ceiling to this + value for Z.AI Coding overload 429s so the 30/60/90/120s waits run. + """ + return short_attempts + len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) + 1