fix(agent): run Z.AI overload adaptive backoff on the overloaded path

Z.AI Coding Plan GLM-5.2 reports server overload as HTTP 429 code 1305
("temporarily overloaded"). classify_api_error routes that to
FailoverReason.overloaded (so a valid credential pool isn't burned), but
the adaptive Z.AI backoff was gated on is_rate_limited — which excludes
overloaded — so it never ran (policy=default) and the request failed after
a few quick short retries.

Two compounding causes, both fixed here:

1. Detect the Z.AI overload 429 directly and let its adaptive backoff run
   on the overloaded path, not only the rate_limit path.
2. Raise the retry ceiling for this narrow case via
   zai_coding_overload_retry_ceiling(). The long-backoff tier
   (30/60/90/120s) starts after short_attempts (3) retries, but the default
   api_max_retries is also 3, so the loop always gave up before the long
   tier could run — leaving the whole long-backoff schedule as dead code.

Scope is limited to the existing narrow is_zai_coding_overload_error match,
so other providers' 429/503/529 handling is unchanged.
This commit is contained in:
xxxigm 2026-07-06 16:25:21 +07:00 committed by kshitijk4poor
parent 05cbddc012
commit 1c702aa73e
2 changed files with 42 additions and 4 deletions

View file

@ -58,7 +58,12 @@ from agent.model_metadata import (
)
from agent.process_bootstrap import _install_safe_stdio
from agent.prompt_caching import apply_anthropic_cache_control
from agent.retry_utils import adaptive_rate_limit_backoff, jittered_backoff
from agent.retry_utils import (
adaptive_rate_limit_backoff,
is_zai_coding_overload_error,
jittered_backoff,
zai_coding_overload_retry_ceiling,
)
from agent.trajectory import has_incomplete_scratchpad
from agent.usage_pricing import estimate_usage_cost, normalize_usage
from hermes_constants import PARTIAL_STREAM_STUB_ID
@ -3142,6 +3147,21 @@ def run_conversation(
FailoverReason.timeout,
FailoverReason.overloaded,
}
# Z.AI Coding Plan GLM-5.2 signals server overload as HTTP 429
# code 1305 ("temporarily overloaded"). classify_api_error routes
# that to `overloaded` (not `rate_limit`) so the credential pool
# isn't burned on a valid key — but `overloaded` is excluded from
# `is_rate_limited`, which is exactly what gates the adaptive Z.AI
# backoff below. Detect the overload 429 directly so its adaptive
# long-backoff schedule still runs, and raise the retry ceiling so
# the long tier (30/60/90/120s) is actually reachable: the default
# ceiling equals the short-retry threshold, so the loop otherwise
# gives up after a few quick retries and the long tier is dead code.
_is_zai_coding_overload = is_zai_coding_overload_error(
base_url=str(_base), model=_model, error=api_error
)
if _is_zai_coding_overload:
max_retries = max(max_retries, zai_coding_overload_retry_ceiling())
_should_fallback = (
is_rate_limited
or (_is_transport_failure and retry_count >= 2)
@ -4092,7 +4112,7 @@ def run_conversation(
pass
wait_time = _retry_after if _retry_after else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0)
_backoff_policy = None
if is_rate_limited and not _retry_after:
if (is_rate_limited or _is_zai_coding_overload) and not _retry_after:
wait_time, _backoff_policy = adaptive_rate_limit_backoff(
retry_count,
base_url=str(_base),
@ -4100,13 +4120,14 @@ def run_conversation(
error=api_error,
default_wait=wait_time,
)
if is_rate_limited:
if is_rate_limited or _is_zai_coding_overload:
_policy_note = ""
if _backoff_policy == "zai_coding_overload_long":
_policy_note = " (Z.AI Coding overload adaptive long backoff)"
elif _backoff_policy == "zai_coding_overload_short":
_policy_note = " (Z.AI Coding overload short retry)"
_rate_limit_status = f"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}..."
_wait_reason = "Provider overloaded" if _is_zai_coding_overload and not is_rate_limited else "Rate limited"
_rate_limit_status = f"⏱️ {_wait_reason}. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}..."
# Normal retries are buffered to avoid noisy transient chatter. Long
# Z.AI Coding waits are different: they can last minutes, so surface
# progress immediately instead of making the TUI look frozen.

View file

@ -127,3 +127,20 @@ def adaptive_rate_limit_backoff(
# A smaller jitter ratio keeps long waits readable while still avoiding
# synchronized retry storms across concurrent Hermes sessions.
return jittered_backoff(1, base_delay=base_delay, max_delay=base_delay, jitter_ratio=0.2), "zai_coding_overload_long"
def zai_coding_overload_retry_ceiling(short_attempts: int = 3) -> int:
"""Retry-loop ceiling needed for the full Z.AI overload backoff schedule.
The adaptive policy runs ``short_attempts`` short retries, then walks the
long-backoff table one entry per subsequent attempt. The retry loop gives
up as soon as ``retry_count >= ceiling`` and that check runs *before* the
attempt's backoff is computed — so the ceiling must sit one past the final
long-backoff entry for every long tier to actually execute.
With the default ``api_max_retries`` (3) equal to ``short_attempts`` (3),
the loop always gave up before reaching the long tier, leaving the whole
long-backoff schedule as dead code. Callers extend the ceiling to this
value for Z.AI Coding overload 429s so the 30/60/90/120s waits run.
"""
return short_attempts + len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) + 1