mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-08 13:12:08 +00:00
fix(agent): run Z.AI overload adaptive backoff on the overloaded path
Z.AI Coding Plan GLM-5.2 reports server overload as HTTP 429 code 1305
("temporarily overloaded"). classify_api_error routes that to
FailoverReason.overloaded (so a valid credential pool isn't burned), but
the adaptive Z.AI backoff was gated on is_rate_limited — which excludes
overloaded — so it never ran (policy=default) and the request failed after
a few quick short retries.
Two compounding causes, both fixed here:
1. Detect the Z.AI overload 429 directly and let its adaptive backoff run
on the overloaded path, not only the rate_limit path.
2. Raise the retry ceiling for this narrow case via
zai_coding_overload_retry_ceiling(). The long-backoff tier
(30/60/90/120s) starts after short_attempts (3) retries, but the default
api_max_retries is also 3, so the loop always gave up before the long
tier could run — leaving the whole long-backoff schedule as dead code.
Scope is limited to the existing narrow is_zai_coding_overload_error match,
so other providers' 429/503/529 handling is unchanged.
This commit is contained in:
parent
05cbddc012
commit
1c702aa73e
2 changed files with 42 additions and 4 deletions
|
|
@ -58,7 +58,12 @@ from agent.model_metadata import (
|
|||
)
|
||||
from agent.process_bootstrap import _install_safe_stdio
|
||||
from agent.prompt_caching import apply_anthropic_cache_control
|
||||
from agent.retry_utils import adaptive_rate_limit_backoff, jittered_backoff
|
||||
from agent.retry_utils import (
|
||||
adaptive_rate_limit_backoff,
|
||||
is_zai_coding_overload_error,
|
||||
jittered_backoff,
|
||||
zai_coding_overload_retry_ceiling,
|
||||
)
|
||||
from agent.trajectory import has_incomplete_scratchpad
|
||||
from agent.usage_pricing import estimate_usage_cost, normalize_usage
|
||||
from hermes_constants import PARTIAL_STREAM_STUB_ID
|
||||
|
|
@ -3142,6 +3147,21 @@ def run_conversation(
|
|||
FailoverReason.timeout,
|
||||
FailoverReason.overloaded,
|
||||
}
|
||||
# Z.AI Coding Plan GLM-5.2 signals server overload as HTTP 429
|
||||
# code 1305 ("temporarily overloaded"). classify_api_error routes
|
||||
# that to `overloaded` (not `rate_limit`) so the credential pool
|
||||
# isn't burned on a valid key — but `overloaded` is excluded from
|
||||
# `is_rate_limited`, which is exactly what gates the adaptive Z.AI
|
||||
# backoff below. Detect the overload 429 directly so its adaptive
|
||||
# long-backoff schedule still runs, and raise the retry ceiling so
|
||||
# the long tier (30/60/90/120s) is actually reachable: the default
|
||||
# ceiling equals the short-retry threshold, so the loop otherwise
|
||||
# gives up after a few quick retries and the long tier is dead code.
|
||||
_is_zai_coding_overload = is_zai_coding_overload_error(
|
||||
base_url=str(_base), model=_model, error=api_error
|
||||
)
|
||||
if _is_zai_coding_overload:
|
||||
max_retries = max(max_retries, zai_coding_overload_retry_ceiling())
|
||||
_should_fallback = (
|
||||
is_rate_limited
|
||||
or (_is_transport_failure and retry_count >= 2)
|
||||
|
|
@ -4092,7 +4112,7 @@ def run_conversation(
|
|||
pass
|
||||
wait_time = _retry_after if _retry_after else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0)
|
||||
_backoff_policy = None
|
||||
if is_rate_limited and not _retry_after:
|
||||
if (is_rate_limited or _is_zai_coding_overload) and not _retry_after:
|
||||
wait_time, _backoff_policy = adaptive_rate_limit_backoff(
|
||||
retry_count,
|
||||
base_url=str(_base),
|
||||
|
|
@ -4100,13 +4120,14 @@ def run_conversation(
|
|||
error=api_error,
|
||||
default_wait=wait_time,
|
||||
)
|
||||
if is_rate_limited:
|
||||
if is_rate_limited or _is_zai_coding_overload:
|
||||
_policy_note = ""
|
||||
if _backoff_policy == "zai_coding_overload_long":
|
||||
_policy_note = " (Z.AI Coding overload adaptive long backoff)"
|
||||
elif _backoff_policy == "zai_coding_overload_short":
|
||||
_policy_note = " (Z.AI Coding overload short retry)"
|
||||
_rate_limit_status = f"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}..."
|
||||
_wait_reason = "Provider overloaded" if _is_zai_coding_overload and not is_rate_limited else "Rate limited"
|
||||
_rate_limit_status = f"⏱️ {_wait_reason}. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}..."
|
||||
# Normal retries are buffered to avoid noisy transient chatter. Long
|
||||
# Z.AI Coding waits are different: they can last minutes, so surface
|
||||
# progress immediately instead of making the TUI look frozen.
|
||||
|
|
|
|||
|
|
@ -127,3 +127,20 @@ def adaptive_rate_limit_backoff(
|
|||
# A smaller jitter ratio keeps long waits readable while still avoiding
|
||||
# synchronized retry storms across concurrent Hermes sessions.
|
||||
return jittered_backoff(1, base_delay=base_delay, max_delay=base_delay, jitter_ratio=0.2), "zai_coding_overload_long"
|
||||
|
||||
|
||||
def zai_coding_overload_retry_ceiling(short_attempts: int = 3) -> int:
|
||||
"""Retry-loop ceiling needed for the full Z.AI overload backoff schedule.
|
||||
|
||||
The adaptive policy runs ``short_attempts`` short retries, then walks the
|
||||
long-backoff table one entry per subsequent attempt. The retry loop gives
|
||||
up as soon as ``retry_count >= ceiling`` — and that check runs *before* the
|
||||
attempt's backoff is computed — so the ceiling must sit one past the final
|
||||
long-backoff entry for every long tier to actually execute.
|
||||
|
||||
With the default ``api_max_retries`` (3) equal to ``short_attempts`` (3),
|
||||
the loop always gave up before reaching the long tier, leaving the whole
|
||||
long-backoff schedule as dead code. Callers extend the ceiling to this
|
||||
value for Z.AI Coding overload 429s so the 30/60/90/120s waits run.
|
||||
"""
|
||||
return short_attempts + len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) + 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue