mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-20 15:33:54 +00:00
fix(plugins): classify stale-call circuit breaker as failover, not retry
The cross-turn stale-call circuit breaker (_check_stale_giveup in agent/chat_completion_helpers.py) raises a RuntimeError when the provider has been unresponsive for N consecutive stale attempts. This error was classified as FailoverReason.unknown (retryable=True, should_fallback=False), causing the retry loop to burn all max_retries against the same dead provider — each retry hitting the circuit breaker instantly with zero network overhead — before fallback was attempted. Add a classification rule (section 7b in classify_api_error) that recognizes the stale-breaker RuntimeError by its signature phrases and classifies it as FailoverReason.timeout with retryable=False, should_fallback=True. This makes the retry loop skip retries and go straight to fallback provider activation on the first hit. Test: test_stale_breaker_runtime_error_triggers_fallback_not_retry
This commit is contained in:
parent
c346f018d4
commit
58033baba2
2 changed files with 47 additions and 2 deletions
|
|
@ -840,12 +840,34 @@ def classify_api_error(
|
|||
)
|
||||
return _result(FailoverReason.timeout, retryable=True)
|
||||
|
||||
# ── 7. Transport / timeout heuristics ───────────────────────────
|
||||
# ── 7b. Stale-call circuit breaker → failover immediately ──────
|
||||
# _check_stale_giveup() in agent/chat_completion_helpers.py raises a
|
||||
# RuntimeError when the provider has been unresponsive for N
|
||||
# consecutive stale attempts (default 5). The error is NOT a transport
|
||||
# timeout — the circuit breaker fires *before* any network call to avoid
|
||||
# an indefinite stall. Without this classification the RuntimeError
|
||||
# falls through to FailoverReason.unknown (retryable=True), which burns
|
||||
# all max_retries against the same dead provider (each retry hitting the
|
||||
# circuit breaker instantly with zero network overhead) before fallback
|
||||
# is attempted. Classify as non-retryable + should_fallback so the
|
||||
# retry loop activates the next fallback provider on the first hit.
|
||||
if (
|
||||
error_type == "RuntimeError"
|
||||
and "consecutive stale attempts" in error_msg
|
||||
and "aborting this call" in error_msg
|
||||
):
|
||||
return _result(
|
||||
FailoverReason.timeout,
|
||||
retryable=False,
|
||||
should_fallback=True,
|
||||
)
|
||||
|
||||
# ── 8. Transport / timeout heuristics ───────────────────────────
|
||||
|
||||
if error_type in _TRANSPORT_ERROR_TYPES or isinstance(error, (TimeoutError, ConnectionError, OSError)):
|
||||
return _result(FailoverReason.timeout, retryable=True)
|
||||
|
||||
# ── 8. Fallback: unknown ────────────────────────────────────────
|
||||
# ── 9. Fallback: unknown ────────────────────────────────────────
|
||||
|
||||
return _result(FailoverReason.unknown, retryable=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -2067,3 +2067,26 @@ class Test408RequestTimeout:
|
|||
assert result.retryable is True
|
||||
assert result.should_compress is False
|
||||
|
||||
def test_stale_breaker_runtime_error_triggers_fallback_not_retry(self):
|
||||
# The cross-turn stale-call circuit breaker (_check_stale_giveup in
|
||||
# chat_completion_helpers.py) raises a RuntimeError when the provider
|
||||
# has been unresponsive for N consecutive stale attempts. This must
|
||||
# be classified as non-retryable + should_fallback so the retry loop
|
||||
# activates the fallback provider immediately instead of burning all
|
||||
# max_retries against the same dead provider (each retry hitting the
|
||||
# circuit breaker instantly with zero network overhead).
|
||||
e = RuntimeError(
|
||||
"Provider has been unresponsive (no response received) for "
|
||||
"6 consecutive stale attempts — aborting this call to "
|
||||
"avoid an indefinite stall. Switch models or start a new "
|
||||
"session, then retry."
|
||||
)
|
||||
result = classify_api_error(
|
||||
e, provider="openrouter", model="anthropic/claude-fable-5",
|
||||
approx_tokens=126327, context_length=200000, num_messages=274,
|
||||
)
|
||||
assert result.reason == FailoverReason.timeout
|
||||
assert result.retryable is False
|
||||
assert result.should_fallback is True
|
||||
assert result.should_compress is False
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue