mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-14 14:12:44 +00:00
fix(auxiliary): recover from stale fallback-candidate credentials instead of aborting
A fallback candidate can itself carry a stale credential (e.g. an expired ANTHROPIC_TOKEN picked up by _try_anthropic). Its 401 previously propagated out of the fallback call site and aborted the auxiliary task — for compression: a 60s cooldown + context marker while the session kept growing past the context cap. Live case: mattalachia debug dump (Jul 2026), Codex timeout → Anthropic 401 x5 → 296K 'Cannot compress further'. Now each fallback candidate call is wrapped: on auth error, refresh the candidate's provider credentials and retry once; if unrefreshable, mark the provider unhealthy and walk the discovery chain again so the next viable candidate serves. Sync + async paths. Non-auth errors still raise unchanged.
This commit is contained in:
parent
f69e3aadf1
commit
d42e9b1788
2 changed files with 296 additions and 17 deletions
|
|
@ -2159,6 +2159,136 @@ class TestCallLlmPaymentFallback:
|
|||
mock_fb.assert_not_called()
|
||||
|
||||
|
||||
class TestStaleFallbackCandidateSkip:
|
||||
"""A fallback candidate with a stale credential must not abort the task.
|
||||
|
||||
Live case (mattalachia debug dump, Jul 2026): Codex compression timed out,
|
||||
the aux chain fell back to Anthropic using an expired ANTHROPIC_TOKEN, and
|
||||
the resulting 401 aborted compression with a 60s cooldown — five times in
|
||||
one session — even though refreshing or skipping the candidate would have
|
||||
let compression proceed.
|
||||
"""
|
||||
|
||||
def _timeout_err(self):
|
||||
# Class name carries "Timeout" — matches _is_connection_error's
|
||||
# type-name detection, like the real Codex stream-deadline error.
|
||||
class _AuxStreamTimeoutError(Exception):
|
||||
pass
|
||||
return _AuxStreamTimeoutError(
|
||||
"Codex auxiliary Responses stream exceeded 120.0s total timeout")
|
||||
|
||||
def test_stale_anthropic_fallback_refreshes_and_retries(self, monkeypatch):
|
||||
"""401 from the fallback candidate → refresh its creds → retry succeeds."""
|
||||
primary_client = MagicMock()
|
||||
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
primary_client.chat.completions.create.side_effect = self._timeout_err()
|
||||
|
||||
stale_fb = MagicMock()
|
||||
stale_fb.base_url = "https://api.anthropic.com"
|
||||
stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token")
|
||||
|
||||
fresh_fb = MagicMock()
|
||||
fresh_fb.base_url = "https://api.anthropic.com"
|
||||
fresh_fb.chat.completions.create.return_value = _DummyResponse("fresh-fallback")
|
||||
|
||||
def _cached_client(provider, model=None, **kw):
|
||||
if provider == "anthropic":
|
||||
return (fresh_fb, "claude-haiku-4-5-20251001")
|
||||
return (primary_client, "gpt-5.5")
|
||||
|
||||
with patch("agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None)), \
|
||||
patch("agent.auxiliary_client._get_cached_client", side_effect=_cached_client), \
|
||||
patch("agent.auxiliary_client._try_configured_fallback_chain",
|
||||
return_value=(None, None, "")), \
|
||||
patch("agent.auxiliary_client._try_main_fallback_chain",
|
||||
return_value=(None, None, "")), \
|
||||
patch("agent.auxiliary_client._try_payment_fallback",
|
||||
return_value=(stale_fb, "claude-haiku-4-5-20251001", "anthropic")), \
|
||||
patch("agent.auxiliary_client._refresh_provider_credentials",
|
||||
return_value=True) as mock_refresh:
|
||||
result = call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "summarize"}],
|
||||
)
|
||||
|
||||
assert result.choices[0].message.content == "fresh-fallback"
|
||||
mock_refresh.assert_called_once_with("anthropic")
|
||||
assert stale_fb.chat.completions.create.call_count == 1
|
||||
assert fresh_fb.chat.completions.create.call_count == 1
|
||||
|
||||
def test_unrefreshable_stale_candidate_is_skipped_to_next(self, monkeypatch):
|
||||
"""Refresh fails (expired setup token) → candidate quarantined, chain
|
||||
walked again, next candidate serves the request."""
|
||||
primary_client = MagicMock()
|
||||
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
primary_client.chat.completions.create.side_effect = self._timeout_err()
|
||||
|
||||
stale_fb = MagicMock()
|
||||
stale_fb.base_url = "https://api.anthropic.com"
|
||||
stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token")
|
||||
|
||||
healthy_fb = MagicMock()
|
||||
healthy_fb.base_url = "https://openrouter.ai/api/v1"
|
||||
healthy_fb.chat.completions.create.return_value = _DummyResponse("openrouter-serves")
|
||||
|
||||
fb_walks = [
|
||||
(stale_fb, "claude-haiku-4-5-20251001", "anthropic"),
|
||||
(healthy_fb, "fallback-model", "openrouter"),
|
||||
]
|
||||
|
||||
with patch("agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None)), \
|
||||
patch("agent.auxiliary_client._get_cached_client",
|
||||
return_value=(primary_client, "gpt-5.5")), \
|
||||
patch("agent.auxiliary_client._try_configured_fallback_chain",
|
||||
return_value=(None, None, "")), \
|
||||
patch("agent.auxiliary_client._try_main_fallback_chain",
|
||||
return_value=(None, None, "")), \
|
||||
patch("agent.auxiliary_client._try_payment_fallback",
|
||||
side_effect=fb_walks) as mock_fb, \
|
||||
patch("agent.auxiliary_client._refresh_provider_credentials",
|
||||
return_value=False), \
|
||||
patch("agent.auxiliary_client._mark_provider_unhealthy") as mock_mark:
|
||||
result = call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "summarize"}],
|
||||
)
|
||||
|
||||
assert result.choices[0].message.content == "openrouter-serves"
|
||||
assert mock_fb.call_count == 2
|
||||
assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential"
|
||||
mock_mark.assert_called_once_with("anthropic")
|
||||
assert stale_fb.chat.completions.create.call_count == 1
|
||||
assert healthy_fb.chat.completions.create.call_count == 1
|
||||
|
||||
def test_non_auth_fallback_error_still_raises(self, monkeypatch):
|
||||
"""A non-auth error from the fallback candidate propagates unchanged."""
|
||||
primary_client = MagicMock()
|
||||
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
primary_client.chat.completions.create.side_effect = self._timeout_err()
|
||||
|
||||
broken_fb = MagicMock()
|
||||
broken_fb.base_url = "https://api.anthropic.com"
|
||||
broken_fb.chat.completions.create.side_effect = ValueError("malformed response")
|
||||
|
||||
with patch("agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None)), \
|
||||
patch("agent.auxiliary_client._get_cached_client",
|
||||
return_value=(primary_client, "gpt-5.5")), \
|
||||
patch("agent.auxiliary_client._try_configured_fallback_chain",
|
||||
return_value=(None, None, "")), \
|
||||
patch("agent.auxiliary_client._try_main_fallback_chain",
|
||||
return_value=(None, None, "")), \
|
||||
patch("agent.auxiliary_client._try_payment_fallback",
|
||||
return_value=(broken_fb, "claude-haiku-4-5-20251001", "anthropic")):
|
||||
with pytest.raises(ValueError, match="malformed response"):
|
||||
call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "summarize"}],
|
||||
)
|
||||
|
||||
|
||||
class TestAuxiliaryFallbackLayering:
|
||||
"""Explicit-provider users get layered fallback: configured_chain → main agent → warn."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue