mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-08 13:12:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix(auxiliary): recover from stale fallback-candidate credentials instead of aborting

Browse source 
A fallback candidate can itself carry a stale credential (e.g. an
expired ANTHROPIC_TOKEN picked up by _try_anthropic). Its 401 previously
propagated out of the fallback call site and aborted the auxiliary task
— for compression: a 60s cooldown + context marker while the session
kept growing past the context cap. Live case: mattalachia debug dump
(Jul 2026), Codex timeout → Anthropic 401 x5 → 296K 'Cannot compress
further'.

Now each fallback candidate call is wrapped: on auth error, refresh the
candidate's provider credentials and retry once; if unrefreshable, mark
the provider unhealthy and walk the discovery chain again so the next
viable candidate serves. Sync + async paths. Non-auth errors still
raise unchanged.
This commit is contained in:
teknium1 2026-07-06 13:26:42 -07:00 • committed by Teknium
parent f69e3aadf1
commit d42e9b1788
2 changed files with 296 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

183
agent/auxiliary_client.py
View file

@ -3482,6 +3482,127 @@ def _auth_refresh_provider_for_route(
return normalized
def _call_fallback_candidate_sync(
fb_client: Any,
fb_model: Optional[str],
fb_label: str,
*,
task: Optional[str],
messages: list,
temperature: Optional[float],
max_tokens: Optional[int],
tools: Optional[list],
effective_timeout: float,
effective_extra_body: dict,
) -> Optional[Any]:
"""Call one fallback candidate with stale-credential recovery.
A fallback candidate can itself carry a stale credential (e.g. an expired
``ANTHROPIC_TOKEN`` picked up by ``_try_anthropic``). Before this helper,
such a 401 propagated out of the fallback site and aborted the auxiliary
task (for compression: a 60s cooldown + context marker) even though other
healthy candidates remained. Live case: a Codex-timeout → Anthropic
fallback 401-looped five times in one session (mattalachia debug dump,
Jul 2026).
On an auth error: refresh the candidate's provider credentials and retry
once with a rebuilt client; if the retry also auth-fails (non-refreshable
expired token), mark the provider unhealthy and return ``None`` so the
caller can continue to the next fallback layer. Non-auth errors raise.
"""
fb_base = str(getattr(fb_client, "base_url", "") or "")
fb_kwargs = _build_call_kwargs(
fb_label, fb_model, messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body, base_url=fb_base)
try:
return _validate_llm_response(
fb_client.chat.completions.create(**fb_kwargs), task)
except Exception as fb_err:
if not _is_auth_error(fb_err):
raise
fb_provider = _auth_refresh_provider_for_route(fb_label, fb_base)
if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider):
retry_client, retry_model = _get_cached_client(fb_provider, fb_model)
if retry_client is not None:
retry_kwargs = _build_call_kwargs(
fb_provider, retry_model or fb_model, messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body,
base_url=str(getattr(retry_client, "base_url", "") or fb_base))
try:
return _validate_llm_response(
retry_client.chat.completions.create(**retry_kwargs), task)
except Exception as retry_err:
if not _is_auth_error(retry_err):
raise
# Refresh unavailable or the refreshed credential still 401s —
# the token is dead (expired setup token with no refresh token).
# Quarantine the candidate so subsequent chain walks skip it, and
# let the caller move on instead of aborting the whole task.
_mark_provider_unhealthy(fb_provider or fb_label)
logger.warning(
"Auxiliary %s: fallback candidate %s has a stale/unrefreshable "
"credential (%s) — skipping to next fallback",
task or "call", fb_label, fb_err,
)
return None
async def _call_fallback_candidate_async(
fb_client: Any,
fb_model: Optional[str],
fb_label: str,
*,
task: Optional[str],
messages: list,
temperature: Optional[float],
max_tokens: Optional[int],
tools: Optional[list],
effective_timeout: float,
effective_extra_body: dict,
) -> Optional[Any]:
"""Async mirror of :func:`_call_fallback_candidate_sync`."""
fb_base = str(getattr(fb_client, "base_url", "") or "")
fb_kwargs = _build_call_kwargs(
fb_label, fb_model, messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body, base_url=fb_base)
try:
return _validate_llm_response(
await fb_client.chat.completions.create(**fb_kwargs), task)
except Exception as fb_err:
if not _is_auth_error(fb_err):
raise
fb_provider = _auth_refresh_provider_for_route(fb_label, fb_base)
if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider):
retry_client, retry_model = _get_cached_client(
fb_provider, fb_model, async_mode=True)
if retry_client is not None:
retry_kwargs = _build_call_kwargs(
fb_provider, retry_model or fb_model, messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body,
base_url=str(getattr(retry_client, "base_url", "") or fb_base))
try:
return _validate_llm_response(
await retry_client.chat.completions.create(**retry_kwargs), task)
except Exception as retry_err:
if not _is_auth_error(retry_err):
raise
_mark_provider_unhealthy(fb_provider or fb_label)
logger.warning(
"Auxiliary %s (async): fallback candidate %s has a stale/unrefreshable "
"credential (%s) — skipping to next fallback",
task or "call", fb_label, fb_err,
)
return None
def _try_payment_fallback(
failed_provider: str,
task: str = None,
@ -6691,14 +6812,28 @@ def call_llm(
resolved_provider, task, reason=reason)
if fb_client is not None:
fb_kwargs = _build_call_kwargs(
fb_label, fb_model, messages,
fb_resp = _call_fallback_candidate_sync(
fb_client, fb_model, fb_label,
task=task, messages=messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body,
base_url=str(getattr(fb_client, "base_url", "") or ""))
return _validate_llm_response(
fb_client.chat.completions.create(**fb_kwargs), task)
tools=tools, effective_timeout=effective_timeout,
effective_extra_body=effective_extra_body)
if fb_resp is not None:
return fb_resp
# The candidate had a stale/unrefreshable credential and was
# quarantined — walk the discovery chain once more; unhealthy
# entries are skipped so the next viable candidate serves.
fb_client, fb_model, fb_label = _try_payment_fallback(
resolved_provider, task, reason="stale fallback credential")
if fb_client is not None:
fb_resp = _call_fallback_candidate_sync(
fb_client, fb_model, fb_label,
task=task, messages=messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, effective_timeout=effective_timeout,
effective_extra_body=effective_extra_body)
if fb_resp is not None:
return fb_resp
# All fallback layers exhausted — emit a single user-visible
# warning so the operator knows aux task is about to fail.
# (#26882) The error itself is re-raised below.
@ -7182,20 +7317,34 @@ async def async_call_llm(
resolved_provider, task, reason=reason)
if fb_client is not None:
fb_kwargs = _build_call_kwargs(
fb_label, fb_model, messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, timeout=effective_timeout,
extra_body=effective_extra_body,
base_url=str(getattr(fb_client, "base_url", "") or ""))
# Convert sync fallback client to async
async_fb, async_fb_model = _to_async_client(
fb_client, fb_model or "", is_vision=(task == "vision")
)
if async_fb_model and async_fb_model != fb_kwargs.get("model"):
fb_kwargs["model"] = async_fb_model
return _validate_llm_response(
await async_fb.chat.completions.create(**fb_kwargs), task)
fb_resp = await _call_fallback_candidate_async(
async_fb, async_fb_model or fb_model, fb_label,
task=task, messages=messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, effective_timeout=effective_timeout,
effective_extra_body=effective_extra_body)
if fb_resp is not None:
return fb_resp
# Stale/unrefreshable candidate credential — quarantined; walk
# the discovery chain once more (unhealthy entries skipped).
fb_client, fb_model, fb_label = _try_payment_fallback(
resolved_provider, task, reason="stale fallback credential")
if fb_client is not None:
async_fb, async_fb_model = _to_async_client(
fb_client, fb_model or "", is_vision=(task == "vision")
)
fb_resp = await _call_fallback_candidate_async(
async_fb, async_fb_model or fb_model, fb_label,
task=task, messages=messages,
temperature=temperature, max_tokens=max_tokens,
tools=tools, effective_timeout=effective_timeout,
effective_extra_body=effective_extra_body)
if fb_resp is not None:
return fb_resp
# All fallback layers exhausted — warn before re-raising. (#26882)
logger.warning(
"Auxiliary %s (async): %s on %s and all fallbacks exhausted "

130
tests/agent/test_auxiliary_client.py
View file

@ -2159,6 +2159,136 @@ class TestCallLlmPaymentFallback:
mock_fb.assert_not_called()
class TestStaleFallbackCandidateSkip:
"""A fallback candidate with a stale credential must not abort the task.
Live case (mattalachia debug dump, Jul 2026): Codex compression timed out,
the aux chain fell back to Anthropic using an expired ANTHROPIC_TOKEN, and
the resulting 401 aborted compression with a 60s cooldown — five times in
one session — even though refreshing or skipping the candidate would have
let compression proceed.
"""
def _timeout_err(self):
# Class name carries "Timeout" — matches _is_connection_error's
# type-name detection, like the real Codex stream-deadline error.
class _AuxStreamTimeoutError(Exception):
pass
return _AuxStreamTimeoutError(
"Codex auxiliary Responses stream exceeded 120.0s total timeout")
def test_stale_anthropic_fallback_refreshes_and_retries(self, monkeypatch):
"""401 from the fallback candidate → refresh its creds → retry succeeds."""
primary_client = MagicMock()
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
primary_client.chat.completions.create.side_effect = self._timeout_err()
stale_fb = MagicMock()
stale_fb.base_url = "https://api.anthropic.com"
stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token")
fresh_fb = MagicMock()
fresh_fb.base_url = "https://api.anthropic.com"
fresh_fb.chat.completions.create.return_value = _DummyResponse("fresh-fallback")
def _cached_client(provider, model=None, **kw):
if provider == "anthropic":
return (fresh_fb, "claude-haiku-4-5-20251001")
return (primary_client, "gpt-5.5")
with patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None)), \
patch("agent.auxiliary_client._get_cached_client", side_effect=_cached_client), \
patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_main_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_payment_fallback",
return_value=(stale_fb, "claude-haiku-4-5-20251001", "anthropic")), \
patch("agent.auxiliary_client._refresh_provider_credentials",
return_value=True) as mock_refresh:
result = call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
assert result.choices[0].message.content == "fresh-fallback"
mock_refresh.assert_called_once_with("anthropic")
assert stale_fb.chat.completions.create.call_count == 1
assert fresh_fb.chat.completions.create.call_count == 1
def test_unrefreshable_stale_candidate_is_skipped_to_next(self, monkeypatch):
"""Refresh fails (expired setup token) → candidate quarantined, chain
walked again, next candidate serves the request."""
primary_client = MagicMock()
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
primary_client.chat.completions.create.side_effect = self._timeout_err()
stale_fb = MagicMock()
stale_fb.base_url = "https://api.anthropic.com"
stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token")
healthy_fb = MagicMock()
healthy_fb.base_url = "https://openrouter.ai/api/v1"
healthy_fb.chat.completions.create.return_value = _DummyResponse("openrouter-serves")
fb_walks = [
(stale_fb, "claude-haiku-4-5-20251001", "anthropic"),
(healthy_fb, "fallback-model", "openrouter"),
]
with patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None)), \
patch("agent.auxiliary_client._get_cached_client",
return_value=(primary_client, "gpt-5.5")), \
patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_main_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_payment_fallback",
side_effect=fb_walks) as mock_fb, \
patch("agent.auxiliary_client._refresh_provider_credentials",
return_value=False), \
patch("agent.auxiliary_client._mark_provider_unhealthy") as mock_mark:
result = call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
assert result.choices[0].message.content == "openrouter-serves"
assert mock_fb.call_count == 2
assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential"
mock_mark.assert_called_once_with("anthropic")
assert stale_fb.chat.completions.create.call_count == 1
assert healthy_fb.chat.completions.create.call_count == 1
def test_non_auth_fallback_error_still_raises(self, monkeypatch):
"""A non-auth error from the fallback candidate propagates unchanged."""
primary_client = MagicMock()
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
primary_client.chat.completions.create.side_effect = self._timeout_err()
broken_fb = MagicMock()
broken_fb.base_url = "https://api.anthropic.com"
broken_fb.chat.completions.create.side_effect = ValueError("malformed response")
with patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None)), \
patch("agent.auxiliary_client._get_cached_client",
return_value=(primary_client, "gpt-5.5")), \
patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_main_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_payment_fallback",
return_value=(broken_fb, "claude-haiku-4-5-20251001", "anthropic")):
with pytest.raises(ValueError, match="malformed response"):
call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
class TestAuxiliaryFallbackLayering:
"""Explicit-provider users get layered fallback: configured_chain → main agent → warn."""