fix(auxiliary): recover from stale fallback-candidate credentials instead of aborting

A fallback candidate can itself carry a stale credential (e.g. an
expired ANTHROPIC_TOKEN picked up by _try_anthropic). Its 401 previously
propagated out of the fallback call site and aborted the auxiliary task
— for compression: a 60s cooldown + context marker while the session
kept growing past the context cap. Live case: mattalachia debug dump
(Jul 2026), Codex timeout → Anthropic 401 x5 → 296K 'Cannot compress
further'.

Now each fallback candidate call is wrapped: on auth error, refresh the
candidate's provider credentials and retry once; if unrefreshable, mark
the provider unhealthy and walk the discovery chain again so the next
viable candidate serves. Sync + async paths. Non-auth errors still
raise unchanged.
This commit is contained in:
teknium1 2026-07-06 13:26:42 -07:00 committed by Teknium
parent f69e3aadf1
commit d42e9b1788
2 changed files with 296 additions and 17 deletions

View file

@ -2159,6 +2159,136 @@ class TestCallLlmPaymentFallback:
mock_fb.assert_not_called()
class TestStaleFallbackCandidateSkip:
"""A fallback candidate with a stale credential must not abort the task.
Live case (mattalachia debug dump, Jul 2026): Codex compression timed out,
the aux chain fell back to Anthropic using an expired ANTHROPIC_TOKEN, and
the resulting 401 aborted compression with a 60s cooldown five times in
one session even though refreshing or skipping the candidate would have
let compression proceed.
"""
def _timeout_err(self):
# Class name carries "Timeout" — matches _is_connection_error's
# type-name detection, like the real Codex stream-deadline error.
class _AuxStreamTimeoutError(Exception):
pass
return _AuxStreamTimeoutError(
"Codex auxiliary Responses stream exceeded 120.0s total timeout")
def test_stale_anthropic_fallback_refreshes_and_retries(self, monkeypatch):
"""401 from the fallback candidate → refresh its creds → retry succeeds."""
primary_client = MagicMock()
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
primary_client.chat.completions.create.side_effect = self._timeout_err()
stale_fb = MagicMock()
stale_fb.base_url = "https://api.anthropic.com"
stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token")
fresh_fb = MagicMock()
fresh_fb.base_url = "https://api.anthropic.com"
fresh_fb.chat.completions.create.return_value = _DummyResponse("fresh-fallback")
def _cached_client(provider, model=None, **kw):
if provider == "anthropic":
return (fresh_fb, "claude-haiku-4-5-20251001")
return (primary_client, "gpt-5.5")
with patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None)), \
patch("agent.auxiliary_client._get_cached_client", side_effect=_cached_client), \
patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_main_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_payment_fallback",
return_value=(stale_fb, "claude-haiku-4-5-20251001", "anthropic")), \
patch("agent.auxiliary_client._refresh_provider_credentials",
return_value=True) as mock_refresh:
result = call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
assert result.choices[0].message.content == "fresh-fallback"
mock_refresh.assert_called_once_with("anthropic")
assert stale_fb.chat.completions.create.call_count == 1
assert fresh_fb.chat.completions.create.call_count == 1
def test_unrefreshable_stale_candidate_is_skipped_to_next(self, monkeypatch):
"""Refresh fails (expired setup token) → candidate quarantined, chain
walked again, next candidate serves the request."""
primary_client = MagicMock()
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
primary_client.chat.completions.create.side_effect = self._timeout_err()
stale_fb = MagicMock()
stale_fb.base_url = "https://api.anthropic.com"
stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token")
healthy_fb = MagicMock()
healthy_fb.base_url = "https://openrouter.ai/api/v1"
healthy_fb.chat.completions.create.return_value = _DummyResponse("openrouter-serves")
fb_walks = [
(stale_fb, "claude-haiku-4-5-20251001", "anthropic"),
(healthy_fb, "fallback-model", "openrouter"),
]
with patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None)), \
patch("agent.auxiliary_client._get_cached_client",
return_value=(primary_client, "gpt-5.5")), \
patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_main_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_payment_fallback",
side_effect=fb_walks) as mock_fb, \
patch("agent.auxiliary_client._refresh_provider_credentials",
return_value=False), \
patch("agent.auxiliary_client._mark_provider_unhealthy") as mock_mark:
result = call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
assert result.choices[0].message.content == "openrouter-serves"
assert mock_fb.call_count == 2
assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential"
mock_mark.assert_called_once_with("anthropic")
assert stale_fb.chat.completions.create.call_count == 1
assert healthy_fb.chat.completions.create.call_count == 1
def test_non_auth_fallback_error_still_raises(self, monkeypatch):
"""A non-auth error from the fallback candidate propagates unchanged."""
primary_client = MagicMock()
primary_client.base_url = "https://chatgpt.com/backend-api/codex"
primary_client.chat.completions.create.side_effect = self._timeout_err()
broken_fb = MagicMock()
broken_fb.base_url = "https://api.anthropic.com"
broken_fb.chat.completions.create.side_effect = ValueError("malformed response")
with patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None)), \
patch("agent.auxiliary_client._get_cached_client",
return_value=(primary_client, "gpt-5.5")), \
patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_main_fallback_chain",
return_value=(None, None, "")), \
patch("agent.auxiliary_client._try_payment_fallback",
return_value=(broken_fb, "claude-haiku-4-5-20251001", "anthropic")):
with pytest.raises(ValueError, match="malformed response"):
call_llm(
task="compression",
messages=[{"role": "user", "content": "summarize"}],
)
class TestAuxiliaryFallbackLayering:
"""Explicit-provider users get layered fallback: configured_chain → main agent → warn."""