From d42e9b1788d9107962d2f00edb98069ab2ed9bdb Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:26:42 -0700 Subject: [PATCH] fix(auxiliary): recover from stale fallback-candidate credentials instead of aborting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fallback candidate can itself carry a stale credential (e.g. an expired ANTHROPIC_TOKEN picked up by _try_anthropic). Its 401 previously propagated out of the fallback call site and aborted the auxiliary task — for compression: a 60s cooldown + context marker while the session kept growing past the context cap. Live case: mattalachia debug dump (Jul 2026), Codex timeout → Anthropic 401 x5 → 296K 'Cannot compress further'. Now each fallback candidate call is wrapped: on auth error, refresh the candidate's provider credentials and retry once; if unrefreshable, mark the provider unhealthy and walk the discovery chain again so the next viable candidate serves. Sync + async paths. Non-auth errors still raise unchanged. --- agent/auxiliary_client.py | 183 ++++++++++++++++++++++++--- tests/agent/test_auxiliary_client.py | 130 +++++++++++++++++++ 2 files changed, 296 insertions(+), 17 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index bc14c69f541..93b7abcef60 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -3482,6 +3482,127 @@ def _auth_refresh_provider_for_route( return normalized +def _call_fallback_candidate_sync( + fb_client: Any, + fb_model: Optional[str], + fb_label: str, + *, + task: Optional[str], + messages: list, + temperature: Optional[float], + max_tokens: Optional[int], + tools: Optional[list], + effective_timeout: float, + effective_extra_body: dict, +) -> Optional[Any]: + """Call one fallback candidate with stale-credential recovery. + + A fallback candidate can itself carry a stale credential (e.g. an expired + ``ANTHROPIC_TOKEN`` picked up by ``_try_anthropic``). Before this helper, + such a 401 propagated out of the fallback site and aborted the auxiliary + task (for compression: a 60s cooldown + context marker) even though other + healthy candidates remained. Live case: a Codex-timeout → Anthropic + fallback 401-looped five times in one session (mattalachia debug dump, + Jul 2026). + + On an auth error: refresh the candidate's provider credentials and retry + once with a rebuilt client; if the retry also auth-fails (non-refreshable + expired token), mark the provider unhealthy and return ``None`` so the + caller can continue to the next fallback layer. Non-auth errors raise. + """ + fb_base = str(getattr(fb_client, "base_url", "") or "") + fb_kwargs = _build_call_kwargs( + fb_label, fb_model, messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, timeout=effective_timeout, + extra_body=effective_extra_body, base_url=fb_base) + try: + return _validate_llm_response( + fb_client.chat.completions.create(**fb_kwargs), task) + except Exception as fb_err: + if not _is_auth_error(fb_err): + raise + fb_provider = _auth_refresh_provider_for_route(fb_label, fb_base) + if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider): + retry_client, retry_model = _get_cached_client(fb_provider, fb_model) + if retry_client is not None: + retry_kwargs = _build_call_kwargs( + fb_provider, retry_model or fb_model, messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, timeout=effective_timeout, + extra_body=effective_extra_body, + base_url=str(getattr(retry_client, "base_url", "") or fb_base)) + try: + return _validate_llm_response( + retry_client.chat.completions.create(**retry_kwargs), task) + except Exception as retry_err: + if not _is_auth_error(retry_err): + raise + # Refresh unavailable or the refreshed credential still 401s — + # the token is dead (expired setup token with no refresh token). + # Quarantine the candidate so subsequent chain walks skip it, and + # let the caller move on instead of aborting the whole task. + _mark_provider_unhealthy(fb_provider or fb_label) + logger.warning( + "Auxiliary %s: fallback candidate %s has a stale/unrefreshable " + "credential (%s) — skipping to next fallback", + task or "call", fb_label, fb_err, + ) + return None + + +async def _call_fallback_candidate_async( + fb_client: Any, + fb_model: Optional[str], + fb_label: str, + *, + task: Optional[str], + messages: list, + temperature: Optional[float], + max_tokens: Optional[int], + tools: Optional[list], + effective_timeout: float, + effective_extra_body: dict, +) -> Optional[Any]: + """Async mirror of :func:`_call_fallback_candidate_sync`.""" + fb_base = str(getattr(fb_client, "base_url", "") or "") + fb_kwargs = _build_call_kwargs( + fb_label, fb_model, messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, timeout=effective_timeout, + extra_body=effective_extra_body, base_url=fb_base) + try: + return _validate_llm_response( + await fb_client.chat.completions.create(**fb_kwargs), task) + except Exception as fb_err: + if not _is_auth_error(fb_err): + raise + fb_provider = _auth_refresh_provider_for_route(fb_label, fb_base) + if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider): + retry_client, retry_model = _get_cached_client( + fb_provider, fb_model, async_mode=True) + if retry_client is not None: + retry_kwargs = _build_call_kwargs( + fb_provider, retry_model or fb_model, messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, timeout=effective_timeout, + extra_body=effective_extra_body, + base_url=str(getattr(retry_client, "base_url", "") or fb_base)) + try: + return _validate_llm_response( + await retry_client.chat.completions.create(**retry_kwargs), task) + except Exception as retry_err: + if not _is_auth_error(retry_err): + raise + _mark_provider_unhealthy(fb_provider or fb_label) + logger.warning( + "Auxiliary %s (async): fallback candidate %s has a stale/unrefreshable " + "credential (%s) — skipping to next fallback", + task or "call", fb_label, fb_err, + ) + return None + + def _try_payment_fallback( failed_provider: str, task: str = None, @@ -6691,14 +6812,28 @@ def call_llm( resolved_provider, task, reason=reason) if fb_client is not None: - fb_kwargs = _build_call_kwargs( - fb_label, fb_model, messages, + fb_resp = _call_fallback_candidate_sync( + fb_client, fb_model, fb_label, + task=task, messages=messages, temperature=temperature, max_tokens=max_tokens, - tools=tools, timeout=effective_timeout, - extra_body=effective_extra_body, - base_url=str(getattr(fb_client, "base_url", "") or "")) - return _validate_llm_response( - fb_client.chat.completions.create(**fb_kwargs), task) + tools=tools, effective_timeout=effective_timeout, + effective_extra_body=effective_extra_body) + if fb_resp is not None: + return fb_resp + # The candidate had a stale/unrefreshable credential and was + # quarantined — walk the discovery chain once more; unhealthy + # entries are skipped so the next viable candidate serves. + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason="stale fallback credential") + if fb_client is not None: + fb_resp = _call_fallback_candidate_sync( + fb_client, fb_model, fb_label, + task=task, messages=messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, effective_timeout=effective_timeout, + effective_extra_body=effective_extra_body) + if fb_resp is not None: + return fb_resp # All fallback layers exhausted — emit a single user-visible # warning so the operator knows aux task is about to fail. # (#26882) The error itself is re-raised below. @@ -7182,20 +7317,34 @@ async def async_call_llm( resolved_provider, task, reason=reason) if fb_client is not None: - fb_kwargs = _build_call_kwargs( - fb_label, fb_model, messages, - temperature=temperature, max_tokens=max_tokens, - tools=tools, timeout=effective_timeout, - extra_body=effective_extra_body, - base_url=str(getattr(fb_client, "base_url", "") or "")) # Convert sync fallback client to async async_fb, async_fb_model = _to_async_client( fb_client, fb_model or "", is_vision=(task == "vision") ) - if async_fb_model and async_fb_model != fb_kwargs.get("model"): - fb_kwargs["model"] = async_fb_model - return _validate_llm_response( - await async_fb.chat.completions.create(**fb_kwargs), task) + fb_resp = await _call_fallback_candidate_async( + async_fb, async_fb_model or fb_model, fb_label, + task=task, messages=messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, effective_timeout=effective_timeout, + effective_extra_body=effective_extra_body) + if fb_resp is not None: + return fb_resp + # Stale/unrefreshable candidate credential — quarantined; walk + # the discovery chain once more (unhealthy entries skipped). + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason="stale fallback credential") + if fb_client is not None: + async_fb, async_fb_model = _to_async_client( + fb_client, fb_model or "", is_vision=(task == "vision") + ) + fb_resp = await _call_fallback_candidate_async( + async_fb, async_fb_model or fb_model, fb_label, + task=task, messages=messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, effective_timeout=effective_timeout, + effective_extra_body=effective_extra_body) + if fb_resp is not None: + return fb_resp # All fallback layers exhausted — warn before re-raising. (#26882) logger.warning( "Auxiliary %s (async): %s on %s and all fallbacks exhausted " diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 59ad0286e23..5e18fa73a1f 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -2159,6 +2159,136 @@ class TestCallLlmPaymentFallback: mock_fb.assert_not_called() +class TestStaleFallbackCandidateSkip: + """A fallback candidate with a stale credential must not abort the task. + + Live case (mattalachia debug dump, Jul 2026): Codex compression timed out, + the aux chain fell back to Anthropic using an expired ANTHROPIC_TOKEN, and + the resulting 401 aborted compression with a 60s cooldown — five times in + one session — even though refreshing or skipping the candidate would have + let compression proceed. + """ + + def _timeout_err(self): + # Class name carries "Timeout" — matches _is_connection_error's + # type-name detection, like the real Codex stream-deadline error. + class _AuxStreamTimeoutError(Exception): + pass + return _AuxStreamTimeoutError( + "Codex auxiliary Responses stream exceeded 120.0s total timeout") + + def test_stale_anthropic_fallback_refreshes_and_retries(self, monkeypatch): + """401 from the fallback candidate → refresh its creds → retry succeeds.""" + primary_client = MagicMock() + primary_client.base_url = "https://chatgpt.com/backend-api/codex" + primary_client.chat.completions.create.side_effect = self._timeout_err() + + stale_fb = MagicMock() + stale_fb.base_url = "https://api.anthropic.com" + stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token") + + fresh_fb = MagicMock() + fresh_fb.base_url = "https://api.anthropic.com" + fresh_fb.chat.completions.create.return_value = _DummyResponse("fresh-fallback") + + def _cached_client(provider, model=None, **kw): + if provider == "anthropic": + return (fresh_fb, "claude-haiku-4-5-20251001") + return (primary_client, "gpt-5.5") + + with patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("auto", None, None, None, None)), \ + patch("agent.auxiliary_client._get_cached_client", side_effect=_cached_client), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_payment_fallback", + return_value=(stale_fb, "claude-haiku-4-5-20251001", "anthropic")), \ + patch("agent.auxiliary_client._refresh_provider_credentials", + return_value=True) as mock_refresh: + result = call_llm( + task="compression", + messages=[{"role": "user", "content": "summarize"}], + ) + + assert result.choices[0].message.content == "fresh-fallback" + mock_refresh.assert_called_once_with("anthropic") + assert stale_fb.chat.completions.create.call_count == 1 + assert fresh_fb.chat.completions.create.call_count == 1 + + def test_unrefreshable_stale_candidate_is_skipped_to_next(self, monkeypatch): + """Refresh fails (expired setup token) → candidate quarantined, chain + walked again, next candidate serves the request.""" + primary_client = MagicMock() + primary_client.base_url = "https://chatgpt.com/backend-api/codex" + primary_client.chat.completions.create.side_effect = self._timeout_err() + + stale_fb = MagicMock() + stale_fb.base_url = "https://api.anthropic.com" + stale_fb.chat.completions.create.side_effect = _AuxAuth401("Invalid bearer token") + + healthy_fb = MagicMock() + healthy_fb.base_url = "https://openrouter.ai/api/v1" + healthy_fb.chat.completions.create.return_value = _DummyResponse("openrouter-serves") + + fb_walks = [ + (stale_fb, "claude-haiku-4-5-20251001", "anthropic"), + (healthy_fb, "fallback-model", "openrouter"), + ] + + with patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("auto", None, None, None, None)), \ + patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "gpt-5.5")), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_payment_fallback", + side_effect=fb_walks) as mock_fb, \ + patch("agent.auxiliary_client._refresh_provider_credentials", + return_value=False), \ + patch("agent.auxiliary_client._mark_provider_unhealthy") as mock_mark: + result = call_llm( + task="compression", + messages=[{"role": "user", "content": "summarize"}], + ) + + assert result.choices[0].message.content == "openrouter-serves" + assert mock_fb.call_count == 2 + assert mock_fb.call_args_list[1].kwargs.get("reason") == "stale fallback credential" + mock_mark.assert_called_once_with("anthropic") + assert stale_fb.chat.completions.create.call_count == 1 + assert healthy_fb.chat.completions.create.call_count == 1 + + def test_non_auth_fallback_error_still_raises(self, monkeypatch): + """A non-auth error from the fallback candidate propagates unchanged.""" + primary_client = MagicMock() + primary_client.base_url = "https://chatgpt.com/backend-api/codex" + primary_client.chat.completions.create.side_effect = self._timeout_err() + + broken_fb = MagicMock() + broken_fb.base_url = "https://api.anthropic.com" + broken_fb.chat.completions.create.side_effect = ValueError("malformed response") + + with patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("auto", None, None, None, None)), \ + patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "gpt-5.5")), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_payment_fallback", + return_value=(broken_fb, "claude-haiku-4-5-20251001", "anthropic")): + with pytest.raises(ValueError, match="malformed response"): + call_llm( + task="compression", + messages=[{"role": "user", "content": "summarize"}], + ) + + class TestAuxiliaryFallbackLayering: """Explicit-provider users get layered fallback: configured_chain → main agent → warn."""