From 766f263bd2453838bb34e98fbe048e09f9fefa25 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 17 May 2026 16:27:06 -0700 Subject: [PATCH] =?UTF-8?q?test(auxiliary):=20cover=20layered=20fallback?= =?UTF-8?q?=20(chain=20=E2=86=92=20main=20agent=20=E2=86=92=20warn)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7 new tests: TestAuxiliaryFallbackLayering (3): - configured_chain succeeds → main agent fallback NOT consulted - chain returns nothing → main agent fallback runs and succeeds - both exhausted → user-visible 'all fallbacks exhausted' warning fires before the original error is re-raised TestTryMainAgentModelFallback (4): - returns (None, None, "") when main provider is 'auto' - returns (None, None, "") when failed provider == main provider (no point retrying the same backend) - resolves the main provider's client when configured correctly - skips when main provider is marked unhealthy --- tests/agent/test_auxiliary_client.py | 134 +++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 49d26825dde..2522fa16197 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -1151,6 +1151,140 @@ class TestCallLlmPaymentFallback: # Fallback client should have been used assert fallback_client.chat.completions.create.called + +class TestAuxiliaryFallbackLayering: + """Explicit-provider users get layered fallback: configured_chain → main agent → warn.""" + + def _make_payment_err(self): + exc = Exception("Payment Required: insufficient credits") + exc.status_code = 402 + return exc + + def test_explicit_provider_uses_configured_chain_first(self, monkeypatch, caplog): + """When a user has fallback_chain configured, it's tried BEFORE the main agent model.""" + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + primary_client = MagicMock() + primary_client.chat.completions.create.side_effect = self._make_payment_err() + + chain_client = MagicMock() + chain_client.chat.completions.create.return_value = MagicMock(choices=[ + MagicMock(message=MagicMock(content="from configured chain")) + ]) + + main_called = MagicMock() + + with patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "glm-4v-flash")), \ + patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("glm", "glm-4v-flash", None, None, None)), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(chain_client, "gpt-4o-mini", "fallback_chain[0](openai)")), \ + patch("agent.auxiliary_client._try_main_agent_model_fallback", + side_effect=main_called): + result = call_llm( + task="vision", + messages=[{"role": "user", "content": "hello"}], + ) + + assert chain_client.chat.completions.create.called + # Main agent fallback should NOT have been consulted — chain succeeded first + main_called.assert_not_called() + + def test_explicit_provider_falls_back_to_main_when_chain_exhausted(self, monkeypatch): + """If configured fallback_chain returns nothing, main agent model is tried next.""" + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + primary_client = MagicMock() + primary_client.chat.completions.create.side_effect = self._make_payment_err() + + main_client = MagicMock() + main_client.chat.completions.create.return_value = MagicMock(choices=[ + MagicMock(message=MagicMock(content="from main agent")) + ]) + + with patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "glm-4v-flash")), \ + patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("glm", "glm-4v-flash", None, None, None)), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_agent_model_fallback", + return_value=(main_client, "claude-sonnet-4", "main-agent(openrouter)")): + result = call_llm( + task="vision", + messages=[{"role": "user", "content": "hello"}], + ) + + assert main_client.chat.completions.create.called + + def test_warning_emitted_when_all_fallbacks_exhausted(self, monkeypatch, caplog): + """When chain AND main model both fail, a user-visible warning fires before re-raise.""" + monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") + + primary_client = MagicMock() + primary_client.chat.completions.create.side_effect = self._make_payment_err() + + with patch("agent.auxiliary_client._get_cached_client", + return_value=(primary_client, "glm-4v-flash")), \ + patch("agent.auxiliary_client._resolve_task_provider_model", + return_value=("glm", "glm-4v-flash", None, None, None)), \ + patch("agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, "")), \ + patch("agent.auxiliary_client._try_main_agent_model_fallback", + return_value=(None, None, "")), \ + caplog.at_level("WARNING", logger="agent.auxiliary_client"): + with pytest.raises(Exception, match="Payment Required"): + call_llm( + task="vision", + messages=[{"role": "user", "content": "hello"}], + ) + + assert any( + "all fallbacks exhausted" in r.message for r in caplog.records + ), f"Expected exhaustion warning, got: {[r.message for r in caplog.records]}" + + +class TestTryMainAgentModelFallback: + """_try_main_agent_model_fallback resolves the user's main provider+model as a safety net.""" + + def test_returns_none_when_main_provider_is_auto(self): + from agent.auxiliary_client import _try_main_agent_model_fallback + with patch("agent.auxiliary_client._read_main_provider", return_value="auto"), \ + patch("agent.auxiliary_client._read_main_model", return_value="some-model"): + client, model, label = _try_main_agent_model_fallback("glm", task="vision") + assert client is None and model is None and label == "" + + def test_returns_none_when_failed_provider_equals_main(self): + """If the thing that failed IS the main model, no point retrying it.""" + from agent.auxiliary_client import _try_main_agent_model_fallback + with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4"): + client, model, label = _try_main_agent_model_fallback("openrouter", task="vision") + assert client is None and label == "" + + def test_resolves_main_provider_client(self): + from agent.auxiliary_client import _try_main_agent_model_fallback + fake_client = MagicMock() + with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4"), \ + patch("agent.auxiliary_client._is_provider_unhealthy", return_value=False), \ + patch("agent.auxiliary_client.resolve_provider_client", + return_value=(fake_client, "anthropic/claude-sonnet-4")): + client, model, label = _try_main_agent_model_fallback("glm", task="vision") + assert client is fake_client + assert model == "anthropic/claude-sonnet-4" + assert label == "main-agent(openrouter)" + + def test_skips_when_main_provider_is_unhealthy(self): + from agent.auxiliary_client import _try_main_agent_model_fallback + with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \ + patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4"), \ + patch("agent.auxiliary_client._is_provider_unhealthy", return_value=True): + client, model, label = _try_main_agent_model_fallback("glm", task="vision") + assert client is None + + # --------------------------------------------------------------------------- # Gate: _resolve_api_key_provider must skip anthropic when not configured # ---------------------------------------------------------------------------