mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-18 04:41:56 +00:00
6ddc48b058
Fallback chain entries with 'api_key_env: ENV_VAR_NAME' weren't being
resolved by either the init-time fallback path (line ~1660) or the
runtime _try_activate_fallback path (line ~8045). Only literal
'api_key' was honored; the snake_case 'api_key_env' alias documented
elsewhere in the config was silently dropped, so a 'provider: custom'
fallback with base_url + api_key_env worked as primary but failed as
fallback with 'no endpoint credentials found' / 401.
Adds 'or fb.get("api_key_env")' to the existing 'key_env' lookup in
both call sites, with empty-string-to-None coercion so unset env vars
don't poison the resolver.
Salvage of #22665's fallback portion. The original PR also bundled
gateway-degrade-on-no-adapters changes (those land via the carve-out
in #22853 which is the same code) and run_agent.py memory-nudge
counter hydration (issue #22357 territory, not mentioned in the
title). Drops both bundled pieces; keeps just the api_key_env fix.
Closes #5392.
511 lines
20 KiB
Python
511 lines
20 KiB
Python
"""Tests for the provider fallback model feature.
|
|
|
|
Verifies that AIAgent can switch to a configured fallback model/provider
|
|
when the primary fails after retries.
|
|
"""
|
|
|
|
import os
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from run_agent import AIAgent
|
|
import run_agent
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_fallback_wait(monkeypatch):
|
|
"""Short-circuit time.sleep in fallback/recovery paths so tests don't
|
|
block on the ``min(3 + retry_count, 8)`` wait before a primary retry."""
|
|
import time as _time
|
|
monkeypatch.setattr(_time, "sleep", lambda *_a, **_k: None)
|
|
monkeypatch.setattr(run_agent, "jittered_backoff", lambda *a, **k: 0.0)
|
|
|
|
|
|
def _make_tool_defs(*names: str) -> list:
|
|
return [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": n,
|
|
"description": f"{n} tool",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
for n in names
|
|
]
|
|
|
|
|
|
def _make_agent(fallback_model=None):
|
|
"""Create a minimal AIAgent with optional fallback config."""
|
|
with (
|
|
patch("run_agent.get_tool_definitions", return_value=_make_tool_defs("web_search")),
|
|
patch("run_agent.check_toolset_requirements", return_value={}),
|
|
patch("run_agent.OpenAI"),
|
|
):
|
|
agent = AIAgent(
|
|
api_key="test-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
fallback_model=fallback_model,
|
|
)
|
|
agent.client = MagicMock()
|
|
return agent
|
|
|
|
|
|
def _mock_resolve(base_url="https://openrouter.ai/api/v1", api_key="test-key"):
|
|
"""Helper to create a mock client for resolve_provider_client."""
|
|
mock_client = MagicMock()
|
|
mock_client.api_key = api_key
|
|
mock_client.base_url = base_url
|
|
return mock_client
|
|
|
|
|
|
# =============================================================================
|
|
# _try_activate_fallback()
|
|
# =============================================================================
|
|
|
|
class TestTryActivateFallback:
|
|
def test_returns_false_when_not_configured(self):
|
|
agent = _make_agent(fallback_model=None)
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_returns_false_for_empty_config(self):
|
|
agent = _make_agent(fallback_model={"provider": "", "model": ""})
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_returns_false_for_missing_provider(self):
|
|
agent = _make_agent(fallback_model={"model": "gpt-4.1"})
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_returns_false_for_missing_model(self):
|
|
agent = _make_agent(fallback_model={"provider": "openrouter"})
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_activates_openrouter_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-fallback-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "anthropic/claude-sonnet-4"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent._fallback_activated is True
|
|
assert agent.model == "anthropic/claude-sonnet-4"
|
|
assert agent.provider == "openrouter"
|
|
assert agent.api_mode == "chat_completions"
|
|
assert agent.client is mock_client
|
|
|
|
def test_activates_zai_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "glm-5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-zai-key",
|
|
base_url="https://open.z.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent.model == "glm-5"
|
|
assert agent.provider == "zai"
|
|
assert agent.client is mock_client
|
|
|
|
def test_fallback_uses_resolved_normalized_model(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "zai/glm-5.1"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-zai-key",
|
|
base_url="https://api.z.ai/api/paas/v4",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5.1"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
|
|
assert result is True
|
|
assert agent.model == "glm-5.1"
|
|
assert agent.provider == "zai"
|
|
assert agent.client is mock_client
|
|
|
|
def test_activates_kimi_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "kimi-coding", "model": "kimi-k2.5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-kimi-key",
|
|
base_url="https://api.moonshot.ai/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "kimi-k2.5"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.model == "kimi-k2.5"
|
|
assert agent.provider == "kimi-coding"
|
|
|
|
def test_activates_minimax_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "minimax", "model": "MiniMax-M2.7"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-mm-key",
|
|
base_url="https://api.minimax.io/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "MiniMax-M2.7"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.model == "MiniMax-M2.7"
|
|
assert agent.provider == "minimax"
|
|
assert agent.client is mock_client
|
|
|
|
def test_only_fires_once(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "anthropic/claude-sonnet-4"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
# Second attempt should return False
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_returns_false_when_no_api_key(self):
|
|
"""Fallback should fail gracefully when the API key env var is unset."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "minimax", "model": "MiniMax-M2.7"},
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(None, None),
|
|
):
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_custom_base_url(self):
|
|
"""Custom base_url in config should override the provider default."""
|
|
agent = _make_agent(
|
|
fallback_model={
|
|
"provider": "custom",
|
|
"model": "my-model",
|
|
"base_url": "http://localhost:8080/v1",
|
|
"api_key_env": "MY_CUSTOM_KEY",
|
|
},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="custom-secret",
|
|
base_url="http://localhost:8080/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "my-model"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.client is mock_client
|
|
assert agent.model == "my-model"
|
|
|
|
def test_prompt_caching_enabled_for_claude_on_openrouter(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "anthropic/claude-sonnet-4"),
|
|
):
|
|
agent._try_activate_fallback()
|
|
assert agent._use_prompt_caching is True
|
|
|
|
def test_prompt_caching_disabled_for_non_claude(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "google/gemini-2.5-flash"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "google/gemini-2.5-flash"),
|
|
):
|
|
agent._try_activate_fallback()
|
|
assert agent._use_prompt_caching is False
|
|
|
|
def test_prompt_caching_disabled_for_non_openrouter(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "glm-5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-zai-key",
|
|
base_url="https://open.z.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5"),
|
|
):
|
|
agent._try_activate_fallback()
|
|
assert agent._use_prompt_caching is False
|
|
|
|
def test_zai_alt_env_var(self):
|
|
"""Z.AI should also check Z_AI_API_KEY as fallback env var."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "glm-5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-alt-key",
|
|
base_url="https://open.z.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.client is mock_client
|
|
|
|
def test_activates_codex_fallback(self):
|
|
"""OpenAI Codex fallback should use OAuth credentials and codex_responses mode."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openai-codex", "model": "gpt-5.3-codex"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="codex-oauth-token",
|
|
base_url="https://chatgpt.com/backend-api/codex",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "gpt-5.3-codex"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent.model == "gpt-5.3-codex"
|
|
assert agent.provider == "openai-codex"
|
|
assert agent.api_mode == "codex_responses"
|
|
assert agent.client is mock_client
|
|
|
|
def test_codex_fallback_fails_gracefully_without_credentials(self):
|
|
"""Codex fallback should return False if no OAuth credentials available."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openai-codex", "model": "gpt-5.3-codex"},
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(None, None),
|
|
):
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_activates_nous_fallback(self):
|
|
"""Nous Portal fallback should use OAuth credentials and chat_completions mode."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "nous", "model": "nous-hermes-3"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="nous-agent-key-abc",
|
|
base_url="https://inference-api.nousresearch.com/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "nous-hermes-3"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent.model == "nous-hermes-3"
|
|
assert agent.provider == "nous"
|
|
assert agent.api_mode == "chat_completions"
|
|
assert agent.client is mock_client
|
|
|
|
def test_nous_fallback_fails_gracefully_without_login(self):
|
|
"""Nous fallback should return False if not logged in."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "nous", "model": "nous-hermes-3"},
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(None, None),
|
|
):
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
|
|
# =============================================================================
|
|
# Fallback config init
|
|
# =============================================================================
|
|
|
|
class TestFallbackInit:
|
|
def test_fallback_stored_when_configured(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
assert agent._fallback_model is not None
|
|
assert agent._fallback_model["provider"] == "openrouter"
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_fallback_none_when_not_configured(self):
|
|
agent = _make_agent(fallback_model=None)
|
|
assert agent._fallback_model is None
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_fallback_none_for_non_dict(self):
|
|
agent = _make_agent(fallback_model="not-a-dict")
|
|
assert agent._fallback_model is None
|
|
|
|
|
|
# =============================================================================
|
|
# Provider credential resolution
|
|
# =============================================================================
|
|
|
|
class TestProviderCredentials:
|
|
"""Verify that each supported provider resolves via the centralized router."""
|
|
|
|
@pytest.mark.parametrize("provider,env_var,base_url_fragment", [
|
|
("openrouter", "OPENROUTER_API_KEY", "openrouter"),
|
|
("zai", "ZAI_API_KEY", "z.ai"),
|
|
("kimi-coding", "KIMI_API_KEY", "moonshot.ai"),
|
|
("minimax", "MINIMAX_API_KEY", "minimax.io"),
|
|
("minimax-cn", "MINIMAX_CN_API_KEY", "minimaxi.com"),
|
|
])
|
|
def test_provider_resolves(self, provider, env_var, base_url_fragment):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": provider, "model": "test-model"},
|
|
)
|
|
mock_client = MagicMock()
|
|
mock_client.api_key = "test-api-key"
|
|
mock_client.base_url = f"https://{base_url_fragment}/v1"
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "test-model"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True, f"Failed to activate fallback for {provider}"
|
|
assert agent.client is mock_client
|
|
assert agent.model == "test-model"
|
|
assert agent.provider == provider
|
|
|
|
|
|
# =============================================================================
|
|
# api_key_env / key_env resolution in fallback entries (#5392)
|
|
# =============================================================================
|
|
|
|
class TestFallbackKeyEnvResolution:
|
|
"""Verify that api_key_env and key_env are both resolved from the
|
|
environment and forwarded to resolve_provider_client as explicit_api_key.
|
|
|
|
Before the fix, _try_activate_fallback only checked ``key_env`` and ignored
|
|
the ``api_key_env`` alias documented in the custom_providers config schema.
|
|
The init-time fallback path never resolved either field.
|
|
"""
|
|
|
|
def test_api_key_env_resolved_at_runtime_fallback(self, monkeypatch):
|
|
"""api_key_env in fallback entry must be read from env and passed
|
|
as explicit_api_key to resolve_provider_client (#5392)."""
|
|
monkeypatch.setenv("MY_GOOGLE_KEY", "google-secret-from-env")
|
|
|
|
agent = _make_agent(
|
|
fallback_model={
|
|
"provider": "custom",
|
|
"model": "gemini-flash",
|
|
"base_url": "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
"api_key_env": "MY_GOOGLE_KEY",
|
|
},
|
|
)
|
|
captured = {}
|
|
|
|
def _fake_resolve(provider, model=None, raw_codex=False,
|
|
explicit_base_url=None, explicit_api_key=None, **kw):
|
|
captured["explicit_api_key"] = explicit_api_key
|
|
captured["explicit_base_url"] = explicit_base_url
|
|
mock = MagicMock()
|
|
mock.api_key = explicit_api_key or "no-key"
|
|
mock.base_url = explicit_base_url or "https://example.com/v1"
|
|
return mock, model
|
|
|
|
with patch("agent.auxiliary_client.resolve_provider_client", side_effect=_fake_resolve):
|
|
result = agent._try_activate_fallback()
|
|
|
|
assert result is True
|
|
assert captured["explicit_api_key"] == "google-secret-from-env", (
|
|
"api_key_env value was not resolved and forwarded as explicit_api_key"
|
|
)
|
|
assert captured["explicit_base_url"] == "https://generativelanguage.googleapis.com/v1beta/openai"
|
|
|
|
def test_key_env_still_works_at_runtime_fallback(self, monkeypatch):
|
|
"""key_env (canonical form) must still be resolved correctly."""
|
|
monkeypatch.setenv("MY_PROVIDER_KEY", "secret-via-key-env")
|
|
|
|
agent = _make_agent(
|
|
fallback_model={
|
|
"provider": "custom",
|
|
"model": "my-model",
|
|
"base_url": "https://api.example.com/v1",
|
|
"key_env": "MY_PROVIDER_KEY",
|
|
},
|
|
)
|
|
captured = {}
|
|
|
|
def _fake_resolve(provider, model=None, raw_codex=False,
|
|
explicit_base_url=None, explicit_api_key=None, **kw):
|
|
captured["explicit_api_key"] = explicit_api_key
|
|
mock = MagicMock()
|
|
mock.api_key = explicit_api_key or "no-key"
|
|
mock.base_url = explicit_base_url or "https://api.example.com/v1"
|
|
return mock, model
|
|
|
|
with patch("agent.auxiliary_client.resolve_provider_client", side_effect=_fake_resolve):
|
|
result = agent._try_activate_fallback()
|
|
|
|
assert result is True
|
|
assert captured["explicit_api_key"] == "secret-via-key-env"
|
|
|
|
def test_api_key_env_unset_does_not_crash(self, monkeypatch):
|
|
"""When api_key_env refers to an unset variable, explicit_api_key is None
|
|
(not an empty string) so the provider can fall through to its default."""
|
|
monkeypatch.delenv("ABSENT_KEY_VAR", raising=False)
|
|
|
|
agent = _make_agent(
|
|
fallback_model={
|
|
"provider": "openrouter",
|
|
"model": "some/model",
|
|
"api_key_env": "ABSENT_KEY_VAR",
|
|
},
|
|
)
|
|
captured = {}
|
|
|
|
def _fake_resolve(provider, model=None, raw_codex=False,
|
|
explicit_base_url=None, explicit_api_key=None, **kw):
|
|
captured["explicit_api_key"] = explicit_api_key
|
|
mock = MagicMock()
|
|
mock.api_key = "fallback-default"
|
|
mock.base_url = "https://openrouter.ai/api/v1"
|
|
return mock, model
|
|
|
|
with patch("agent.auxiliary_client.resolve_provider_client", side_effect=_fake_resolve):
|
|
agent._try_activate_fallback()
|
|
|
|
assert captured["explicit_api_key"] is None, (
|
|
"Unset api_key_env should yield None, not empty string"
|
|
)
|