mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-25 00:51:20 +00:00
3207b9bda0
Cuts shard-3 local runtime in half by neutralizing real wall-clock waits across three classes of slow test: ## 1. Retry backoff mocks - tests/run_agent/conftest.py (NEW): autouse fixture mocks jittered_backoff to 0.0 so the `while time.time() < sleep_end` busy-loop exits immediately. No global time.sleep mock (would break threading tests). - test_anthropic_error_handling, test_413_compression, test_run_agent_codex_responses, test_fallback_model: per-file fixtures mock time.sleep / asyncio.sleep for retry / compression paths. - test_retaindb_plugin: cap the retaindb module's bound time.sleep to 0.05s via a per-test shim (background writer-thread retries sleep 2s after errors; tests don't care about exact duration). Plus replace arbitrary time.sleep(N) waits with short polling loops bounded by deadline. ## 2. Subprocess sleeps in production code - test_update_gateway_restart: mock time.sleep. Production code does time.sleep(3) after `systemctl restart` to verify the service survived. Tests mock subprocess.run \u2014 nothing actually restarts \u2014 so the wait is dead time. ## 3. Network / IMDS timeouts (biggest single win) - tests/conftest.py: add AWS_EC2_METADATA_DISABLED=true plus AWS_METADATA_SERVICE_TIMEOUT=1 and ATTEMPTS=1. boto3 falls back to IMDS (169.254.169.254) when no AWS creds are set. Any test hitting has_aws_credentials() / resolve_aws_auth_env_var() (e.g. test_status, test_setup_copilot_acp, anything that touches provider auto-detect) burned ~2-4s waiting for that to time out. - test_exit_cleanup_interrupt: explicitly mock resolve_runtime_provider which was doing real network auto-detect (~4s). Tests don't care about provider resolution \u2014 the agent is already mocked. - test_timezone: collapse the 3-test "TZ env in subprocess" suite into 2 tests by checking both injection AND no-leak in the same subprocess spawn (was 3 \u00d7 3.2s, now 2 \u00d7 4s). ## Validation | Test | Before | After | |---|---|---| | test_anthropic_error_handling (8 tests) | ~80s | ~15s | | test_413_compression (14 tests) | ~18s | 2.3s | | test_retaindb_plugin (67 tests) | ~13s | 1.3s | | test_status_includes_tavily_key | 4.0s | 0.05s | | test_setup_copilot_acp_skips_same_provider_pool_step | 8.0s | 0.26s | | test_update_gateway_restart (5 tests) | ~18s total | ~0.35s total | | test_exit_cleanup_interrupt (2 tests) | 8s | 1.5s | | **Matrix shard 3 local** | **108s** | **50s** | No behavioral contract changed \u2014 tests still verify retry happens, service restart logic runs, etc.; they just don't burn real seconds waiting for it. Supersedes PR #11779 (those changes are included here).
407 lines
15 KiB
Python
407 lines
15 KiB
Python
"""Tests for the provider fallback model feature.
|
|
|
|
Verifies that AIAgent can switch to a configured fallback model/provider
|
|
when the primary fails after retries.
|
|
"""
|
|
|
|
import os
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from run_agent import AIAgent
|
|
import run_agent
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_fallback_wait(monkeypatch):
|
|
"""Short-circuit time.sleep in fallback/recovery paths so tests don't
|
|
block on the ``min(3 + retry_count, 8)`` wait before a primary retry."""
|
|
import time as _time
|
|
monkeypatch.setattr(_time, "sleep", lambda *_a, **_k: None)
|
|
monkeypatch.setattr(run_agent, "jittered_backoff", lambda *a, **k: 0.0)
|
|
|
|
|
|
def _make_tool_defs(*names: str) -> list:
|
|
return [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": n,
|
|
"description": f"{n} tool",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
for n in names
|
|
]
|
|
|
|
|
|
def _make_agent(fallback_model=None):
|
|
"""Create a minimal AIAgent with optional fallback config."""
|
|
with (
|
|
patch("run_agent.get_tool_definitions", return_value=_make_tool_defs("web_search")),
|
|
patch("run_agent.check_toolset_requirements", return_value={}),
|
|
patch("run_agent.OpenAI"),
|
|
):
|
|
agent = AIAgent(
|
|
api_key="test-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
fallback_model=fallback_model,
|
|
)
|
|
agent.client = MagicMock()
|
|
return agent
|
|
|
|
|
|
def _mock_resolve(base_url="https://openrouter.ai/api/v1", api_key="test-key"):
|
|
"""Helper to create a mock client for resolve_provider_client."""
|
|
mock_client = MagicMock()
|
|
mock_client.api_key = api_key
|
|
mock_client.base_url = base_url
|
|
return mock_client
|
|
|
|
|
|
# =============================================================================
|
|
# _try_activate_fallback()
|
|
# =============================================================================
|
|
|
|
class TestTryActivateFallback:
|
|
def test_returns_false_when_not_configured(self):
|
|
agent = _make_agent(fallback_model=None)
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_returns_false_for_empty_config(self):
|
|
agent = _make_agent(fallback_model={"provider": "", "model": ""})
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_returns_false_for_missing_provider(self):
|
|
agent = _make_agent(fallback_model={"model": "gpt-4.1"})
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_returns_false_for_missing_model(self):
|
|
agent = _make_agent(fallback_model={"provider": "openrouter"})
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_activates_openrouter_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-fallback-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "anthropic/claude-sonnet-4"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent._fallback_activated is True
|
|
assert agent.model == "anthropic/claude-sonnet-4"
|
|
assert agent.provider == "openrouter"
|
|
assert agent.api_mode == "chat_completions"
|
|
assert agent.client is mock_client
|
|
|
|
def test_activates_zai_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "glm-5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-zai-key",
|
|
base_url="https://open.z.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent.model == "glm-5"
|
|
assert agent.provider == "zai"
|
|
assert agent.client is mock_client
|
|
|
|
def test_fallback_uses_resolved_normalized_model(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "zai/glm-5.1"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-zai-key",
|
|
base_url="https://api.z.ai/api/paas/v4",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5.1"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
|
|
assert result is True
|
|
assert agent.model == "glm-5.1"
|
|
assert agent.provider == "zai"
|
|
assert agent.client is mock_client
|
|
|
|
def test_activates_kimi_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "kimi-coding", "model": "kimi-k2.5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-kimi-key",
|
|
base_url="https://api.moonshot.ai/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "kimi-k2.5"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.model == "kimi-k2.5"
|
|
assert agent.provider == "kimi-coding"
|
|
|
|
def test_activates_minimax_fallback(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "minimax", "model": "MiniMax-M2.7"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-mm-key",
|
|
base_url="https://api.minimax.io/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "MiniMax-M2.7"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.model == "MiniMax-M2.7"
|
|
assert agent.provider == "minimax"
|
|
assert agent.client is mock_client
|
|
|
|
def test_only_fires_once(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "anthropic/claude-sonnet-4"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
# Second attempt should return False
|
|
assert agent._try_activate_fallback() is False
|
|
|
|
def test_returns_false_when_no_api_key(self):
|
|
"""Fallback should fail gracefully when the API key env var is unset."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "minimax", "model": "MiniMax-M2.7"},
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(None, None),
|
|
):
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_custom_base_url(self):
|
|
"""Custom base_url in config should override the provider default."""
|
|
agent = _make_agent(
|
|
fallback_model={
|
|
"provider": "custom",
|
|
"model": "my-model",
|
|
"base_url": "http://localhost:8080/v1",
|
|
"api_key_env": "MY_CUSTOM_KEY",
|
|
},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="custom-secret",
|
|
base_url="http://localhost:8080/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "my-model"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.client is mock_client
|
|
assert agent.model == "my-model"
|
|
|
|
def test_prompt_caching_enabled_for_claude_on_openrouter(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "anthropic/claude-sonnet-4"),
|
|
):
|
|
agent._try_activate_fallback()
|
|
assert agent._use_prompt_caching is True
|
|
|
|
def test_prompt_caching_disabled_for_non_claude(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "google/gemini-2.5-flash"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-or-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "google/gemini-2.5-flash"),
|
|
):
|
|
agent._try_activate_fallback()
|
|
assert agent._use_prompt_caching is False
|
|
|
|
def test_prompt_caching_disabled_for_non_openrouter(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "glm-5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-zai-key",
|
|
base_url="https://open.z.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5"),
|
|
):
|
|
agent._try_activate_fallback()
|
|
assert agent._use_prompt_caching is False
|
|
|
|
def test_zai_alt_env_var(self):
|
|
"""Z.AI should also check Z_AI_API_KEY as fallback env var."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "zai", "model": "glm-5"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="sk-alt-key",
|
|
base_url="https://open.z.ai/api/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "glm-5"),
|
|
):
|
|
assert agent._try_activate_fallback() is True
|
|
assert agent.client is mock_client
|
|
|
|
def test_activates_codex_fallback(self):
|
|
"""OpenAI Codex fallback should use OAuth credentials and codex_responses mode."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openai-codex", "model": "gpt-5.3-codex"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="codex-oauth-token",
|
|
base_url="https://chatgpt.com/backend-api/codex",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "gpt-5.3-codex"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent.model == "gpt-5.3-codex"
|
|
assert agent.provider == "openai-codex"
|
|
assert agent.api_mode == "codex_responses"
|
|
assert agent.client is mock_client
|
|
|
|
def test_codex_fallback_fails_gracefully_without_credentials(self):
|
|
"""Codex fallback should return False if no OAuth credentials available."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openai-codex", "model": "gpt-5.3-codex"},
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(None, None),
|
|
):
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_activates_nous_fallback(self):
|
|
"""Nous Portal fallback should use OAuth credentials and chat_completions mode."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "nous", "model": "nous-hermes-3"},
|
|
)
|
|
mock_client = _mock_resolve(
|
|
api_key="nous-agent-key-abc",
|
|
base_url="https://inference-api.nousresearch.com/v1",
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "nous-hermes-3"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True
|
|
assert agent.model == "nous-hermes-3"
|
|
assert agent.provider == "nous"
|
|
assert agent.api_mode == "chat_completions"
|
|
assert agent.client is mock_client
|
|
|
|
def test_nous_fallback_fails_gracefully_without_login(self):
|
|
"""Nous fallback should return False if not logged in."""
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "nous", "model": "nous-hermes-3"},
|
|
)
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(None, None),
|
|
):
|
|
assert agent._try_activate_fallback() is False
|
|
assert agent._fallback_activated is False
|
|
|
|
|
|
# =============================================================================
|
|
# Fallback config init
|
|
# =============================================================================
|
|
|
|
class TestFallbackInit:
|
|
def test_fallback_stored_when_configured(self):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
|
)
|
|
assert agent._fallback_model is not None
|
|
assert agent._fallback_model["provider"] == "openrouter"
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_fallback_none_when_not_configured(self):
|
|
agent = _make_agent(fallback_model=None)
|
|
assert agent._fallback_model is None
|
|
assert agent._fallback_activated is False
|
|
|
|
def test_fallback_none_for_non_dict(self):
|
|
agent = _make_agent(fallback_model="not-a-dict")
|
|
assert agent._fallback_model is None
|
|
|
|
|
|
# =============================================================================
|
|
# Provider credential resolution
|
|
# =============================================================================
|
|
|
|
class TestProviderCredentials:
|
|
"""Verify that each supported provider resolves via the centralized router."""
|
|
|
|
@pytest.mark.parametrize("provider,env_var,base_url_fragment", [
|
|
("openrouter", "OPENROUTER_API_KEY", "openrouter"),
|
|
("zai", "ZAI_API_KEY", "z.ai"),
|
|
("kimi-coding", "KIMI_API_KEY", "moonshot.ai"),
|
|
("minimax", "MINIMAX_API_KEY", "minimax.io"),
|
|
("minimax-cn", "MINIMAX_CN_API_KEY", "minimaxi.com"),
|
|
])
|
|
def test_provider_resolves(self, provider, env_var, base_url_fragment):
|
|
agent = _make_agent(
|
|
fallback_model={"provider": provider, "model": "test-model"},
|
|
)
|
|
mock_client = MagicMock()
|
|
mock_client.api_key = "test-api-key"
|
|
mock_client.base_url = f"https://{base_url_fragment}/v1"
|
|
with patch(
|
|
"agent.auxiliary_client.resolve_provider_client",
|
|
return_value=(mock_client, "test-model"),
|
|
):
|
|
result = agent._try_activate_fallback()
|
|
assert result is True, f"Failed to activate fallback for {provider}"
|
|
assert agent.client is mock_client
|
|
assert agent.model == "test-model"
|
|
assert agent.provider == provider
|