mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
110 lines
3.9 KiB
Python
110 lines
3.9 KiB
Python
"""Regression tests for #60955: gateway must not freeze fallback_providers.
|
|
|
|
Cron reloads ``fallback_providers`` from disk on every job. The gateway used to
|
|
freeze ``self._fallback_model`` at process start, so a chain configured (or
|
|
edited) after ``hermes gateway`` was already running never reached messaging
|
|
sessions — even though cron in the same process fell back correctly.
|
|
|
|
These tests pin the reload + cached-agent apply helpers without driving the
|
|
full Feishu session path.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
|
|
def test_refresh_fallback_model_rereads_config(tmp_path, monkeypatch):
|
|
from gateway.run import GatewayRunner
|
|
|
|
monkeypatch.setattr("gateway.run._hermes_home", tmp_path)
|
|
cfg = tmp_path / "config.yaml"
|
|
cfg.write_text(
|
|
"fallback_providers:\n"
|
|
" - provider: deepseek\n"
|
|
" model: deepseek-v4-flash\n"
|
|
)
|
|
|
|
runner = SimpleNamespace(
|
|
_fallback_model=None,
|
|
)
|
|
runner._load_fallback_model = GatewayRunner._load_fallback_model
|
|
bound = GatewayRunner._refresh_fallback_model.__get__(runner)
|
|
chain = bound()
|
|
|
|
assert chain == [{"provider": "deepseek", "model": "deepseek-v4-flash"}]
|
|
assert runner._fallback_model == chain
|
|
|
|
cfg.write_text(
|
|
"fallback_providers:\n"
|
|
" - provider: openrouter\n"
|
|
" model: anthropic/claude-sonnet-4.6\n"
|
|
)
|
|
updated = bound()
|
|
assert updated == [
|
|
{"provider": "openrouter", "model": "anthropic/claude-sonnet-4.6"}
|
|
]
|
|
assert runner._fallback_model == updated
|
|
|
|
|
|
def test_apply_fallback_chain_skips_while_cooldown_holds_fallback():
|
|
"""Do not clobber a live fallback activation during its cooldown window."""
|
|
from gateway.run import GatewayRunner
|
|
|
|
live = [{"provider": "deepseek", "model": "deepseek-v4-flash"}]
|
|
agent = SimpleNamespace(
|
|
_fallback_chain=live,
|
|
_fallback_model=live[0],
|
|
_fallback_index=1,
|
|
_fallback_activated=True,
|
|
_rate_limited_until=time.monotonic() + 30,
|
|
)
|
|
GatewayRunner._apply_fallback_chain_to_agent(
|
|
agent,
|
|
[{"provider": "openrouter", "model": "anthropic/claude-sonnet-4.6"}],
|
|
)
|
|
|
|
assert agent._fallback_chain == live
|
|
assert agent._fallback_index == 1
|
|
assert agent._fallback_activated is True
|
|
|
|
|
|
def test_background_and_main_agent_paths_call_refresh():
|
|
"""Both AIAgent construction sites must pass a refreshed chain, not the
|
|
startup snapshot, and the cached-agent reuse path must apply the refreshed
|
|
chain. Source-level invariant for call sites that resist unit testing.
|
|
"""
|
|
from pathlib import Path
|
|
|
|
source = (
|
|
Path(__file__).resolve().parent.parent.parent / "gateway" / "run.py"
|
|
).read_text(encoding="utf-8")
|
|
assert "fallback_model=self._refresh_fallback_model()" in source
|
|
assert source.count("fallback_model=self._refresh_fallback_model()") >= 2
|
|
# The cached-agent reuse path (the load-bearing fix for a long-lived
|
|
# session in a running gateway) must apply the refreshed chain.
|
|
assert "self._apply_fallback_chain_to_agent(" in source
|
|
# The stale startup-snapshot form must not remain at create sites.
|
|
assert "fallback_model=self._fallback_model," not in source
|
|
|
|
|
|
def test_load_fallback_model_static_unchanged_contract(tmp_path, monkeypatch):
|
|
"""_load_fallback_model remains a pure static reader used by refresh."""
|
|
from gateway.run import GatewayRunner
|
|
|
|
monkeypatch.setattr("gateway.run._hermes_home", tmp_path)
|
|
(tmp_path / "config.yaml").write_text(
|
|
"fallback_providers:\n"
|
|
" - provider: deepseek\n"
|
|
" model: deepseek-v4-flash\n"
|
|
"fallback_model:\n"
|
|
" provider: nous\n"
|
|
" model: Hermes-4\n"
|
|
)
|
|
|
|
chain = GatewayRunner._load_fallback_model()
|
|
assert chain == [
|
|
{"provider": "deepseek", "model": "deepseek-v4-flash"},
|
|
{"provider": "nous", "model": "Hermes-4"},
|
|
]
|