mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
180 lines
7.4 KiB
Python
180 lines
7.4 KiB
Python
"""Tests for the Brave Search (free tier) web search provider.
|
|
|
|
Covers:
|
|
- BraveFreeWebSearchProvider.is_available() env var gating
|
|
- BraveFreeWebSearchProvider.search() — happy path, HTTP error, request error, bad JSON
|
|
- Result normalization (title, url, description, position)
|
|
- Limit truncation + Brave's count cap (20)
|
|
- _is_backend_available("brave-free") integration
|
|
- _get_backend() recognizes "brave-free" as a valid configured backend
|
|
- check_web_api_key() includes brave-free in availability check
|
|
- web_extract returns a search-only error when brave-free is active
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from tests.tools.conftest import register_all_web_providers
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BraveFreeWebSearchProvider unit tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBraveFreeProviderIsConfigured:
|
|
def test_configured_when_key_set(self, monkeypatch):
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
|
|
assert BraveFreeWebSearchProvider().is_available() is True
|
|
|
|
|
|
def test_implements_web_search_provider(self):
|
|
from agent.web_search_provider import WebSearchProvider
|
|
from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
|
|
assert issubclass(BraveFreeWebSearchProvider, WebSearchProvider)
|
|
|
|
|
|
class TestBraveFreeProviderSearch:
|
|
_SAMPLE_RESPONSE = {
|
|
"web": {
|
|
"results": [
|
|
{"title": "A", "url": "https://a.example.com", "description": "desc A"},
|
|
{"title": "B", "url": "https://b.example.com", "description": "desc B"},
|
|
{"title": "C", "url": "https://c.example.com", "description": "desc C"},
|
|
]
|
|
}
|
|
}
|
|
|
|
@staticmethod
|
|
def _mock_resp(json_data, status_code=200):
|
|
m = MagicMock()
|
|
m.status_code = status_code
|
|
m.json.return_value = json_data
|
|
m.raise_for_status = MagicMock()
|
|
return m
|
|
|
|
def test_happy_path_normalizes_results(self, monkeypatch):
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
|
|
|
|
with patch("httpx.get", return_value=self._mock_resp(self._SAMPLE_RESPONSE)):
|
|
result = BraveFreeWebSearchProvider().search("test query", limit=5)
|
|
|
|
assert result["success"] is True
|
|
web = result["data"]["web"]
|
|
assert len(web) == 3
|
|
assert web[0] == {"title": "A", "url": "https://a.example.com", "description": "desc A", "position": 1}
|
|
assert web[2]["position"] == 3
|
|
|
|
def test_sends_subscription_token_header_and_count(self, monkeypatch):
|
|
"""Brave uses X-Subscription-Token; count maps from limit."""
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
|
|
|
|
captured = {}
|
|
|
|
def fake_get(url, **kwargs):
|
|
captured["url"] = url
|
|
captured["headers"] = kwargs.get("headers", {})
|
|
captured["params"] = kwargs.get("params", {})
|
|
return self._mock_resp({"web": {"results": []}})
|
|
|
|
with patch("httpx.get", side_effect=fake_get):
|
|
BraveFreeWebSearchProvider().search("q", limit=5)
|
|
|
|
assert captured["url"] == "https://api.search.brave.com/res/v1/web/search"
|
|
assert captured["headers"].get("X-Subscription-Token") == "BSAkey123"
|
|
assert captured["params"].get("q") == "q"
|
|
assert captured["params"].get("count") == 5
|
|
|
|
|
|
def test_missing_web_key_returns_empty(self, monkeypatch):
|
|
"""Responses without a ``web`` block should produce an empty result set, not crash."""
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
|
|
|
|
with patch("httpx.get", return_value=self._mock_resp({})):
|
|
result = BraveFreeWebSearchProvider().search("q", limit=5)
|
|
|
|
assert result["success"] is True
|
|
assert result["data"]["web"] == []
|
|
|
|
|
|
def test_missing_key_returns_failure(self, monkeypatch):
|
|
monkeypatch.delenv("BRAVE_SEARCH_API_KEY", raising=False)
|
|
from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
|
|
|
|
result = BraveFreeWebSearchProvider().search("q", limit=5)
|
|
assert result["success"] is False
|
|
assert "BRAVE_SEARCH_API_KEY" in result["error"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Integration: _is_backend_available / _get_backend / check_web_api_key
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBraveFreeBackendWiring:
|
|
def test_is_backend_available_true_when_key_set(self, monkeypatch):
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
from tools.web_tools import _is_backend_available
|
|
assert _is_backend_available("brave-free") is True
|
|
|
|
|
|
def test_brave_free_does_not_override_paid_provider(self, monkeypatch):
|
|
"""Tavily (higher priority) should win in auto-detect."""
|
|
from tools import web_tools
|
|
monkeypatch.setattr(web_tools, "_load_web_config", lambda: {})
|
|
for key in ("FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY", "EXA_API_KEY", "SEARXNG_URL"):
|
|
monkeypatch.delenv(key, raising=False)
|
|
monkeypatch.setenv("TAVILY_API_KEY", "tvly")
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False)
|
|
assert web_tools._get_backend() == "tavily"
|
|
|
|
def test_check_web_api_key_true_when_brave_free_configured(self, monkeypatch):
|
|
from tools import web_tools
|
|
monkeypatch.setattr(web_tools, "_load_web_config", lambda: {"backend": "brave-free"})
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
assert web_tools.check_web_api_key() is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# brave-free is search-only: web_extract returns a clear error
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBraveFreeSearchOnlyErrors:
|
|
_register_providers = staticmethod(register_all_web_providers)
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _populate_web_registry(self):
|
|
self._register_providers()
|
|
yield
|
|
from agent.web_search_registry import _reset_for_tests
|
|
_reset_for_tests()
|
|
|
|
def test_web_extract_returns_search_only_error(self, monkeypatch):
|
|
import asyncio
|
|
from tools import web_tools
|
|
|
|
monkeypatch.setattr(web_tools, "_load_web_config", lambda: {"backend": "brave-free"})
|
|
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123")
|
|
monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False)
|
|
async def _allow_ssrf(_url: str) -> bool:
|
|
return True
|
|
|
|
monkeypatch.setattr(web_tools, "async_is_safe_url", _allow_ssrf)
|
|
monkeypatch.setattr("tools.interrupt.is_interrupted", lambda: False, raising=False)
|
|
|
|
result_str = asyncio.get_event_loop().run_until_complete(
|
|
web_tools.web_extract_tool(["https://example.com"])
|
|
)
|
|
result = json.loads(result_str)
|
|
assert result["success"] is False
|
|
assert "search-only" in result["error"].lower()
|
|
assert "brave" in result["error"].lower()
|