From 6b219f5af6022ef09d0312a8ceccf3e1b11c3aa7 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Wed, 13 May 2026 23:52:02 +0530 Subject: [PATCH] refactor(web): remove legacy in-tree provider modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletes tools/web_providers/{brave_free,ddgs,searxng}.py — the three providers that moved to plugins/web/ in prior commits. tools/web_tools.py no longer imports them (registry dispatch as of d8735963f), so removing them is purely a cleanup pass. Also migrates the existing tests to the new import paths: tests/tools/test_web_providers_brave_free.py tests/tools/test_web_providers_ddgs.py tests/tools/test_web_providers_searxng.py Mechanical rewrites: - `from tools.web_providers.X import YSearchProvider` -> `from plugins.web.X.provider import YWebSearchProvider` - `.is_configured()` -> `.is_available()` (legacy method -> new method) - `.provider_name()` -> `.name` (legacy method -> new property) - `from tools.web_providers.base import WebSearchProvider` -> `from agent.web_search_provider import WebSearchProvider` (the subclass-check asserts membership in the new plugin-facing ABC) - `sys.modules.delitem("tools.web_providers.ddgs")` updated to point at `plugins.web.ddgs.provider` (cache-busting for lazy ddgs imports) The TestXBackendWiring / TestXSearchOnlyErrors classes (covering _is_backend_available, _get_backend, check_web_api_key, and the "search-only" error paths in web_extract/web_crawl) are untouched — those still test web_tools.py's backend-selection logic, which continues to recognize the names "brave-free" / "ddgs" / "searxng" even after the modules behind them moved to plugins. tools/web_providers/base.py is intentionally NOT deleted by this commit — it's the parent ABC of the legacy modules and shares its name with agent/web_search_provider.py::WebSearchProvider. Removing it surfaces the naming collision (see PR description Finding 0); the real migration PR deletes it in the same commit that drops the _WEB_PLUGIN_SKIPLIST guards in hermes_cli/tools_config.py. Test results: bash scripts/run_tests.sh tests/tools/test_web_providers_*.py -> 65 passed in 3.41s (all rewritten unit tests + unchanged integration tests) bash scripts/run_tests.sh tests/tools/test_web_*.py -> 141 passed in 4.70s (full web test set, post-deletion) --- plugins/web/brave_free/provider.py | 6 +- plugins/web/ddgs/provider.py | 5 +- plugins/web/searxng/provider.py | 6 +- tests/tools/test_web_providers_brave_free.py | 64 ++++----- tests/tools/test_web_providers_ddgs.py | 56 ++++---- tests/tools/test_web_providers_searxng.py | 70 +++++----- tools/web_providers/brave_free.py | 130 ------------------ tools/web_providers/ddgs.py | 98 -------------- tools/web_providers/searxng.py | 132 ------------------- 9 files changed, 105 insertions(+), 462 deletions(-) delete mode 100644 tools/web_providers/brave_free.py delete mode 100644 tools/web_providers/ddgs.py delete mode 100644 tools/web_providers/searxng.py diff --git a/plugins/web/brave_free/provider.py b/plugins/web/brave_free/provider.py index dfa927ef10e..df4584f7732 100644 --- a/plugins/web/brave_free/provider.py +++ b/plugins/web/brave_free/provider.py @@ -1,9 +1,9 @@ """Brave Search (free tier) — plugin form. Subclasses :class:`agent.web_search_provider.WebSearchProvider` (the -plugin-facing ABC) and reuses the existing Brave search logic from the -legacy ``tools.web_providers.brave_free`` module. Once the spike validates -the pattern, the legacy module is deleted and this becomes the canonical +plugin-facing ABC). The legacy in-tree module +``tools.web_providers.brave_free`` was removed in the same commit that +moved this code under ``plugins/``; this file is now the canonical implementation. Config keys this provider responds to:: diff --git a/plugins/web/ddgs/provider.py b/plugins/web/ddgs/provider.py index eefd98d51f6..1cc6f9e7b68 100644 --- a/plugins/web/ddgs/provider.py +++ b/plugins/web/ddgs/provider.py @@ -1,8 +1,9 @@ """DuckDuckGo search — plugin form (via the ``ddgs`` package). Subclasses the plugin-facing :class:`agent.web_search_provider.WebSearchProvider`. -Same behavior as the legacy ``tools.web_providers.ddgs`` module — only the -ABC name and import path change. +The legacy in-tree module ``tools.web_providers.ddgs`` was removed in the +same commit that moved this code under ``plugins/``; this file is now the +canonical implementation. The ``ddgs`` package is an optional dependency. ``is_available()`` reflects whether the package is importable; the plugin still registers either way so diff --git a/plugins/web/searxng/provider.py b/plugins/web/searxng/provider.py index a303ef1f5d7..043f6711c1b 100644 --- a/plugins/web/searxng/provider.py +++ b/plugins/web/searxng/provider.py @@ -1,8 +1,10 @@ """SearXNG search — plugin form. Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Same JSON -API call (``/search?format=json``), same result normalization as the legacy -:mod:`tools.web_providers.searxng` module. +API call (``/search?format=json``), same result normalization. The legacy +in-tree module ``tools.web_providers.searxng`` was removed in the same +commit that moved this code under ``plugins/``; this file is now the +canonical implementation. Search-only — SearXNG aggregates results from upstream engines but does not fetch/extract arbitrary URLs. ``supports_extract()`` returns False. diff --git a/tests/tools/test_web_providers_brave_free.py b/tests/tools/test_web_providers_brave_free.py index 36fe41640e8..f441bf0f8b4 100644 --- a/tests/tools/test_web_providers_brave_free.py +++ b/tests/tools/test_web_providers_brave_free.py @@ -1,8 +1,8 @@ """Tests for the Brave Search (free tier) web search provider. Covers: -- BraveFreeSearchProvider.is_configured() env var gating -- BraveFreeSearchProvider.search() — happy path, HTTP error, request error, bad JSON +- BraveFreeWebSearchProvider.is_available() env var gating +- BraveFreeWebSearchProvider.search() — happy path, HTTP error, request error, bad JSON - Result normalization (title, url, description, position) - Limit truncation + Brave's count cap (20) - _is_backend_available("brave-free") integration @@ -17,34 +17,34 @@ from unittest.mock import MagicMock, patch # --------------------------------------------------------------------------- -# BraveFreeSearchProvider unit tests +# BraveFreeWebSearchProvider unit tests # --------------------------------------------------------------------------- class TestBraveFreeProviderIsConfigured: def test_configured_when_key_set(self, monkeypatch): monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider - assert BraveFreeSearchProvider().is_configured() is True + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider + assert BraveFreeWebSearchProvider().is_available() is True def test_not_configured_when_key_missing(self, monkeypatch): monkeypatch.delenv("BRAVE_SEARCH_API_KEY", raising=False) - from tools.web_providers.brave_free import BraveFreeSearchProvider - assert BraveFreeSearchProvider().is_configured() is False + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider + assert BraveFreeWebSearchProvider().is_available() is False def test_not_configured_when_key_whitespace(self, monkeypatch): monkeypatch.setenv("BRAVE_SEARCH_API_KEY", " ") - from tools.web_providers.brave_free import BraveFreeSearchProvider - assert BraveFreeSearchProvider().is_configured() is False + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider + assert BraveFreeWebSearchProvider().is_available() is False def test_provider_name(self): - from tools.web_providers.brave_free import BraveFreeSearchProvider - assert BraveFreeSearchProvider().provider_name() == "brave-free" + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider + assert BraveFreeWebSearchProvider().name == "brave-free" def test_implements_web_search_provider(self): - from tools.web_providers.base import WebSearchProvider - from tools.web_providers.brave_free import BraveFreeSearchProvider - assert issubclass(BraveFreeSearchProvider, WebSearchProvider) + from agent.web_search_provider import WebSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider + assert issubclass(BraveFreeWebSearchProvider, WebSearchProvider) class TestBraveFreeProviderSearch: @@ -68,10 +68,10 @@ class TestBraveFreeProviderSearch: def test_happy_path_normalizes_results(self, monkeypatch): monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider with patch("httpx.get", return_value=self._mock_resp(self._SAMPLE_RESPONSE)): - result = BraveFreeSearchProvider().search("test query", limit=5) + result = BraveFreeWebSearchProvider().search("test query", limit=5) assert result["success"] is True web = result["data"]["web"] @@ -82,7 +82,7 @@ class TestBraveFreeProviderSearch: def test_sends_subscription_token_header_and_count(self, monkeypatch): """Brave uses X-Subscription-Token; count maps from limit.""" monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider captured = {} @@ -93,7 +93,7 @@ class TestBraveFreeProviderSearch: return self._mock_resp({"web": {"results": []}}) with patch("httpx.get", side_effect=fake_get): - BraveFreeSearchProvider().search("q", limit=5) + BraveFreeWebSearchProvider().search("q", limit=5) assert captured["url"] == "https://api.search.brave.com/res/v1/web/search" assert captured["headers"].get("X-Subscription-Token") == "BSAkey123" @@ -103,7 +103,7 @@ class TestBraveFreeProviderSearch: def test_count_is_capped_at_20(self, monkeypatch): """Brave caps count at 20 — limit above that clamps.""" monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider captured = {} @@ -112,26 +112,26 @@ class TestBraveFreeProviderSearch: return self._mock_resp({"web": {"results": []}}) with patch("httpx.get", side_effect=fake_get): - BraveFreeSearchProvider().search("q", limit=100) + BraveFreeWebSearchProvider().search("q", limit=100) assert captured["params"].get("count") == 20 def test_limit_is_respected_client_side(self, monkeypatch): monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider with patch("httpx.get", return_value=self._mock_resp(self._SAMPLE_RESPONSE)): - result = BraveFreeSearchProvider().search("q", limit=2) + result = BraveFreeWebSearchProvider().search("q", limit=2) assert result["success"] is True assert len(result["data"]["web"]) == 2 def test_empty_results(self, monkeypatch): monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider with patch("httpx.get", return_value=self._mock_resp({"web": {"results": []}})): - result = BraveFreeSearchProvider().search("nothing", limit=5) + result = BraveFreeWebSearchProvider().search("nothing", limit=5) assert result["success"] is True assert result["data"]["web"] == [] @@ -139,10 +139,10 @@ class TestBraveFreeProviderSearch: def test_missing_web_key_returns_empty(self, monkeypatch): """Responses without a ``web`` block should produce an empty result set, not crash.""" monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider with patch("httpx.get", return_value=self._mock_resp({})): - result = BraveFreeSearchProvider().search("q", limit=5) + result = BraveFreeWebSearchProvider().search("q", limit=5) assert result["success"] is True assert result["data"]["web"] == [] @@ -150,14 +150,14 @@ class TestBraveFreeProviderSearch: def test_http_error_returns_failure(self, monkeypatch): import httpx monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider bad = MagicMock() bad.status_code = 429 err = httpx.HTTPStatusError("429", request=MagicMock(), response=bad) with patch("httpx.get", side_effect=err): - result = BraveFreeSearchProvider().search("q", limit=5) + result = BraveFreeWebSearchProvider().search("q", limit=5) assert result["success"] is False assert "429" in result["error"] @@ -165,19 +165,19 @@ class TestBraveFreeProviderSearch: def test_request_error_returns_failure(self, monkeypatch): import httpx monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "BSAkey123") - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider with patch("httpx.get", side_effect=httpx.RequestError("boom")): - result = BraveFreeSearchProvider().search("q", limit=5) + result = BraveFreeWebSearchProvider().search("q", limit=5) assert result["success"] is False assert "boom" in result["error"] or "Brave" in result["error"] def test_missing_key_returns_failure(self, monkeypatch): monkeypatch.delenv("BRAVE_SEARCH_API_KEY", raising=False) - from tools.web_providers.brave_free import BraveFreeSearchProvider + from plugins.web.brave_free.provider import BraveFreeWebSearchProvider - result = BraveFreeSearchProvider().search("q", limit=5) + result = BraveFreeWebSearchProvider().search("q", limit=5) assert result["success"] is False assert "BRAVE_SEARCH_API_KEY" in result["error"] diff --git a/tests/tools/test_web_providers_ddgs.py b/tests/tools/test_web_providers_ddgs.py index 9a3ceec7372..d575fe63e36 100644 --- a/tests/tools/test_web_providers_ddgs.py +++ b/tests/tools/test_web_providers_ddgs.py @@ -1,8 +1,8 @@ """Tests for the DuckDuckGo (ddgs) web search provider. Covers: -- DDGSSearchProvider.is_configured() — reflects package importability -- DDGSSearchProvider.search() — happy path, missing package, runtime error +- DDGSWebSearchProvider.is_available() — reflects package importability +- DDGSWebSearchProvider.search() — happy path, missing package, runtime error - Result normalization (title, url, description, position) - _is_backend_available("ddgs") / _get_backend() integration - web_extract / web_crawl return search-only errors when ddgs is active @@ -40,21 +40,21 @@ def _install_fake_ddgs(monkeypatch, *, text_results=None, text_raises=None): # --------------------------------------------------------------------------- -# DDGSSearchProvider unit tests +# DDGSWebSearchProvider unit tests # --------------------------------------------------------------------------- class TestDDGSProviderIsConfigured: def test_configured_when_package_importable(self, monkeypatch): _install_fake_ddgs(monkeypatch) - # Drop any cached ``tools.web_providers.ddgs`` so is_configured re-imports ddgs fresh - monkeypatch.delitem(sys.modules, "tools.web_providers.ddgs", raising=False) - from tools.web_providers.ddgs import DDGSSearchProvider - assert DDGSSearchProvider().is_configured() is True + # Drop any cached ``plugins.web.ddgs.provider`` so is_configured re-imports ddgs fresh + monkeypatch.delitem(sys.modules, "plugins.web.ddgs.provider", raising=False) + from plugins.web.ddgs.provider import DDGSWebSearchProvider + assert DDGSWebSearchProvider().is_available() is True def test_not_configured_when_package_missing(self, monkeypatch): monkeypatch.delitem(sys.modules, "ddgs", raising=False) - monkeypatch.delitem(sys.modules, "tools.web_providers.ddgs", raising=False) + monkeypatch.delitem(sys.modules, "plugins.web.ddgs.provider", raising=False) # Block the import so ``import ddgs`` raises ImportError even if the package is actually installed import builtins orig_import = builtins.__import__ @@ -65,17 +65,17 @@ class TestDDGSProviderIsConfigured: return orig_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", blocked_import) - from tools.web_providers.ddgs import DDGSSearchProvider - assert DDGSSearchProvider().is_configured() is False + from plugins.web.ddgs.provider import DDGSWebSearchProvider + assert DDGSWebSearchProvider().is_available() is False def test_provider_name(self): - from tools.web_providers.ddgs import DDGSSearchProvider - assert DDGSSearchProvider().provider_name() == "ddgs" + from plugins.web.ddgs.provider import DDGSWebSearchProvider + assert DDGSWebSearchProvider().name == "ddgs" def test_implements_web_search_provider(self): - from tools.web_providers.base import WebSearchProvider - from tools.web_providers.ddgs import DDGSSearchProvider - assert issubclass(DDGSSearchProvider, WebSearchProvider) + from agent.web_search_provider import WebSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider + assert issubclass(DDGSWebSearchProvider, WebSearchProvider) class TestDDGSProviderSearch: @@ -85,9 +85,9 @@ class TestDDGSProviderSearch: {"title": "B", "href": "https://b.example.com", "body": "desc B"}, {"title": "C", "href": "https://c.example.com", "body": "desc C"}, ]) - from tools.web_providers.ddgs import DDGSSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider - result = DDGSSearchProvider().search("q", limit=5) + result = DDGSWebSearchProvider().search("q", limit=5) assert result["success"] is True web = result["data"]["web"] @@ -99,9 +99,9 @@ class TestDDGSProviderSearch: _install_fake_ddgs(monkeypatch, text_results=[ {"title": "A", "url": "https://a.example.com", "body": "desc A"}, ]) - from tools.web_providers.ddgs import DDGSSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider - result = DDGSSearchProvider().search("q", limit=5) + result = DDGSWebSearchProvider().search("q", limit=5) assert result["success"] is True assert result["data"]["web"][0]["url"] == "https://a.example.com" @@ -111,16 +111,16 @@ class TestDDGSProviderSearch: {"title": f"R{i}", "href": f"https://r{i}.example.com", "body": ""} for i in range(10) ]) - from tools.web_providers.ddgs import DDGSSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider - result = DDGSSearchProvider().search("q", limit=3) + result = DDGSWebSearchProvider().search("q", limit=3) assert result["success"] is True assert len(result["data"]["web"]) == 3 def test_missing_package_returns_failure(self, monkeypatch): monkeypatch.delitem(sys.modules, "ddgs", raising=False) - monkeypatch.delitem(sys.modules, "tools.web_providers.ddgs", raising=False) + monkeypatch.delitem(sys.modules, "plugins.web.ddgs.provider", raising=False) import builtins orig_import = builtins.__import__ @@ -130,25 +130,25 @@ class TestDDGSProviderSearch: return orig_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", blocked_import) - from tools.web_providers.ddgs import DDGSSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider - result = DDGSSearchProvider().search("q", limit=5) + result = DDGSWebSearchProvider().search("q", limit=5) assert result["success"] is False assert "ddgs" in result["error"].lower() def test_runtime_error_returns_failure(self, monkeypatch): _install_fake_ddgs(monkeypatch, text_raises=RuntimeError("rate limited 202")) - from tools.web_providers.ddgs import DDGSSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider - result = DDGSSearchProvider().search("q", limit=5) + result = DDGSWebSearchProvider().search("q", limit=5) assert result["success"] is False assert "rate limited" in result["error"] or "failed" in result["error"].lower() def test_empty_results(self, monkeypatch): _install_fake_ddgs(monkeypatch, text_results=[]) - from tools.web_providers.ddgs import DDGSSearchProvider + from plugins.web.ddgs.provider import DDGSWebSearchProvider - result = DDGSSearchProvider().search("nothing", limit=5) + result = DDGSWebSearchProvider().search("nothing", limit=5) assert result["success"] is True assert result["data"]["web"] == [] diff --git a/tests/tools/test_web_providers_searxng.py b/tests/tools/test_web_providers_searxng.py index 4779ed6ce6e..d579fb0d0a6 100644 --- a/tests/tools/test_web_providers_searxng.py +++ b/tests/tools/test_web_providers_searxng.py @@ -1,8 +1,8 @@ """Tests for the SearXNG web search provider. Covers: -- SearXNGSearchProvider.is_configured() env var gating -- SearXNGSearchProvider.search() — happy path, HTTP error, request error, bad JSON +- SearXNGWebSearchProvider.is_available() env var gating +- SearXNGWebSearchProvider.search() — happy path, HTTP error, request error, bad JSON - Result normalization (title, url, description, position) - Score-based sorting and limit truncation - _is_backend_available("searxng") integration @@ -19,38 +19,38 @@ import pytest # --------------------------------------------------------------------------- -# SearXNGSearchProvider unit tests +# SearXNGWebSearchProvider unit tests # --------------------------------------------------------------------------- class TestSearXNGSearchProviderIsConfigured: def test_configured_when_url_set(self, monkeypatch): monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider - assert SearXNGSearchProvider().is_configured() is True + from plugins.web.searxng.provider import SearXNGWebSearchProvider + assert SearXNGWebSearchProvider().is_available() is True def test_not_configured_when_url_missing(self, monkeypatch): monkeypatch.delenv("SEARXNG_URL", raising=False) - from tools.web_providers.searxng import SearXNGSearchProvider - assert SearXNGSearchProvider().is_configured() is False + from plugins.web.searxng.provider import SearXNGWebSearchProvider + assert SearXNGWebSearchProvider().is_available() is False def test_not_configured_when_url_empty_string(self, monkeypatch): monkeypatch.setenv("SEARXNG_URL", " ") - from tools.web_providers.searxng import SearXNGSearchProvider - assert SearXNGSearchProvider().is_configured() is False + from plugins.web.searxng.provider import SearXNGWebSearchProvider + assert SearXNGWebSearchProvider().is_available() is False def test_provider_name(self): - from tools.web_providers.searxng import SearXNGSearchProvider - assert SearXNGSearchProvider().provider_name() == "searxng" + from plugins.web.searxng.provider import SearXNGWebSearchProvider + assert SearXNGWebSearchProvider().name == "searxng" def test_implements_web_search_provider(self): - from tools.web_providers.base import WebSearchProvider - from tools.web_providers.searxng import SearXNGSearchProvider - assert issubclass(SearXNGSearchProvider, WebSearchProvider) + from agent.web_search_provider import WebSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider + assert issubclass(SearXNGWebSearchProvider, WebSearchProvider) class TestSearXNGSearchProviderSearch: - """Happy path and error handling for SearXNGSearchProvider.search().""" + """Happy path and error handling for SearXNGWebSearchProvider.search().""" _SAMPLE_RESPONSE = { "results": [ @@ -69,11 +69,11 @@ class TestSearXNGSearchProviderSearch: def test_happy_path_returns_normalized_results(self, monkeypatch): monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider mock_resp = self._make_mock_response(self._SAMPLE_RESPONSE) with patch("httpx.get", return_value=mock_resp): - result = SearXNGSearchProvider().search("test query", limit=5) + result = SearXNGWebSearchProvider().search("test query", limit=5) assert result["success"] is True web = result["data"]["web"] @@ -86,7 +86,7 @@ class TestSearXNGSearchProviderSearch: def test_results_sorted_by_score_descending(self, monkeypatch): """Results should be sorted by score before limit is applied.""" monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider unordered = { "results": [ {"title": "Low", "url": "https://low.example.com", "content": "", "score": 0.1}, @@ -97,7 +97,7 @@ class TestSearXNGSearchProviderSearch: mock_resp = self._make_mock_response(unordered) with patch("httpx.get", return_value=mock_resp): - result = SearXNGSearchProvider().search("query", limit=5) + result = SearXNGWebSearchProvider().search("query", limit=5) assert result["success"] is True assert result["data"]["web"][0]["title"] == "High" @@ -106,33 +106,33 @@ class TestSearXNGSearchProviderSearch: def test_limit_is_respected(self, monkeypatch): monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider mock_resp = self._make_mock_response(self._SAMPLE_RESPONSE) with patch("httpx.get", return_value=mock_resp): - result = SearXNGSearchProvider().search("query", limit=2) + result = SearXNGWebSearchProvider().search("query", limit=2) assert result["success"] is True assert len(result["data"]["web"]) == 2 def test_position_is_one_indexed(self, monkeypatch): monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider mock_resp = self._make_mock_response(self._SAMPLE_RESPONSE) with patch("httpx.get", return_value=mock_resp): - result = SearXNGSearchProvider().search("query", limit=5) + result = SearXNGWebSearchProvider().search("query", limit=5) positions = [r["position"] for r in result["data"]["web"]] assert positions == [1, 2, 3] def test_empty_results(self, monkeypatch): monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider mock_resp = self._make_mock_response({"results": []}) with patch("httpx.get", return_value=mock_resp): - result = SearXNGSearchProvider().search("nothing", limit=5) + result = SearXNGWebSearchProvider().search("nothing", limit=5) assert result["success"] is True assert result["data"]["web"] == [] @@ -140,7 +140,7 @@ class TestSearXNGSearchProviderSearch: def test_missing_score_falls_back_to_zero(self, monkeypatch): """Results without a score field should sort to the bottom.""" monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider data = { "results": [ {"title": "No score", "url": "https://noscore.example.com", "content": ""}, @@ -150,7 +150,7 @@ class TestSearXNGSearchProviderSearch: mock_resp = self._make_mock_response(data) with patch("httpx.get", return_value=mock_resp): - result = SearXNGSearchProvider().search("query", limit=5) + result = SearXNGWebSearchProvider().search("query", limit=5) assert result["success"] is True # Has score should sort first (0.8 > 0) @@ -159,14 +159,14 @@ class TestSearXNGSearchProviderSearch: def test_http_error_returns_failure(self, monkeypatch): import httpx monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider mock_resp = MagicMock() mock_resp.status_code = 500 http_err = httpx.HTTPStatusError("500", request=MagicMock(), response=mock_resp) with patch("httpx.get", side_effect=http_err): - result = SearXNGSearchProvider().search("query", limit=5) + result = SearXNGWebSearchProvider().search("query", limit=5) assert result["success"] is False assert "500" in result["error"] @@ -174,26 +174,26 @@ class TestSearXNGSearchProviderSearch: def test_request_error_returns_failure(self, monkeypatch): import httpx monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider with patch("httpx.get", side_effect=httpx.RequestError("connection refused")): - result = SearXNGSearchProvider().search("query", limit=5) + result = SearXNGWebSearchProvider().search("query", limit=5) assert result["success"] is False assert "localhost:8080" in result["error"] or "connection" in result["error"].lower() def test_missing_url_returns_failure(self, monkeypatch): monkeypatch.delenv("SEARXNG_URL", raising=False) - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider - result = SearXNGSearchProvider().search("query", limit=5) + result = SearXNGWebSearchProvider().search("query", limit=5) assert result["success"] is False assert "SEARXNG_URL" in result["error"] def test_trailing_slash_stripped_from_url(self, monkeypatch): """Base URL trailing slash should not produce double-slash in endpoint.""" monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080/") - from tools.web_providers.searxng import SearXNGSearchProvider + from plugins.web.searxng.provider import SearXNGWebSearchProvider mock_resp = self._make_mock_response({"results": []}) calls = [] @@ -202,7 +202,7 @@ class TestSearXNGSearchProviderSearch: return mock_resp with patch("httpx.get", side_effect=capture_get): - SearXNGSearchProvider().search("query", limit=5) + SearXNGWebSearchProvider().search("query", limit=5) assert calls[0] == "http://localhost:8080/search", f"Got: {calls[0]}" diff --git a/tools/web_providers/brave_free.py b/tools/web_providers/brave_free.py deleted file mode 100644 index 52d02dec2a1..00000000000 --- a/tools/web_providers/brave_free.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Brave Search web search provider (free tier). - -Brave Search's Data-for-Search API offers a free tier (2,000 queries/mo at the -time of writing) after signing up at https://brave.com/search/api/. This -provider implements ``WebSearchProvider`` only — the Data-for-Search endpoint -returns search results, it does not extract/crawl arbitrary URLs. - -Configuration:: - - # ~/.hermes/.env - BRAVE_SEARCH_API_KEY=your-subscription-token - - # ~/.hermes/config.yaml - web: - search_backend: "brave-free" - extract_backend: "firecrawl" # pair with an extract provider if needed - -The API uses the ``X-Subscription-Token`` header. Free-tier keys are rate -limited (1 qps) and capped at 2k queries/month; see the Brave dashboard for -current quotas. -""" - -from __future__ import annotations - -import logging -import os -from typing import Any, Dict - -from tools.web_providers.base import WebSearchProvider - -logger = logging.getLogger(__name__) - -_BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search" - - -class BraveFreeSearchProvider(WebSearchProvider): - """Search via the Brave Search API (free tier). - - Requires ``BRAVE_SEARCH_API_KEY`` to be set. The value is passed as the - ``X-Subscription-Token`` header. No extract capability — pair with - Firecrawl/Tavily/Exa/Parallel when you also need ``web_extract``. - """ - - def provider_name(self) -> str: - return "brave-free" - - def is_configured(self) -> bool: - """Return True when ``BRAVE_SEARCH_API_KEY`` is set to a non-empty value.""" - return bool(os.getenv("BRAVE_SEARCH_API_KEY", "").strip()) - - def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a search against the Brave Search API. - - Returns normalized results:: - - { - "success": True, - "data": { - "web": [ - { - "title": str, - "url": str, - "description": str, - "position": int, - }, - ... - ] - } - } - - On failure returns ``{"success": False, "error": str}``. - """ - import httpx - - api_key = os.getenv("BRAVE_SEARCH_API_KEY", "").strip() - if not api_key: - return {"success": False, "error": "BRAVE_SEARCH_API_KEY is not set"} - - # Brave's `count` is capped at 20. - count = max(1, min(int(limit), 20)) - - try: - resp = httpx.get( - _BRAVE_ENDPOINT, - params={"q": query, "count": count}, - headers={ - "X-Subscription-Token": api_key, - "Accept": "application/json", - }, - timeout=15, - ) - resp.raise_for_status() - except httpx.HTTPStatusError as exc: - logger.warning("Brave Search HTTP error: %s", exc) - return { - "success": False, - "error": f"Brave Search returned HTTP {exc.response.status_code}", - } - except httpx.RequestError as exc: - logger.warning("Brave Search request error: %s", exc) - return {"success": False, "error": f"Could not reach Brave Search: {exc}"} - - try: - data = resp.json() - except Exception as exc: # noqa: BLE001 - logger.warning("Brave Search response parse error: %s", exc) - return {"success": False, "error": "Could not parse Brave Search response as JSON"} - - raw_results = (data.get("web") or {}).get("results", []) or [] - truncated = raw_results[:limit] - - web_results = [ - { - "title": str(r.get("title", "")), - "url": str(r.get("url", "")), - "description": str(r.get("description", "")), - "position": i + 1, - } - for i, r in enumerate(truncated) - ] - - logger.info( - "Brave Search '%s': %d results (from %d raw, limit %d)", - query, - len(web_results), - len(raw_results), - limit, - ) - - return {"success": True, "data": {"web": web_results}} diff --git a/tools/web_providers/ddgs.py b/tools/web_providers/ddgs.py deleted file mode 100644 index b81b97de2cb..00000000000 --- a/tools/web_providers/ddgs.py +++ /dev/null @@ -1,98 +0,0 @@ -"""DuckDuckGo web search provider via the ``ddgs`` Python package. - -DuckDuckGo does not provide an official programmatic search API. The -community-maintained `ddgs `_ package (the -renamed successor of ``duckduckgo-search``) scrapes DuckDuckGo's HTML results -page and normalizes them. It implements ``WebSearchProvider`` only — there is -no extract capability. - -Configuration:: - - # No API key required. Enable by installing the package and pointing the - # web backend at ddgs: - pip install ddgs - - # ~/.hermes/config.yaml - web: - search_backend: "ddgs" - extract_backend: "firecrawl" # pair with an extract provider if needed - -Rate limits are enforced server-side by DuckDuckGo. Expect intermittent -``DuckDuckGoSearchException`` / 202 responses under heavy use; this provider -surfaces them as ``{"success": False, "error": ...}`` rather than crashing -the tool call. - -See https://duckduckgo.com/?q=duckduckgo+tos for terms of use. -""" - -from __future__ import annotations - -import logging -from typing import Any, Dict - -from tools.web_providers.base import WebSearchProvider - -logger = logging.getLogger(__name__) - - -class DDGSSearchProvider(WebSearchProvider): - """Search via the ``ddgs`` package (DuckDuckGo HTML scrape). - - No API key required. The provider is considered "configured" when the - ``ddgs`` package is importable — there is nothing else to set up. - """ - - def provider_name(self) -> str: - return "ddgs" - - def is_configured(self) -> bool: - """Return True when the ``ddgs`` package is importable. - - Called at tool-registration time; must not perform network I/O. - """ - try: - import ddgs # noqa: F401 - return True - except ImportError: - return False - - def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a DuckDuckGo search and return normalized results. - - Returns ``{"success": True, "data": {"web": [...]}}`` on success or - ``{"success": False, "error": str}`` on failure (missing package, - rate-limited, network error, etc.). - """ - try: - from ddgs import DDGS # type: ignore - except ImportError: - return { - "success": False, - "error": "ddgs package is not installed — run `pip install ddgs`", - } - - # DDGS().text yields at most `max_results` items; we cap defensively - # in case the package ignores the hint. - safe_limit = max(1, int(limit)) - - try: - web_results = [] - with DDGS() as client: - for i, hit in enumerate(client.text(query, max_results=safe_limit)): - if i >= safe_limit: - break - url = str(hit.get("href") or hit.get("url") or "") - web_results.append( - { - "title": str(hit.get("title", "")), - "url": url, - "description": str(hit.get("body", "")), - "position": i + 1, - } - ) - except Exception as exc: # noqa: BLE001 — ddgs raises its own exceptions - logger.warning("DDGS search error: %s", exc) - return {"success": False, "error": f"DuckDuckGo search failed: {exc}"} - - logger.info("DDGS search '%s': %d results (limit %d)", query, len(web_results), limit) - return {"success": True, "data": {"web": web_results}} diff --git a/tools/web_providers/searxng.py b/tools/web_providers/searxng.py deleted file mode 100644 index 589b0a2b337..00000000000 --- a/tools/web_providers/searxng.py +++ /dev/null @@ -1,132 +0,0 @@ -"""SearXNG web search provider. - -SearXNG is a free, self-hosted, privacy-respecting metasearch engine. -It implements ``WebSearchProvider`` only — there is no extract capability. - -Configuration:: - - # ~/.hermes/.env - SEARXNG_URL=http://localhost:8080 - - # Use SearXNG for search, pair with any extract provider: - # ~/.hermes/config.yaml - web: - search_backend: "searxng" - extract_backend: "firecrawl" - -Public SearXNG instances are listed at https://searx.space/ but self-hosting -is recommended for production use (rate limits and availability vary per -public instance). -""" - -from __future__ import annotations - -import logging -import os -from typing import Any, Dict - -from tools.web_providers.base import WebSearchProvider - -logger = logging.getLogger(__name__) - - -class SearXNGSearchProvider(WebSearchProvider): - """Search via a SearXNG instance. - - Requires ``SEARXNG_URL`` to be set (e.g. ``http://localhost:8080``). - No API key needed — SearXNG is open-source and self-hosted. - - Uses the SearXNG JSON API (``/search?format=json``). Results are - sorted by SearXNG's own score and truncated to *limit*. - """ - - def provider_name(self) -> str: - return "searxng" - - def is_configured(self) -> bool: - """Return True when ``SEARXNG_URL`` is set to a non-empty value.""" - return bool(os.getenv("SEARXNG_URL", "").strip()) - - def search(self, query: str, limit: int = 5) -> Dict[str, Any]: - """Execute a search against the configured SearXNG instance. - - Returns normalized results:: - - { - "success": True, - "data": { - "web": [ - { - "title": str, - "url": str, - "description": str, - "position": int, - }, - ... - ] - } - } - - On failure returns ``{"success": False, "error": str}``. - """ - import httpx - - base_url = os.getenv("SEARXNG_URL", "").strip().rstrip("/") - if not base_url: - return {"success": False, "error": "SEARXNG_URL is not set"} - - params: Dict[str, Any] = { - "q": query, - "format": "json", - "pageno": 1, - } - - try: - resp = httpx.get( - f"{base_url}/search", - params=params, - timeout=15, - headers={"Accept": "application/json"}, - ) - resp.raise_for_status() - except httpx.HTTPStatusError as exc: - logger.warning("SearXNG HTTP error: %s", exc) - return {"success": False, "error": f"SearXNG returned HTTP {exc.response.status_code}"} - except httpx.RequestError as exc: - logger.warning("SearXNG request error: %s", exc) - return {"success": False, "error": f"Could not reach SearXNG at {base_url}: {exc}"} - - try: - data = resp.json() - except Exception as exc: # noqa: BLE001 - logger.warning("SearXNG response parse error: %s", exc) - return {"success": False, "error": "Could not parse SearXNG response as JSON"} - - raw_results = data.get("results", []) - - # SearXNG may return a score field; sort descending and cap to limit. - sorted_results = sorted( - raw_results, - key=lambda r: float(r.get("score", 0)), - reverse=True, - )[:limit] - - web_results = [ - { - "title": str(r.get("title", "")), - "url": str(r.get("url", "")), - "description": str(r.get("content", "")), - "position": i + 1, - } - for i, r in enumerate(sorted_results) - ] - - logger.info( - "SearXNG search '%s': %d results (from %d raw, limit %d)", - query, - len(web_results), - len(raw_results), - limit, - ) - - return {"success": True, "data": {"web": web_results}}