mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-25 00:51:20 +00:00
fix(browser): runtime fallback to local Chromium when cloud provider fails
Wraps provider.create_session() in _get_session_info() with try/except to catch cloud provider runtime failures (timeouts, auth errors, rate limits, invalid responses). Falls back to _create_local_session() so browser automation continues working when cloud APIs are down. Marks fallback sessions with fallback_from_cloud, fallback_reason, and fallback_provider metadata for observability. If both cloud and local fail, raises RuntimeError with chained context from both errors. Closes #10883 Co-authored-by: konsisumer <konsisumer@users.noreply.github.com>
This commit is contained in:
parent
e0532be8ae
commit
f726b9b843
2 changed files with 197 additions and 6 deletions
166
tests/tools/test_browser_cloud_fallback.py
Normal file
166
tests/tools/test_browser_cloud_fallback.py
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
"""Tests for cloud browser provider runtime fallback to local Chromium.
|
||||||
|
|
||||||
|
Covers the fallback logic in _get_session_info() when a cloud provider
|
||||||
|
is configured but fails at runtime (issue #10883).
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import tools.browser_tool as browser_tool
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_session_state(monkeypatch):
|
||||||
|
"""Clear caches so each test starts fresh."""
|
||||||
|
monkeypatch.setattr(browser_tool, "_active_sessions", {})
|
||||||
|
monkeypatch.setattr(browser_tool, "_cached_cloud_provider", None)
|
||||||
|
monkeypatch.setattr(browser_tool, "_cloud_provider_resolved", False)
|
||||||
|
monkeypatch.setattr(browser_tool, "_start_browser_cleanup_thread", lambda: None)
|
||||||
|
monkeypatch.setattr(browser_tool, "_update_session_activity", lambda t: None)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCloudProviderRuntimeFallback:
|
||||||
|
"""Tests for _get_session_info cloud → local fallback."""
|
||||||
|
|
||||||
|
def test_cloud_failure_falls_back_to_local(self, monkeypatch):
|
||||||
|
"""When cloud provider.create_session raises, fall back to local."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
provider = Mock()
|
||||||
|
provider.create_session.side_effect = RuntimeError("401 Unauthorized")
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
|
||||||
|
session = browser_tool._get_session_info("task-1")
|
||||||
|
|
||||||
|
assert session["fallback_from_cloud"] is True
|
||||||
|
assert "401 Unauthorized" in session["fallback_reason"]
|
||||||
|
assert session["fallback_provider"] == "Mock"
|
||||||
|
assert session["features"]["local"] is True
|
||||||
|
assert session["cdp_url"] is None
|
||||||
|
|
||||||
|
def test_cloud_success_no_fallback(self, monkeypatch):
|
||||||
|
"""When cloud succeeds, no fallback markers are present."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
provider = Mock()
|
||||||
|
provider.create_session.return_value = {
|
||||||
|
"session_name": "cloud-sess",
|
||||||
|
"bb_session_id": "bb_123",
|
||||||
|
"cdp_url": None,
|
||||||
|
"features": {"browser_use": True},
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
|
||||||
|
session = browser_tool._get_session_info("task-2")
|
||||||
|
|
||||||
|
assert session["session_name"] == "cloud-sess"
|
||||||
|
assert "fallback_from_cloud" not in session
|
||||||
|
assert "fallback_reason" not in session
|
||||||
|
|
||||||
|
def test_cloud_and_local_both_fail(self, monkeypatch):
|
||||||
|
"""When both cloud and local fail, raise RuntimeError with both contexts."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
provider = Mock()
|
||||||
|
provider.create_session.side_effect = RuntimeError("cloud boom")
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
browser_tool, "_create_local_session",
|
||||||
|
Mock(side_effect=OSError("no chromium")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="cloud boom.*local.*no chromium"):
|
||||||
|
browser_tool._get_session_info("task-3")
|
||||||
|
|
||||||
|
def test_no_provider_uses_local_directly(self, monkeypatch):
|
||||||
|
"""When no cloud provider is configured, local mode is used with no fallback markers."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: None)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
|
||||||
|
session = browser_tool._get_session_info("task-4")
|
||||||
|
|
||||||
|
assert session["features"]["local"] is True
|
||||||
|
assert "fallback_from_cloud" not in session
|
||||||
|
|
||||||
|
def test_cdp_override_bypasses_provider(self, monkeypatch):
|
||||||
|
"""CDP override takes priority — cloud provider is never consulted."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
provider = Mock()
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: "ws://host:9222/devtools/browser/abc")
|
||||||
|
|
||||||
|
session = browser_tool._get_session_info("task-5")
|
||||||
|
|
||||||
|
provider.create_session.assert_not_called()
|
||||||
|
assert session["cdp_url"] == "ws://host:9222/devtools/browser/abc"
|
||||||
|
|
||||||
|
def test_fallback_logs_warning_with_provider_name(self, monkeypatch, caplog):
|
||||||
|
"""Fallback emits a warning log with the provider class name and error."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
BrowserUseProviderFake = type("BrowserUseProvider", (), {
|
||||||
|
"create_session": Mock(side_effect=ConnectionError("timeout")),
|
||||||
|
})
|
||||||
|
provider = BrowserUseProviderFake()
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING, logger="tools.browser_tool"):
|
||||||
|
session = browser_tool._get_session_info("task-6")
|
||||||
|
|
||||||
|
assert session["fallback_from_cloud"] is True
|
||||||
|
assert any("BrowserUseProvider" in r.message and "timeout" in r.message
|
||||||
|
for r in caplog.records)
|
||||||
|
|
||||||
|
def test_cloud_failure_does_not_poison_next_task(self, monkeypatch):
|
||||||
|
"""A fallback for one task_id doesn't affect a new task_id when cloud recovers."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
call_count = 0
|
||||||
|
|
||||||
|
def create_session_flaky(task_id):
|
||||||
|
nonlocal call_count
|
||||||
|
call_count += 1
|
||||||
|
if call_count == 1:
|
||||||
|
raise RuntimeError("transient failure")
|
||||||
|
return {
|
||||||
|
"session_name": "cloud-ok",
|
||||||
|
"bb_session_id": "bb_999",
|
||||||
|
"cdp_url": None,
|
||||||
|
"features": {"browser_use": True},
|
||||||
|
}
|
||||||
|
|
||||||
|
provider = Mock()
|
||||||
|
provider.create_session.side_effect = create_session_flaky
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
|
||||||
|
# First call fails → fallback
|
||||||
|
s1 = browser_tool._get_session_info("task-a")
|
||||||
|
assert s1["fallback_from_cloud"] is True
|
||||||
|
|
||||||
|
# Second call (different task) → cloud succeeds
|
||||||
|
s2 = browser_tool._get_session_info("task-b")
|
||||||
|
assert "fallback_from_cloud" not in s2
|
||||||
|
assert s2["session_name"] == "cloud-ok"
|
||||||
|
|
||||||
|
def test_cloud_returns_invalid_session_triggers_fallback(self, monkeypatch):
|
||||||
|
"""Cloud provider returning None or empty dict triggers fallback."""
|
||||||
|
_reset_session_state(monkeypatch)
|
||||||
|
|
||||||
|
provider = Mock()
|
||||||
|
provider.create_session.return_value = None
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cloud_provider", lambda: provider)
|
||||||
|
monkeypatch.setattr(browser_tool, "_get_cdp_override", lambda: None)
|
||||||
|
|
||||||
|
session = browser_tool._get_session_info("task-7")
|
||||||
|
|
||||||
|
assert session["fallback_from_cloud"] is True
|
||||||
|
assert "invalid session" in session["fallback_reason"]
|
||||||
|
|
@ -873,12 +873,37 @@ def _get_session_info(task_id: Optional[str] = None) -> Dict[str, str]:
|
||||||
if provider is None:
|
if provider is None:
|
||||||
session_info = _create_local_session(task_id)
|
session_info = _create_local_session(task_id)
|
||||||
else:
|
else:
|
||||||
session_info = provider.create_session(task_id)
|
try:
|
||||||
if session_info.get("cdp_url"):
|
session_info = provider.create_session(task_id)
|
||||||
# Some cloud providers (including Browser-Use v3) return an HTTP
|
# Validate cloud provider returned a usable session
|
||||||
# CDP discovery URL instead of a raw websocket endpoint.
|
if not session_info or not isinstance(session_info, dict):
|
||||||
session_info = dict(session_info)
|
raise ValueError(f"Cloud provider returned invalid session: {session_info!r}")
|
||||||
session_info["cdp_url"] = _resolve_cdp_override(str(session_info["cdp_url"]))
|
if session_info.get("cdp_url"):
|
||||||
|
# Some cloud providers (including Browser-Use v3) return an HTTP
|
||||||
|
# CDP discovery URL instead of a raw websocket endpoint.
|
||||||
|
session_info = dict(session_info)
|
||||||
|
session_info["cdp_url"] = _resolve_cdp_override(str(session_info["cdp_url"]))
|
||||||
|
except Exception as e:
|
||||||
|
provider_name = type(provider).__name__
|
||||||
|
logger.warning(
|
||||||
|
"Cloud provider %s failed (%s); attempting fallback to local "
|
||||||
|
"Chromium for task %s",
|
||||||
|
provider_name, e, task_id,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
session_info = _create_local_session(task_id)
|
||||||
|
except Exception as local_error:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Cloud provider {provider_name} failed ({e}) and local "
|
||||||
|
f"fallback also failed ({local_error})"
|
||||||
|
) from e
|
||||||
|
# Mark session as degraded for observability
|
||||||
|
if isinstance(session_info, dict):
|
||||||
|
session_info = dict(session_info)
|
||||||
|
session_info["fallback_from_cloud"] = True
|
||||||
|
session_info["fallback_reason"] = str(e)
|
||||||
|
session_info["fallback_provider"] = provider_name
|
||||||
|
|
||||||
with _cleanup_lock:
|
with _cleanup_lock:
|
||||||
# Double-check: another thread may have created a session while we
|
# Double-check: another thread may have created a session while we
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue