mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
0f102fa4dc
* feat(browser): store full snapshots on truncation; make eval denylist opt-in Two harness fixes motivated by BU_Bench results where fixed-verb + lossy observation cost Hermes heavily vs code-driven browser agents: 1. Snapshot truncation no longer loses content. When a snapshot exceeds the 8000-char threshold, the complete accessibility tree is saved to cache/web (same truncate-and-store pattern as web_extract) and the truncated view / LLM summary includes the file path plus a ready-made read_file call. Element refs beyond the cut are recoverable without re-snapshotting. Stored copies are force-redacted and capped at 2MB; content-hash filenames dedupe repeated snapshots of the same page. 2. The browser_console(expression=...) sensitive-primitive denylist is now opt-in via browser.restrict_evaluate (default false). The names-based denylist blocked legitimate DOM extraction — any selector or expression containing 'fetch', 'cookie', 'input', etc. — which crippled the agent's only programmatic page-inspection path. The SSRF/private-URL egress guards in _browser_eval are independent of this policy and remain always-on. browser.allow_unsafe_evaluate keeps its meaning (bypass the denylist) for configs that already set it. * test: update None-guard test for stored-snapshot pointer in _extract_relevant_content test_normal_content_returned pinned the exact return value; the summary now carries a pointer to the stored full snapshot. Assert the summary passes through and the pointer is present instead. * feat(browser): align snapshot threshold with web_extract's 15k char budget SNAPSHOT_SUMMARIZE_THRESHOLD 8000 -> 15000, matching web_tools.DEFAULT_EXTRACT_CHAR_LIMIT so the snapshot and web_extract truncate-and-store paths give the model the same per-page budget. _truncate_snapshot's default max_chars now follows the constant. Invariant test added; docs (en+zh) and CLI tip updated.
111 lines
5.1 KiB
Python
111 lines
5.1 KiB
Python
"""Tests for None guard on browser_tool LLM response content.
|
|
|
|
browser_tool.py has two call sites that access response.choices[0].message.content
|
|
without checking for None — _extract_relevant_content (line 996) and
|
|
browser_vision (line 1626). When reasoning-only models (DeepSeek-R1, QwQ)
|
|
return content=None, these produce null snapshots or null analysis.
|
|
|
|
These tests verify both sites are guarded.
|
|
"""
|
|
|
|
import types
|
|
from unittest.mock import patch
|
|
|
|
|
|
|
|
# ── helpers ────────────────────────────────────────────────────────────────
|
|
|
|
def _make_response(content):
|
|
"""Build a minimal OpenAI-compatible ChatCompletion response stub."""
|
|
message = types.SimpleNamespace(content=content)
|
|
choice = types.SimpleNamespace(message=message)
|
|
return types.SimpleNamespace(choices=[choice])
|
|
|
|
|
|
# ── _extract_relevant_content (line 996) ──────────────────────────────────
|
|
|
|
class TestExtractRelevantContentNoneGuard:
|
|
"""tools/browser_tool.py — _extract_relevant_content()"""
|
|
|
|
def test_none_content_falls_back_to_truncated(self):
|
|
"""When LLM returns None content, should fall back to truncated snapshot."""
|
|
with patch("tools.browser_tool.call_llm", return_value=_make_response(None)), \
|
|
patch("tools.browser_tool._get_extraction_model", return_value="test-model"):
|
|
from tools.browser_tool import _extract_relevant_content
|
|
result = _extract_relevant_content("This is a long snapshot text", "find the button")
|
|
|
|
assert result is not None
|
|
assert isinstance(result, str)
|
|
assert len(result) > 0
|
|
|
|
def test_normal_content_returned(self):
|
|
"""Normal string content should pass through (plus the stored-full-snapshot pointer)."""
|
|
with patch("tools.browser_tool.call_llm", return_value=_make_response("Extracted content here")), \
|
|
patch("tools.browser_tool._get_extraction_model", return_value="test-model"):
|
|
from tools.browser_tool import _extract_relevant_content
|
|
result = _extract_relevant_content("snapshot text", "task")
|
|
|
|
# The summary itself passes through unchanged; a pointer to the stored
|
|
# full snapshot is appended (see _store_full_snapshot).
|
|
assert result.startswith("Extracted content here")
|
|
assert "Full snapshot saved to" in result
|
|
|
|
def test_empty_string_content_falls_back(self):
|
|
"""Empty string content should also fall back to truncated."""
|
|
with patch("tools.browser_tool.call_llm", return_value=_make_response(" ")), \
|
|
patch("tools.browser_tool._get_extraction_model", return_value="test-model"):
|
|
from tools.browser_tool import _extract_relevant_content
|
|
result = _extract_relevant_content("This is a long snapshot text", "task")
|
|
|
|
assert result is not None
|
|
assert len(result) > 0
|
|
|
|
|
|
# ── browser_vision (line 1626) ────────────────────────────────────────────
|
|
|
|
class TestBrowserVisionNoneGuard:
|
|
"""tools/browser_tool.py — browser_vision() analysis extraction"""
|
|
|
|
def test_none_content_produces_fallback_message(self):
|
|
"""When LLM returns None content, analysis should have a fallback message."""
|
|
response = _make_response(None)
|
|
analysis = (response.choices[0].message.content or "").strip()
|
|
fallback = analysis or "Vision analysis returned no content."
|
|
|
|
assert fallback == "Vision analysis returned no content."
|
|
|
|
def test_normal_content_passes_through(self):
|
|
"""Normal analysis content should pass through unchanged."""
|
|
response = _make_response(" The page shows a login form. ")
|
|
analysis = (response.choices[0].message.content or "").strip()
|
|
fallback = analysis or "Vision analysis returned no content."
|
|
|
|
assert fallback == "The page shows a login form."
|
|
|
|
|
|
# ── source line verification ──────────────────────────────────────────────
|
|
|
|
class TestBrowserSourceLinesAreGuarded:
|
|
"""Verify the actual source file has the fix applied."""
|
|
|
|
@staticmethod
|
|
def _read_file() -> str:
|
|
import os
|
|
base = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
|
with open(os.path.join(base, "tools", "browser_tool.py")) as f:
|
|
return f.read()
|
|
|
|
def test_extract_relevant_content_guarded(self):
|
|
src = self._read_file()
|
|
# The old unguarded pattern should NOT exist
|
|
assert "return response.choices[0].message.content\n" not in src, (
|
|
"browser_tool.py _extract_relevant_content still has unguarded "
|
|
".content return — apply None guard"
|
|
)
|
|
|
|
def test_browser_vision_guarded(self):
|
|
src = self._read_file()
|
|
assert "analysis = response.choices[0].message.content\n" not in src, (
|
|
"browser_tool.py browser_vision still has unguarded "
|
|
".content assignment — apply None guard"
|
|
)
|