mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
357 lines
13 KiB
Python
357 lines
13 KiB
Python
"""Tests for _reap_orphaned_browser_sessions() — kills orphaned agent-browser
|
|
daemons whose Python parent exited without cleaning up."""
|
|
|
|
import os
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_tmpdir(tmp_path):
|
|
"""Patch _socket_safe_tmpdir to return a temp dir we control."""
|
|
with patch("tools.browser_tool._socket_safe_tmpdir", return_value=str(tmp_path)):
|
|
yield tmp_path
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_sessions():
|
|
"""Ensure _active_sessions is empty for each test."""
|
|
import tools.browser_tool as bt
|
|
orig = bt._active_sessions.copy()
|
|
bt._active_sessions.clear()
|
|
yield
|
|
bt._active_sessions.clear()
|
|
bt._active_sessions.update(orig)
|
|
|
|
|
|
def _make_socket_dir(tmpdir, session_name, pid=None, owner_pid=None):
|
|
"""Create a fake agent-browser socket directory with optional PID files.
|
|
|
|
Args:
|
|
tmpdir: base temp directory
|
|
session_name: name like "h_abc1234567" or "cdp_abc1234567"
|
|
pid: daemon PID to write to <session>.pid (None = no file)
|
|
owner_pid: owning hermes PID to write to <session>.owner_pid
|
|
(None = no file; tests the legacy path)
|
|
"""
|
|
d = tmpdir / f"agent-browser-{session_name}"
|
|
d.mkdir()
|
|
if pid is not None:
|
|
(d / f"{session_name}.pid").write_text(str(pid))
|
|
if owner_pid is not None:
|
|
(d / f"{session_name}.owner_pid").write_text(str(owner_pid))
|
|
return d
|
|
|
|
|
|
class TestReapOrphanedBrowserSessions:
|
|
"""Tests for the orphan reaper function."""
|
|
|
|
def test_no_socket_dirs_is_noop(self, fake_tmpdir):
|
|
"""No socket dirs => nothing happens, no errors."""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
_reap_orphaned_browser_sessions() # should not raise
|
|
|
|
def test_stale_dir_without_pid_file_is_removed(self, fake_tmpdir):
|
|
"""Socket dir with no PID file is cleaned up."""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
d = _make_socket_dir(fake_tmpdir, "h_abc1234567")
|
|
assert d.exists()
|
|
_reap_orphaned_browser_sessions()
|
|
assert not d.exists()
|
|
|
|
|
|
def test_alive_legacy_daemon_is_reaped(self, fake_tmpdir):
|
|
"""Alive, untracked, legacy (no owner_pid) daemon is reaped.
|
|
|
|
Post-#21561 the liveness probe goes through
|
|
``gateway.status._pid_exists`` (which wraps ``psutil.pid_exists``
|
|
because ``os.kill(pid, 0)`` is a footgun on Windows — bpo-14484).
|
|
With no owner_pid file and no tracked-name entry, the reaper
|
|
terminates the daemon (and its process tree) and removes its socket
|
|
dir regardless of whether termination succeeded (best-effort
|
|
semantics).
|
|
"""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
|
|
d = _make_socket_dir(fake_tmpdir, "h_perm1234567", pid=12345)
|
|
|
|
terminate_calls = []
|
|
|
|
def mock_terminate(pid):
|
|
terminate_calls.append(pid)
|
|
|
|
with patch("gateway.status._pid_exists", return_value=True), \
|
|
patch("tools.browser_tool._verify_reapable_browser_daemon", return_value=True), \
|
|
patch("tools.process_registry.ProcessRegistry._terminate_host_pid", side_effect=mock_terminate):
|
|
_reap_orphaned_browser_sessions()
|
|
|
|
assert 12345 in terminate_calls
|
|
assert not d.exists()
|
|
|
|
|
|
def test_corrupt_pid_file_is_cleaned(self, fake_tmpdir):
|
|
"""PID file with non-integer content is cleaned up."""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
|
|
d = _make_socket_dir(fake_tmpdir, "h_corrupt1234")
|
|
(d / "h_corrupt1234.pid").write_text("not-a-number")
|
|
|
|
_reap_orphaned_browser_sessions()
|
|
assert not d.exists()
|
|
|
|
|
|
class TestOwnerPidCrossProcess:
|
|
"""Tests for owner_pid-based cross-process safe reaping.
|
|
|
|
The owner_pid file records which hermes process owns a daemon so that
|
|
concurrent hermes processes don't reap each other's active browser
|
|
sessions. Added to fix orphan accumulation from crashed processes.
|
|
"""
|
|
|
|
def test_alive_owner_is_not_reaped_even_when_untracked(self, fake_tmpdir):
|
|
"""Daemon with alive owner_pid is NOT reaped, even if not in our _active_sessions.
|
|
|
|
This is the core cross-process safety check: Process B scanning while
|
|
Process A is using a browser must not kill A's daemon.
|
|
"""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
|
|
# Use our own PID as the "owner" — guaranteed alive
|
|
d = _make_socket_dir(
|
|
fake_tmpdir, "h_alive_owner", pid=12345, owner_pid=os.getpid()
|
|
)
|
|
|
|
kill_calls = []
|
|
|
|
def mock_terminate(pid):
|
|
kill_calls.append(pid)
|
|
|
|
# Owner alive → reaper skips without ever probing the daemon.
|
|
with patch("gateway.status._pid_exists", return_value=True), \
|
|
patch("tools.process_registry.ProcessRegistry._terminate_host_pid", side_effect=mock_terminate):
|
|
_reap_orphaned_browser_sessions()
|
|
|
|
assert 12345 not in kill_calls
|
|
assert d.exists()
|
|
|
|
|
|
def test_owner_pid_permission_error_treated_as_alive(self, fake_tmpdir):
|
|
"""Owner PID owned by another user → treat as alive.
|
|
|
|
Post-#21561 this is handled inside ``gateway.status._pid_exists``
|
|
(via psutil's ``OpenProcess`` returning ``ERROR_ACCESS_DENIED`` on
|
|
Windows, or via the POSIX fallback's ``except PermissionError``
|
|
branch). Exposed to callers as ``alive=True``.
|
|
"""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
|
|
d = _make_socket_dir(
|
|
fake_tmpdir, "h_perm_owner1", pid=12345, owner_pid=22222
|
|
)
|
|
|
|
kill_calls = []
|
|
|
|
def mock_terminate(pid):
|
|
kill_calls.append(pid)
|
|
|
|
# Owner 22222 reported alive (PermissionError collapses to True
|
|
# inside _pid_exists). Daemon never probed, never terminated.
|
|
with patch("gateway.status._pid_exists", return_value=True), \
|
|
patch("tools.process_registry.ProcessRegistry._terminate_host_pid", side_effect=mock_terminate):
|
|
_reap_orphaned_browser_sessions()
|
|
|
|
assert 12345 not in kill_calls
|
|
assert d.exists()
|
|
|
|
|
|
def test_write_owner_pid_swallows_oserror(self, fake_tmpdir, monkeypatch):
|
|
"""OSError (e.g. permission denied) doesn't propagate — the reaper
|
|
falls back to the legacy tracked_names heuristic in that case.
|
|
"""
|
|
import tools.browser_tool as bt
|
|
|
|
def raise_oserror(*a, **kw):
|
|
raise OSError("permission denied")
|
|
|
|
monkeypatch.setattr("builtins.open", raise_oserror)
|
|
|
|
# Must not raise
|
|
bt._write_owner_pid(str(fake_tmpdir), "h_readonly123")
|
|
|
|
def test_run_browser_command_calls_write_owner_pid(
|
|
self, fake_tmpdir, monkeypatch
|
|
):
|
|
"""_run_browser_command wires _write_owner_pid after mkdir."""
|
|
import tools.browser_tool as bt
|
|
|
|
session_name = "h_wiringtest1"
|
|
|
|
# Short-circuit Popen so we exit after the owner_pid write
|
|
class _FakePopen:
|
|
def __init__(self, *a, **kw):
|
|
raise RuntimeError("short-circuit after owner_pid")
|
|
|
|
monkeypatch.setattr(bt.subprocess, "Popen", _FakePopen)
|
|
monkeypatch.setattr(bt, "_find_agent_browser", lambda: "/bin/true")
|
|
monkeypatch.setattr(
|
|
bt, "_requires_real_termux_browser_install", lambda *a: False
|
|
)
|
|
monkeypatch.setattr(bt, "_chromium_installed", lambda: True)
|
|
monkeypatch.setattr(
|
|
bt, "_get_session_info",
|
|
lambda task_id: {"session_name": session_name},
|
|
)
|
|
|
|
calls = []
|
|
orig_write = bt._write_owner_pid
|
|
|
|
def _spy(*a, **kw):
|
|
calls.append(a)
|
|
orig_write(*a, **kw)
|
|
|
|
monkeypatch.setattr(bt, "_write_owner_pid", _spy)
|
|
|
|
with patch("tools.browser_tool._socket_safe_tmpdir", return_value=str(fake_tmpdir)):
|
|
try:
|
|
bt._run_browser_command(task_id="test_task", command="goto", args=[])
|
|
except Exception:
|
|
pass
|
|
|
|
assert calls, "_run_browser_command must call _write_owner_pid"
|
|
# First positional arg is the socket_dir, second is the session_name
|
|
socket_dir_arg, session_name_arg = calls[0][0], calls[0][1]
|
|
assert session_name_arg == session_name
|
|
assert session_name in socket_dir_arg
|
|
|
|
|
|
class TestReaperIdentityGuard:
|
|
"""Tests for _verify_reapable_browser_daemon — the #14073 fix.
|
|
|
|
The reaper reads daemon PIDs from world-writable, predictably-named temp
|
|
dirs. Before tree-killing a live PID it must confirm the process really is
|
|
*this* session's agent-browser daemon, defeating planted pid files and
|
|
recycled PIDs that would otherwise become an arbitrary same-user DoS.
|
|
"""
|
|
|
|
class _FakeProc:
|
|
def __init__(self, name="agent-browser", cmdline=None, environ=None,
|
|
raise_environ=False):
|
|
self._name = name
|
|
self._cmdline = cmdline if cmdline is not None else []
|
|
self._environ = environ or {}
|
|
self._raise_environ = raise_environ
|
|
|
|
def name(self):
|
|
return self._name
|
|
|
|
def cmdline(self):
|
|
return self._cmdline
|
|
|
|
def environ(self):
|
|
if self._raise_environ:
|
|
import psutil
|
|
raise psutil.AccessDenied()
|
|
return self._environ
|
|
|
|
def _run(self, fake_proc, socket_dir, session_name="h_sess123456",
|
|
daemon_pid=12345, no_such=False, access_denied=False):
|
|
import psutil
|
|
from tools.browser_tool import _verify_reapable_browser_daemon
|
|
|
|
def _factory(pid):
|
|
if no_such:
|
|
raise psutil.NoSuchProcess(pid)
|
|
if access_denied:
|
|
raise psutil.AccessDenied(pid)
|
|
return fake_proc
|
|
|
|
with patch("psutil.Process", side_effect=_factory):
|
|
return _verify_reapable_browser_daemon(
|
|
daemon_pid, socket_dir, session_name)
|
|
|
|
def test_real_daemon_bound_via_cmdline_is_reapable(self):
|
|
socket_dir = "/tmp/agent-browser-h_sess123456"
|
|
proc = self._FakeProc(
|
|
name="agent-browser",
|
|
cmdline=["agent-browser", "open", "--session", "h_sess123456",
|
|
"--socket-dir", socket_dir],
|
|
)
|
|
assert self._run(proc, socket_dir) is True
|
|
|
|
def test_daemon_bound_via_environ_is_reapable(self):
|
|
socket_dir = "/tmp/agent-browser-h_sess123456"
|
|
proc = self._FakeProc(
|
|
name="agent-browser-linux-x64",
|
|
cmdline=["agent-browser-linux-x64", "daemon"], # no dir in cmd
|
|
environ={"AGENT_BROWSER_SOCKET_DIR": socket_dir},
|
|
)
|
|
assert self._run(proc, socket_dir) is True
|
|
|
|
|
|
def test_recycled_pid_browser_not_bound_to_our_dir_is_refused(self):
|
|
"""An agent-browser process for a DIFFERENT session must not be reaped.
|
|
|
|
Models PID reuse / a concurrent unrelated daemon: it looks like
|
|
agent-browser but is bound to another socket dir.
|
|
"""
|
|
socket_dir = "/tmp/agent-browser-h_sess123456"
|
|
proc = self._FakeProc(
|
|
name="agent-browser",
|
|
cmdline=["agent-browser", "open", "--session", "h_OTHER999",
|
|
"--socket-dir", "/tmp/agent-browser-h_OTHER999"],
|
|
environ={"AGENT_BROWSER_SOCKET_DIR":
|
|
"/tmp/agent-browser-h_OTHER999"},
|
|
)
|
|
assert self._run(proc, socket_dir) is False
|
|
|
|
|
|
def test_planted_pid_survives_full_reaper_path(self, fake_tmpdir):
|
|
"""End-to-end through the reaper: a planted non-browser PID is spared.
|
|
|
|
No owner_pid (legacy path), not tracked, PID 'alive' — but the live
|
|
process is `sleep`, not agent-browser, so it must be left alone and the
|
|
socket dir retained.
|
|
"""
|
|
from tools.browser_tool import _reap_orphaned_browser_sessions
|
|
|
|
d = _make_socket_dir(fake_tmpdir, "h_planted9999", pid=12345)
|
|
|
|
terminate_calls = []
|
|
proc = self._FakeProc(name="sleep", cmdline=["/bin/sleep", "600"])
|
|
|
|
with patch("gateway.status._pid_exists", return_value=True), \
|
|
patch("psutil.Process", return_value=proc), \
|
|
patch("tools.process_registry.ProcessRegistry._terminate_host_pid",
|
|
side_effect=lambda pid: terminate_calls.append(pid)):
|
|
_reap_orphaned_browser_sessions()
|
|
|
|
assert terminate_calls == [], "planted non-browser PID must not be killed"
|
|
assert d.exists(), "socket dir retained for a later sweep"
|
|
|
|
|
|
class TestEmergencyCleanupRunsReaper:
|
|
"""Verify atexit-registered cleanup sweeps orphans even without an active session."""
|
|
|
|
def test_emergency_cleanup_calls_reaper(self, fake_tmpdir, monkeypatch):
|
|
"""_emergency_cleanup_all_sessions must call _reap_orphaned_browser_sessions."""
|
|
import tools.browser_tool as bt
|
|
|
|
# Reset the _cleanup_done flag so the cleanup actually runs
|
|
monkeypatch.setattr(bt, "_cleanup_done", False)
|
|
|
|
reaper_called = []
|
|
orig_reaper = bt._reap_orphaned_browser_sessions
|
|
|
|
def _spy_reaper():
|
|
reaper_called.append(True)
|
|
orig_reaper()
|
|
|
|
monkeypatch.setattr(bt, "_reap_orphaned_browser_sessions", _spy_reaper)
|
|
|
|
# No active sessions — reaper should still run
|
|
bt._emergency_cleanup_all_sessions()
|
|
|
|
assert reaper_called, (
|
|
"Reaper must run on exit even with no active sessions"
|
|
)
|