mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
412 lines
15 KiB
Python
412 lines
15 KiB
Python
"""Tests for backend-specific bulk download implementations and cleanup() wiring."""
|
|
|
|
import asyncio
|
|
import subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from tools.environments import ssh as ssh_env
|
|
from tools.environments import modal as modal_env
|
|
from tools.environments import daytona as daytona_env
|
|
from tools.environments.ssh import SSHEnvironment
|
|
|
|
|
|
# ── SSH helpers ──────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.fixture
|
|
def ssh_mock_env(monkeypatch):
|
|
"""Create an SSHEnvironment with mocked connection/sync."""
|
|
monkeypatch.setattr(ssh_env.shutil, "which", lambda _name: "/usr/bin/ssh")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_establish_connection", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_detect_remote_home", lambda self: "/home/testuser")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_ensure_remote_dirs", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "init_session", lambda self: None)
|
|
monkeypatch.setattr(
|
|
ssh_env, "FileSyncManager",
|
|
lambda **kw: type("M", (), {
|
|
"sync": lambda self, **k: None,
|
|
"sync_back": lambda self: None,
|
|
})(),
|
|
)
|
|
return SSHEnvironment(host="example.com", user="testuser")
|
|
|
|
|
|
# ── Modal helpers ────────────────────────────────────────────────────
|
|
|
|
|
|
def _make_mock_modal_env():
|
|
"""Create a minimal ModalEnvironment without calling __init__."""
|
|
env = object.__new__(modal_env.ModalEnvironment)
|
|
env._sandbox = MagicMock()
|
|
env._worker = MagicMock()
|
|
env._persistent = False
|
|
env._task_id = "test"
|
|
env._sync_manager = None
|
|
return env
|
|
|
|
|
|
def _wire_modal_download(env, *, tar_bytes=b"fake-tar-data", exit_code=0):
|
|
"""Wire sandbox.exec.aio to return mock tar output for download tests.
|
|
|
|
Returns the exec_calls list for assertion.
|
|
"""
|
|
exec_calls = []
|
|
|
|
async def mock_exec_fn(*args, **kwargs):
|
|
exec_calls.append(args)
|
|
proc = MagicMock()
|
|
proc.stdout = MagicMock()
|
|
proc.stdout.read = MagicMock()
|
|
proc.stdout.read.aio = AsyncMock(return_value=tar_bytes)
|
|
proc.wait = MagicMock()
|
|
proc.wait.aio = AsyncMock(return_value=exit_code)
|
|
return proc
|
|
|
|
env._sandbox.exec = MagicMock()
|
|
env._sandbox.exec.aio = mock_exec_fn
|
|
|
|
def real_run_coroutine(coro, **kwargs):
|
|
loop = asyncio.new_event_loop()
|
|
try:
|
|
return loop.run_until_complete(coro)
|
|
finally:
|
|
loop.close()
|
|
|
|
env._worker.run_coroutine = real_run_coroutine
|
|
return exec_calls
|
|
|
|
|
|
# ── Daytona helpers ──────────────────────────────────────────────────
|
|
|
|
|
|
def _make_mock_daytona_env():
|
|
"""Create a minimal DaytonaEnvironment without calling __init__."""
|
|
env = object.__new__(daytona_env.DaytonaEnvironment)
|
|
env._sandbox = MagicMock()
|
|
env._remote_home = "/root"
|
|
env._sync_manager = None
|
|
env._lock = __import__("threading").Lock()
|
|
env._persistent = True
|
|
env._task_id = "test"
|
|
env._daytona = MagicMock()
|
|
return env
|
|
|
|
|
|
# =====================================================================
|
|
# SSH bulk download
|
|
# =====================================================================
|
|
|
|
|
|
class TestSSHBulkDownload:
|
|
"""Unit tests for _ssh_bulk_download."""
|
|
|
|
def test_ssh_bulk_download_runs_tar_over_ssh(self, ssh_mock_env, tmp_path):
|
|
"""subprocess.run command should include tar cf - over SSH."""
|
|
dest = tmp_path / "backup.tar"
|
|
|
|
with patch.object(subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as mock_run:
|
|
# open() will be called to write stdout; mock it to avoid actual file I/O
|
|
ssh_mock_env._ssh_bulk_download(dest)
|
|
|
|
mock_run.assert_called_once()
|
|
cmd = mock_run.call_args[0][0]
|
|
cmd_str = " ".join(cmd)
|
|
assert "tar cf -" in cmd_str
|
|
assert "-C /" in cmd_str
|
|
assert "home/testuser/.hermes" in cmd_str
|
|
assert "ssh" in cmd_str
|
|
assert "testuser@example.com" in cmd_str
|
|
|
|
|
|
def test_ssh_bulk_download_uses_120s_timeout(self, ssh_mock_env, tmp_path):
|
|
"""The subprocess.run call should use a 120s timeout."""
|
|
dest = tmp_path / "backup.tar"
|
|
|
|
with patch.object(subprocess, "run", return_value=subprocess.CompletedProcess([], 0)) as mock_run:
|
|
ssh_mock_env._ssh_bulk_download(dest)
|
|
|
|
call_kwargs = mock_run.call_args
|
|
assert call_kwargs.kwargs.get("timeout") == 120 or call_kwargs[1].get("timeout") == 120
|
|
|
|
|
|
class TestSSHCleanup:
|
|
"""Verify SSH cleanup() calls sync_back() before closing ControlMaster."""
|
|
|
|
def test_ssh_cleanup_calls_sync_back(self, monkeypatch):
|
|
"""cleanup() should call sync_back() before SSH control socket teardown."""
|
|
monkeypatch.setattr(ssh_env.shutil, "which", lambda _name: "/usr/bin/ssh")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_establish_connection", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_detect_remote_home", lambda self: "/home/u")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_ensure_remote_dirs", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "init_session", lambda self: None)
|
|
|
|
call_order = []
|
|
|
|
class TrackingSyncManager:
|
|
def __init__(self, **kwargs):
|
|
pass
|
|
|
|
def sync(self, **kw):
|
|
pass
|
|
|
|
def sync_back(self):
|
|
call_order.append("sync_back")
|
|
|
|
monkeypatch.setattr(ssh_env, "FileSyncManager", TrackingSyncManager)
|
|
|
|
env = SSHEnvironment(host="h", user="u")
|
|
# Ensure control_socket does not exist so cleanup skips the SSH exit call
|
|
env.control_socket = Path("/nonexistent/socket")
|
|
|
|
env.cleanup()
|
|
|
|
assert "sync_back" in call_order
|
|
|
|
def test_ssh_cleanup_calls_sync_back_before_control_exit(self, monkeypatch):
|
|
"""sync_back() must run before the ControlMaster exit command."""
|
|
monkeypatch.setattr(ssh_env.shutil, "which", lambda _name: "/usr/bin/ssh")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_establish_connection", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_detect_remote_home", lambda self: "/home/u")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_ensure_remote_dirs", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "init_session", lambda self: None)
|
|
|
|
call_order = []
|
|
|
|
class TrackingSyncManager:
|
|
def __init__(self, **kwargs):
|
|
pass
|
|
|
|
def sync(self, **kw):
|
|
pass
|
|
|
|
def sync_back(self):
|
|
call_order.append("sync_back")
|
|
|
|
monkeypatch.setattr(ssh_env, "FileSyncManager", TrackingSyncManager)
|
|
|
|
env = SSHEnvironment(host="h", user="u")
|
|
|
|
# Create a fake control socket so cleanup tries the SSH exit
|
|
import tempfile
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".sock") as tmp:
|
|
env.control_socket = Path(tmp.name)
|
|
|
|
def mock_run(cmd, **kwargs):
|
|
cmd_str = " ".join(cmd)
|
|
if "-O" in cmd and "exit" in cmd_str:
|
|
call_order.append("control_exit")
|
|
return subprocess.CompletedProcess([], 0)
|
|
|
|
with patch.object(subprocess, "run", side_effect=mock_run):
|
|
env.cleanup()
|
|
|
|
assert call_order.index("sync_back") < call_order.index("control_exit")
|
|
|
|
|
|
# =====================================================================
|
|
# Modal bulk download
|
|
# =====================================================================
|
|
|
|
|
|
class TestModalBulkDownload:
|
|
"""Unit tests for _modal_bulk_download."""
|
|
|
|
def test_modal_bulk_download_command(self, tmp_path):
|
|
"""exec should be called with tar cf - -C /root/.hermes ."""
|
|
env = _make_mock_modal_env()
|
|
exec_calls = _wire_modal_download(env, tar_bytes=b"tar-content")
|
|
dest = tmp_path / "backup.tar"
|
|
|
|
env._modal_bulk_download(dest)
|
|
|
|
assert len(exec_calls) == 1
|
|
args = exec_calls[0]
|
|
assert args[0] == "bash"
|
|
assert args[1] == "-c"
|
|
assert "tar cf -" in args[2]
|
|
assert "-C / root/.hermes" in args[2]
|
|
|
|
|
|
def test_modal_bulk_download_uses_120s_timeout(self, tmp_path):
|
|
"""run_coroutine should be called with timeout=120."""
|
|
env = _make_mock_modal_env()
|
|
_wire_modal_download(env, tar_bytes=b"data")
|
|
|
|
run_kwargs = {}
|
|
original_run = env._worker.run_coroutine
|
|
|
|
def tracking_run(coro, **kwargs):
|
|
run_kwargs.update(kwargs)
|
|
return original_run(coro, **kwargs)
|
|
|
|
env._worker.run_coroutine = tracking_run
|
|
dest = tmp_path / "backup.tar"
|
|
|
|
env._modal_bulk_download(dest)
|
|
|
|
assert run_kwargs.get("timeout") == 120
|
|
|
|
|
|
class TestModalCleanup:
|
|
"""Verify Modal cleanup() calls sync_back() before terminate."""
|
|
|
|
def test_modal_cleanup_calls_sync_back(self):
|
|
"""cleanup() should call sync_back() before sandbox.terminate."""
|
|
env = _make_mock_modal_env()
|
|
|
|
call_order = []
|
|
sync_mgr = MagicMock()
|
|
sync_mgr.sync_back = lambda: call_order.append("sync_back")
|
|
env._sync_manager = sync_mgr
|
|
|
|
# Mock terminate to track call order
|
|
async def mock_terminate():
|
|
pass
|
|
|
|
env._sandbox.terminate = MagicMock()
|
|
env._sandbox.terminate.aio = mock_terminate
|
|
env._worker.run_coroutine = lambda coro, **kw: (
|
|
call_order.append("terminate"),
|
|
asyncio.new_event_loop().run_until_complete(coro),
|
|
)
|
|
env._worker.stop = lambda: None
|
|
|
|
env.cleanup()
|
|
|
|
assert "sync_back" in call_order
|
|
assert call_order.index("sync_back") < call_order.index("terminate")
|
|
|
|
|
|
# =====================================================================
|
|
# Daytona bulk download
|
|
# =====================================================================
|
|
|
|
|
|
class TestDaytonaBulkDownload:
|
|
"""Unit tests for _daytona_bulk_download."""
|
|
|
|
def test_daytona_bulk_download_creates_tar_and_downloads(self, tmp_path):
|
|
"""exec and download_file should both be called."""
|
|
env = _make_mock_daytona_env()
|
|
dest = tmp_path / "backup.tar"
|
|
|
|
env._daytona_bulk_download(dest)
|
|
|
|
# exec called twice: tar creation + rm cleanup
|
|
assert env._sandbox.process.exec.call_count == 2
|
|
tar_cmd = env._sandbox.process.exec.call_args_list[0][0][0]
|
|
assert "tar cf" in tar_cmd
|
|
# PID-suffixed temp path avoids collisions on sync_back retry
|
|
assert "/tmp/.hermes_sync." in tar_cmd
|
|
assert ".tar" in tar_cmd
|
|
assert ".hermes" in tar_cmd
|
|
|
|
cleanup_cmd = env._sandbox.process.exec.call_args_list[1][0][0]
|
|
assert "rm -f" in cleanup_cmd
|
|
assert "/tmp/.hermes_sync." in cleanup_cmd
|
|
|
|
# download_file called once with the same PID-suffixed path
|
|
env._sandbox.fs.download_file.assert_called_once()
|
|
download_args = env._sandbox.fs.download_file.call_args[0]
|
|
assert download_args[0].startswith("/tmp/.hermes_sync.")
|
|
assert download_args[0].endswith(".tar")
|
|
assert download_args[1] == str(dest)
|
|
|
|
def test_daytona_bulk_download_uses_remote_home(self, tmp_path):
|
|
"""The tar command should use the env's _remote_home."""
|
|
env = _make_mock_daytona_env()
|
|
env._remote_home = "/home/daytona"
|
|
dest = tmp_path / "backup.tar"
|
|
|
|
env._daytona_bulk_download(dest)
|
|
|
|
tar_cmd = env._sandbox.process.exec.call_args_list[0][0][0]
|
|
assert "home/daytona/.hermes" in tar_cmd
|
|
|
|
|
|
class TestDaytonaCleanup:
|
|
"""Verify Daytona cleanup() calls sync_back() before stop."""
|
|
|
|
def test_daytona_cleanup_calls_sync_back(self):
|
|
"""cleanup() should call sync_back() before sandbox.stop()."""
|
|
env = _make_mock_daytona_env()
|
|
|
|
call_order = []
|
|
sync_mgr = MagicMock()
|
|
sync_mgr.sync_back = lambda: call_order.append("sync_back")
|
|
env._sync_manager = sync_mgr
|
|
env._sandbox.stop = lambda: call_order.append("stop")
|
|
|
|
env.cleanup()
|
|
|
|
assert "sync_back" in call_order
|
|
assert "stop" in call_order
|
|
assert call_order.index("sync_back") < call_order.index("stop")
|
|
|
|
|
|
# =====================================================================
|
|
# FileSyncManager wiring: bulk_download_fn passed by each backend
|
|
# =====================================================================
|
|
|
|
|
|
class TestBulkDownloadWiring:
|
|
"""Verify each backend passes bulk_download_fn to FileSyncManager."""
|
|
|
|
def test_ssh_passes_bulk_download_fn(self, monkeypatch):
|
|
"""SSHEnvironment should pass _ssh_bulk_download to FileSyncManager."""
|
|
monkeypatch.setattr(ssh_env.shutil, "which", lambda _name: "/usr/bin/ssh")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_establish_connection", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_detect_remote_home", lambda self: "/root")
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "_ensure_remote_dirs", lambda self: None)
|
|
monkeypatch.setattr(ssh_env.SSHEnvironment, "init_session", lambda self: None)
|
|
|
|
captured_kwargs = {}
|
|
|
|
class CaptureSyncManager:
|
|
def __init__(self, **kwargs):
|
|
captured_kwargs.update(kwargs)
|
|
|
|
def sync(self, **kw):
|
|
pass
|
|
|
|
monkeypatch.setattr(ssh_env, "FileSyncManager", CaptureSyncManager)
|
|
|
|
SSHEnvironment(host="h", user="u")
|
|
|
|
assert "bulk_download_fn" in captured_kwargs
|
|
assert callable(captured_kwargs["bulk_download_fn"])
|
|
|
|
|
|
def test_daytona_passes_bulk_download_fn(self, monkeypatch):
|
|
"""DaytonaEnvironment should pass _daytona_bulk_download to FileSyncManager."""
|
|
captured_kwargs = {}
|
|
|
|
def capture_fsm(**kwargs):
|
|
captured_kwargs.update(kwargs)
|
|
return type("M", (), {"sync": lambda self, **k: None})()
|
|
|
|
monkeypatch.setattr(daytona_env, "FileSyncManager", capture_fsm)
|
|
|
|
env = object.__new__(daytona_env.DaytonaEnvironment)
|
|
env._sandbox = MagicMock()
|
|
env._remote_home = "/root"
|
|
env._lock = __import__("threading").Lock()
|
|
env._persistent = True
|
|
env._task_id = "test"
|
|
env._daytona = MagicMock()
|
|
|
|
# Replicate the wiring done in __init__
|
|
from tools.environments.file_sync import iter_sync_files
|
|
env._sync_manager = daytona_env.FileSyncManager(
|
|
get_files_fn=lambda: iter_sync_files(f"{env._remote_home}/.hermes"),
|
|
upload_fn=env._daytona_upload,
|
|
delete_fn=env._daytona_delete,
|
|
bulk_upload_fn=env._daytona_bulk_upload,
|
|
bulk_download_fn=env._daytona_bulk_download,
|
|
)
|
|
|
|
assert "bulk_download_fn" in captured_kwargs
|
|
assert callable(captured_kwargs["bulk_download_fn"])
|