mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
325 lines
12 KiB
Python
325 lines
12 KiB
Python
"""Unit tests for the Daytona cloud sandbox environment backend."""
|
|
|
|
import threading
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers to build mock Daytona SDK objects
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _make_exec_response(result="", exit_code=0):
|
|
return SimpleNamespace(result=result, exit_code=exit_code)
|
|
|
|
|
|
def _make_sandbox(sandbox_id="sb-123", state="started"):
|
|
sb = MagicMock()
|
|
sb.id = sandbox_id
|
|
sb.state = state
|
|
sb.process.exec.return_value = _make_exec_response()
|
|
return sb
|
|
|
|
|
|
def _patch_daytona_imports(monkeypatch):
|
|
"""Patch the daytona SDK so DaytonaEnvironment can be imported without it."""
|
|
import types as _types
|
|
|
|
import enum
|
|
|
|
class _SandboxState(str, enum.Enum):
|
|
STARTED = "started"
|
|
STOPPED = "stopped"
|
|
ARCHIVED = "archived"
|
|
ERROR = "error"
|
|
|
|
daytona_mod = _types.ModuleType("daytona")
|
|
daytona_mod.Daytona = MagicMock
|
|
daytona_mod.CreateSandboxFromImageParams = MagicMock
|
|
daytona_mod.DaytonaError = type("DaytonaError", (Exception,), {})
|
|
daytona_mod.Resources = MagicMock(name="Resources")
|
|
daytona_mod.SandboxState = _SandboxState
|
|
|
|
monkeypatch.setitem(__import__("sys").modules, "daytona", daytona_mod)
|
|
return daytona_mod
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.fixture()
|
|
def daytona_sdk(monkeypatch):
|
|
"""Provide a mock daytona SDK module and return it for assertions."""
|
|
return _patch_daytona_imports(monkeypatch)
|
|
|
|
|
|
@pytest.fixture()
|
|
def make_env(daytona_sdk, monkeypatch):
|
|
"""Factory that creates a DaytonaEnvironment with a mocked SDK."""
|
|
# Prevent is_interrupted from interfering — patch where it's used (base.py)
|
|
monkeypatch.setattr("tools.environments.base.is_interrupted", lambda: False)
|
|
# Prevent skills/credential sync from consuming mock exec calls
|
|
monkeypatch.setattr("tools.credential_files.get_credential_file_mounts", lambda: [])
|
|
monkeypatch.setattr("tools.credential_files.get_skills_directory_mount", lambda **kw: None)
|
|
monkeypatch.setattr("tools.credential_files.iter_skills_files", lambda **kw: [])
|
|
|
|
def _factory(
|
|
sandbox=None,
|
|
get_side_effect=None,
|
|
list_return=None,
|
|
home_dir="/root",
|
|
persistent=True,
|
|
**kwargs,
|
|
):
|
|
sandbox = sandbox or _make_sandbox()
|
|
# Mock the $HOME detection
|
|
sandbox.process.exec.return_value = _make_exec_response(result=home_dir)
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.create.return_value = sandbox
|
|
|
|
if get_side_effect is not None:
|
|
mock_client.get.side_effect = get_side_effect
|
|
else:
|
|
# Default: no existing sandbox found via get()
|
|
mock_client.get.side_effect = daytona_sdk.DaytonaError("not found")
|
|
|
|
# Default: no legacy sandbox found via list()
|
|
if list_return is not None:
|
|
mock_client.list.return_value = list_return
|
|
else:
|
|
mock_client.list.return_value = iter([])
|
|
|
|
daytona_sdk.Daytona = MagicMock(return_value=mock_client)
|
|
|
|
from tools.environments.daytona import DaytonaEnvironment
|
|
|
|
kwargs.setdefault("disk", 10240)
|
|
env = DaytonaEnvironment(
|
|
image="test-image:latest",
|
|
persistent_filesystem=persistent,
|
|
**kwargs,
|
|
)
|
|
env._mock_client = mock_client # expose for assertions
|
|
return env
|
|
|
|
return _factory
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constructor / cwd resolution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestCwdResolution:
|
|
def test_default_cwd_resolves_home(self, make_env):
|
|
env = make_env(home_dir="/home/testuser")
|
|
assert env.cwd == "/home/testuser"
|
|
|
|
|
|
def test_empty_home_keeps_default_cwd(self, make_env):
|
|
env = make_env(home_dir="")
|
|
assert env.cwd == "/home/daytona" # keeps constructor default
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sandbox persistence / resume
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestPersistence:
|
|
def test_persistent_resumes_via_get(self, make_env):
|
|
existing = _make_sandbox(sandbox_id="sb-existing")
|
|
existing.process.exec.return_value = _make_exec_response(result="/root")
|
|
env = make_env(get_side_effect=lambda name: existing, persistent=True,
|
|
task_id="mytask")
|
|
existing.start.assert_called_once()
|
|
env._mock_client.get.assert_called_once_with("hermes-mytask")
|
|
env._mock_client.create.assert_not_called()
|
|
|
|
|
|
def test_non_persistent_skips_lookup(self, make_env):
|
|
env = make_env(persistent=False)
|
|
env._mock_client.get.assert_not_called()
|
|
env._mock_client.list.assert_not_called()
|
|
env._mock_client.create.assert_called_once()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestCleanup:
|
|
def test_persistent_cleanup_stops_sandbox(self, make_env):
|
|
env = make_env(persistent=True)
|
|
sb = env._sandbox
|
|
env.cleanup()
|
|
sb.stop.assert_called_once()
|
|
|
|
|
|
def test_cleanup_swallows_errors(self, make_env):
|
|
env = make_env(persistent=True)
|
|
env._sandbox.stop.side_effect = RuntimeError("stop failed")
|
|
env.cleanup() # should not raise
|
|
assert env._sandbox is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Execute
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestExecute:
|
|
def test_basic_command(self, make_env):
|
|
sb = _make_sandbox()
|
|
# Calls: (1) $HOME detection, (2) init_session bootstrap, (3) actual command
|
|
sb.process.exec.side_effect = [
|
|
_make_exec_response(result="/root"), # $HOME
|
|
_make_exec_response(result="", exit_code=0), # init_session
|
|
_make_exec_response(result="hello", exit_code=0), # actual cmd
|
|
]
|
|
sb.state = "started"
|
|
env = make_env(sandbox=sb)
|
|
|
|
result = env.execute("echo hello")
|
|
assert "hello" in result["output"]
|
|
assert result["returncode"] == 0
|
|
|
|
|
|
def test_daytona_error_triggers_retry(self, make_env, daytona_sdk):
|
|
sb = _make_sandbox()
|
|
sb.state = "started"
|
|
sb.process.exec.side_effect = [
|
|
_make_exec_response(result="/root"), # $HOME
|
|
_make_exec_response(result="", exit_code=0), # init_session
|
|
daytona_sdk.DaytonaError("transient"), # first attempt fails
|
|
_make_exec_response(result="ok", exit_code=0), # retry succeeds
|
|
]
|
|
env = make_env(sandbox=sb)
|
|
|
|
result = env.execute("echo retry")
|
|
# DaytonaError now surfaces directly through _ThreadedProcessHandle
|
|
# (no retry logic) — the error becomes returncode=1
|
|
assert result["returncode"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resource conversion
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestResourceConversion:
|
|
def _get_resources_kwargs(self, daytona_sdk):
|
|
return daytona_sdk.Resources.call_args.kwargs
|
|
|
|
def test_memory_converted_to_gib(self, make_env, daytona_sdk):
|
|
env = make_env(memory=5120)
|
|
assert self._get_resources_kwargs(daytona_sdk)["memory"] == 5
|
|
|
|
|
|
def test_small_values_clamped_to_1(self, make_env, daytona_sdk):
|
|
env = make_env(memory=100, disk=100)
|
|
kw = self._get_resources_kwargs(daytona_sdk)
|
|
assert kw["memory"] == 1
|
|
assert kw["disk"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Ensure sandbox ready
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestInterrupt:
|
|
def test_interrupt_stops_sandbox_and_returns_130(self, make_env, monkeypatch):
|
|
sb = _make_sandbox()
|
|
sb.state = "started"
|
|
event = threading.Event()
|
|
calls = {"n": 0}
|
|
|
|
def exec_side_effect(*args, **kwargs):
|
|
calls["n"] += 1
|
|
if calls["n"] == 1:
|
|
return _make_exec_response(result="/root") # $HOME detection
|
|
if calls["n"] == 2:
|
|
return _make_exec_response(result="", exit_code=0) # init_session
|
|
event.wait(timeout=5) # simulate long-running command
|
|
return _make_exec_response(result="done", exit_code=0)
|
|
|
|
sb.process.exec.side_effect = exec_side_effect
|
|
env = make_env(sandbox=sb)
|
|
|
|
# is_interrupted is checked by base.py's _wait_for_process,
|
|
# patch where it's actually referenced (base.py's local binding)
|
|
monkeypatch.setattr(
|
|
"tools.environments.base.is_interrupted", lambda: True
|
|
)
|
|
try:
|
|
result = env.execute("sleep 10")
|
|
assert result["returncode"] == 130
|
|
sb.stop.assert_called()
|
|
finally:
|
|
event.set()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DaytonaError surfaces directly (no retry)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestRetryExhausted:
|
|
def test_both_attempts_fail(self, make_env, daytona_sdk):
|
|
"""DaytonaError surfaces directly as rc=1 (retry logic was removed)."""
|
|
sb = _make_sandbox()
|
|
sb.state = "started"
|
|
sb.process.exec.side_effect = [
|
|
_make_exec_response(result="/root"), # $HOME
|
|
_make_exec_response(result="", exit_code=0), # init_session
|
|
daytona_sdk.DaytonaError("fail1"), # actual command fails
|
|
]
|
|
env = make_env(sandbox=sb)
|
|
|
|
result = env.execute("echo x")
|
|
# Error surfaces directly through _ThreadedProcessHandle (rc=1)
|
|
assert result["returncode"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Ensure sandbox ready
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestEnsureSandboxReady:
|
|
def test_restarts_stopped_sandbox(self, make_env):
|
|
env = make_env()
|
|
env._sandbox.state = "stopped"
|
|
env._ensure_sandbox_ready()
|
|
env._sandbox.start.assert_called()
|
|
|
|
def test_no_restart_when_running(self, make_env):
|
|
env = make_env()
|
|
env._sandbox.state = "started"
|
|
env._ensure_sandbox_ready()
|
|
env._sandbox.start.assert_not_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sync safety: shell-metacharacter quoting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSyncSafety:
|
|
def test_single_upload_quotes_parent_path(self, make_env, tmp_path):
|
|
"""A remote path with shell metacharacters must be quoted, not injected."""
|
|
env = make_env()
|
|
env._sandbox.process.exec.reset_mock()
|
|
|
|
host_file = tmp_path / "token.txt"
|
|
host_file.write_text("secret", encoding="utf-8")
|
|
remote_path = "/root/.hermes/skills/evil; touch /tmp/daytona-owned/file.txt"
|
|
|
|
env._daytona_upload(str(host_file), remote_path)
|
|
|
|
mkdir_cmd = env._sandbox.process.exec.call_args_list[0][0][0]
|
|
# The whole parent dir is a single quoted argument — the ';' cannot
|
|
# break out into a second command.
|
|
assert mkdir_cmd == (
|
|
"mkdir -p '/root/.hermes/skills/evil; touch /tmp/daytona-owned'"
|
|
)
|
|
assert "; touch" not in mkdir_cmd.replace(
|
|
"'/root/.hermes/skills/evil; touch /tmp/daytona-owned'", ""
|
|
)
|