mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
104 lines
3.7 KiB
Python
104 lines
3.7 KiB
Python
"""Reproducible perf benchmark for file sync overhead.
|
|
|
|
Measures actual env.execute() wall-clock time, no LLM in the loop.
|
|
Run with: uv run pytest tests/tools/test_file_sync_perf.py -v -o "addopts=" -s
|
|
|
|
Requires backends to be configured (SSH host, Modal creds, etc).
|
|
Skip markers gate each backend.
|
|
"""
|
|
|
|
import statistics
|
|
import time
|
|
|
|
import pytest
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Backend fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.fixture
|
|
def local_env():
|
|
from tools.environments.local import LocalEnvironment
|
|
env = LocalEnvironment(cwd="/tmp", timeout=30)
|
|
yield env
|
|
env.cleanup()
|
|
|
|
|
|
@pytest.fixture
|
|
def ssh_env():
|
|
import os
|
|
host = os.environ.get("TERMINAL_SSH_HOST")
|
|
user = os.environ.get("TERMINAL_SSH_USER")
|
|
if not host or not user:
|
|
pytest.skip("TERMINAL_SSH_HOST and TERMINAL_SSH_USER required")
|
|
from tools.environments.ssh import SSHEnvironment
|
|
env = SSHEnvironment(host=host, user=user, cwd="/tmp", timeout=30)
|
|
yield env
|
|
env.cleanup()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _time_executions(env, command: str, n: int = 10) -> list[float]:
|
|
"""Run *command* n times and return per-call wall-clock durations."""
|
|
durations = []
|
|
for _ in range(n):
|
|
t0 = time.monotonic()
|
|
result = env.execute(command, timeout=10)
|
|
elapsed = time.monotonic() - t0
|
|
durations.append(elapsed)
|
|
assert result.get("returncode", result.get("exit_code", -1)) == 0, \
|
|
f"command failed: {result}"
|
|
return durations
|
|
|
|
|
|
def _report(label: str, durations: list[float]):
|
|
"""Print timing stats."""
|
|
med = statistics.median(durations)
|
|
mean = statistics.mean(durations)
|
|
p95 = sorted(durations)[int(len(durations) * 0.95)]
|
|
print(f"\n {label}:")
|
|
print(f" n={len(durations)} median={med*1000:.0f}ms mean={mean*1000:.0f}ms p95={p95*1000:.0f}ms")
|
|
print(f" raw: {[f'{d*1000:.0f}ms' for d in durations]}")
|
|
return med
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestLocalPerf:
|
|
"""Local baseline — no file sync, no network. Sets the floor."""
|
|
|
|
def test_echo_latency(self, local_env):
|
|
durations = _time_executions(local_env, "echo hello", n=20)
|
|
med = _report("local echo", durations)
|
|
# Spawn-per-call overhead should be < 500ms
|
|
assert med < 0.5, f"local echo median {med*1000:.0f}ms exceeds 500ms"
|
|
|
|
|
|
@pytest.mark.ssh
|
|
class TestSSHPerf:
|
|
"""SSH with FileSyncManager — mtime skip should make sync ~0ms."""
|
|
|
|
def test_echo_latency(self, ssh_env):
|
|
"""Sequential echo commands — measures per-command overhead including sync check."""
|
|
durations = _time_executions(ssh_env, "echo hello", n=20)
|
|
med = _report("ssh echo (with sync check)", durations)
|
|
# SSH round-trip + spawn-per-call, but sync should be ~0ms (rate limited)
|
|
assert med < 2.0, f"ssh echo median {med*1000:.0f}ms exceeds 2000ms"
|
|
|
|
|
|
def test_no_sync_within_interval(self, ssh_env):
|
|
"""Rapid sequential commands within 5s window — no sync at all."""
|
|
# First command triggers sync
|
|
ssh_env.execute("echo prime", timeout=10)
|
|
|
|
# Immediately run 10 more — all within rate-limit window
|
|
durations = _time_executions(ssh_env, "echo rapid", n=10)
|
|
med = _report("ssh echo (within interval, no sync)", durations)
|
|
|
|
# Should be pure SSH overhead, no sync
|
|
assert med < 1.5, f"within-interval median {med*1000:.0f}ms exceeds 1500ms"
|