mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
451 lines
17 KiB
Python
451 lines
17 KiB
Python
"""Regression tests for the machine-dashboard multi-profile unification.
|
|
|
|
The dashboard is ONE machine-level management surface: config, env, MCP,
|
|
model, and chat-PTY endpoints accept an optional ``profile`` so the global
|
|
profile switcher can target any profile's HERMES_HOME. These tests pin:
|
|
reads/writes land in the REQUESTED profile, the dashboard's own profile
|
|
stays untouched, and the chat PTY env is scoped via HERMES_HOME.
|
|
"""
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_profiles(tmp_path, monkeypatch, _isolate_hermes_home):
|
|
"""Isolated default home + one named profile, each with config + .env."""
|
|
from hermes_constants import get_hermes_home
|
|
from hermes_cli import profiles
|
|
|
|
default_home = get_hermes_home()
|
|
profiles_root = default_home / "profiles"
|
|
worker_home = profiles_root / "worker_beta"
|
|
for home in (default_home, worker_home):
|
|
home.mkdir(parents=True, exist_ok=True)
|
|
(home / "config.yaml").write_text("{}\n", encoding="utf-8")
|
|
(worker_home / ".env").write_text("", encoding="utf-8")
|
|
|
|
monkeypatch.setattr(profiles, "_get_default_hermes_home", lambda: default_home)
|
|
monkeypatch.setattr(profiles, "_get_profiles_root", lambda: profiles_root)
|
|
return {"default": default_home, "worker_beta": worker_home}
|
|
|
|
|
|
@pytest.fixture
|
|
def client(monkeypatch, isolated_profiles):
|
|
try:
|
|
from starlette.testclient import TestClient
|
|
except ImportError:
|
|
pytest.skip("fastapi/starlette not installed")
|
|
|
|
import hermes_state
|
|
from hermes_constants import get_hermes_home
|
|
from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN
|
|
|
|
monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db")
|
|
c = TestClient(app)
|
|
c.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN
|
|
return c
|
|
|
|
|
|
def _cfg(home):
|
|
return yaml.safe_load((home / "config.yaml").read_text()) or {}
|
|
|
|
|
|
class TestProfileScopedConfig:
|
|
|
|
|
|
def test_config_query_param_equivalent_to_body(self, client, isolated_profiles):
|
|
"""The SPA's fetchJSON injects ?profile= — must scope like body.profile."""
|
|
resp = client.put(
|
|
"/api/config?profile=worker_beta",
|
|
json={"config": {"timezone": "Pluto/Far"}},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert _cfg(isolated_profiles["worker_beta"]).get("timezone") == "Pluto/Far"
|
|
assert _cfg(isolated_profiles["default"]).get("timezone") != "Pluto/Far"
|
|
|
|
|
|
|
|
def test_unknown_profile_404(self, client, isolated_profiles):
|
|
resp = client.get("/api/config", params={"profile": "ghost"})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestProfileScopedEnv:
|
|
def test_env_set_lands_in_target_profile_only(self, client, isolated_profiles):
|
|
resp = client.put(
|
|
"/api/env",
|
|
json={"key": "FAL_KEY", "value": "test-fal-123", "profile": "worker_beta"},
|
|
)
|
|
assert resp.status_code == 200
|
|
worker_env = (isolated_profiles["worker_beta"] / ".env").read_text()
|
|
assert "test-fal-123" in worker_env
|
|
default_env_path = isolated_profiles["default"] / ".env"
|
|
if default_env_path.exists():
|
|
assert "test-fal-123" not in default_env_path.read_text()
|
|
|
|
|
|
def test_env_delete_scoped(self, client, isolated_profiles):
|
|
(isolated_profiles["worker_beta"] / ".env").write_text(
|
|
"FAL_KEY=doomed\n", encoding="utf-8"
|
|
)
|
|
resp = client.request(
|
|
"DELETE",
|
|
"/api/env",
|
|
json={"key": "FAL_KEY", "profile": "worker_beta"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert "doomed" not in (isolated_profiles["worker_beta"] / ".env").read_text()
|
|
|
|
|
|
class TestProfileScopedMcp:
|
|
|
|
def test_mcp_bearer_secret_is_profile_scoped(self, client, isolated_profiles):
|
|
secret = "worker-only-secret"
|
|
response = client.post(
|
|
"/api/mcp/servers",
|
|
params={"profile": "worker_beta"},
|
|
json={
|
|
"name": "profile-bearer",
|
|
"url": "https://example.com/mcp",
|
|
"auth": "header",
|
|
"bearer_token": secret,
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
worker_cfg = _cfg(isolated_profiles["worker_beta"])
|
|
assert worker_cfg["mcp_servers"]["profile-bearer"]["headers"] == {
|
|
"Authorization": "Bearer ${MCP_PROFILE_BEARER_API_KEY}",
|
|
}
|
|
assert secret in (isolated_profiles["worker_beta"] / ".env").read_text()
|
|
assert not (isolated_profiles["default"] / ".env").exists()
|
|
assert "profile-bearer" not in _cfg(isolated_profiles["default"]).get(
|
|
"mcp_servers", {}
|
|
)
|
|
|
|
|
|
|
|
def test_mcp_test_oauth_server_without_token_is_not_ok(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
"""An `auth: oauth` server that serves tools/list anonymously must not
|
|
false-green: a successful probe with no token on disk reports needs-auth."""
|
|
import hermes_cli.mcp_config as mcp_config
|
|
|
|
(isolated_profiles["worker_beta"] / "config.yaml").write_text(
|
|
"mcp_servers:\n oauth-srv:\n url: http://x/sse\n auth: oauth\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(
|
|
mcp_config,
|
|
"_probe_single_server",
|
|
lambda name, config, connect_timeout=30, details=None: [("tool-a", "desc")],
|
|
)
|
|
monkeypatch.setattr(mcp_config, "_oauth_tokens_present", lambda name: False)
|
|
|
|
resp = client.post(
|
|
"/api/mcp/servers/oauth-srv/test", params={"profile": "worker_beta"}
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["ok"] is False
|
|
assert "oauth" in body["error"].lower()
|
|
|
|
# With a token present, the same probe is genuinely authenticated.
|
|
monkeypatch.setattr(mcp_config, "_oauth_tokens_present", lambda name: True)
|
|
resp = client.post(
|
|
"/api/mcp/servers/oauth-srv/test", params={"profile": "worker_beta"}
|
|
)
|
|
assert resp.json()["ok"] is True
|
|
|
|
|
|
class TestProfileScopedModel:
|
|
def test_model_set_main_scoped(self, client, isolated_profiles):
|
|
resp = client.post(
|
|
"/api/model/set",
|
|
json={
|
|
"scope": "main",
|
|
"provider": "openrouter",
|
|
"model": "test/model-1",
|
|
"confirm_expensive_model": True,
|
|
"profile": "worker_beta",
|
|
},
|
|
)
|
|
assert resp.status_code == 200
|
|
worker_cfg = _cfg(isolated_profiles["worker_beta"])
|
|
model_cfg = worker_cfg.get("model", {})
|
|
assert isinstance(model_cfg, dict)
|
|
assert model_cfg.get("provider") == "openrouter"
|
|
default_model = _cfg(isolated_profiles["default"]).get("model", {})
|
|
if isinstance(default_model, dict):
|
|
assert default_model.get("default") != "test/model-1"
|
|
|
|
|
|
|
|
|
|
|
|
def test_model_info_unknown_profile_404(self, client, isolated_profiles):
|
|
"""Regression: the broad except used to convert the 404 into a 200
|
|
with empty model info ("no model set" — silently wrong)."""
|
|
resp = client.get("/api/model/info", params={"profile": "ghost"})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestProfileScopedPostSetup:
|
|
def test_post_setup_spawns_with_profile_flag(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
"""Post-setup runs in a -p scoped subprocess so hooks that read
|
|
config / write per-profile state see the same HERMES_HOME the rest
|
|
of the drawer's writes targeted."""
|
|
import hermes_cli.web_server as web_server
|
|
|
|
calls = []
|
|
|
|
class _FakeProc:
|
|
pid = 777
|
|
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"_spawn_hermes_action",
|
|
lambda subcommand, name: calls.append(list(subcommand)) or _FakeProc(),
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.tools_config.valid_post_setup_keys",
|
|
lambda: {"agent_browser"},
|
|
)
|
|
resp = client.post(
|
|
"/api/tools/toolsets/browser/post-setup",
|
|
json={"key": "agent_browser", "profile": "worker_beta"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert calls == [
|
|
["-p", "worker_beta", "tools", "post-setup", "agent_browser"]
|
|
]
|
|
|
|
def test_post_setup_without_profile_keeps_legacy_argv(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
import hermes_cli.web_server as web_server
|
|
|
|
calls = []
|
|
|
|
class _FakeProc:
|
|
pid = 777
|
|
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"_spawn_hermes_action",
|
|
lambda subcommand, name: calls.append(list(subcommand)) or _FakeProc(),
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.tools_config.valid_post_setup_keys",
|
|
lambda: {"agent_browser"},
|
|
)
|
|
resp = client.post(
|
|
"/api/tools/toolsets/browser/post-setup",
|
|
json={"key": "agent_browser"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert calls == [["tools", "post-setup", "agent_browser"]]
|
|
|
|
|
|
class TestProfileScopedGateway:
|
|
|
|
def test_status_reads_requested_profile_home(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
import hermes_cli.web_server as web_server
|
|
from hermes_constants import get_hermes_home
|
|
|
|
seen_homes = []
|
|
|
|
def fake_get_running_pid(*args, **kwargs):
|
|
# /api/status?profile= now passes pid_path= explicitly (the TTL
|
|
# cache would otherwise serve another profile's PID) — accept it.
|
|
seen_homes.append(str(get_hermes_home()))
|
|
return None
|
|
|
|
monkeypatch.setattr(web_server, "check_config_version", lambda: (1, 1))
|
|
# get_status probes via the TTL-cached wrapper (PR #53511 salvage);
|
|
# patch the cached name so the fake still intercepts the probe.
|
|
monkeypatch.setattr(web_server, "get_running_pid_cached", fake_get_running_pid)
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"read_runtime_status",
|
|
lambda *a, **k: {"gateway_state": "startup_failed", "platforms": {}},
|
|
)
|
|
monkeypatch.setattr(web_server, "_GATEWAY_HEALTH_URL", None)
|
|
|
|
resp = client.get("/api/status", params={"profile": "worker_beta"})
|
|
|
|
assert resp.status_code == 200
|
|
assert seen_homes[0] == str(isolated_profiles["worker_beta"])
|
|
assert resp.json()["hermes_home"] == str(isolated_profiles["worker_beta"])
|
|
|
|
def test_status_uses_runtime_pid_when_profile_pid_file_is_missing(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
import hermes_cli.web_server as web_server
|
|
|
|
worker_home = isolated_profiles["worker_beta"]
|
|
(worker_home / ".env").write_text(
|
|
"TELEGRAM_BOT_TOKEN=worker-token\n", encoding="utf-8"
|
|
)
|
|
(worker_home / "config.yaml").write_text(
|
|
yaml.safe_dump({"platforms": {"telegram": {"enabled": True}}}),
|
|
encoding="utf-8",
|
|
)
|
|
runtime = {
|
|
"pid": 4242,
|
|
"gateway_state": "running",
|
|
"platforms": {"telegram": {"state": "connected"}},
|
|
"exit_reason": None,
|
|
"updated_at": "2026-06-17T00:00:00+00:00",
|
|
}
|
|
monkeypatch.setattr(web_server, "check_config_version", lambda: (1, 1))
|
|
monkeypatch.setattr(
|
|
web_server, "get_running_pid_cached", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(web_server, "read_runtime_status", lambda *a, **k: runtime)
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"get_runtime_status_running_pid",
|
|
lambda payload, **k: 4242,
|
|
)
|
|
monkeypatch.setattr(web_server, "_GATEWAY_HEALTH_URL", None)
|
|
from gateway.config import Platform
|
|
|
|
class _FakeGatewayConfig:
|
|
def get_connected_platforms(self):
|
|
return [Platform.TELEGRAM]
|
|
|
|
monkeypatch.setattr(
|
|
"gateway.config.load_gateway_config", lambda: _FakeGatewayConfig()
|
|
)
|
|
|
|
resp = client.get("/api/status", params={"profile": "worker_beta"})
|
|
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["gateway_running"] is True
|
|
assert data["gateway_pid"] == 4242
|
|
assert data["gateway_state"] == "running"
|
|
assert data["gateway_platforms"] == {"telegram": {"state": "connected"}}
|
|
|
|
|
|
class TestProfileScopedTelegramOnboarding:
|
|
def test_apply_writes_target_profile_and_restarts_target(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
import time
|
|
import hermes_cli.web_server as web_server
|
|
|
|
with web_server._telegram_onboarding_lock:
|
|
web_server._telegram_onboarding_pairings.clear()
|
|
web_server._telegram_onboarding_pairings["pair-worker"] = (
|
|
web_server._TelegramOnboardingPairing(
|
|
poll_token="poll-secret",
|
|
expires_at="2027-05-18T00:00:00.000Z",
|
|
expires_at_ts=time.time() + 600,
|
|
bot_token="123456:SECRET",
|
|
bot_username="worker_bot",
|
|
owner_user_id="123456789",
|
|
)
|
|
)
|
|
|
|
calls = []
|
|
|
|
class _FakeProc:
|
|
pid = 889
|
|
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"_spawn_hermes_action",
|
|
lambda subcommand, name: calls.append((list(subcommand), name)) or _FakeProc(),
|
|
)
|
|
web_server._ACTION_PROCS.pop("gateway-restart", None)
|
|
web_server._ACTION_COMMANDS.pop("gateway-restart", None)
|
|
|
|
resp = client.post(
|
|
"/api/messaging/telegram/onboarding/pair-worker/apply",
|
|
params={"profile": "worker_beta"},
|
|
json={"allowed_user_ids": ["123456789"]},
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
assert resp.json()["restart_started"] is True
|
|
assert calls == [
|
|
(["-p", "worker_beta", "gateway", "restart"], "gateway-restart")
|
|
]
|
|
|
|
worker_env = (isolated_profiles["worker_beta"] / ".env").read_text()
|
|
assert "TELEGRAM_BOT_TOKEN=123456:SECRET" in worker_env
|
|
assert "TELEGRAM_ALLOWED_USERS=123456789" in worker_env
|
|
default_env_path = isolated_profiles["default"] / ".env"
|
|
if default_env_path.exists():
|
|
assert "TELEGRAM_BOT_TOKEN" not in default_env_path.read_text()
|
|
|
|
worker_cfg = _cfg(isolated_profiles["worker_beta"])
|
|
default_cfg = _cfg(isolated_profiles["default"])
|
|
assert worker_cfg["platforms"]["telegram"]["enabled"] is True
|
|
assert default_cfg.get("platforms", {}).get("telegram", {}).get("enabled") is not True
|
|
|
|
|
|
class TestProfileScopedChatPty:
|
|
def test_chat_argv_scopes_hermes_home(self, isolated_profiles, monkeypatch):
|
|
import hermes_cli.web_server as web_server
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.main._make_tui_argv",
|
|
lambda root, tui_dev=False: (["cat"], None),
|
|
raising=False,
|
|
)
|
|
argv, cwd, env = web_server._resolve_chat_argv(profile="worker_beta")
|
|
assert env is not None
|
|
assert env["HERMES_HOME"] == str(isolated_profiles["worker_beta"])
|
|
# Scoped chat must NOT attach to the dashboard's in-memory gateway.
|
|
assert "HERMES_TUI_GATEWAY_URL" not in env
|
|
|
|
|
|
class TestProfileScopedAudio:
|
|
"""Audio endpoints must honor ``profile`` like the rest of the dashboard.
|
|
|
|
Historically /api/audio/transcribe|speak|elevenlabs/voices took no profile
|
|
and always resolved the dashboard's own config/.env, so a non-default
|
|
profile's TTS/STT settings were silently ignored (#53441 #45506 #66012
|
|
#64057).
|
|
"""
|
|
|
|
|
|
|
|
def test_transcribe_runs_inside_target_profile_home(
|
|
self, client, isolated_profiles, monkeypatch
|
|
):
|
|
import base64
|
|
|
|
import tools.voice_mode as voice_mode
|
|
|
|
seen = {}
|
|
|
|
def _fake_transcribe(path):
|
|
from hermes_constants import get_hermes_home
|
|
|
|
seen["home"] = str(get_hermes_home())
|
|
return {"success": True, "transcript": "hi", "provider": "fake"}
|
|
|
|
monkeypatch.setattr(voice_mode, "transcribe_recording", _fake_transcribe)
|
|
payload = base64.b64encode(b"\x00fakeaudio").decode("ascii")
|
|
resp = client.post(
|
|
"/api/audio/transcribe?profile=worker_beta",
|
|
json={"data_url": f"data:audio/webm;base64,{payload}"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["transcript"] == "hi"
|
|
assert seen["home"] == str(isolated_profiles["worker_beta"])
|
|
|
|
def test_audio_endpoints_unknown_profile_404(self, client, isolated_profiles):
|
|
resp = client.get("/api/audio/elevenlabs/voices?profile=ghost")
|
|
assert resp.status_code == 404
|
|
resp = client.post("/api/audio/speak?profile=ghost", json={"text": "x"})
|
|
assert resp.status_code == 404
|