mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
288 lines
11 KiB
Python
288 lines
11 KiB
Python
"""Tests for the memory/skill write-approval gate (tools/write_approval.py)
|
|
and the shared slash-command handlers (hermes_cli/write_approval_commands.py).
|
|
|
|
Covers the boolean write_approval gate (off by default = write freely; on =
|
|
require approval) for both subsystems, the foreground-vs-background staging
|
|
split, pending store CRUD, and the list/approve/reject/diff/approval
|
|
subcommand dispatch.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import shutil
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def hermes_home(monkeypatch):
|
|
d = tempfile.mkdtemp(prefix="hermes_wa_test_")
|
|
home = os.path.join(d, ".hermes")
|
|
os.makedirs(home)
|
|
monkeypatch.setenv("HERMES_HOME", home)
|
|
yield home
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
|
|
|
|
def _set_approval(subsystem, enabled):
|
|
import hermes_cli.config as cfg
|
|
c = cfg.load_config()
|
|
c.setdefault(subsystem, {})["write_approval"] = enabled
|
|
cfg.save_config(c)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config resolution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_default_gate_is_off(hermes_home):
|
|
from tools import write_approval as wa
|
|
# Default: gate off → writes flow freely.
|
|
assert wa.write_approval_enabled("memory") is False
|
|
assert wa.write_approval_enabled("skills") is False
|
|
|
|
|
|
def test_invalid_subsystem_is_off(hermes_home):
|
|
from tools import write_approval as wa
|
|
assert wa.write_approval_enabled("bogus") is False
|
|
|
|
|
|
def test_normalize_enabled_coerces_values():
|
|
from tools import write_approval as wa
|
|
# Real bools pass through.
|
|
assert wa._normalize_enabled(True) is True
|
|
assert wa._normalize_enabled(False) is False
|
|
# Truthy strings → True (incl. legacy 'approve').
|
|
assert wa._normalize_enabled("on") is True
|
|
assert wa._normalize_enabled("approve") is True
|
|
assert wa._normalize_enabled("true") is True
|
|
# Everything else → False (gate off is the safe default).
|
|
assert wa._normalize_enabled("off") is False
|
|
assert wa._normalize_enabled("garbage") is False
|
|
assert wa._normalize_enabled(None) is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Memory gate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_memory_gate_off_allows_write(hermes_home):
|
|
# Default (gate off) → write straight through, no staging.
|
|
from tools.memory_tool import memory_tool, MemoryStore
|
|
from tools import write_approval as wa
|
|
store = MemoryStore(); store.load_from_disk()
|
|
r = json.loads(memory_tool("add", "user", "save me", store=store))
|
|
assert r["success"] is True
|
|
assert r["entry_count"] == 1
|
|
assert wa.pending_count("memory") == 0
|
|
|
|
|
|
def test_cli_memory_approve_without_live_agent_uses_fresh_store(hermes_home, capsys):
|
|
"""#46783: ``/memory approve`` from a context with no live agent (e.g. the
|
|
Desktop GUI) passed ``memory_store=None`` into the shared handler, which
|
|
returned "memory store unavailable" and applied nothing. The CLI handler must
|
|
fall back to a freshly loaded on-disk store, like the gateway path does."""
|
|
import json
|
|
from tools.memory_tool import memory_tool, MemoryStore
|
|
from tools import write_approval as wa
|
|
from hermes_cli.cli_commands_mixin import CLICommandsMixin
|
|
|
|
_set_approval("memory", True)
|
|
staging = MemoryStore(); staging.load_from_disk()
|
|
r = json.loads(memory_tool("add", "memory", "remember the launch date", store=staging))
|
|
assert r.get("pending_id"), r
|
|
assert wa.pending_count("memory") == 1
|
|
|
|
# Bare CLI handler with no live agent → store resolves to None pre-fix.
|
|
handler = CLICommandsMixin.__new__(CLICommandsMixin)
|
|
handler.agent = None
|
|
handler._handle_memory_command("/memory approve all")
|
|
|
|
out = capsys.readouterr().out
|
|
assert "memory store unavailable" not in out, out
|
|
assert "Approved 1" in out, out
|
|
assert wa.pending_count("memory") == 0
|
|
# The approved write landed in a freshly loaded on-disk store (MEMORY.md).
|
|
reloaded = MemoryStore(); reloaded.load_from_disk()
|
|
assert any("remember the launch date" in e for e in reloaded.memory_entries)
|
|
|
|
|
|
def test_load_on_disk_store_honors_configured_char_limits(hermes_home, monkeypatch):
|
|
"""load_on_disk_store() must read memory.memory_char_limit /
|
|
user_char_limit from config so approvals applied without a live agent
|
|
enforce the SAME caps as the live agent (agent_init.py). Falls back to
|
|
defaults when config can't be loaded.
|
|
"""
|
|
from tools.memory_tool import load_on_disk_store
|
|
|
|
# Config override path: helper picks up the configured limits.
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {"memory": {"memory_char_limit": 999, "user_char_limit": 444}},
|
|
)
|
|
store = load_on_disk_store()
|
|
assert store.memory_char_limit == 999
|
|
assert store.user_char_limit == 444
|
|
|
|
# Failure path: config raises → defaults, never blows up.
|
|
def _boom():
|
|
raise RuntimeError("no config")
|
|
|
|
monkeypatch.setattr("hermes_cli.config.load_config", _boom)
|
|
fallback = load_on_disk_store()
|
|
assert fallback.memory_char_limit == 2200
|
|
assert fallback.user_char_limit == 1375
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Skill gate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SKILL = (
|
|
"---\nname: test-skill\ndescription: A test skill\nversion: 1.0.0\n---\n"
|
|
"# Test\nbody\n"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pending store CRUD
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared command handler
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_handle_approve_all(hermes_home):
|
|
from hermes_cli.write_approval_commands import handle_pending_subcommand
|
|
from tools.memory_tool import MemoryStore
|
|
from tools import write_approval as wa
|
|
store = MemoryStore(); store.load_from_disk()
|
|
wa.stage_write("memory", {"action": "add", "target": "user", "content": "a"},
|
|
summary="a", origin="foreground")
|
|
wa.stage_write("memory", {"action": "add", "target": "user", "content": "b"},
|
|
summary="b", origin="foreground")
|
|
out = handle_pending_subcommand(wa.MEMORY, ["approve", "all"], memory_store=store)
|
|
assert "Approved 2" in out
|
|
assert wa.pending_count("memory") == 0
|
|
assert len(store.user_entries) == 2
|
|
|
|
|
|
def test_handle_approval_on(hermes_home):
|
|
from hermes_cli.write_approval_commands import handle_pending_subcommand
|
|
from tools import write_approval as wa
|
|
captured = {}
|
|
out = handle_pending_subcommand(
|
|
wa.MEMORY, ["approval", "on"],
|
|
set_mode_fn=lambda enabled: captured.update(enabled=enabled),
|
|
)
|
|
assert captured["enabled"] is True
|
|
assert "on" in out
|
|
|
|
|
|
def test_handle_approval_off(hermes_home):
|
|
from hermes_cli.write_approval_commands import handle_pending_subcommand
|
|
from tools import write_approval as wa
|
|
captured = {}
|
|
out = handle_pending_subcommand(
|
|
wa.SKILLS, ["approval", "off"],
|
|
set_mode_fn=lambda enabled: captured.update(enabled=enabled),
|
|
)
|
|
assert captured["enabled"] is False
|
|
assert "off" in out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Inline (interactive CLI) approval path — regression for the bug where the
|
|
# per-thread approval callback was never passed to prompt_dangerous_approval,
|
|
# so every gated foreground memory write was silently denied.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.fixture
|
|
def approval_callback_cleanup():
|
|
yield
|
|
from tools.terminal_tool import set_approval_callback
|
|
set_approval_callback(None)
|
|
|
|
|
|
def test_memory_inline_approve_writes(hermes_home, approval_callback_cleanup):
|
|
from tools.memory_tool import memory_tool, MemoryStore
|
|
from tools.terminal_tool import set_approval_callback
|
|
from tools import write_approval as wa
|
|
_set_approval("memory", True)
|
|
|
|
calls = []
|
|
def approve_cb(command, description, **kw):
|
|
calls.append((command, description))
|
|
return "once"
|
|
set_approval_callback(approve_cb)
|
|
|
|
store = MemoryStore(); store.load_from_disk()
|
|
r = json.loads(memory_tool("add", "memory", "approved fact", store=store))
|
|
assert r["success"] is True
|
|
assert r.get("staged") is None # real write, not staged
|
|
assert store.memory_entries == ["approved fact"]
|
|
assert wa.pending_count("memory") == 0
|
|
# The registered callback must actually be invoked (not the input() path).
|
|
assert len(calls) == 1
|
|
assert "approved fact" in calls[0][0]
|
|
|
|
|
|
def test_memory_inline_deny_blocks(hermes_home, approval_callback_cleanup):
|
|
from tools.memory_tool import memory_tool, MemoryStore
|
|
from tools.terminal_tool import set_approval_callback
|
|
from tools import write_approval as wa
|
|
_set_approval("memory", True)
|
|
set_approval_callback(lambda command, description, **kw: "deny")
|
|
|
|
store = MemoryStore(); store.load_from_disk()
|
|
r = json.loads(memory_tool("add", "memory", "denied fact", store=store))
|
|
assert r["success"] is False
|
|
assert "denied" in r["error"].lower()
|
|
assert store.memory_entries == []
|
|
assert wa.pending_count("memory") == 0 # denied, not staged
|
|
|
|
|
|
def test_memory_invalid_params_rejected_before_staging(hermes_home):
|
|
# Param validation must run BEFORE the gate so a broken write is rejected
|
|
# immediately instead of staged and failing at approve time.
|
|
from tools.memory_tool import memory_tool, MemoryStore
|
|
from tools import write_approval as wa
|
|
_set_approval("memory", True)
|
|
store = MemoryStore(); store.load_from_disk()
|
|
r = json.loads(memory_tool("add", "memory", None, store=store))
|
|
assert r["success"] is False
|
|
assert wa.pending_count("memory") == 0
|
|
|
|
|
|
class TestSkillGist:
|
|
"""skill_gist builds a heuristic one-line summary for a pending skill write.
|
|
|
|
Pure, no model call — every branch is verifiable from the function source.
|
|
"""
|
|
|
|
def test_create_with_frontmatter_description(self):
|
|
from tools import write_approval as wa
|
|
content = "---\ndescription: My cool skill\n---\nprint('hi')\n"
|
|
assert (
|
|
wa.skill_gist("create", "demo", content=content)
|
|
== f"create 'demo' — My cool skill ({len(content)} chars)"
|
|
)
|
|
|
|
def test_edit_without_description_uses_size_only(self):
|
|
from tools import write_approval as wa
|
|
content = "no frontmatter here"
|
|
assert (
|
|
wa.skill_gist("edit", "demo", content=content)
|
|
== f"rewrite 'demo' ({len(content)} chars)"
|
|
)
|
|
|
|
|
|
def test_file_actions_and_unknown_fallback(self):
|
|
from tools import write_approval as wa
|
|
assert wa.skill_gist("write_file", "demo", file_path="a.py") == "write a.py in 'demo'"
|
|
assert wa.skill_gist("remove_file", "demo", file_path="a.py") == "remove a.py from 'demo'"
|
|
assert wa.skill_gist("delete", "demo") == "delete skill 'demo'"
|
|
assert wa.skill_gist("unknown", "demo") == "unknown 'demo'"
|