mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
484 lines
16 KiB
Python
484 lines
16 KiB
Python
"""Tests for the bundled observability/nemo_relay plugin."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextvars
|
|
import gc
|
|
import importlib
|
|
import json
|
|
import sys
|
|
import warnings
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from hermes_cli import lifecycle, plugins as plugin_api
|
|
from hermes_cli.observability import relay_runtime, relay_shared_metrics
|
|
from hermes_cli.plugins import PluginManager
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
PLUGIN_DIR = REPO_ROOT / "plugins" / "observability" / "nemo_relay"
|
|
|
|
|
|
class _FakeNemoRelay:
|
|
def __init__(self):
|
|
self.events = []
|
|
self._callbacks = {}
|
|
self._llm_starts = {}
|
|
self._scope_serial = 0
|
|
self._scope_context = contextvars.ContextVar(
|
|
"fake_nemo_relay_scope", default=None
|
|
)
|
|
self.ScopeType = SimpleNamespace(Agent="agent", Function="function")
|
|
self.scope = SimpleNamespace(
|
|
push=self._scope_push,
|
|
pop=self._scope_pop,
|
|
event=self._scope_event,
|
|
)
|
|
self.llm = SimpleNamespace(
|
|
call=self._llm_call,
|
|
call_end=self._llm_call_end,
|
|
execute=self._llm_execute,
|
|
)
|
|
self.tools = SimpleNamespace(
|
|
call=self._tool_call,
|
|
call_end=self._tool_call_end,
|
|
execute=self._tool_execute,
|
|
request_intercepts=self._tool_request_intercepts,
|
|
)
|
|
self.plugin = SimpleNamespace(
|
|
initialize=self._plugin_initialize,
|
|
clear=self._plugin_clear,
|
|
initialize_with_dynamic_plugins=self._plugin_initialize_with_dynamic,
|
|
)
|
|
self.subscribers = SimpleNamespace(
|
|
register=self._register_subscriber,
|
|
deregister=self._deregister_subscriber,
|
|
flush=self._flush_subscribers,
|
|
)
|
|
self.LLMRequest = _FakeLLMRequest
|
|
self.AtofExporterConfig = _FakeAtofExporterConfig
|
|
self.AtofExporterMode = SimpleNamespace(Append="append", Overwrite="overwrite")
|
|
self.AtofExporter = self._make_atof_exporter
|
|
self.AtifExporter = self._make_atif_exporter
|
|
self.get_scope_stack = self._get_scope_stack
|
|
|
|
def _scope_push(self, name, scope_type, **kwargs):
|
|
self._scope_serial += 1
|
|
handle = ("scope", name, self._scope_serial)
|
|
self._scope_context.set(handle)
|
|
self.events.append(("scope.push", name, scope_type, kwargs))
|
|
return handle
|
|
|
|
def _scope_pop(self, handle, **kwargs):
|
|
self.events.append(("scope.pop", handle, kwargs))
|
|
|
|
def _scope_event(self, name, **kwargs):
|
|
self.events.append(("scope.event", name, kwargs))
|
|
|
|
def _get_scope_stack(self):
|
|
current = self._scope_context.get()
|
|
self.events.append(("scope.sync", current))
|
|
return current
|
|
|
|
def _llm_call(self, name, request, **kwargs):
|
|
handle = ("llm", name)
|
|
self._llm_starts[handle] = kwargs
|
|
self.events.append(("llm.call", name, request.content, kwargs))
|
|
return handle
|
|
|
|
def _llm_call_end(self, handle, response, **kwargs):
|
|
self.events.append(("llm.call_end", handle, response, kwargs))
|
|
start = self._llm_starts.pop(handle, {})
|
|
event = SimpleNamespace(
|
|
kind="scope",
|
|
category="llm",
|
|
name=handle[1],
|
|
scope_category="end",
|
|
category_profile={"model_name": start.get("model_name")},
|
|
metadata={
|
|
**(start.get("metadata") or {}),
|
|
**(kwargs.get("metadata") or {}),
|
|
"otel.status_code": "OK",
|
|
},
|
|
data=response,
|
|
)
|
|
for callback in list(self._callbacks.values()):
|
|
callback(event)
|
|
|
|
def _llm_execute(self, name, request, func, **kwargs):
|
|
self.events.append(("llm.execute.start", name, request.content, kwargs))
|
|
handle = self._llm_call(name, request, **kwargs)
|
|
result = func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content}))
|
|
self._llm_call_end(
|
|
handle,
|
|
result,
|
|
**{key: value for key, value in kwargs.items() if key != "handle"},
|
|
)
|
|
self.events.append(("llm.execute.end", name, result, kwargs))
|
|
return result
|
|
|
|
def _tool_call(self, name, args, **kwargs):
|
|
handle = ("tool", name)
|
|
self.events.append(("tool.call", name, args, kwargs))
|
|
return handle
|
|
|
|
def _tool_call_end(self, handle, result, **kwargs):
|
|
self.events.append(("tool.call_end", handle, result, kwargs))
|
|
|
|
def _tool_execute(self, name, args, func, **kwargs):
|
|
self.events.append(("tool.execute.start", name, args, kwargs))
|
|
handle = self._tool_call(name, args, **kwargs)
|
|
result = func(args)
|
|
self._tool_call_end(
|
|
handle,
|
|
result,
|
|
**{key: value for key, value in kwargs.items() if key != "handle"},
|
|
)
|
|
self.events.append(("tool.execute.end", name, result, kwargs))
|
|
return result
|
|
|
|
def _tool_request_intercepts(self, name, args):
|
|
self.events.append(("tool.request_intercepts", name, args))
|
|
return {"intercepted": True, **args}
|
|
|
|
def _make_atof_exporter(self, config):
|
|
return _FakeAtofExporter(self.events, config)
|
|
|
|
def _make_atif_exporter(self, session_id, agent_name, agent_version, **kwargs):
|
|
return _FakeAtifExporter(self.events, session_id, agent_name, agent_version, kwargs)
|
|
|
|
async def _plugin_initialize(self, config):
|
|
self.events.append(("plugin.initialize", config))
|
|
return {"diagnostics": []}
|
|
|
|
async def _plugin_clear(self):
|
|
self.events.append(("plugin.clear",))
|
|
|
|
async def _plugin_initialize_with_dynamic(self, config, dynamic_plugins):
|
|
self.events.append(("plugin.activate_dynamic", config, dynamic_plugins))
|
|
return _FakePluginActivation(self.events)
|
|
|
|
def _register_subscriber(self, name, callback):
|
|
self._callbacks[name] = callback
|
|
self.events.append(("subscribers.register", name))
|
|
|
|
def _deregister_subscriber(self, name):
|
|
self._callbacks.pop(name, None)
|
|
self.events.append(("subscribers.deregister", name))
|
|
|
|
def _flush_subscribers(self):
|
|
self.events.append(("subscribers.flush",))
|
|
|
|
|
|
class _FakePluginActivation:
|
|
def __init__(self, events):
|
|
self.events = events
|
|
self.report = {"diagnostics": []}
|
|
|
|
async def close(self):
|
|
self.events.append(("plugin.activation.close",))
|
|
|
|
|
|
class _FakeLLMRequest:
|
|
def __init__(self, headers, content):
|
|
self.headers = headers
|
|
self.content = content
|
|
|
|
|
|
class _FakeAtofExporterConfig:
|
|
def __init__(self):
|
|
self.output_directory = ""
|
|
self.filename = "events.jsonl"
|
|
self.mode = "append"
|
|
|
|
|
|
class _FakeAtofExporter:
|
|
def __init__(self, events, config):
|
|
self.events = events
|
|
self.config = config
|
|
|
|
def register(self, name):
|
|
self.events.append(("atof.register", name, self.config.output_directory, self.config.filename))
|
|
|
|
def deregister(self, name):
|
|
self.events.append(("atof.deregister", name, self.config.output_directory, self.config.filename))
|
|
return True
|
|
|
|
|
|
class _FakeAtifExporter:
|
|
def __init__(self, events, session_id, agent_name, agent_version, kwargs):
|
|
self.events = events
|
|
self.session_id = session_id
|
|
self.agent_name = agent_name
|
|
self.agent_version = agent_version
|
|
self.kwargs = kwargs
|
|
|
|
def register(self, name):
|
|
self.events.append(("atif.register", name, self.session_id))
|
|
|
|
def deregister(self, name):
|
|
self.events.append(("atif.deregister", name, self.session_id))
|
|
return True
|
|
|
|
def export_json(self):
|
|
self.events.append(("atif.export", self.session_id))
|
|
return json.dumps({"session_id": self.session_id, "agent_name": self.agent_name})
|
|
|
|
|
|
def _fresh_plugin(monkeypatch, fake):
|
|
existing = sys.modules.get("plugins.observability.nemo_relay")
|
|
if existing is not None:
|
|
existing.reset_for_tests()
|
|
relay_shared_metrics._reset_for_tests()
|
|
relay_runtime._reset_for_tests()
|
|
monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
|
|
monkeypatch.setitem(sys.modules, "nemo_relay", fake)
|
|
sys.modules.pop("plugins.observability.nemo_relay", None)
|
|
plugin = importlib.import_module("plugins.observability.nemo_relay")
|
|
plugin.reset_for_tests()
|
|
return plugin
|
|
|
|
|
|
def _enable_dynamic_plugin(tmp_path, monkeypatch) -> Path:
|
|
plugins_toml = tmp_path / "plugins.toml"
|
|
plugins_toml.write_text(
|
|
f"""
|
|
version = 1
|
|
|
|
[[dynamic_plugins]]
|
|
plugin_id = "fixture"
|
|
kind = "rust_dynamic"
|
|
manifest_ref = "{(tmp_path / "fixture" / "relay-plugin.toml").as_posix()}"
|
|
|
|
[dynamic_plugins.config]
|
|
mode = "test"
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
|
return plugins_toml
|
|
|
|
|
|
def test_manifest_fields():
|
|
data = yaml.safe_load((PLUGIN_DIR / "plugin.yaml").read_text())
|
|
assert data["name"] == "nemo_relay"
|
|
assert set(data["hooks"]) == {
|
|
"on_session_start",
|
|
"on_session_end",
|
|
"on_session_finalize",
|
|
"on_session_reset",
|
|
"pre_llm_call",
|
|
"post_llm_call",
|
|
"pre_approval_request",
|
|
"post_approval_response",
|
|
"subagent_start",
|
|
"subagent_stop",
|
|
}
|
|
|
|
|
|
def test_nemo_relay_plugin_is_discoverable_as_bundled_plugin(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes_test"))
|
|
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
|
|
loaded = manager._plugins["observability/nemo_relay"]
|
|
assert loaded.manifest.name == "nemo_relay"
|
|
assert loaded.manifest.source == "bundled"
|
|
assert not loaded.enabled
|
|
|
|
|
|
def test_shared_metrics_and_rich_plugin_share_one_core_session(
|
|
tmp_path,
|
|
monkeypatch,
|
|
):
|
|
from agent import relay_llm
|
|
|
|
fake = _FakeNemoRelay()
|
|
hermes_home = tmp_path / "hermes-home"
|
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
|
monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1")
|
|
monkeypatch.setenv(
|
|
"HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "atif")
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.read_raw_config_readonly",
|
|
lambda: {"telemetry": {"shared_metrics": {"enabled": True}}},
|
|
)
|
|
plugin = _fresh_plugin(monkeypatch, fake)
|
|
manager = PluginManager()
|
|
|
|
class _Context:
|
|
def register_hook(self, name, callback):
|
|
manager._hooks.setdefault(name, []).append(callback)
|
|
|
|
plugin.register(_Context())
|
|
monkeypatch.setattr(plugin_api, "_plugin_manager", manager)
|
|
|
|
event = {
|
|
"session_id": "s1",
|
|
"task_id": "t1",
|
|
"api_request_id": "api-1",
|
|
"provider": "anthropic",
|
|
"model": "claude-sonnet",
|
|
"platform": "cli",
|
|
}
|
|
coordinator = relay_runtime.SESSION_COORDINATOR
|
|
lease = coordinator.acquire_conversation(
|
|
profile_key=relay_runtime.current_profile_key(),
|
|
session_id="s1",
|
|
platform="cli",
|
|
model=event["model"],
|
|
)
|
|
lifecycle.invoke_hook("on_session_start", **event)
|
|
turn = coordinator.begin_turn(
|
|
lease,
|
|
turn_id="turn-1",
|
|
task_id="t1",
|
|
)
|
|
lifecycle.invoke_hook(
|
|
"pre_api_request",
|
|
**event,
|
|
request={"body": {"messages": [{"role": "user", "content": "hi"}]}},
|
|
)
|
|
relay_llm.execute(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
lambda _request: {
|
|
"assistant_message": {"role": "assistant", "content": "hello"}
|
|
},
|
|
session_id="s1",
|
|
name="anthropic",
|
|
model_name="claude-sonnet",
|
|
metadata={"api_request_id": "api-1", "api_mode": "custom"},
|
|
)
|
|
lifecycle.invoke_hook(
|
|
"post_api_request",
|
|
**event,
|
|
response={"assistant_message": {"role": "assistant", "content": "hello"}},
|
|
)
|
|
coordinator.end_turn(turn, outcome="success")
|
|
coordinator.release_conversation(lease)
|
|
lifecycle.finalize_session(session_id="s1")
|
|
|
|
session_pushes = [
|
|
item
|
|
for item in fake.events
|
|
if item[0] == "scope.push" and item[1] == relay_runtime.SESSION_SCOPE
|
|
]
|
|
assert len(session_pushes) == 1
|
|
register_metrics = next(
|
|
index
|
|
for index, item in enumerate(fake.events)
|
|
if item[0] == "subscribers.register"
|
|
and item[1].startswith("hermes.nemo_relay.shared_metrics.")
|
|
)
|
|
register_atif = next(
|
|
index for index, item in enumerate(fake.events) if item[0] == "atif.register"
|
|
)
|
|
open_session = fake.events.index(session_pushes[0])
|
|
assert register_metrics < register_atif < open_session
|
|
|
|
plugin_runtime = plugin._get_runtime()
|
|
assert plugin_runtime is not None
|
|
assert not plugin_runtime.sessions
|
|
assert relay_runtime.get_session_handle("s1") is None
|
|
packages = list(
|
|
(hermes_home / "telemetry" / "shared_metrics" / "outbox").glob("*.json")
|
|
)
|
|
assert len(packages) == 1
|
|
package = json.loads(packages[0].read_text(encoding="utf-8"))
|
|
assert package["metrics"][0]["name"] == "hermes.model_call.count"
|
|
assert package["metrics"][0]["value"] == 1
|
|
assert (tmp_path / "atif" / "hermes-atif-s1.json").exists()
|
|
|
|
|
|
def test_real_binding_shares_plugin_configuration_across_two_profiles(
|
|
tmp_path,
|
|
monkeypatch,
|
|
):
|
|
relay = pytest.importorskip("nemo_relay")
|
|
if getattr(relay, "_native", None) is None:
|
|
pytest.skip("NeMo Relay native binding is unavailable on this platform")
|
|
plugin = _fresh_plugin(monkeypatch, relay)
|
|
original_initialize = relay.plugin.initialize
|
|
original_clear = relay.plugin.clear
|
|
original_clear()
|
|
initialize_calls = []
|
|
clear_calls = 0
|
|
|
|
async def _initialize(config):
|
|
initialize_calls.append(config)
|
|
return await original_initialize(config)
|
|
|
|
def _clear():
|
|
nonlocal clear_calls
|
|
clear_calls += 1
|
|
return original_clear()
|
|
|
|
monkeypatch.setattr(relay.plugin, "initialize", _initialize)
|
|
monkeypatch.setattr(relay.plugin, "clear", _clear)
|
|
monkeypatch.setattr(
|
|
plugin,
|
|
"_load_settings",
|
|
lambda: plugin._Settings(plugins_config={"version": 1}),
|
|
)
|
|
profile_a = str(tmp_path / "profile-a")
|
|
profile_b = str(tmp_path / "profile-b")
|
|
host_a = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_a)
|
|
host_b = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_b)
|
|
|
|
try:
|
|
runtime_a = plugin._get_runtime(profile_key=profile_a, host=host_a)
|
|
runtime_b = plugin._get_runtime(profile_key=profile_b, host=host_b)
|
|
assert runtime_a is not None
|
|
assert runtime_b is not None
|
|
runtime_a.ensure_session({"session_id": "session-a"})
|
|
runtime_b.ensure_session({"session_id": "session-b"})
|
|
|
|
assert initialize_calls == [{"version": 1}]
|
|
assert relay.plugin.report() is not None
|
|
|
|
runtime_a.close_session({"session_id": "session-a"})
|
|
|
|
assert clear_calls == 0
|
|
assert relay.plugin.report() is not None
|
|
assert runtime_b.host.get_session("session-b") is not None
|
|
|
|
runtime_b.close_session({"session_id": "session-b"})
|
|
|
|
assert clear_calls == 1
|
|
assert relay.plugin.report() is None
|
|
finally:
|
|
plugin.reset_for_tests()
|
|
host_a.shutdown()
|
|
host_b.shutdown()
|
|
original_clear()
|
|
|
|
|
|
def test_relay_tool_request_rewrite_precedes_hermes_authorization_boundary(
|
|
tmp_path,
|
|
monkeypatch,
|
|
):
|
|
from hermes_cli.middleware import apply_tool_request_middleware
|
|
|
|
fake = _FakeNemoRelay()
|
|
plugin = _fresh_plugin(monkeypatch, fake)
|
|
_enable_dynamic_plugin(tmp_path, monkeypatch)
|
|
plugin.on_session_start(session_id="s1")
|
|
|
|
result = apply_tool_request_middleware(
|
|
"fixture-tool",
|
|
{"value": 1},
|
|
session_id="s1",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert result.payload == {"intercepted": True, "value": 1}
|
|
assert result.trace[0] == {"source": "nemo_relay"}
|
|
|
|
|