mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
test(telemetry): end-to-end plugin dispatch coverage
The existing hook tests call the plugin's _on_* callbacks directly, which passes
even if the bundled plugin stops auto-loading or a hook name drifts from what core
fires — real runs would go dark while the suite stays green.
Add test_plugin_e2e.py, which drives the real dispatch chain through public entry
points only (discover_plugins -> invoke_hook -> registered callback -> emitter ->
tel_* tables), exactly as core does:
- one completed turn produces tel_runs / tel_model_calls / tel_tool_calls rows
with real provider/model/tool values and correct counts;
- telemetry.local=false means the plugin does not load and nothing is written.
Verified robust against test ordering (singleton resets for the plugin manager and
the emitter in the fixture).
This commit is contained in:
parent
1e5e61b4be
commit
9737728872
1 changed files with 127 additions and 0 deletions
127
tests/telemetry/test_plugin_e2e.py
Normal file
127
tests/telemetry/test_plugin_e2e.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""End-to-end telemetry wiring test.
|
||||
|
||||
Unlike test_plugin_hooks.py (which calls the plugin's ``_on_*`` callbacks directly),
|
||||
this drives the REAL dispatch chain that core uses at runtime:
|
||||
|
||||
discover_plugins() -> plugin registers hooks -> invoke_hook(name, **kwargs)
|
||||
-> registered callback -> emitter -> tel_* tables
|
||||
|
||||
If the bundled plugin stops auto-loading, stops registering a hook, or the hook name
|
||||
drifts from what core fires, the hand-written hook tests still pass but real runs go
|
||||
dark. This test is the guard against that — it only touches public entry points
|
||||
(``discover_plugins`` / ``invoke_hook``), exactly as core does.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import hermes_state
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runtime(tmp_path, monkeypatch):
|
||||
"""A clean HERMES_HOME with state.db, a fresh plugin manager, and a fresh emitter."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
db = tmp_path / "state.db"
|
||||
hermes_state.SessionDB(db_path=db)
|
||||
|
||||
# Reset the global plugin-manager singleton so discovery re-runs in this HERMES_HOME.
|
||||
import hermes_cli.plugins as plugins_mod
|
||||
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
|
||||
|
||||
# Reset the emitter singleton so it binds to this state.db (and tear it down after).
|
||||
from agent.telemetry import emitter as emitter_mod
|
||||
emitter_mod.reset_emitter_for_tests(None)
|
||||
# Clear the plugin's per-run accumulators between tests.
|
||||
import plugins.telemetry as plug
|
||||
plug._runs.clear()
|
||||
|
||||
yield db, plugins_mod, emitter_mod, plug
|
||||
|
||||
try:
|
||||
emitter_mod.get_emitter().flush()
|
||||
except Exception:
|
||||
pass
|
||||
emitter_mod.reset_emitter_for_tests(None)
|
||||
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
|
||||
|
||||
|
||||
def _fire_one_turn(invoke_hook):
|
||||
"""Fire the hook sequence of a single completed turn, as core does."""
|
||||
invoke_hook("on_session_start", session_id="s1",
|
||||
model="anthropic/claude-opus-4", platform="cli")
|
||||
invoke_hook("post_api_request", session_id="s1", platform="cli",
|
||||
provider="anthropic", base_url=None, model="claude-opus-4",
|
||||
api_duration=0.9,
|
||||
usage={"input_tokens": 1000, "output_tokens": 120,
|
||||
"cache_read_tokens": 0, "cache_write_tokens": 0,
|
||||
"reasoning_tokens": 0})
|
||||
invoke_hook("post_tool_call", session_id="s1", platform="cli",
|
||||
function_name="web_search", duration_ms=210, result='{"data": "ok"}')
|
||||
invoke_hook("on_session_finalize", session_id="s1", platform="cli",
|
||||
turn_exit_reason="completed", estimated_cost_usd=0.01, cost_status="known")
|
||||
|
||||
|
||||
def test_real_dispatch_writes_tel_rows(runtime):
|
||||
"""The bundled plugin, loaded via discover_plugins, captures a turn end to end."""
|
||||
db, plugins_mod, emitter_mod, _plug = runtime
|
||||
|
||||
plugins_mod.discover_plugins(force=True)
|
||||
|
||||
# The plugin must have registered the lifecycle hooks core fires.
|
||||
mgr = plugins_mod.get_plugin_manager()
|
||||
registered = {k for k, v in getattr(mgr, "_hooks", {}).items() if v}
|
||||
for hook in ("on_session_start", "post_api_request", "post_tool_call",
|
||||
"on_session_finalize"):
|
||||
assert hook in registered, f"core hook {hook!r} not registered by the plugin"
|
||||
|
||||
_fire_one_turn(plugins_mod.invoke_hook)
|
||||
|
||||
time.sleep(0.5) # let the background writer drain
|
||||
emitter_mod.get_emitter().flush()
|
||||
|
||||
conn = sqlite3.connect(db)
|
||||
conn.row_factory = sqlite3.Row
|
||||
assert conn.execute("SELECT COUNT(*) c FROM tel_runs").fetchone()["c"] == 1
|
||||
assert conn.execute("SELECT COUNT(*) c FROM tel_model_calls").fetchone()["c"] == 1
|
||||
assert conn.execute("SELECT COUNT(*) c FROM tel_tool_calls").fetchone()["c"] == 1
|
||||
|
||||
# Real values, not buckets.
|
||||
mc = conn.execute("SELECT provider, model FROM tel_model_calls").fetchone()
|
||||
assert mc["provider"] == "anthropic"
|
||||
assert mc["model"] == "claude-opus-4"
|
||||
tc = conn.execute("SELECT tool_name FROM tel_tool_calls").fetchone()
|
||||
assert tc["tool_name"] == "web_search"
|
||||
run = conn.execute("SELECT end_reason, model_call_count, tool_call_count "
|
||||
"FROM tel_runs").fetchone()
|
||||
assert run["end_reason"] == "completed"
|
||||
assert run["model_call_count"] == 1
|
||||
assert run["tool_call_count"] == 1
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_local_disabled_writes_nothing(runtime, monkeypatch):
|
||||
"""telemetry.local=false: the plugin does not auto-load, so no rows are written."""
|
||||
db, plugins_mod, emitter_mod, _plug = runtime
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.load_config",
|
||||
lambda: {"telemetry": {"local": False}},
|
||||
raising=False,
|
||||
)
|
||||
|
||||
plugins_mod.discover_plugins(force=True)
|
||||
_fire_one_turn(plugins_mod.invoke_hook)
|
||||
time.sleep(0.3)
|
||||
try:
|
||||
emitter_mod.get_emitter().flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
conn = sqlite3.connect(db)
|
||||
assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0
|
||||
assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 0
|
||||
conn.close()
|
||||
Loading…
Add table
Add a link
Reference in a new issue