test(telemetry): end-to-end plugin dispatch coverage

The existing hook tests call the plugin's _on_* callbacks directly, which passes
even if the bundled plugin stops auto-loading or a hook name drifts from what core
fires — real runs would go dark while the suite stays green.

Add test_plugin_e2e.py, which drives the real dispatch chain through public entry
points only (discover_plugins -> invoke_hook -> registered callback -> emitter ->
tel_* tables), exactly as core does:

  - one completed turn produces tel_runs / tel_model_calls / tel_tool_calls rows
    with real provider/model/tool values and correct counts;
  - telemetry.local=false means the plugin does not load and nothing is written.

Verified robust against test ordering (singleton resets for the plugin manager and
the emitter in the fixture).
This commit is contained in:
emozilla 2026-06-26 15:01:10 -04:00
parent 1e5e61b4be
commit 9737728872

View file

@ -0,0 +1,127 @@
"""End-to-end telemetry wiring test.
Unlike test_plugin_hooks.py (which calls the plugin's ``_on_*`` callbacks directly),
this drives the REAL dispatch chain that core uses at runtime:
discover_plugins() -> plugin registers hooks -> invoke_hook(name, **kwargs)
-> registered callback -> emitter -> tel_* tables
If the bundled plugin stops auto-loading, stops registering a hook, or the hook name
drifts from what core fires, the hand-written hook tests still pass but real runs go
dark. This test is the guard against that — it only touches public entry points
(``discover_plugins`` / ``invoke_hook``), exactly as core does.
"""
from __future__ import annotations
import sqlite3
import time
import pytest
import hermes_state
@pytest.fixture
def runtime(tmp_path, monkeypatch):
"""A clean HERMES_HOME with state.db, a fresh plugin manager, and a fresh emitter."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
db = tmp_path / "state.db"
hermes_state.SessionDB(db_path=db)
# Reset the global plugin-manager singleton so discovery re-runs in this HERMES_HOME.
import hermes_cli.plugins as plugins_mod
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
# Reset the emitter singleton so it binds to this state.db (and tear it down after).
from agent.telemetry import emitter as emitter_mod
emitter_mod.reset_emitter_for_tests(None)
# Clear the plugin's per-run accumulators between tests.
import plugins.telemetry as plug
plug._runs.clear()
yield db, plugins_mod, emitter_mod, plug
try:
emitter_mod.get_emitter().flush()
except Exception:
pass
emitter_mod.reset_emitter_for_tests(None)
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
def _fire_one_turn(invoke_hook):
"""Fire the hook sequence of a single completed turn, as core does."""
invoke_hook("on_session_start", session_id="s1",
model="anthropic/claude-opus-4", platform="cli")
invoke_hook("post_api_request", session_id="s1", platform="cli",
provider="anthropic", base_url=None, model="claude-opus-4",
api_duration=0.9,
usage={"input_tokens": 1000, "output_tokens": 120,
"cache_read_tokens": 0, "cache_write_tokens": 0,
"reasoning_tokens": 0})
invoke_hook("post_tool_call", session_id="s1", platform="cli",
function_name="web_search", duration_ms=210, result='{"data": "ok"}')
invoke_hook("on_session_finalize", session_id="s1", platform="cli",
turn_exit_reason="completed", estimated_cost_usd=0.01, cost_status="known")
def test_real_dispatch_writes_tel_rows(runtime):
"""The bundled plugin, loaded via discover_plugins, captures a turn end to end."""
db, plugins_mod, emitter_mod, _plug = runtime
plugins_mod.discover_plugins(force=True)
# The plugin must have registered the lifecycle hooks core fires.
mgr = plugins_mod.get_plugin_manager()
registered = {k for k, v in getattr(mgr, "_hooks", {}).items() if v}
for hook in ("on_session_start", "post_api_request", "post_tool_call",
"on_session_finalize"):
assert hook in registered, f"core hook {hook!r} not registered by the plugin"
_fire_one_turn(plugins_mod.invoke_hook)
time.sleep(0.5) # let the background writer drain
emitter_mod.get_emitter().flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
assert conn.execute("SELECT COUNT(*) c FROM tel_runs").fetchone()["c"] == 1
assert conn.execute("SELECT COUNT(*) c FROM tel_model_calls").fetchone()["c"] == 1
assert conn.execute("SELECT COUNT(*) c FROM tel_tool_calls").fetchone()["c"] == 1
# Real values, not buckets.
mc = conn.execute("SELECT provider, model FROM tel_model_calls").fetchone()
assert mc["provider"] == "anthropic"
assert mc["model"] == "claude-opus-4"
tc = conn.execute("SELECT tool_name FROM tel_tool_calls").fetchone()
assert tc["tool_name"] == "web_search"
run = conn.execute("SELECT end_reason, model_call_count, tool_call_count "
"FROM tel_runs").fetchone()
assert run["end_reason"] == "completed"
assert run["model_call_count"] == 1
assert run["tool_call_count"] == 1
conn.close()
def test_local_disabled_writes_nothing(runtime, monkeypatch):
"""telemetry.local=false: the plugin does not auto-load, so no rows are written."""
db, plugins_mod, emitter_mod, _plug = runtime
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"telemetry": {"local": False}},
raising=False,
)
plugins_mod.discover_plugins(force=True)
_fire_one_turn(plugins_mod.invoke_hook)
time.sleep(0.3)
try:
emitter_mod.get_emitter().flush()
except Exception:
pass
conn = sqlite3.connect(db)
assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0
assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 0
conn.close()