mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
0ebdd48f9d
Self-review after the #51714 feedback found the reviewer's dead-table finding was not isolated — the schema advertised far more than the code populates, and our own tests hid it by hand-feeding fields production never sends. Make the surface honest by subtraction. Schema (10 tel_* tables -> 5): - Delete tel_gateway_events, tel_cron_events, tel_skill_events, tel_memory_events, tel_feedback_events — declared, never written, never read. - Drop columns nothing populates: tel_runs.{profile_id,estimated_cost_usd, cost_status}; tel_model_calls.{ttft_ms,estimated_cost_usd,cost_status, cost_source,end_reason,retry_count}; tel_tool_calls.{backend,retry_count, approval}; tel_spans.attrs_json. Cost duplicated the existing sessions billing columns and was always NULL here. - events.py / emitter _TABLE_COLUMNS / OTLP _span_attrs / rollup / preview display all trimmed to match. Correctness: - end_reason no longer hardcodes "completed". Production finalize callers pass `reason` (shutdown/session_expired/session_reset); _coarse_end_reason now reads it and maps accordingly. - Fix a latent bug the trim exposed: the model_call hook passed end_reason= to ModelCallEvent, which the @_safe wrapper was silently swallowing — so tel_model_calls dropped every row in real runs. Now writes correctly. Tests: - Stop hand-feeding estimated_cost_usd / turn_exit_reason that no production call site sends. Finalize is now driven with the real `reason` kwarg, and assertions cover only fields that are actually populated. This is what let the model_call drop hide — the suite graded on a fictional contract. Net: a smaller system that does what it says. Verified end-to-end over the real dispatch path (runs + connected span tree + model/tool rows populate; dead tables gone). 160 telemetry/state/insights tests green.
109 lines
4 KiB
Python
109 lines
4 KiB
Python
"""Trace/span layer: tel_spans is populated as a connected run -> calls tree.
|
|
|
|
Drives the real dispatch chain (discover_plugins -> invoke_hook) and asserts the
|
|
timing/lineage backbone in tel_spans:
|
|
- one root span per run (kind="run", parent_span_id NULL),
|
|
- one child span per model/tool call parented to the root,
|
|
- a single trace_id across the run,
|
|
- call detail rows (tel_model_calls / tel_tool_calls) JOIN to their span by span_id,
|
|
- reconstructed durations match the reported latency/duration.
|
|
|
|
This is the regression guard for the waterfall a desktop trace viewer renders.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import time
|
|
|
|
import pytest
|
|
|
|
import hermes_state
|
|
|
|
|
|
@pytest.fixture
|
|
def runtime(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
db = tmp_path / "state.db"
|
|
hermes_state.SessionDB(db_path=db)
|
|
import hermes_cli.plugins as plugins_mod
|
|
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
|
|
from agent.telemetry import emitter as emitter_mod
|
|
emitter_mod.reset_emitter_for_tests(None)
|
|
import plugins.telemetry as plug
|
|
plug._runs.clear()
|
|
yield db, plugins_mod, emitter_mod
|
|
try:
|
|
emitter_mod.get_emitter().flush()
|
|
except Exception:
|
|
pass
|
|
emitter_mod.reset_emitter_for_tests(None)
|
|
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
|
|
|
|
|
|
def _one_turn(invoke_hook):
|
|
invoke_hook("on_session_start", session_id="s1",
|
|
model="anthropic/claude-opus-4", platform="cli")
|
|
invoke_hook("post_api_request", session_id="s1", platform="cli",
|
|
provider="anthropic", model="claude-opus-4", api_duration=0.9,
|
|
usage={"input_tokens": 1000, "output_tokens": 120})
|
|
invoke_hook("post_tool_call", session_id="s1", platform="cli",
|
|
function_name="web_search", duration_ms=210, result='{"data": "ok"}')
|
|
invoke_hook("on_session_finalize", session_id="s1", platform="cli",
|
|
reason="shutdown")
|
|
|
|
|
|
def test_tel_spans_forms_connected_trace(runtime):
|
|
db, plugins_mod, emitter_mod = runtime
|
|
plugins_mod.discover_plugins(force=True)
|
|
_one_turn(plugins_mod.invoke_hook)
|
|
time.sleep(0.5)
|
|
emitter_mod.get_emitter().flush()
|
|
|
|
conn = sqlite3.connect(db)
|
|
conn.row_factory = sqlite3.Row
|
|
spans = conn.execute(
|
|
"SELECT span_id, parent_span_id, kind, name, start_ns, end_ns, status, trace_id "
|
|
"FROM tel_spans"
|
|
).fetchall()
|
|
|
|
# root + model + tool
|
|
assert len(spans) == 3
|
|
roots = [s for s in spans if s["parent_span_id"] is None]
|
|
children = [s for s in spans if s["parent_span_id"] is not None]
|
|
assert len(roots) == 1
|
|
assert roots[0]["kind"] == "run"
|
|
assert len(children) == 2
|
|
|
|
# single trace, all children parented to the root
|
|
assert len({s["trace_id"] for s in spans}) == 1
|
|
assert all(c["parent_span_id"] == roots[0]["span_id"] for c in children)
|
|
|
|
# spans are time-ordered and carry real durations
|
|
by_kind = {s["kind"]: s for s in spans}
|
|
assert (by_kind["model"]["end_ns"] - by_kind["model"]["start_ns"]) == 900 * 1_000_000
|
|
assert (by_kind["tool"]["end_ns"] - by_kind["tool"]["start_ns"]) == 210 * 1_000_000
|
|
assert by_kind["run"]["end_ns"] >= by_kind["run"]["start_ns"]
|
|
|
|
|
|
def test_detail_rows_join_to_spans(runtime):
|
|
db, plugins_mod, emitter_mod = runtime
|
|
plugins_mod.discover_plugins(force=True)
|
|
_one_turn(plugins_mod.invoke_hook)
|
|
time.sleep(0.5)
|
|
emitter_mod.get_emitter().flush()
|
|
|
|
conn = sqlite3.connect(db)
|
|
conn.row_factory = sqlite3.Row
|
|
mc = conn.execute(
|
|
"SELECT m.model, s.kind, s.trace_id FROM tel_model_calls m "
|
|
"JOIN tel_spans s ON m.span_id = s.span_id"
|
|
).fetchone()
|
|
assert mc is not None and mc["model"] == "claude-opus-4" and mc["kind"] == "model"
|
|
|
|
tc = conn.execute(
|
|
"SELECT t.tool_name, s.kind FROM tel_tool_calls t "
|
|
"JOIN tel_spans s ON t.span_id = s.span_id"
|
|
).fetchone()
|
|
assert tc is not None and tc["tool_name"] == "web_search" and tc["kind"] == "tool"
|
|
conn.close()
|