mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
edac3a7bae
Self-review after the #51714 feedback found the reviewer's dead-table finding was not isolated — the schema advertised far more than the code populates, and our own tests hid it by hand-feeding fields production never sends. Make the surface honest by subtraction. Schema (10 tel_* tables -> 5): - Delete tel_gateway_events, tel_cron_events, tel_skill_events, tel_memory_events, tel_feedback_events — declared, never written, never read. - Drop columns nothing populates: tel_runs.{profile_id,estimated_cost_usd, cost_status}; tel_model_calls.{ttft_ms,estimated_cost_usd,cost_status, cost_source,end_reason,retry_count}; tel_tool_calls.{backend,retry_count, approval}; tel_spans.attrs_json. Cost duplicated the existing sessions billing columns and was always NULL here. - events.py / emitter _TABLE_COLUMNS / OTLP _span_attrs / rollup / preview display all trimmed to match. Correctness: - end_reason no longer hardcodes "completed". Production finalize callers pass `reason` (shutdown/session_expired/session_reset); _coarse_end_reason now reads it and maps accordingly. - Fix a latent bug the trim exposed: the model_call hook passed end_reason= to ModelCallEvent, which the @_safe wrapper was silently swallowing — so tel_model_calls dropped every row in real runs. Now writes correctly. Tests: - Stop hand-feeding estimated_cost_usd / turn_exit_reason that no production call site sends. Finalize is now driven with the real `reason` kwarg, and assertions cover only fields that are actually populated. This is what let the model_call drop hide — the suite graded on a fictional contract. Net: a smaller system that does what it says. Verified end-to-end over the real dispatch path (runs + connected span tree + model/tool rows populate; dead tables gone). 160 telemetry/state/insights tests green.
108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
"""`hermes telemetry` handler smoke tests (local-only; no upload)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import time
|
|
import types
|
|
|
|
import pytest
|
|
|
|
import hermes_state
|
|
|
|
|
|
@pytest.fixture
|
|
def home(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
# state.db with tel_* + seeded data
|
|
db = tmp_path / "state.db"
|
|
hermes_state.SessionDB(db_path=db)
|
|
from agent.telemetry.emitter import TelemetryEmitter
|
|
from agent.telemetry.events import RunEvent, ModelCallEvent, ToolCallEvent
|
|
em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "e.jsonl", db_path=db)
|
|
now = time.time_ns()
|
|
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed",
|
|
start_ns=now - 60_000_000, end_ns=now, model_call_count=1,
|
|
tool_call_count=1))
|
|
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic",
|
|
model="claude-opus-4",
|
|
input_tokens=20000, output_tokens=2000))
|
|
em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search",
|
|
result_class="ok"))
|
|
em.flush()
|
|
em.close()
|
|
yield tmp_path
|
|
|
|
|
|
def _run(action, **kw):
|
|
from hermes_cli.main import cmd_telemetry
|
|
args = types.SimpleNamespace(telemetry_action=action, days=30, limit=10, json=False)
|
|
for k, v in kw.items():
|
|
setattr(args, k, v)
|
|
cmd_telemetry(args)
|
|
|
|
|
|
def test_status_runs(home, capsys):
|
|
_run("status")
|
|
out = capsys.readouterr().out
|
|
assert "Telemetry status" in out
|
|
assert "Upload:" in out and "DISABLED" in out
|
|
assert "Local data:" in out
|
|
|
|
|
|
def test_preview_shows_real_values(home, capsys):
|
|
_run("preview")
|
|
out = capsys.readouterr().out
|
|
assert "NOT uploaded" in out
|
|
assert "workflow_completed" in out
|
|
# real model + tool names ARE shown (this is the user's own local data)
|
|
assert "claude-opus-4" in out
|
|
assert "web_search" in out
|
|
|
|
|
|
def test_status_reflects_consent_set_via_config(home, capsys):
|
|
# Opting in is a plain config write now (no `enable` verb). status should
|
|
# reflect consent_state=aggregate as aggregate metrics being on.
|
|
from hermes_cli.config import load_config, save_config
|
|
cfg = load_config()
|
|
cfg.setdefault("telemetry", {})["consent_state"] = "aggregate"
|
|
save_config(cfg)
|
|
_run("status")
|
|
out = capsys.readouterr().out
|
|
assert "consent_state=aggregate" in out
|
|
assert "Aggregate metrics: on" in out
|
|
|
|
|
|
def test_status_shows_optin_hint_when_unknown(home, capsys):
|
|
_run("status")
|
|
out = capsys.readouterr().out
|
|
assert "Aggregate metrics: off" in out
|
|
assert "config set telemetry.consent_state aggregate" in out
|
|
|
|
|
|
def test_allow_aggregate_false_keeps_metrics_off_in_status(home, capsys):
|
|
# Even with consent opted in, a managed allow_aggregate:false wins.
|
|
from hermes_cli.config import load_config, save_config
|
|
cfg = load_config()
|
|
tel = cfg.setdefault("telemetry", {})
|
|
tel["consent_state"] = "aggregate"
|
|
tel["allow_aggregate"] = False
|
|
save_config(cfg)
|
|
_run("status")
|
|
out = capsys.readouterr().out
|
|
assert "Aggregate metrics: off" in out
|
|
assert "allow_aggregate is false" in out
|
|
|
|
|
|
def test_local_off_with_consent_shows_inert_in_status(home, capsys):
|
|
# local off + opted in: aggregate is off and the status explains why.
|
|
from hermes_cli.config import load_config, save_config
|
|
cfg = load_config()
|
|
tel = cfg.setdefault("telemetry", {})
|
|
tel["local"] = False
|
|
tel["consent_state"] = "aggregate"
|
|
save_config(cfg)
|
|
_run("status")
|
|
out = capsys.readouterr().out
|
|
assert "Aggregate metrics: off" in out
|
|
assert "inert: local telemetry is off" in out
|