mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
3e28eaccde
Add a built-in telemetry system that records what the agent does — workflows,
model calls, tool calls, errors — to the local machine, powers `/insights`, and
can export to an operator-chosen destination. Default-on locally; nothing leaves
the machine unless the user exports it or opts into the aggregate plane.
Three planes with a hard wall between them:
- local: full-fidelity observability (real model/provider/tool names), on by
default, never leaves the machine.
- aggregate: opt-in metadata, default off. No uploader ships — consent is
recorded via telemetry.consent_state, and `preview` shows what would be
produced, computed locally.
- trajectories: full message content, opt-in, exported only to the operator's
own destination.
Mechanism:
- Bundled `telemetry` plugin registers observational lifecycle hooks
(on_session_start / post_api_request / post_tool_call / on_session_finalize).
No core call sites are edited; hooks already carry the data.
- Fire-and-forget emitter: emit() returns in microseconds, never blocks or
raises into a model/tool call. A daemon thread writes events to an
append-only JSONL log and the tel_* tables in state.db (its own sqlite
connection, separate from SessionDB).
- tel_runs / tel_model_calls / tel_tool_calls live in the declarative
SCHEMA_SQL and are reconciled automatically; SCHEMA_VERSION 16 -> 17.
- metrics derives rollups for /usage and /insights; rollup builds per-run
summaries for `hermes telemetry preview`.
Consent is config, not a parallel command surface. The config file is the root
of trust: set telemetry.consent_state with `hermes config set`, or pin any
telemetry.* key (including allow_aggregate) via managed scope, which overrides
the user's value per key. `hermes telemetry` exposes only what config cannot:
status (report), preview (query), and export.
Export:
- exporter_bulk writes telemetry (and, when the trajectories plane is enabled,
session content) to ndjson/json.
- otlp_exporter streams spans to a configured OpenTelemetry Collector over
OTLP/HTTP. The SDK is an optional extra (hermes-agent[otlp]), lazily
installed via tools.lazy_deps on first use.
- Secrets are always redacted on every export path
(redact_sensitive_text(force=True)); content export is gated by the
trajectories plane, and PII scrubbing follows telemetry.content_redaction.
OTLP auth headers reference environment variable names, never inline values.
No outbound emission to Nous. The aggregate uploader is intentionally not built.
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
"""rollup tests: tel_* -> per-run summary events with REAL values (local only)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import time
|
|
|
|
import hermes_state
|
|
from agent.telemetry import rollup
|
|
from agent.telemetry.emitter import TelemetryEmitter
|
|
from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent
|
|
|
|
|
|
def _seed(tmp_path):
|
|
db = tmp_path / "state.db"
|
|
conn = sqlite3.connect(db)
|
|
conn.executescript(hermes_state.SCHEMA_SQL)
|
|
conn.close()
|
|
em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db)
|
|
now = time.time_ns()
|
|
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway",
|
|
platform="telegram", end_reason="completed",
|
|
start_ns=now - 90_000_000, end_ns=now,
|
|
model_call_count=2, tool_call_count=2, estimated_cost_usd=2.1))
|
|
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic",
|
|
model="claude-opus-4", input_tokens=60000, output_tokens=8000))
|
|
em.emit(ModelCallEvent(span_id="m2", run_id="r1", provider="anthropic",
|
|
model="claude-opus-4", input_tokens=5000, output_tokens=500))
|
|
em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search",
|
|
result_class="ok"))
|
|
em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate",
|
|
result_class="ok"))
|
|
# an in-progress run (no end_ns) must be excluded
|
|
em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", start_ns=now))
|
|
em.flush()
|
|
em.close()
|
|
return db
|
|
|
|
|
|
def test_builds_one_event_per_completed_run_with_real_values(tmp_path):
|
|
db = _seed(tmp_path)
|
|
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db,
|
|
include_heartbeat=False)
|
|
wf = [e for e in events if e["event_name"] == "workflow_completed"]
|
|
assert len(wf) == 1 # r2 (no end_ns) excluded
|
|
e = wf[0]
|
|
assert e["entrypoint"] == "gateway"
|
|
assert e["platform"] == "telegram"
|
|
# REAL model id + provider, not a bucket/class
|
|
models = {m["model"] for m in e["models_used"]}
|
|
assert models == {"claude-opus-4"}
|
|
assert e["models_used"][0]["provider"] == "anthropic"
|
|
assert sorted(e["tools_used"]) == ["browser_navigate", "web_search"]
|
|
# real token totals, not buckets
|
|
assert e["input_tokens"] == 65000
|
|
assert e["output_tokens"] == 8500
|
|
|
|
|
|
def test_real_model_and_tool_names_present(tmp_path):
|
|
db = _seed(tmp_path)
|
|
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db)
|
|
blob = " ".join(str(v) for e in events for v in e.values())
|
|
assert "claude-opus-4" in blob
|
|
assert "web_search" in blob
|
|
|
|
|
|
def test_heartbeat_included_by_default(tmp_path):
|
|
db = _seed(tmp_path)
|
|
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db)
|
|
assert any(e["event_name"] == "heartbeat" for e in events)
|
|
|
|
|
|
def test_summarize_counts_by_event_name(tmp_path):
|
|
db = _seed(tmp_path)
|
|
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db)
|
|
s = rollup.summarize(events)
|
|
assert s["total"] == len(events)
|
|
assert s["by_event_name"]["workflow_completed"] == 1
|
|
assert s["by_event_name"]["heartbeat"] == 1
|
|
|
|
|
|
def test_empty_db_yields_only_heartbeat(tmp_path):
|
|
db = tmp_path / "state.db"
|
|
conn = sqlite3.connect(db)
|
|
conn.executescript(hermes_state.SCHEMA_SQL)
|
|
conn.close()
|
|
events = rollup.build_aggregate_events(install_id="x", db_path=db)
|
|
assert [e["event_name"] for e in events] == ["heartbeat"]
|