From 3e28eaccded920f2f3db86593b0589aa9b662eea Mon Sep 17 00:00:00 2001 From: emozilla Date: Wed, 24 Jun 2026 02:14:02 -0400 Subject: [PATCH 01/29] feat(telemetry): local-first telemetry & observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a built-in telemetry system that records what the agent does — workflows, model calls, tool calls, errors — to the local machine, powers `/insights`, and can export to an operator-chosen destination. Default-on locally; nothing leaves the machine unless the user exports it or opts into the aggregate plane. Three planes with a hard wall between them: - local: full-fidelity observability (real model/provider/tool names), on by default, never leaves the machine. - aggregate: opt-in metadata, default off. No uploader ships — consent is recorded via telemetry.consent_state, and `preview` shows what would be produced, computed locally. - trajectories: full message content, opt-in, exported only to the operator's own destination. Mechanism: - Bundled `telemetry` plugin registers observational lifecycle hooks (on_session_start / post_api_request / post_tool_call / on_session_finalize). No core call sites are edited; hooks already carry the data. - Fire-and-forget emitter: emit() returns in microseconds, never blocks or raises into a model/tool call. A daemon thread writes events to an append-only JSONL log and the tel_* tables in state.db (its own sqlite connection, separate from SessionDB). - tel_runs / tel_model_calls / tel_tool_calls live in the declarative SCHEMA_SQL and are reconciled automatically; SCHEMA_VERSION 16 -> 17. - metrics derives rollups for /usage and /insights; rollup builds per-run summaries for `hermes telemetry preview`. Consent is config, not a parallel command surface. The config file is the root of trust: set telemetry.consent_state with `hermes config set`, or pin any telemetry.* key (including allow_aggregate) via managed scope, which overrides the user's value per key. `hermes telemetry` exposes only what config cannot: status (report), preview (query), and export. Export: - exporter_bulk writes telemetry (and, when the trajectories plane is enabled, session content) to ndjson/json. - otlp_exporter streams spans to a configured OpenTelemetry Collector over OTLP/HTTP. The SDK is an optional extra (hermes-agent[otlp]), lazily installed via tools.lazy_deps on first use. - Secrets are always redacted on every export path (redact_sensitive_text(force=True)); content export is gated by the trajectories plane, and PII scrubbing follows telemetry.content_redaction. OTLP auth headers reference environment variable names, never inline values. No outbound emission to Nous. The aggregate uploader is intentionally not built. --- agent/insights.py | 116 +++++++ agent/telemetry/__init__.py | 30 ++ agent/telemetry/emitter.py | 317 ++++++++++++++++++ agent/telemetry/events.py | 99 ++++++ agent/telemetry/exporter_bulk.py | 139 ++++++++ agent/telemetry/metrics.py | 219 +++++++++++++ agent/telemetry/otlp_exporter.py | 282 ++++++++++++++++ agent/telemetry/policy.py | 107 +++++++ agent/telemetry/redaction.py | 187 +++++++++++ agent/telemetry/rollup.py | 145 +++++++++ agent/telemetry/spans.py | 83 +++++ cli-config.yaml.example | 43 +++ docs/observability/README.md | 7 + docs/observability/telemetry.md | 209 ++++++++++++ hermes_cli/config.py | 56 +++- hermes_cli/main.py | 186 ++++++++++- hermes_cli/subcommands/telemetry.py | 54 ++++ plugins/telemetry/__init__.py | 319 +++++++++++++++++++ plugins/telemetry/plugin.yaml | 12 + pyproject.toml | 4 + tests/hermes_cli/test_plugins.py | 6 +- tests/telemetry/__init__.py | 0 tests/telemetry/test_cli_telemetry.py | 94 ++++++ tests/telemetry/test_emitter.py | 107 +++++++ tests/telemetry/test_export_redaction.py | 113 +++++++ tests/telemetry/test_exporter_bulk.py | 102 ++++++ tests/telemetry/test_governance.py | 98 ++++++ tests/telemetry/test_insights_integration.py | 82 +++++ tests/telemetry/test_otlp_exporter.py | 171 ++++++++++ tests/telemetry/test_plugin_autoload.py | 53 +++ tests/telemetry/test_plugin_hooks.py | 135 ++++++++ tests/telemetry/test_policy_consent.py | 61 ++++ tests/telemetry/test_rollup.py | 88 +++++ tests/telemetry/test_schema_migration.py | 46 +++ tools/lazy_deps.py | 9 + 35 files changed, 3776 insertions(+), 3 deletions(-) create mode 100644 agent/telemetry/__init__.py create mode 100644 agent/telemetry/emitter.py create mode 100644 agent/telemetry/events.py create mode 100644 agent/telemetry/exporter_bulk.py create mode 100644 agent/telemetry/metrics.py create mode 100644 agent/telemetry/otlp_exporter.py create mode 100644 agent/telemetry/policy.py create mode 100644 agent/telemetry/redaction.py create mode 100644 agent/telemetry/rollup.py create mode 100644 agent/telemetry/spans.py create mode 100644 docs/observability/telemetry.md create mode 100644 hermes_cli/subcommands/telemetry.py create mode 100644 plugins/telemetry/__init__.py create mode 100644 plugins/telemetry/plugin.yaml create mode 100644 tests/telemetry/__init__.py create mode 100644 tests/telemetry/test_cli_telemetry.py create mode 100644 tests/telemetry/test_emitter.py create mode 100644 tests/telemetry/test_export_redaction.py create mode 100644 tests/telemetry/test_exporter_bulk.py create mode 100644 tests/telemetry/test_governance.py create mode 100644 tests/telemetry/test_insights_integration.py create mode 100644 tests/telemetry/test_otlp_exporter.py create mode 100644 tests/telemetry/test_plugin_autoload.py create mode 100644 tests/telemetry/test_plugin_hooks.py create mode 100644 tests/telemetry/test_policy_consent.py create mode 100644 tests/telemetry/test_rollup.py create mode 100644 tests/telemetry/test_schema_migration.py diff --git a/agent/insights.py b/agent/insights.py index 086150c279e..304410707ed 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -82,6 +82,19 @@ def _bar_chart(values: List[int], max_width: int = 20) -> List[str]: return ["█" * max(1, int(v / peak * max_width)) if v > 0 else "" for v in values] +def _fmt_ms(ms: float) -> str: + """Compact human duration from milliseconds (e.g. 850ms, 2.4s, 1.5m).""" + try: + ms = float(ms or 0) + except (TypeError, ValueError): + return "0ms" + if ms < 1000: + return f"{int(ms)}ms" + if ms < 60_000: + return f"{ms / 1000:.1f}s" + return f"{ms / 60_000:.1f}m" + + class InsightsEngine: """ Analyzes session history and produces usage insights. @@ -139,6 +152,7 @@ class InsightsEngine: }, "activity": {}, "top_sessions": [], + "telemetry": {}, } # Compute insights @@ -149,6 +163,7 @@ class InsightsEngine: skills = self._compute_skill_breakdown(skill_usage) activity = self._compute_activity_patterns(sessions) top_sessions = self._compute_top_sessions(sessions) + telemetry = self._compute_telemetry(cutoff) return { "days": days, @@ -162,8 +177,37 @@ class InsightsEngine: "skills": skills, "activity": activity, "top_sessions": top_sessions, + "telemetry": telemetry, } + # ========================================================================= + # Telemetry (observability) — from the tel_* tables (local plane) + # ========================================================================= + + def _compute_telemetry(self, cutoff: float) -> Dict[str, Any]: + """Roll up the local telemetry tables for the same window. + + Reuses the engine's existing connection. Fully fail-soft: if the tel_* + tables are empty or absent (telemetry.local disabled, fresh install), this + returns an empty dict and the renderer skips the section. + """ + try: + from agent.telemetry import metrics + except Exception: + return {} + try: + since_ns = int(cutoff * 1e9) + if not metrics.has_data(conn=self._conn): + return {} + return { + "workflows": metrics.workflow_summary(since_ns=since_ns, conn=self._conn), + "model_calls": metrics.model_call_summary(since_ns=since_ns, conn=self._conn), + "tool_calls": metrics.tool_call_summary(conn=self._conn), + "errors": metrics.error_summary(conn=self._conn), + } + except Exception: + return {} + # ========================================================================= # Data gathering (SQL queries) # ========================================================================= @@ -1023,8 +1067,80 @@ class InsightsEngine: lines.append(f" {ts['label']:<20} {ts['value']:<18} ({ts['date']}, {ts['session_id']})") lines.append("") + # Telemetry / observability (local plane) — only when data exists + tel = report.get("telemetry") or {} + if tel: + self._append_telemetry_section(lines, tel) + return "\n".join(lines) + def _append_telemetry_section(self, lines: List[str], tel: Dict[str, Any]) -> None: + """Render the observability rollups (workflows, tools, providers, errors).""" + wf = tel.get("workflows", {}) + mc = tel.get("model_calls", {}) + tc = tel.get("tool_calls", {}) + errs = tel.get("errors", {}).get("by_class", {}) + + lines.append(" 📡 Observability (local telemetry)") + lines.append(" " + "─" * 56) + + total_runs = wf.get("total_runs", 0) + if total_runs: + sr = wf.get("success_rate", 0.0) * 100 + p50 = wf.get("duration_ms_p50", 0) + p95 = wf.get("duration_ms_p95", 0) + lines.append( + f" Workflows: {total_runs:,} Success: {sr:.1f}% " + f"Duration p50/p95: {_fmt_ms(p50)} / {_fmt_ms(p95)}" + ) + by_entry = wf.get("by_entrypoint", {}) + if by_entry: + entry_str = ", ".join( + f"{k}: {v}" for k, v in sorted(by_entry.items(), key=lambda x: -x[1]) + ) + lines.append(f" Entrypoints: {entry_str}") + + # Tool reliability + if tc.get("total"): + fail_pct = tc.get("failure_rate", 0.0) * 100 + lines.append( + f" Tool calls: {tc['total']:,} Failure rate: {fail_pct:.1f}%" + ) + tools = tc.get("by_tool", {}) + fails = tc.get("failures_by_tool", {}) + top = sorted(tools.items(), key=lambda x: -x[1])[:6] + if top: + parts = [] + for name, n in top: + f = fails.get(name, 0) + parts.append(f"{name}: {n}" + (f" ({f} failed)" if f else "")) + lines.append(" " + " ".join(parts)) + + # Provider / model mix + cache (real names) + by_provider = mc.get("by_provider", {}) + if by_provider: + prov_str = ", ".join( + f"{k}: {v}" for k, v in sorted(by_provider.items(), key=lambda x: -x[1]) + ) + lines.append(f" Providers: {prov_str}") + by_model = mc.get("by_model", {}) + if by_model: + model_str = ", ".join( + f"{k}: {v}" for k, v in sorted(by_model.items(), key=lambda x: -x[1])[:8] + ) + cache = mc.get("cache_hit_rate", 0.0) * 100 + suffix = f" Cache hit: {cache:.1f}%" if cache else "" + lines.append(f" Models: {model_str}{suffix}") + + # Error classes + if errs: + err_str = ", ".join( + f"{k}: {v}" for k, v in sorted(errs.items(), key=lambda x: -x[1])[:6] + ) + lines.append(f" Errors: {err_str}") + + lines.append("") + def format_gateway(self, report: Dict) -> str: """Format the insights report for gateway/messaging (shorter).""" if report.get("empty"): diff --git a/agent/telemetry/__init__.py b/agent/telemetry/__init__.py new file mode 100644 index 00000000000..4c79dd329c0 --- /dev/null +++ b/agent/telemetry/__init__.py @@ -0,0 +1,30 @@ +"""Hermes telemetry & observability. + +Local-first observability, on by default. The ``telemetry`` plugin registers Hermes +lifecycle hooks and hands typed events to the fire-and-forget ``emitter`` (queue -> +background writer -> JSONL + state.db ``tel_*`` index). The emitter never blocks or +raises into a model/tool call (the hot-path invariant). + +Events record the observed model ids, provider names, and tool names. ``metrics`` +derives rollups for /usage and /insights; ``rollup`` builds the per-run summaries shown +by ``hermes telemetry preview``. ``redaction`` + ``exporter_bulk`` + ``otlp_exporter`` +handle export to an operator-chosen destination. ``policy`` holds the consent state +machine for the opt-in aggregate plane (no uploader ships). +""" + +from __future__ import annotations + +from . import emitter, events, metrics, policy, spans + +emit = emitter.emit +get_emitter = emitter.get_emitter + +__all__ = [ + "emitter", + "events", + "metrics", + "policy", + "spans", + "emit", + "get_emitter", +] diff --git a/agent/telemetry/emitter.py b/agent/telemetry/emitter.py new file mode 100644 index 00000000000..d6f3788cca0 --- /dev/null +++ b/agent/telemetry/emitter.py @@ -0,0 +1,317 @@ +"""Local-plane telemetry emitter: fire-and-forget queue + background writer. + +The emitter is the single seam between instrumentation (the telemetry plugin's hook +callbacks) and durable storage. Its contract is the hot-path invariant: + + ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, and + MUST NEVER raise into the caller. A telemetry failure is logged locally and + dropped — it can never affect a model call, a tool call, or a session. + +Mechanism: + * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare except. + On a full queue it drops the *oldest* event and counts the drop. + * A daemon thread drains the queue and writes each event to two places: + 1. the append-only JSONL log (source of truth) + 2. the ``tel_*`` SQLite tables in state.db (rebuildable index) + * The writer uses its own sqlite connection to state.db, separate from SessionDB, + so telemetry writes never contend with or corrupt session writes. + +Local plane only. Nothing here uploads anywhere. +""" + +from __future__ import annotations + +import json +import logging +import queue +import sqlite3 +import threading +import time +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full +_DRAIN_BATCH = 256 + + +def _default_dir() -> Path: + """Resolve the telemetry dir under the active HERMES_HOME (profile-safe).""" + from hermes_constants import get_hermes_home + return get_hermes_home() / "telemetry" + + +def _default_db_path() -> Path: + """Resolve state.db under the active HERMES_HOME (profile-safe).""" + from hermes_constants import get_hermes_home + return get_hermes_home() / "state.db" + + +# Map a telemetry event dict (its "event" tag) to (table, column-ordered insert). +# Only the columns the indexer knows about are written; unknown keys are ignored, +# so an event carrying extra fields never breaks the insert. +_TABLE_COLUMNS: Dict[str, tuple] = { + "run": ( + "tel_runs", + ("run_id", "trace_id", "session_id", "profile_id", "entrypoint", + "platform", "start_ns", "end_ns", "end_reason", + "model_call_count", "tool_call_count", "error_count", + "estimated_cost_usd", "cost_status"), + ), + "model_call": ( + "tel_model_calls", + ("span_id", "run_id", "provider", "model", "base_url", + "input_tokens", "output_tokens", "cache_read_tokens", + "cache_write_tokens", "reasoning_tokens", "latency_ms", "ttft_ms", + "estimated_cost_usd", "cost_status", "cost_source", "end_reason", + "retry_count"), + ), + "tool_call": ( + "tel_tool_calls", + ("span_id", "run_id", "tool_name", "backend", + "duration_ms", "result_class", "retry_count", "approval"), + ), + "error": ( + "tel_error_events", + ("run_id", "error_class", "subsystem", "recovery", "ts_ns"), + ), +} + + +class TelemetryEmitter: + """Owns the queue, the writer thread, and the telemetry sqlite connection.""" + + def __init__( + self, + *, + events_path: Optional[Path] = None, + db_path: Optional[Path] = None, + enabled: bool = True, + ) -> None: + self._dir = (events_path.parent if events_path else _default_dir()) + self._events_path = events_path or (self._dir / "events.jsonl") + self._db_path = db_path or _default_db_path() + self._enabled = enabled + self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) + self._dropped = 0 + self._written = 0 + self._stop = threading.Event() + self._started = False + self._lock = threading.Lock() + self._conn: Optional[sqlite3.Connection] = None + self._thread: Optional[threading.Thread] = None + # Optional live subscribers (e.g. OTLP exporter). Called from the writer + # thread AFTER durable writes, fully fail-isolated — a subscriber that + # raises or blocks can never affect the JSONL/SQLite source of truth or + # the hot path. Each subscriber is callable(batch: list[dict]). + self._subscribers: list = [] + + # ── public API (hot path) ─────────────────────────────────────────────── + def emit(self, event: Any) -> None: + """Enqueue an event. Never blocks, never raises. + + ``event`` may be a dataclass with ``to_dict()`` or a plain dict. + """ + if not self._enabled: + return + try: + payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) + payload.setdefault("ts_ns", time.time_ns()) + self._ensure_started() + try: + self._q.put_nowait(payload) + except queue.Full: + # Drop oldest to make room — bounded memory, newest-wins. + try: + self._q.get_nowait() + self._dropped += 1 + self._q.put_nowait(payload) + except Exception: + self._dropped += 1 + except Exception: # the hot-path invariant: never propagate + logger.debug("telemetry emit failed", exc_info=True) + + # ── lifecycle ─────────────────────────────────────────────────────────── + def _ensure_started(self) -> None: + if self._started: + return + with self._lock: + if self._started: + return + try: + self._dir.mkdir(parents=True, exist_ok=True) + except Exception: + logger.debug("telemetry dir create failed", exc_info=True) + self._thread = threading.Thread( + target=self._run, name="hermes-telemetry-writer", daemon=True + ) + self._thread.start() + self._started = True + + def _open_conn(self) -> Optional[sqlite3.Connection]: + if self._conn is not None: + return self._conn + try: + conn = sqlite3.connect(str(self._db_path), isolation_level=None, timeout=5.0) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=5000") + self._conn = conn + except Exception: + logger.debug("telemetry db open failed", exc_info=True) + self._conn = None + return self._conn + + def _run(self) -> None: + while not self._stop.is_set(): + try: + first = self._q.get(timeout=0.5) + except queue.Empty: + continue + batch = [first] + while len(batch) < _DRAIN_BATCH: + try: + batch.append(self._q.get_nowait()) + except queue.Empty: + break + self._write_batch(batch) + + def _write_batch(self, batch) -> None: + # JSONL append (source of truth) — best effort. + try: + with open(self._events_path, "a", encoding="utf-8") as fh: + for ev in batch: + fh.write(json.dumps(ev, ensure_ascii=False) + "\n") + except Exception: + logger.debug("telemetry jsonl append failed", exc_info=True) + + # SQLite index — best effort, per-event so one bad row can't lose the batch. + conn = self._open_conn() + if conn is None: + return + for ev in batch: + try: + self._index_one(conn, ev) + self._written += 1 + except Exception: + logger.debug("telemetry index row failed", exc_info=True) + + # Live fan-out (e.g. OTLP) — AFTER durable writes, fully fail-isolated. + # A slow/raising subscriber never affects JSONL/SQLite or the hot path. + for sub in self._subscribers: + try: + sub(batch) + except Exception: + logger.debug("telemetry subscriber failed", exc_info=True) + + def subscribe(self, callback) -> None: + """Register a live batch subscriber (callable(batch: list[dict])). + + Called from the writer thread after durable writes. Used by the OTLP + exporter for continuous streaming. Fail-isolated; never on the hot path. + """ + if callback not in self._subscribers: + self._subscribers.append(callback) + + def unsubscribe(self, callback) -> None: + try: + self._subscribers.remove(callback) + except ValueError: + pass + + def _index_one(self, conn: sqlite3.Connection, ev: Dict[str, Any]) -> None: + kind = ev.get("event") + spec = _TABLE_COLUMNS.get(kind) + if spec is None: + return + table, cols = spec + values = [ev.get(c) for c in cols] + placeholders = ", ".join("?" for _ in cols) + collist = ", ".join(cols) + conn.execute( + f"INSERT OR REPLACE INTO {table} ({collist}) VALUES ({placeholders})", + values, + ) + + # ── introspection / shutdown (tests, CLI) ─────────────────────────────── + def flush(self, timeout: float = 2.0) -> None: + """Block until the queue drains (test/CLI helper, NOT the hot path).""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._q.empty(): + # give the writer a tick to finish the in-flight batch + time.sleep(0.05) + if self._q.empty(): + return + time.sleep(0.02) + + def stats(self) -> Dict[str, int]: + return { + "queued": self._q.qsize(), + "written": self._written, + "dropped": self._dropped, + } + + def close(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + if self._conn is not None: + try: + self._conn.close() + except Exception: + pass + self._conn = None + self._started = False + + +# ── process-wide singleton ────────────────────────────────────────────────── +_EMITTER: Optional[TelemetryEmitter] = None +_EMITTER_LOCK = threading.Lock() + + +def get_emitter() -> TelemetryEmitter: + """Return the process-wide emitter, honoring telemetry.local config.""" + global _EMITTER + if _EMITTER is not None: + return _EMITTER + with _EMITTER_LOCK: + if _EMITTER is None: + enabled = _local_enabled() + _EMITTER = TelemetryEmitter(enabled=enabled) + return _EMITTER + + +def _local_enabled() -> bool: + try: + from hermes_cli.config import load_config + cfg = load_config() + tel = cfg.get("telemetry") if isinstance(cfg, dict) else {} + return bool((tel or {}).get("local", True)) + except Exception: + return True + + +def emit(event: Any) -> None: + """Module-level convenience: emit via the singleton.""" + get_emitter().emit(event) + + +def reset_emitter_for_tests(emitter: Optional[TelemetryEmitter] = None) -> None: + """Swap the singleton (tests only).""" + global _EMITTER + with _EMITTER_LOCK: + if _EMITTER is not None and emitter is not _EMITTER: + try: + _EMITTER.close() + except Exception: + pass + _EMITTER = emitter + + +__all__ = [ + "TelemetryEmitter", + "get_emitter", + "emit", + "reset_emitter_for_tests", +] diff --git a/agent/telemetry/events.py b/agent/telemetry/events.py new file mode 100644 index 00000000000..f084659f726 --- /dev/null +++ b/agent/telemetry/events.py @@ -0,0 +1,99 @@ +"""Typed local-plane telemetry events. + +These dataclasses are the rows written to the local JSONL log and the ``tel_*`` +SQLite tables. They record the values observed for each run — model id, provider, tool +name, token counts, durations — and stay on the machine unless explicitly exported. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field, asdict +from typing import Any, Dict, Optional + +# ── local-plane events (real values) ──────────────────────────────────────── + + +def _now_ns() -> int: + return time.time_ns() + + +@dataclass(slots=True) +class RunEvent: + """One top-level workflow execution (a trace root).""" + run_id: str + trace_id: str + entrypoint: str + session_id: Optional[str] = None + profile_id: Optional[str] = None + platform: Optional[str] = None + start_ns: int = field(default_factory=_now_ns) + end_ns: Optional[int] = None + end_reason: Optional[str] = None + model_call_count: int = 0 + tool_call_count: int = 0 + error_count: int = 0 + estimated_cost_usd: Optional[float] = None + cost_status: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return {"event": "run", **asdict(self)} + + +@dataclass(slots=True) +class ModelCallEvent: + span_id: str + run_id: str + provider: Optional[str] = None # raw provider, e.g. "anthropic" + model: Optional[str] = None # raw model id, e.g. "claude-opus-4" + base_url: Optional[str] = None + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_tokens: int = 0 + cache_write_tokens: int = 0 + reasoning_tokens: int = 0 + latency_ms: Optional[int] = None + ttft_ms: Optional[int] = None + estimated_cost_usd: Optional[float] = None + cost_status: Optional[str] = None + cost_source: Optional[str] = None + end_reason: Optional[str] = None + retry_count: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return {"event": "model_call", **asdict(self)} + + +@dataclass(slots=True) +class ToolCallEvent: + span_id: str + run_id: str + tool_name: Optional[str] = None # raw tool name, e.g. "web_search" + backend: Optional[str] = None + duration_ms: Optional[int] = None + result_class: Optional[str] = None + retry_count: int = 0 + approval: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return {"event": "tool_call", **asdict(self)} + + +@dataclass(slots=True) +class ErrorEvent: + run_id: Optional[str] + error_class: str + subsystem: str + recovery: Optional[str] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "error", **asdict(self)} + + +__all__ = [ + "RunEvent", + "ModelCallEvent", + "ToolCallEvent", + "ErrorEvent", +] diff --git a/agent/telemetry/exporter_bulk.py b/agent/telemetry/exporter_bulk.py new file mode 100644 index 00000000000..e0dd1e26d04 --- /dev/null +++ b/agent/telemetry/exporter_bulk.py @@ -0,0 +1,139 @@ +"""Export telemetry (and optionally session content) to a file or stream. + +Two data domains, both written to an operator-chosen destination: + + * Telemetry: the tel_* rows + events.jsonl (structural observability). + * Content (opt-in via the trajectories plane): sessions + messages, with every + content field (message body, reasoning, raw tool-call args) passed through the + redaction pipeline (secrets always stripped; PII per content_redaction). + +Formats: ndjson (default) and json. OTLP streaming export lives in otlp_exporter.py. + +Content export is gated by ``redaction.content_export_enabled``. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, TextIO + +from . import redaction + +_TEL_TABLES = ( + "tel_runs", "tel_model_calls", "tel_tool_calls", "tel_error_events", +) + + +def _open(db_path: Optional[Path]) -> sqlite3.Connection: + if db_path is None: + from hermes_constants import get_hermes_home + db_path = get_hermes_home() / "state.db" + c = sqlite3.connect(str(db_path), timeout=5.0) + c.row_factory = sqlite3.Row + return c + + +def _iter_telemetry(conn: sqlite3.Connection, since_ns: Optional[int]) -> Iterator[Dict[str, Any]]: + for table in _TEL_TABLES: + # only tel_runs has start_ns; window the rest by run join when needed. + if table == "tel_runs" and since_ns: + rows = conn.execute( + f"SELECT * FROM {table} WHERE start_ns >= ?", (int(since_ns),) + ).fetchall() + else: + rows = conn.execute(f"SELECT * FROM {table}").fetchall() + for r in rows: + d = dict(r) + d["_kind"] = table + yield d + + +def _iter_content( + db_path: Optional[Path], + *, + config: Optional[Dict[str, Any]], + include_content: bool, +) -> Iterator[Dict[str, Any]]: + """Yield session records. Message bodies included only when trajectories on.""" + from hermes_state import SessionDB + + content_mode = redaction.content_mode_for(config) + db = SessionDB(db_path=db_path) if db_path else SessionDB() + try: + for session in db.export_all(): + msgs = session.get("messages", []) or [] + red_msgs = [ + redaction.redact_message( + m, content_mode=content_mode, include_content=include_content + ) + for m in msgs + ] + # Session-level metadata is structural; keep ids/model/counts, drop + # any free-text title only when content is excluded. + out = { + "_kind": "session", + "id": session.get("id"), + "source": session.get("source"), + "model": session.get("model"), + "started_at": session.get("started_at"), + "ended_at": session.get("ended_at"), + "message_count": session.get("message_count"), + "tool_call_count": session.get("tool_call_count"), + "messages": red_msgs, + } + if include_content and session.get("title"): + out["title"] = redaction.redact_for_export( + session["title"], content_mode=content_mode + ) + yield out + finally: + db.close() + + +def export( + out: TextIO, + *, + fmt: str = "ndjson", + since_ns: Optional[int] = None, + include_content: bool = False, + config: Optional[Dict[str, Any]] = None, + db_path: Optional[Path] = None, +) -> Dict[str, int]: + """Write telemetry (+ optional content) to ``out``. Returns counts. + + ``include_content`` is honored only when the trajectories plane is enabled in + ``config``; otherwise content is forced off and only structural data is written. + """ + # Trajectories gate: a flag cannot override the consent plane. + content_allowed = include_content and redaction.content_export_enabled(config) + counts = {"telemetry": 0, "sessions": 0, "content_included": int(content_allowed)} + + conn = _open(db_path) + records: List[Dict[str, Any]] = [] + try: + for rec in _iter_telemetry(conn, since_ns): + counts["telemetry"] += 1 + if fmt == "ndjson": + out.write(json.dumps(rec, ensure_ascii=False) + "\n") + else: + records.append(rec) + finally: + conn.close() + + # Content/session domain (separate connection via SessionDB). + for rec in _iter_content(db_path, config=config, include_content=content_allowed): + counts["sessions"] += 1 + if fmt == "ndjson": + out.write(json.dumps(rec, ensure_ascii=False) + "\n") + else: + records.append(rec) + + if fmt != "ndjson": + json.dump({"records": records}, out, ensure_ascii=False, indent=2) + + return counts + + +__all__ = ["export"] diff --git a/agent/telemetry/metrics.py b/agent/telemetry/metrics.py new file mode 100644 index 00000000000..389a9ad5e79 --- /dev/null +++ b/agent/telemetry/metrics.py @@ -0,0 +1,219 @@ +"""Derive metric rollups from the local telemetry tables. + +Reads the ``tel_*`` tables in state.db and returns aggregates for /usage, /insights, +and local dashboards. Metrics are computed by querying the event log rather than being +emitted on the hot path. + +Each function accepts either an open caller-owned ``conn`` (reused, not closed) or a +``db_path`` (opened and closed internally). InsightsEngine passes its existing +connection; a standalone dashboard passes a path. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional + + +@contextmanager +def _cursor( + conn: Optional[sqlite3.Connection], db_path: Optional[Path] +) -> Iterator[sqlite3.Connection]: + """Yield a Row-factory connection. Closes it only if we opened it.""" + if conn is not None: + prev_factory = conn.row_factory + conn.row_factory = sqlite3.Row + try: + yield conn + finally: + conn.row_factory = prev_factory + return + if db_path is None: + from hermes_constants import get_hermes_home + db_path = get_hermes_home() / "state.db" + c = sqlite3.connect(str(db_path), timeout=5.0) + c.row_factory = sqlite3.Row + try: + yield c + finally: + c.close() + + +def _since_clause(since_ns: Optional[int], col: str = "start_ns") -> str: + return f" WHERE {col} >= {int(since_ns)}" if since_ns else "" + + +def workflow_summary( + db_path: Optional[Path] = None, + since_ns: Optional[int] = None, + *, + conn: Optional[sqlite3.Connection] = None, +) -> Dict[str, Any]: + """Run-level counters + duration percentiles (local plane, exact).""" + with _cursor(conn, db_path) as c: + where = _since_clause(since_ns) + total = c.execute(f"SELECT COUNT(*) n FROM tel_runs{where}").fetchone()["n"] + by_reason = { + r["end_reason"] or "unknown": r["n"] + for r in c.execute( + f"SELECT end_reason, COUNT(*) n FROM tel_runs{where} GROUP BY end_reason" + ).fetchall() + } + by_entry = { + r["entrypoint"] or "unknown": r["n"] + for r in c.execute( + f"SELECT entrypoint, COUNT(*) n FROM tel_runs{where} GROUP BY entrypoint" + ).fetchall() + } + dur_where = (where + " AND end_ns IS NOT NULL") if where else " WHERE end_ns IS NOT NULL" + durations = [ + (r["end_ns"] - r["start_ns"]) / 1e6 + for r in c.execute( + f"SELECT start_ns, end_ns FROM tel_runs{dur_where}" + ).fetchall() + ] + return { + "total_runs": total, + "by_end_reason": by_reason, + "by_entrypoint": by_entry, + "duration_ms_p50": _pct(durations, 50), + "duration_ms_p95": _pct(durations, 95), + "success_rate": round(by_reason.get("completed", 0) / total, 4) if total else 0.0, + } + + +def model_call_summary( + db_path: Optional[Path] = None, + since_ns: Optional[int] = None, + *, + conn: Optional[sqlite3.Connection] = None, +) -> Dict[str, Any]: + with _cursor(conn, db_path) as c: + rows = c.execute( + "SELECT provider, model, COUNT(*) n, " + "SUM(input_tokens) inp, SUM(output_tokens) outp, " + "SUM(cache_read_tokens) cache, AVG(latency_ms) avg_latency " + "FROM tel_model_calls GROUP BY provider, model" + ).fetchall() + by_provider: Dict[str, int] = {} + by_model: Dict[str, int] = {} + tokens = {"input": 0, "output": 0, "cache_read": 0} + breakdown: List[Dict[str, Any]] = [] + for r in rows: + prov = r["provider"] or "unknown" + mdl = r["model"] or "unknown" + by_provider[prov] = by_provider.get(prov, 0) + r["n"] + by_model[mdl] = by_model.get(mdl, 0) + r["n"] + tokens["input"] += r["inp"] or 0 + tokens["output"] += r["outp"] or 0 + tokens["cache_read"] += r["cache"] or 0 + breakdown.append({ + "provider": r["provider"], + "model": r["model"], + "calls": r["n"], + "avg_latency_ms": round(r["avg_latency"] or 0, 1), + }) + cache_total = tokens["cache_read"] + tokens["input"] + return { + "by_provider": by_provider, + "by_model": by_model, + "tokens": tokens, + "cache_hit_rate": round(tokens["cache_read"] / cache_total, 4) if cache_total else 0.0, + "breakdown": breakdown, + } + + +def tool_call_summary( + db_path: Optional[Path] = None, + *, + conn: Optional[sqlite3.Connection] = None, +) -> Dict[str, Any]: + with _cursor(conn, db_path) as c: + by_tool = { + r["tool_name"] or "unknown": r["n"] + for r in c.execute( + "SELECT tool_name, COUNT(*) n FROM tel_tool_calls GROUP BY tool_name" + ).fetchall() + } + fails = { + r["tool_name"] or "unknown": r["n"] + for r in c.execute( + "SELECT tool_name, COUNT(*) n FROM tel_tool_calls " + "WHERE result_class IN ('error','timeout','blocked') GROUP BY tool_name" + ).fetchall() + } + total = sum(by_tool.values()) + total_fail = sum(fails.values()) + return { + "by_tool": by_tool, + "failures_by_tool": fails, + "total": total, + "failure_rate": round(total_fail / total, 4) if total else 0.0, + } + + +def error_summary( + db_path: Optional[Path] = None, + *, + conn: Optional[sqlite3.Connection] = None, +) -> Dict[str, Any]: + with _cursor(conn, db_path) as c: + return { + "by_class": { + r["error_class"] or "unknown": r["n"] + for r in c.execute( + "SELECT error_class, COUNT(*) n FROM tel_error_events GROUP BY error_class" + ).fetchall() + }, + } + + +def _pct(values: List[float], p: int) -> float: + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * (p / 100) + lo = int(k) + hi = min(lo + 1, len(s) - 1) + frac = k - lo + return round(s[lo] + (s[hi] - s[lo]) * frac, 2) + + +def overview( + db_path: Optional[Path] = None, + since_ns: Optional[int] = None, + *, + conn: Optional[sqlite3.Connection] = None, +) -> Dict[str, Any]: + """One call for a dashboard: all the rollups.""" + return { + "workflows": workflow_summary(db_path, since_ns, conn=conn), + "model_calls": model_call_summary(db_path, since_ns, conn=conn), + "tool_calls": tool_call_summary(db_path, conn=conn), + "errors": error_summary(db_path, conn=conn), + } + + +def has_data( + db_path: Optional[Path] = None, + *, + conn: Optional[sqlite3.Connection] = None, +) -> bool: + """True when any telemetry runs exist (cheap guard for /insights rendering).""" + try: + with _cursor(conn, db_path) as c: + return c.execute("SELECT 1 FROM tel_runs LIMIT 1").fetchone() is not None + except Exception: + return False + + +__all__ = [ + "workflow_summary", + "model_call_summary", + "tool_call_summary", + "error_summary", + "overview", + "has_data", +] diff --git a/agent/telemetry/otlp_exporter.py b/agent/telemetry/otlp_exporter.py new file mode 100644 index 00000000000..141c10b0164 --- /dev/null +++ b/agent/telemetry/otlp_exporter.py @@ -0,0 +1,282 @@ +"""Export telemetry to an OpenTelemetry Collector over OTLP/HTTP. + +Maps telemetry events (which carry trace_id/run_id/span_id/parent_span_id) to OTel +spans and sends them to the endpoint configured under ``telemetry.export.otlp``. Lets +an operator stream Hermes telemetry into their own observability stack. + +Notes: + * The destination is operator-configured; this module only sends to that endpoint. + It does not import or interact with any aggregate-metrics path. + * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an optional + extra (``pip install hermes-agent[otlp]``), imported lazily so the dependency is + only required when OTLP export is actually used. + * ``headers_env`` maps a header name to an environment variable name; values are read + from the environment at export time and never logged or stored. + * The continuous subscriber runs in the emitter's writer thread after durable writes + and is fail-isolated, so an export error cannot affect a run. + +Spans carry structural telemetry by default. Message content is included only when the +trajectories plane is enabled, and always passes through the export redaction pipeline. +""" + +from __future__ import annotations + +import logging +import os +import sqlite3 +from pathlib import Path +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class OTLPUnavailable(RuntimeError): + """Raised when the optional OpenTelemetry SDK isn't installed.""" + + +def _require_sdk(*, auto_install: bool = True, prompt: bool = True): + """Import the OTel SDK, lazily installing it on first use if needed. + + Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK + triggers the standard venv install flow — same as every other optional + backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back + to OTLPUnavailable (with a manual install hint) when the SDK can't be made + importable (lazy installs disabled, install failed, or auto_install=False). + + ``auto_install``: attempt the lazy install when missing (default True). + ``prompt``: ask before installing when interactive (default True); pass + False from non-interactive contexts like the continuous streamer. + """ + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except ImportError: + pass # lazy_deps unavailable — fall through to the import attempt + except Exception: + # FeatureUnavailable (lazy installs disabled / declined / failed) — + # fall through; the import below raises OTLPUnavailable with the hint. + pass + try: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.resources import Resource + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.trace import SpanKind + return { + "TracerProvider": TracerProvider, + "BatchSpanProcessor": BatchSpanProcessor, + "Resource": Resource, + "OTLPSpanExporter": OTLPSpanExporter, + "SpanKind": SpanKind, + } + except Exception as e: # ImportError or partial install + raise OTLPUnavailable( + "OTLP export requires the optional dependency. Install with:\n" + " pip install 'hermes-agent[otlp]'\n" + f"(import error: {e})" + ) + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env. + + The config stores environment variable names, not secret values; values are read + from the environment here. Missing variables are skipped (and noted at debug level + without the value). + """ + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + else: + logger.debug("OTLP header %s: env var %s not set; skipping", + header_name, env_name) + return resolved + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + tel = (config or {}).get("telemetry") or {} + export = tel.get("export") or {} + return export.get("otlp") or {} + + +def build_exporter(config: Dict[str, Any]): + """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK.""" + sdk = _require_sdk() + otlp = _otlp_config(config) + endpoint = otlp.get("endpoint") + if not endpoint: + raise ValueError("telemetry.export.otlp.endpoint is not set") + headers = _resolve_headers(otlp.get("headers_env")) + return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) + + +def _make_provider(config: Dict[str, Any]): + sdk = _require_sdk() + resource = sdk["Resource"].create({ + "service.name": "hermes-agent", + "telemetry.plane": "local", # never aggregate + }) + provider = sdk["TracerProvider"](resource=resource) + processor = sdk["BatchSpanProcessor"](build_exporter(config)) + provider.add_span_processor(processor) + return provider, processor + + +# ── event -> span attribute mapping (real values) ─────────────────────────── +def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: + """Span attributes for an event — the real recorded values (local plane).""" + kind = ev.get("event") + attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} + keep_by_kind = { + "run": ("entrypoint", "platform", "end_reason", + "model_call_count", "tool_call_count", "error_count", + "estimated_cost_usd", "cost_status"), + "model_call": ("provider", "model", "base_url", + "input_tokens", "output_tokens", "cache_read_tokens", + "cache_write_tokens", "reasoning_tokens", "latency_ms", + "ttft_ms", "end_reason"), + "tool_call": ("tool_name", "backend", "duration_ms", "result_class"), + "error": ("error_class", "subsystem", "recovery"), + } + for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] + v = ev.get(col) + if v is not None: + attrs[f"hermes.{col}"] = v + return attrs + + +def export_batch(provider, batch: List[Dict[str, Any]]) -> int: + """Map a batch of events to OTel spans. Returns spans created.""" + tracer = provider.get_tracer("hermes.telemetry") + n = 0 + for ev in batch: + try: + name = f"hermes.{ev.get('event', 'event')}" + span = tracer.start_span(name, attributes=_span_attrs(ev)) + span.end() + n += 1 + except Exception: + logger.debug("OTLP span map failed", exc_info=True) + return n + + +# ── one-shot drain (export current local rows) ────────────────────────────── +def export_once( + config: Dict[str, Any], + *, + db_path: Optional[Path] = None, + since_ns: Optional[int] = None, +) -> int: + """Drain the local tel_* tables to the configured OTLP endpoint once.""" + provider, processor = _make_provider(config) + try: + rows = _read_events(db_path, since_ns) + total = export_batch(provider, rows) + processor.force_flush() + return total + finally: + try: + provider.shutdown() + except Exception: + pass + + +def _read_events(db_path: Optional[Path], since_ns: Optional[int]) -> List[Dict[str, Any]]: + if db_path is None: + from hermes_constants import get_hermes_home + db_path = get_hermes_home() / "state.db" + c = sqlite3.connect(str(db_path), timeout=5.0) + c.row_factory = sqlite3.Row + out: List[Dict[str, Any]] = [] + try: + table_event = { + "tel_runs": "run", "tel_model_calls": "model_call", + "tel_tool_calls": "tool_call", "tel_error_events": "error", + } + for table, evkind in table_event.items(): + where = "" + if table == "tel_runs" and since_ns: + where = f" WHERE start_ns >= {int(since_ns)}" + for r in c.execute(f"SELECT * FROM {table}{where}").fetchall(): + d = dict(r) + d["event"] = evkind + out.append(d) + finally: + c.close() + return out + + +# ── continuous streaming subscriber ───────────────────────────────────────── +class OTLPStreamer: + """A live subscriber that pushes each emitter batch to OTLP as it lands. + + Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter. + """ + + def __init__(self, config: Dict[str, Any]): + self._provider, self._processor = _make_provider(config) + self.exported = 0 + + def __call__(self, batch: List[Dict[str, Any]]) -> None: + self.exported += export_batch(self._provider, batch) + + def shutdown(self) -> None: + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def is_available() -> bool: + """True when the OTel SDK is already importable. Does NOT auto-install — + this is a pure check (e.g. for status display).""" + try: + _require_sdk(auto_install=False) + return True + except OTLPUnavailable: + return False + + +def is_enabled(config: Dict[str, Any]) -> bool: + otlp = _otlp_config(config) + return bool(otlp.get("enabled") and otlp.get("endpoint")) + + +def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]: + """If OTLP is enabled, attach a streamer to the singleton emitter. + + Non-interactive context (startup): attempts a lazy install with prompt=False + so a configured-but-missing SDK is installed once (gated by + security.allow_lazy_installs), then streams. If it still can't load, logs and + no-ops — never blocks or raises into startup. + """ + if not is_enabled(config): + return None + try: + _require_sdk(prompt=False) + except OTLPUnavailable: + logger.warning("telemetry.export.otlp.enabled but the OTel SDK could not " + "be installed/imported; install 'hermes-agent[otlp]'") + return None + from agent.telemetry.emitter import get_emitter + streamer = OTLPStreamer(config) + get_emitter().subscribe(streamer) + return streamer + + +__all__ = [ + "OTLPUnavailable", + "OTLPStreamer", + "build_exporter", + "export_once", + "export_batch", + "is_available", + "is_enabled", + "start_streaming", +] diff --git a/agent/telemetry/policy.py b/agent/telemetry/policy.py new file mode 100644 index 00000000000..f1c001b4c9a --- /dev/null +++ b/agent/telemetry/policy.py @@ -0,0 +1,107 @@ +"""Telemetry consent posture and the aggregate-plane gate. + +Consent is a single field, ``telemetry.consent_state``: + + * "unknown" — no choice recorded; never uploads (the default). + * "local" — declined the aggregate plane; local plane only. + * "aggregate" — opted in to the aggregate plane. + +The config file is the source of truth: set ``telemetry.consent_state`` with +``hermes config set`` (or a managed-scope pin). There is no separate boolean mirror — +a single field cannot drift out of sync with itself, so a stray value can't +accidentally imply consent. + +``allow_aggregate`` is the hard gate. An administrator pins +``telemetry.allow_aggregate: false`` through the managed-scope layer +(``/etc/hermes/config.yaml``), which takes precedence over the user's config; when it +is false, the aggregate plane is off regardless of ``consent_state``. + +This module makes the decisions; it performs no I/O and contains no uploader. A future +uploader must call :func:`may_upload_aggregate` at its boundary. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import Any, Dict + +CONSENT_UNKNOWN = "unknown" +CONSENT_LOCAL = "local" +CONSENT_AGGREGATE = "aggregate" +_VALID_STATES = {CONSENT_UNKNOWN, CONSENT_LOCAL, CONSENT_AGGREGATE} + + +@dataclass(slots=True) +class TelemetryDecision: + """The resolved telemetry posture for the current process.""" + local_enabled: bool + aggregate_enabled: bool + consent_state: str + install_id: str + allow_aggregate: bool + + def may_upload_aggregate(self) -> bool: + """The single gate the uploader must consult before any network send.""" + return self.allow_aggregate and self.consent_state == CONSENT_AGGREGATE + + +def _telemetry_cfg(config: Dict[str, Any]) -> Dict[str, Any]: + cfg = config.get("telemetry") if isinstance(config, dict) else None + return cfg if isinstance(cfg, dict) else {} + + +def ensure_install_id(config: Dict[str, Any]) -> str: + """Return a stable install id, minting one if the config slot is empty. + + Does not persist — the caller writes the returned value back to config.yaml. A + fresh uuid4 is used; clearing ``telemetry.install_id`` (e.g. with + ``hermes config set telemetry.install_id ""``) causes the next call to mint anew. + """ + tel = _telemetry_cfg(config) + existing = tel.get("install_id") + if isinstance(existing, str) and existing.strip(): + return existing + return str(uuid.uuid4()) + + +def resolve(config: Dict[str, Any]) -> TelemetryDecision: + """Resolve the effective telemetry posture from config. + + ``consent_state`` is the single source of truth for the aggregate opt-in. + ``allow_aggregate`` (admin-pinnable via managed scope) hard-disables the aggregate + plane regardless of consent. + """ + tel = _telemetry_cfg(config) + + local_enabled = bool(tel.get("local", True)) + allow_aggregate = bool(tel.get("allow_aggregate", True)) + state = tel.get("consent_state", CONSENT_UNKNOWN) + if state not in _VALID_STATES: + state = CONSENT_UNKNOWN + + aggregate_enabled = allow_aggregate and state == CONSENT_AGGREGATE + + return TelemetryDecision( + local_enabled=local_enabled, + aggregate_enabled=aggregate_enabled, + consent_state=state, + install_id=ensure_install_id(config), + allow_aggregate=allow_aggregate, + ) + + +def may_upload_aggregate(config: Dict[str, Any]) -> bool: + """Convenience gate for the uploader boundary.""" + return resolve(config).may_upload_aggregate() + + +__all__ = [ + "CONSENT_UNKNOWN", + "CONSENT_LOCAL", + "CONSENT_AGGREGATE", + "TelemetryDecision", + "resolve", + "may_upload_aggregate", + "ensure_install_id", +] diff --git a/agent/telemetry/redaction.py b/agent/telemetry/redaction.py new file mode 100644 index 00000000000..a65ea695172 --- /dev/null +++ b/agent/telemetry/redaction.py @@ -0,0 +1,187 @@ +"""Redaction applied to telemetry data on export. + +Two independent controls: + + * Secrets are always redacted, on every export and in every mode; no setting + disables this. Wraps ``agent/redact.py::redact_sensitive_text(force=True)``. + + * Whether message bodies, reasoning, and raw tool arguments are exportable at all is + governed by the trajectories plane (``telemetry.trajectories.enabled``, default + off, admin-pinnable), not by a redaction mode. With trajectories off, content is + dropped. With it on, content is exportable and ``content_redaction`` (none|pii) + controls how much is scrubbed; secrets are still always stripped. + +This applies to the local and trajectory export paths. It is unrelated to any +aggregate-metrics path. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +# Content-redaction strengths for any content that IS exported. +CONTENT_NONE = "none" # drop content entirely (structural telemetry only) +CONTENT_PII = "pii" # codec-aware PII redaction on exported content +CONTENT_MODES = {CONTENT_NONE, CONTENT_PII} + +# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ── +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +# E.164-ish and common separators; conservative to avoid nuking code/IDs. +_PHONE_RE = re.compile( + r"(? Optional[str]: + """Always-on secret redaction. force=True so user config can't disable it.""" + if text is None: + return None + try: + from agent.redact import redact_sensitive_text + return redact_sensitive_text(str(text), force=True) + except Exception: + # Fail CLOSED: if the redactor can't run, do not emit the raw string. + return "[redaction-unavailable]" + + +def _pii_redact(text: str) -> str: + text = _EMAIL_RE.sub("[email]", text) + text = _UUID_RE.sub("[id]", text) + text = _PHONE_RE.sub("[phone]", text) + return text + + +def redact_for_export( + text: Optional[str], + *, + content_mode: str = CONTENT_NONE, +) -> Optional[str]: + """Redact a single content string for export. + + Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. + Callers gate *whether content is exported at all* via the trajectories plane + (see ``content_export_enabled``); this function only scrubs content that the + caller has already decided to export. + """ + redacted = _secret_redact(text) + if redacted is None: + return None + if content_mode == CONTENT_PII: + redacted = _pii_redact(redacted) + return redacted + + +def content_export_enabled(config: Optional[Dict[str, Any]]) -> bool: + """True only when the trajectories plane is explicitly enabled. + + This is the consent gate for exporting message bodies / reasoning / raw tool + args. Default off. Admin-pinnable via managed scope (telemetry.trajectories.enabled). + """ + try: + tel = (config or {}).get("telemetry") or {} + traj = tel.get("trajectories") or {} + return bool(traj.get("enabled", False)) + except Exception: + return False + + +def content_mode_for(config: Optional[Dict[str, Any]]) -> str: + try: + tel = (config or {}).get("telemetry") or {} + mode = tel.get("content_redaction", CONTENT_NONE) + return mode if mode in CONTENT_MODES else CONTENT_NONE + except Exception: + return CONTENT_NONE + + +# ── Codec-aware message redaction (NeMo pattern) ───────────────────────────── +# Redact the right fields of a provider message shape rather than regex-blasting +# the whole blob. Structure (roles, names, counts) is preserved; only the +# free-text content fields are scrubbed. + +def redact_message( + msg: Dict[str, Any], + *, + content_mode: str = CONTENT_NONE, + include_content: bool = False, +) -> Dict[str, Any]: + """Redact one chat message dict for export. + + When include_content is False (trajectories off), content/reasoning/tool-arg + fields are dropped — only structural fields (role, tool name, counts) remain. + When True, those fields are kept but passed through redact_for_export. + """ + role = msg.get("role") + out: Dict[str, Any] = {"role": role} + + # Always-structural fields. + if msg.get("tool_name") is not None: + out["tool_name"] = msg.get("tool_name") + if msg.get("name") is not None: + out["name"] = msg.get("name") + + if not include_content: + # Structural only: record presence/size, not bytes. + c = msg.get("content") + if c is not None: + out["content_chars"] = len(str(c)) + if msg.get("reasoning_content"): + out["reasoning_chars"] = len(str(msg["reasoning_content"])) + if msg.get("tool_calls"): + out["tool_call_count"] = _count_tool_calls(msg["tool_calls"]) + return out + + # Content included (trajectories enabled): scrub then keep. + if msg.get("content") is not None: + out["content"] = redact_for_export(msg["content"], content_mode=content_mode) + if msg.get("reasoning_content"): + out["reasoning_content"] = redact_for_export( + msg["reasoning_content"], content_mode=content_mode + ) + if msg.get("tool_calls"): + out["tool_calls"] = _redact_tool_calls(msg["tool_calls"], content_mode=content_mode) + return out + + +def _count_tool_calls(tool_calls: Any) -> int: + try: + import json + tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls + return len(tc) if isinstance(tc, list) else (1 if tc else 0) + except Exception: + return 0 + + +def _redact_tool_calls(tool_calls: Any, *, content_mode: str) -> Any: + """Redact raw tool-call arguments (free text) while keeping function names.""" + import json + try: + tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls + except Exception: + return "[unparseable-tool-calls]" + if not isinstance(tc, list): + return [] + out: List[Dict[str, Any]] = [] + for call in tc: + if not isinstance(call, dict): + continue + fn = (call.get("function") or {}) if isinstance(call.get("function"), dict) else {} + name = fn.get("name") or call.get("name") + args = fn.get("arguments") + red_args = redact_for_export(args, content_mode=content_mode) if args is not None else None + out.append({"name": name, "arguments": red_args}) + return out + + +__all__ = [ + "CONTENT_NONE", + "CONTENT_PII", + "CONTENT_MODES", + "redact_for_export", + "content_export_enabled", + "content_mode_for", + "redact_message", +] diff --git a/agent/telemetry/rollup.py b/agent/telemetry/rollup.py new file mode 100644 index 00000000000..8f30304f92b --- /dev/null +++ b/agent/telemetry/rollup.py @@ -0,0 +1,145 @@ +"""Build per-run summary events from the local telemetry tables. + +Reads the ``tel_*`` tables and projects each completed run into a summary dict holding +the recorded values: provider, models used, tool names, token totals, duration, and +cost. Powers ``hermes telemetry preview``. No aggregation or bucketing is applied here. +""" + +from __future__ import annotations + +import platform +import sqlite3 +from pathlib import Path +from typing import Any, Dict, List, Optional + + +def _os_family() -> str: + s = platform.system().lower() + if s.startswith("lin"): + return "linux" + if s == "darwin": + return "macos" + if s.startswith("win"): + return "windows" + return "other" + + +def _hermes_version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "0.0.0" + + +def _open(db_path: Optional[Path], conn: Optional[sqlite3.Connection]): + if conn is not None: + prev = conn.row_factory + conn.row_factory = sqlite3.Row + return conn, prev, False + if db_path is None: + from hermes_constants import get_hermes_home + db_path = get_hermes_home() / "state.db" + c = sqlite3.connect(str(db_path), timeout=5.0) + c.row_factory = sqlite3.Row + return c, None, True + + +def _run_events(c: sqlite3.Connection, since_ns: Optional[int]) -> List[Dict[str, Any]]: + """Project completed runs into per-run summary dicts.""" + where = " WHERE end_ns IS NOT NULL" + if since_ns: + where += f" AND start_ns >= {int(since_ns)}" + rows = c.execute( + "SELECT run_id, entrypoint, platform, end_reason, start_ns, end_ns, " + "model_call_count, tool_call_count, error_count, estimated_cost_usd " + "FROM tel_runs" + where + ).fetchall() + + events: List[Dict[str, Any]] = [] + for r in rows: + # Models actually used in this run (real ids), with token totals. + models = [ + {"provider": m["provider"], "model": m["model"], + "calls": m["n"], "input_tokens": int(m["inp"] or 0), + "output_tokens": int(m["outp"] or 0)} + for m in c.execute( + "SELECT provider, model, COUNT(*) n, SUM(input_tokens) inp, " + "SUM(output_tokens) outp FROM tel_model_calls WHERE run_id = ? " + "GROUP BY provider, model ORDER BY n DESC", + (r["run_id"],), + ).fetchall() + ] + tools = [ + row["tool_name"] + for row in c.execute( + "SELECT DISTINCT tool_name FROM tel_tool_calls WHERE run_id = ?", + (r["run_id"],), + ).fetchall() + if row["tool_name"] + ] + trow = c.execute( + "SELECT SUM(input_tokens) inp, SUM(output_tokens) outp " + "FROM tel_model_calls WHERE run_id = ?", + (r["run_id"],), + ).fetchone() + duration_ms = (r["end_ns"] - r["start_ns"]) / 1e6 if r["end_ns"] else None + events.append({ + "event_name": "workflow_completed", + "run_id": r["run_id"], + "entrypoint": r["entrypoint"] or "cli", + "platform": r["platform"], + "end_reason": r["end_reason"] or "completed", + "models_used": models, + "tools_used": tools, + "model_call_count": r["model_call_count"] or 0, + "tool_call_count": r["tool_call_count"] or 0, + "error_count": r["error_count"] or 0, + "duration_ms": round(duration_ms, 1) if duration_ms is not None else None, + "input_tokens": int((trow["inp"] if trow else 0) or 0), + "output_tokens": int((trow["outp"] if trow else 0) or 0), + "estimated_cost_usd": r["estimated_cost_usd"], + }) + return events + + +def build_aggregate_events( + *, + install_id: str, + db_path: Optional[Path] = None, + since_ns: Optional[int] = None, + conn: Optional[sqlite3.Connection] = None, + include_heartbeat: bool = True, +) -> List[Dict[str, Any]]: + """Return per-run summary events plus an optional heartbeat.""" + c, prev_factory, owned = _open(db_path, conn) + try: + events = _run_events(c, since_ns) + if include_heartbeat: + events.append({ + "event_name": "heartbeat", + "install_id": install_id, + "hermes_version": _hermes_version(), + "os_family": _os_family(), + "entrypoint": "cli", + }) + return events + finally: + if owned: + c.close() + elif prev_factory is not None: + c.row_factory = prev_factory + + +def summarize(events: List[Dict[str, Any]]) -> Dict[str, Any]: + """Counts by event_name + field coverage, for status/preview output.""" + by_name: Dict[str, int] = {} + fields = set() + for e in events: + name = e.get("event_name", "?") + by_name[name] = by_name.get(name, 0) + 1 + fields.update(e.keys()) + return {"total": len(events), "by_event_name": by_name, "fields_present": sorted(fields)} + + +__all__ = ["build_aggregate_events", "summarize"] diff --git a/agent/telemetry/spans.py b/agent/telemetry/spans.py new file mode 100644 index 00000000000..bfd876f8a11 --- /dev/null +++ b/agent/telemetry/spans.py @@ -0,0 +1,83 @@ +"""Trace / run / span id propagation via contextvars. + +Telemetry events share IDs so a workflow can be reconstructed: one ``trace_id`` per +workflow, one ``run_id`` per top-level execution, ``span_id`` per timed operation, and +``parent_span_id`` for nesting. These live in contextvars so async tool calls and +spawned subagents inherit the lineage automatically. + +Provides helpers to start/clear a run context and mint child span ids. The telemetry +plugin sets the run context on session start and reads it in each hook callback. +Nothing here writes to storage — it only carries ids. +""" + +from __future__ import annotations + +import contextvars +import uuid +from dataclasses import dataclass +from typing import Optional + +_trace_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "hermes_tel_trace_id", default=None +) +_run_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "hermes_tel_run_id", default=None +) +_parent_span_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "hermes_tel_parent_span_id", default=None +) + + +def new_id() -> str: + return uuid.uuid4().hex + + +@dataclass(slots=True) +class RunContext: + trace_id: str + run_id: str + + +def start_run(trace_id: Optional[str] = None, run_id: Optional[str] = None) -> RunContext: + """Begin a run context, minting ids when not supplied. Sets contextvars.""" + tid = trace_id or new_id() + rid = run_id or new_id() + _trace_id.set(tid) + _run_id.set(rid) + _parent_span_id.set(None) + return RunContext(trace_id=tid, run_id=rid) + + +def current_trace_id() -> Optional[str]: + return _trace_id.get() + + +def current_run_id() -> Optional[str]: + return _run_id.get() + + +def current_parent_span_id() -> Optional[str]: + return _parent_span_id.get() + + +def new_span_id() -> str: + """Mint a span id (does not alter the parent pointer).""" + return new_id() + + +def clear_run() -> None: + _trace_id.set(None) + _run_id.set(None) + _parent_span_id.set(None) + + +__all__ = [ + "RunContext", + "new_id", + "start_run", + "current_trace_id", + "current_run_id", + "current_parent_span_id", + "new_span_id", + "clear_run", +] diff --git a/cli-config.yaml.example b/cli-config.yaml.example index d52e3ec009c..89ace2dd657 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1414,6 +1414,49 @@ display: # # Routing/delivery still uses the original values internally. # redact_pii: false +# ============================================================================= +# Telemetry & Observability +# ============================================================================= +# Three planes with a hard wall between them: +# local — full-fidelity observability you own. Default ON. +# aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. +# trajectories — content trajectories for training. Separate consent (later). +# +# The local plane records real values (actual models, providers, tool names) — +# your own data, on your machine. The aggregate plane is opt-in and has no +# uploader today; if one ships it would summarize at that egress boundary. +# +# Enterprise locking: any telemetry.* key can be pinned by an administrator via +# the managed-scope layer (/etc/hermes/config.yaml), which wins over the user's +# value. To hard-forbid egress on a locked-down deployment, pin +# `telemetry.allow_aggregate: false` there. +# telemetry: +# # Local plane: event log + SQLite index in state.db. Never leaves your +# # machine unless you export it or opt into the aggregate plane. +# local: true +# # Hard gate for the aggregate plane. When false, the aggregate plane is off +# # regardless of consent_state. Pin false via managed scope to forbid egress. +# allow_aggregate: true +# # Aggregate-plane consent (the opt-in). No uploader ships yet. +# # unknown (no choice — never uploads) | local (declined) | aggregate (opted in) +# consent_state: unknown +# # Stable install id (aggregate plane only). Empty = mint on first use; +# # clear it to rotate. +# install_id: "" +# # Local event-log retention before rotation (days). +# retention_days: 90 +# # Keep secret redaction on even at full local capture. +# redact_secrets: true +# # Content redaction for exports / support bundles: none | pii. +# content_redaction: none +# # Exporters. The OTLP exporter sends spans to a configured Collector endpoint; +# # header values reference environment variable names, not inline secrets. +# export: +# otlp: +# enabled: false +# endpoint: null +# headers_env: {} # e.g. {Authorization: MY_OTLP_TOKEN_ENVVAR} + # ============================================================================= # Shell-script hooks # ============================================================================= diff --git a/docs/observability/README.md b/docs/observability/README.md index 9040929ca48..941236c0d10 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -307,6 +307,13 @@ nested agent work or security lifecycle events. ## Existing Consumers +The bundled **`telemetry`** plugin is the built-in local-first telemetry plane: it +records runs, model calls, and tool calls to a local event log + `state.db`, powers +`hermes insights`, and supports export to your own file or OpenTelemetry Collector. It +is on by default and never sends data to Nous unless you opt in. See +[`telemetry.md`](./telemetry.md) for the full feature, the `hermes telemetry` commands, +and enterprise export. + The bundled Langfuse plugin demonstrates direct hook-based observability for turns, provider requests, and tool calls. diff --git a/docs/observability/telemetry.md b/docs/observability/telemetry.md new file mode 100644 index 00000000000..cdffc43922b --- /dev/null +++ b/docs/observability/telemetry.md @@ -0,0 +1,209 @@ +# Telemetry & Observability + +Hermes ships with a built-in, local-first telemetry system. It records what your +agent does — workflows, model calls, tool calls, errors — to your own machine, powers +`/insights`, and (when *you* enable it) exports everything to your own observability +stack. It is private by default and never sends your data to Nous unless you explicitly +opt in to anonymous aggregate metrics. + +This page explains the whole feature: the three planes, what's captured, the +`hermes telemetry` commands, and how an enterprise streams or exports all of its data +to its own infrastructure. + +> Looking for the **plugin hook contract** (how to write your own observer plugin)? +> That's in [`README.md`](./README.md). This page is about the built-in telemetry +> plane and its CLI. + +## The three planes + +Telemetry is organized into three planes with a hard wall between them: + +| Plane | What it holds | Default | Destination | +| --- | --- | --- | --- | +| **local** | Full-fidelity observability — runs, model/tool calls (real model & provider names), durations, errors | **on** | your machine only | +| **aggregate** | Opt-in metadata to Nous (no uploader ships yet) | **off** (opt-in) | Nous, only if you enable it | +| **trajectories** | Full message content / reasoning / raw tool args | **off** (opt-in) | your own export destinations only | + +The local plane is the one you'll use day to day — it records the real values that +happened (actual model ids, providers, tool names). The aggregate plane is the only +thing that could ever leave for Nous; it is opt-in, default-off, and has no uploader +today. The trajectories plane unlocks full-content export to *your own* destinations — +it is never wired to Nous. + +## Local plane — always-on observability + +The local plane is implemented as a bundled `telemetry` plugin that listens to Hermes +lifecycle hooks (model calls, tool calls, session start/finalize) and writes events to: + +- an append-only JSONL log at `~/.hermes/telemetry/events.jsonl` (the source of truth) +- indexed `tel_*` tables in `state.db` (for fast queries and rollups) + +Writes are fire-and-forget on a background thread: telemetry can never block, slow, or +fail a model call or tool call. If the local plane is disabled (`telemetry.local: false`) +the plugin does not load at all. + +### Seeing your local data + +```bash +hermes insights # usage report — now includes an "Observability" section +hermes telemetry status # planes, consent, export posture, local data volume +``` + +The `insights` Observability section shows workflow counts and success rate, duration +p50/p95, tool failure rates by category, provider/model-class mix, and cache hit rate — +all computed locally with exact values. + +## `hermes telemetry` commands + +```text +hermes telemetry status Show planes, consent state, export posture, local volume +hermes telemetry preview Show the aggregate events that would be produced (local) +hermes telemetry export Export local telemetry to a file/stream or OTLP endpoint +``` + +Consent and the install id are plain config, not separate verbs — set them with +`hermes config set` (or a managed-scope pin): + +```bash +hermes config set telemetry.consent_state aggregate # opt in to the aggregate plane +hermes config set telemetry.consent_state local # opt out (local plane stays on) +hermes config set telemetry.install_id "" # reset the install id (mints a new one) +``` + +### Aggregate plane (opt-in) + +The aggregate plane is **off by default** and has **no uploader today** — nothing is +sent to Nous. Consent lives in `telemetry.consent_state` (`unknown` / `local` / +`aggregate`); setting it to `aggregate` records the opt-in for if/when an uploader ships. +If one is built, it would summarize at that egress boundary. + +`hermes telemetry preview` shows your recent runs as they'd be summarized — computed and +shown **locally only**, with the real model and tool names from your own telemetry. It's +a local inspection surface, not an upload. + +## Enterprise: getting all of your data + +Everything below sends data to **your own** destination — a file, your SIEM, or your own +OpenTelemetry Collector. None of it goes to Nous. + +### Bulk export to a file + +```bash +# Structural telemetry only (default — no message content) +hermes telemetry export --out telemetry.ndjson + +# JSON instead of NDJSON, last 7 days only +hermes telemetry export --out dump.json --format json --since 7 +``` + +By default the export is **structural** — runs, model/tool-call metadata, session shells +with message *counts* but no message bodies. + +### Including content (trajectories plane) + +To export full message content, enable the trajectories plane. This is a deliberate, +separate consent — it's how an enterprise opts into exporting work-product content to its +own store: + +```yaml +# config.yaml +telemetry: + trajectories: + enabled: true # unlocks content export to YOUR destination + content_redaction: pii # "none" | "pii" +``` + +```bash +hermes telemetry export --out full.ndjson --include-content +``` + +`--include-content` is a no-op unless the trajectories plane is enabled — the consent +plane governs, not the flag. + +### Live streaming to your OpenTelemetry Collector / SIEM (OTLP) + +Hermes can stream telemetry to your own OTLP endpoint. This requires the optional `otlp` +extra: + +```bash +pip install 'hermes-agent[otlp]' +``` + +```yaml +# config.yaml +telemetry: + export: + otlp: + enabled: true + endpoint: "https://collector.your-corp.internal:4318/v1/traces" + headers_env: # secrets by reference — env var NAMES, not values + Authorization: CORP_OTLP_TOKEN +``` + +```bash +export CORP_OTLP_TOKEN="..." # the actual token lives in the environment +hermes telemetry export --otlp # drain current telemetry to your collector +``` + +Span attributes are structural by default. For authentication, the config holds the +*name* of an environment variable rather than the secret itself; the value is read at +export time and is never written to config or logged. + +## Redaction + +Two independent controls govern what content looks like on export: + +| Control | Values | Effect | +| --- | --- | --- | +| Secret redaction | always on | API keys, tokens, auth headers, connection strings are **always** stripped on every export path. Cannot be disabled. | +| `content_redaction` | `none` \| `pii` | When content is exported, `pii` additionally redacts emails, phone numbers, and id-shaped strings. | + +Secret redaction is always on — even at full content fidelity — because a SIEM or +warehouse full of live credentials is a bigger attack target than the data it holds. It +fails closed: if the redactor can't run, the raw string is not emitted. + +## Configuration reference + +```yaml +telemetry: + local: true # local plane (default on) + allow_aggregate: true # hard gate; pin false to forbid the aggregate plane entirely + consent_state: unknown # aggregate opt-in: unknown | local | aggregate + install_id: "" # stable anon id; "" mints one; clear to rotate + retention_days: 90 # local event-log retention + redact_secrets: true # always-on secret redaction (kept on by design) + content_redaction: none # none | pii + trajectories: + enabled: false # unlocks full-content export to your destination + export: + otlp: + enabled: false + endpoint: null + headers_env: {} # {HeaderName: ENV_VAR_NAME} +``` + +### Enterprise policy via managed scope + +Any `telemetry.*` key can be pinned by an administrator through Hermes' managed-scope +layer (`/etc/hermes/config.yaml`), which wins over the user's value on a per-key basis. +There is no telemetry-specific policy block — to lock down a fleet, pin the keys you care +about. Common examples: + +- `telemetry.allow_aggregate: false` — the aggregate plane stays off even if + `consent_state` is set to `aggregate`. +- `telemetry.export.otlp.endpoint` — point every install at the corporate collector. +- `telemetry.trajectories.enabled` — centrally decide whether content export is allowed. + +When a key is managed, attempts to change it are rejected by managed scope with a message +naming the source. `hermes telemetry status` shows the current export posture (endpoint +host, whether the auth env var is set, content gate, redaction modes) — it never prints +secret values. + +## Privacy summary + +- Local telemetry never leaves your machine. +- The aggregate plane (the only thing that could go to Nous) is opt-in, default-off, + and has no uploader today — nothing is sent. +- All export surfaces (file, OTLP) point at *your* destinations. +- Secrets are always redacted on export; content export is off until you enable the + trajectories plane; PII redaction is a knob. diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 5f27dee3bcb..18247ca48e4 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2174,7 +2174,61 @@ DEFAULT_CONFIG = { "privacy": { "redact_pii": False, # When True, hash user IDs and strip phone numbers from LLM context }, - + + # Telemetry & observability. Three planes with a hard wall between them: + # local — full-fidelity local observability the user owns (real + # models, providers, tool names). Default ON. + # aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. + # trajectories — content trajectories for training. Separate consent (not here yet). + # + # Enterprise locking: any telemetry.* key can be pinned by an administrator via + # the managed-scope layer (/etc/hermes/config.yaml), which wins over the user's + # value on a per-key basis. There is no telemetry-specific policy block — pin + # e.g. `telemetry.allow_aggregate: false` in managed scope to hard-forbid egress. + "telemetry": { + # Local plane: event log + SQLite index in state.db. The user's own data; + # never leaves the machine unless they export it or opt into aggregate. + "local": True, + # Hard gate for the aggregate plane. When False, the aggregate plane is off + # regardless of consent_state. Intended to be pinned False by an administrator + # via managed scope on locked-down or air-gapped deployments. Default True + # (consent_state still governs the opt-in). + "allow_aggregate": True, + # Aggregate-plane consent — the single source of truth for the opt-in: + # "unknown" (no choice made — never uploads), "local" (declined), + # "aggregate" (opted in). Set with `hermes config set telemetry.consent_state` + # or a managed-scope pin. Non-interactive installs sit at "unknown". + "consent_state": "unknown", + # Stable install identifier (aggregate plane only). Empty string means "mint a + # fresh UUID on first use"; clear it to rotate. Never sent on the local plane. + "install_id": "", + # Local event-log retention before rotation (days). Local plane only. + "retention_days": 90, + # Keep secret redaction on even at full local capture (a SIEM full of live + # credentials is a bigger attack target). Admin may override via managed scope. + "redact_secrets": True, + # Content redaction for exports / support bundles: "none" | "pii". + "content_redaction": "none", + # Trajectories plane: full message content / reasoning / raw tool args. + # Off by default. When enabled, full content becomes exportable to the + # configured destination — always secret-redacted, and PII-redacted per + # content_redaction. This is the consent gate for content export and is + # admin-pinnable via managed scope. It does not enable any upload. + "trajectories": { + "enabled": False, + }, + # Exporters. The OTLP exporter sends to a configured Collector endpoint; + # endpoint headers reference environment variable names rather than inline + # secrets. + "export": { + "otlp": { + "enabled": False, + "endpoint": None, + "headers_env": {}, # {"Authorization": "MY_OTLP_TOKEN_ENVVAR"} + }, + }, + }, + # Text-to-speech configuration # Each provider supports an optional `max_text_length:` override for the # per-request input-character cap. Omit it to use the provider's documented diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 5ae40df077d..46017aaa13f 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -463,6 +463,7 @@ from hermes_cli.subcommands.memory import build_memory_parser from hermes_cli.subcommands.acp import build_acp_parser from hermes_cli.subcommands.tools import build_tools_parser from hermes_cli.subcommands.insights import build_insights_parser +from hermes_cli.subcommands.telemetry import build_telemetry_parser from hermes_cli.subcommands.skills import build_skills_parser from hermes_cli.subcommands.pairing import build_pairing_parser from hermes_cli.subcommands.plugins import build_plugins_parser @@ -13862,7 +13863,7 @@ _BUILTIN_SUBCOMMANDS = frozenset( "project", "proxy", "prompt-size", "send", "sessions", "setup", - "skin", "skills", "slack", "status", "tools", "uninstall", "update", + "skin", "skills", "slack", "status", "telemetry", "tools", "uninstall", "update", "version", "webhook", "whatsapp", "whatsapp-cloud", "chat", "secrets", "security", # Help-ish invocations — plugin commands not being listed in # top-level --help is an acceptable trade-off for skipping an @@ -14311,6 +14312,186 @@ def cmd_insights(args): print(f"Error generating insights: {e}") +def cmd_telemetry(args): + """Local-only telemetry control + inspection. No uploader exists yet.""" + import json as _json + import time + + from hermes_cli.config import load_config, save_config + from agent.telemetry import policy, rollup, metrics + + action = getattr(args, "telemetry_action", None) or "status" + config = load_config() + decision = policy.resolve(config) + + def _persist_install_id(): + # Make sure a minted id is written back so it stays stable. + config.setdefault("telemetry", {})["install_id"] = decision.install_id + save_config(config) + + if action == "status": + print("Telemetry status") + print("─" * 56) + print(f" Local plane: {'on' if decision.local_enabled else 'off'} " + f"(telemetry.local)") + print(f" Aggregate plane: {'on' if decision.aggregate_enabled else 'off'} " + f"(opt-in; consent_state={decision.consent_state})") + if decision.consent_state != policy.CONSENT_AGGREGATE and decision.allow_aggregate: + print(" opt in: hermes config set telemetry.consent_state aggregate") + if not decision.allow_aggregate: + print(" ⚠ allow_aggregate is false (egress hard-disabled)") + print(f" Install id: {decision.install_id}") + print(" Upload: DISABLED — no server yet. Aggregate is computed " + "locally only.") + print() + # Export posture — where YOUR data can flow (never Nous). Values only; + # lock-state is managed scope's concern, surfaced by its own tooling. + try: + from agent.telemetry import otlp_exporter, redaction + otlp = otlp_exporter._otlp_config(config) + print(" Export") + if otlp.get("enabled") and otlp.get("endpoint"): + _host = str(otlp.get("endpoint")) + print(f" OTLP: enabled → {_host}") + # Show the header name + env var + whether it's set, never the value. + hdrs = otlp.get("headers_env") or {} + if hdrs: + import os as _os + for _h, _env in hdrs.items(): + _state = "set" if _os.environ.get(str(_env)) else "NOT set" + print(f" auth: header '{_h}' ← ${_env} ({_state})") + _sdk = "installed" if otlp_exporter.is_available() else "MISSING — pip install 'hermes-agent[otlp]'" + print(f" SDK: {_sdk}") + else: + print(" OTLP: disabled (telemetry.export.otlp.enabled)") + # Content gate (trajectories plane) + redaction posture. + if redaction.content_export_enabled(config): + print(f" Content export: on (trajectories plane) — message " + f"content exportable") + else: + print(" Content export: off (trajectories plane) — structural " + "telemetry only") + print(" Secret redaction: on (always)") + print(f" PII redaction: {redaction.content_mode_for(config)}") + print(" Bulk export: hermes telemetry export --out FILE [--otlp]") + except Exception: + pass + print() + # Local data volume + try: + ov = metrics.overview() + runs = ov["workflows"]["total_runs"] + mcs = sum(ov["model_calls"]["by_provider"].values()) + tcs = ov["tool_calls"]["total"] + print(f" Local data: {runs:,} workflows · {mcs:,} model calls · {tcs:,} tool calls") + except Exception: + print(" Local data: (none yet)") + print("\n Inspect what would be shared: hermes telemetry preview") + return + + if action == "preview": + _persist_install_id() + since_ns = int((time.time() - args.days * 86400) * 1e9) + events = rollup.build_aggregate_events( + install_id=decision.install_id, since_ns=since_ns + ) + summary = rollup.summarize(events) + print("Telemetry preview — computed locally, NOT uploaded") + print("─" * 56) + print(f" Window: last {args.days} days Events: {summary['total']}") + for name, n in sorted(summary["by_event_name"].items()): + print(f" {name}: {n}") + print(" Shows the actual models, tools, and counts from local telemetry.") + print() + shown = events[: args.limit] + if getattr(args, "json", False): + print(_json.dumps(shown, indent=2)) + else: + for e in shown: + if e.get("event_name") == "heartbeat": + print(f" • heartbeat hermes={e.get('hermes_version')} os={e.get('os_family')}") + continue + bits = [f"{e.get('event_name')}"] + for k in ("entrypoint", "platform", "end_reason", "duration_ms", + "estimated_cost_usd"): + if e.get(k) is not None: + bits.append(f"{k}={e[k]}") + models = e.get("models_used") or [] + if models: + bits.append("models=" + ",".join( + f"{m.get('model') or m.get('provider') or '?'}" for m in models)) + if e.get("tools_used"): + bits.append("tools=" + ",".join(e["tools_used"])) + print(" • " + " ".join(bits)) + if len(events) > len(shown): + print(f" ... and {len(events) - len(shown)} more (use --limit)") + return + + if action == "export": + import sys as _sys + from agent.telemetry import exporter_bulk, redaction + + since_ns = int((time.time() - args.since * 86400) * 1e9) if getattr(args, "since", 0) else None + want_content = getattr(args, "include_content", False) + + # OTLP path: stream spans to the configured Collector endpoint. + if getattr(args, "otlp", False): + from agent.telemetry import otlp_exporter + if not otlp_exporter.is_enabled(config): + print("telemetry.export.otlp is not enabled/endpoint not set. " + "Set telemetry.export.otlp.enabled + .endpoint.", file=_sys.stderr) + return + # The OTel SDK is an optional dep; export_once() lazily installs it on + # first use (gated by security.allow_lazy_installs, TTY-prompted), + # same as every other optional backend. OTLPUnavailable is the + # fallback when it can't be installed. + try: + n = otlp_exporter.export_once(config, since_ns=since_ns) + except otlp_exporter.OTLPUnavailable as e: + print(str(e), file=_sys.stderr) + return + except Exception as e: + print(f"OTLP export failed: {e}", file=_sys.stderr) + return + print(f"Exported {n} spans to the OTLP endpoint " + f"({(otlp_exporter._otlp_config(config) or {}).get('endpoint')}).", + file=_sys.stderr) + return + + content_ok = redaction.content_export_enabled(config) + if want_content and not content_ok: + print("⚠ --include-content ignored: telemetry.trajectories.enabled is false.") + print(" Enable the trajectories plane (admin/config) to export message content.") + print(" Exporting structural telemetry only.") + + if not getattr(args, "out", None): + print("--out is required (or use --otlp).", file=_sys.stderr) + return + out_path = args.out + if out_path == "-": + counts = exporter_bulk.export( + _sys.stdout, fmt=args.fmt, since_ns=since_ns, + include_content=want_content, config=config, + ) + else: + with open(out_path, "w", encoding="utf-8") as fh: + counts = exporter_bulk.export( + fh, fmt=args.fmt, since_ns=since_ns, + include_content=want_content, config=config, + ) + # status to stderr so stdout stays clean for `--out -` + msg = (f"Exported {counts['telemetry']} telemetry records" + + (f" + {counts['sessions']} sessions" if counts["sessions"] else "") + + (" (content INCLUDED, redacted)" if counts["content_included"] + else " (structural only)") + + (f" -> {out_path}" if out_path != "-" else "")) + print(msg, file=_sys.stderr) + return + + print(f"Unknown telemetry action: {action}") + print("Use: status | preview | export") + + def cmd_skills(args): # Route 'config' action to skills_config module if getattr(args, "skills_action", None) == "config": @@ -16258,6 +16439,9 @@ def main(): # ========================================================================= build_insights_parser(subparsers, cmd_insights=cmd_insights) + # telemetry command (parser in hermes_cli/subcommands/telemetry.py) + build_telemetry_parser(subparsers, cmd_telemetry=cmd_telemetry) + # ========================================================================= # claw command (parser built in hermes_cli/subcommands/claw.py) # ========================================================================= diff --git a/hermes_cli/subcommands/telemetry.py b/hermes_cli/subcommands/telemetry.py new file mode 100644 index 00000000000..65ff344fbb9 --- /dev/null +++ b/hermes_cli/subcommands/telemetry.py @@ -0,0 +1,54 @@ +"""``hermes telemetry`` subcommand parser. + +Telemetry control and inspection. ``preview`` shows the per-run summary events that +would be produced for the aggregate plane; there is no uploader, so it terminates as a +local view. + +The handler is injected to avoid importing ``main`` (mirrors the insights subcommand). +""" + +from __future__ import annotations + +from typing import Callable + + +def build_telemetry_parser(subparsers, *, cmd_telemetry: Callable) -> None: + """Attach the ``telemetry`` subcommand (with actions) to ``subparsers``.""" + p = subparsers.add_parser( + "telemetry", + help="Inspect local telemetry and export it", + description=( + "Local-first telemetry. The local plane records observability on this " + "machine. The aggregate plane is opt-in (set telemetry.consent_state via " + "`hermes config set`); it has no uploader and is shown only via `preview`." + ), + ) + sub = p.add_subparsers(dest="telemetry_action") + + sub.add_parser("status", help="Show telemetry planes, consent state, and local data volume") + + prev = sub.add_parser( + "preview", + help="Show the aggregate events that would be produced (computed locally, not uploaded)", + ) + prev.add_argument("--days", type=int, default=30, help="Window to roll up (default: 30)") + prev.add_argument("--limit", type=int, default=10, help="Max events to print (default: 10)") + prev.add_argument("--json", action="store_true", help="Print raw JSON events") + + exp = sub.add_parser( + "export", + help="Export local telemetry (and optional content) to a file, stream, or OTLP endpoint", + ) + exp.add_argument("--out", help="Output file path (use - for stdout). Not needed with --otlp.") + exp.add_argument("--format", dest="fmt", choices=["ndjson", "json"], default="ndjson", + help="Output format (default: ndjson)") + exp.add_argument("--since", type=int, default=0, + help="Only telemetry from the last N days (0 = all)") + exp.add_argument("--include-content", action="store_true", + help="Include session/message content (requires telemetry.trajectories.enabled). " + "Secrets always redacted; PII per telemetry.content_redaction.") + exp.add_argument("--otlp", action="store_true", + help="Export to the configured OTLP endpoint (telemetry.export.otlp.*) " + "instead of a file. Requires the optional 'otlp' extra.") + + p.set_defaults(func=cmd_telemetry) diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py new file mode 100644 index 00000000000..ba61d65054b --- /dev/null +++ b/plugins/telemetry/__init__.py @@ -0,0 +1,319 @@ +"""Telemetry plugin — wires Hermes lifecycle hooks to the local-plane emitter. + +This is the *only* instrumentation seam. It registers observational hooks (which core +already invokes fail-open) and translates each into a typed local-plane telemetry +event handed to ``agent.telemetry.emitter``. There are zero edits to core call sites: +the hooks already carry model/provider/usage/duration/tool data. + +Everything here is best-effort and fail-open — a raised exception in a hook callback is +swallowed by core, and we additionally guard each callback so a telemetry bug can never +disturb a session. No content, no network: local plane only. + +Hooks consumed: + on_session_start -> begin a run context (trace_id/run_id), buffer a run row + post_api_request -> one model_call event (tokens, latency, raw provider/model) + api_request_error -> one error event + post_tool_call -> one tool_call event (raw tool name, duration, result class) + on_session_finalize -> finalize the run row (end_reason, counts, cost) + subagent_start/stop -> (reserved) lineage markers +""" + +from __future__ import annotations + +import logging +import threading +import time +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +# Per-run accumulators keyed by run_id, so on_session_finalize can roll up counts. +_runs: Dict[str, Dict[str, Any]] = {} +_runs_lock = threading.Lock() + + +def _safe(fn): + """Decorator: never let a telemetry hook raise into core.""" + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except Exception: + logger.debug("telemetry hook %s failed", getattr(fn, "__name__", "?"), exc_info=True) + return None + wrapper.__name__ = getattr(fn, "__name__", "wrapper") + return wrapper + + +def _run_key(session_id: Optional[str], task_id: Optional[str]) -> str: + return session_id or task_id or "default" + + +# ── on_session_start ──────────────────────────────────────────────────────── +@_safe +def _on_session_start(**kw: Any) -> None: + from agent.telemetry import spans + + session_id = kw.get("session_id") or "" + platform = kw.get("platform") or kw.get("source") or "" + ctx = spans.start_run() + key = _run_key(session_id, kw.get("task_id")) + with _runs_lock: + _runs[key] = { + "run_id": ctx.run_id, + "trace_id": ctx.trace_id, + "session_id": session_id or None, + "entrypoint": _entrypoint_for(platform, kw.get("source")), + "platform": platform or None, + "start_ns": time.time_ns(), + "model_call_count": 0, + "tool_call_count": 0, + "error_count": 0, + } + + +def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str = "") -> Dict[str, Any]: + """Return the run accumulator, lazily creating one if session_start was missed.""" + from agent.telemetry import spans + + key = _run_key(session_id, task_id) + with _runs_lock: + run = _runs.get(key) + if run is None: + rid = spans.current_run_id() or spans.new_id() + tid = spans.current_trace_id() or spans.new_id() + run = { + "run_id": rid, + "trace_id": tid, + "session_id": session_id or None, + "entrypoint": _entrypoint_for(platform), + "platform": platform or None, + "start_ns": time.time_ns(), + "model_call_count": 0, + "tool_call_count": 0, + "error_count": 0, + } + _runs[key] = run + return run + + +def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str: + """Coarse entrypoint label (cli / gateway / tui / api / cron …). + + This is a workflow *surface* label, not model/tool anonymization — it answers + "where did this run come from", which is genuinely categorical. + """ + s = (source or platform or "").lower() + if s in ("", "chat", "interactive", "cli", "desktop"): + return "cli" + if s in ("telegram", "discord", "slack", "whatsapp", "signal", "matrix", + "email", "sms", "teams", "feishu", "wecom", "line", "google_chat"): + return "gateway" + if s == "tui": + return "tui" + if s in ("api", "api_server", "openai_api"): + return "api" + if s == "cron": + return "cron" + if s == "batch": + return "batch" + if s == "acp": + return "acp" + return "cli" + + +# ── post_api_request -> model_call event ──────────────────────────────────── +@_safe +def _on_post_api_request(**kw: Any) -> None: + from agent.telemetry import emitter, spans + from agent.telemetry.events import ModelCallEvent + + session_id = kw.get("session_id") or "" + platform = kw.get("platform") or "" + run = _ensure_run(session_id, kw.get("task_id"), platform) + + usage = kw.get("usage") or {} + duration = kw.get("api_duration") + latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None + + evt = ModelCallEvent( + span_id=spans.new_span_id(), + run_id=run["run_id"], + provider=kw.get("provider"), # raw + model=kw.get("model"), # raw + base_url=kw.get("base_url"), + input_tokens=int(usage.get("input_tokens") or 0), + output_tokens=int(usage.get("output_tokens") or 0), + cache_read_tokens=int(usage.get("cache_read_tokens") or 0), + cache_write_tokens=int(usage.get("cache_write_tokens") or 0), + reasoning_tokens=int(usage.get("reasoning_tokens") or 0), + latency_ms=latency_ms, + end_reason="completed", + ) + with _runs_lock: + run["model_call_count"] += 1 + emitter.emit(evt) + + +# ── api_request_error -> error event ──────────────────────────────────────── +@_safe +def _on_api_request_error(**kw: Any) -> None: + from agent.telemetry import emitter + from agent.telemetry.events import ErrorEvent + + session_id = kw.get("session_id") or "" + run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") + error_class = _coarse_error_class(kw.get("error_type") or kw.get("error") or "") + with _runs_lock: + run["error_count"] += 1 + emitter.emit(ErrorEvent( + run_id=run["run_id"], + error_class=error_class, + subsystem="model_api", + recovery=None, + )) + + +def _coarse_error_class(raw: Any) -> str: + s = str(raw).lower() + if "timeout" in s: + return "provider_timeout" + if "rate" in s and "limit" in s: + return "rate_limit" + if any(k in s for k in ("auth", "401", "403", "unauthorized", "forbidden")): + return "auth" + if any(k in s for k in ("connection", "network", "dns", "socket")): + return "network" + if "context" in s and ("length" in s or "overflow" in s or "token" in s): + return "context_overflow" + if any(k in s for k in ("500", "502", "503", "server error", "provider")): + return "provider_error" + return "unknown" + + +# ── post_tool_call -> tool_call event ─────────────────────────────────────── +@_safe +def _on_post_tool_call(**kw: Any) -> None: + from agent.telemetry import emitter, spans + from agent.telemetry.events import ToolCallEvent + + session_id = kw.get("session_id") or "" + run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") + + function_name = kw.get("function_name") or kw.get("tool_name") + duration_ms = kw.get("duration_ms") + result = kw.get("result") + result_class = _tool_result_class(result) + + with _runs_lock: + run["tool_call_count"] += 1 + if result_class == "error": + run["error_count"] += 1 + + emitter.emit(ToolCallEvent( + span_id=spans.new_span_id(), + run_id=run["run_id"], + tool_name=function_name, # raw tool name + duration_ms=int(duration_ms) if isinstance(duration_ms, (int, float)) else None, + result_class=result_class, + )) + + +def _tool_result_class(result: Any) -> str: + """Classify a tool result without retaining content — error vs ok vs blocked.""" + try: + import json + if isinstance(result, str): + r = result.strip() + if r.startswith("{"): + obj = json.loads(r) + if isinstance(obj, dict): + if obj.get("error") or obj.get("blocked"): + return "blocked" if obj.get("blocked") else "error" + if obj.get("timeout"): + return "timeout" + return "ok" + if isinstance(result, dict): + if result.get("error"): + return "error" + return "ok" + except Exception: + return "ok" + return "ok" + + +# ── on_session_finalize -> finalize the run row ───────────────────────────── +@_safe +def _on_session_finalize(**kw: Any) -> None: + from agent.telemetry import emitter, spans + from agent.telemetry.events import RunEvent + + session_id = kw.get("session_id") or "" + key = _run_key(session_id, kw.get("task_id")) + with _runs_lock: + run = _runs.pop(key, None) + if run is None: + run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") + with _runs_lock: + _runs.pop(key, None) + + end_reason = _coarse_end_reason(kw) + emitter.emit(RunEvent( + run_id=run["run_id"], + trace_id=run["trace_id"], + entrypoint=run.get("entrypoint", "cli"), + session_id=run.get("session_id"), + platform=run.get("platform"), + start_ns=run.get("start_ns", time.time_ns()), + end_ns=time.time_ns(), + end_reason=end_reason, + model_call_count=run.get("model_call_count", 0), + tool_call_count=run.get("tool_call_count", 0), + error_count=run.get("error_count", 0), + estimated_cost_usd=_as_float(kw.get("estimated_cost_usd")), + cost_status=kw.get("cost_status"), + )) + spans.clear_run() + + +def _coarse_end_reason(kw: Dict[str, Any]) -> str: + if kw.get("interrupted"): + return "interrupted" + if kw.get("failed"): + return "failed" + reason = str(kw.get("turn_exit_reason") or "").lower() + if "max_iteration" in reason: + return "max_iterations" + if "timeout" in reason: + return "timeout" + return "completed" + + +def _as_float(v: Any) -> Optional[float]: + try: + return float(v) if v is not None else None + except (TypeError, ValueError): + return None + + +# ── subagent lineage (reserved) ───────────────────────────────────────────── +@_safe +def _on_subagent_start(**kw: Any) -> None: + # Subagents inherit the run context via contextvars; no explicit handling needed. + return None + + +@_safe +def _on_subagent_stop(**kw: Any) -> None: + return None + + +# ── registration ──────────────────────────────────────────────────────────── +def register(ctx) -> None: + ctx.register_hook("on_session_start", _on_session_start) + ctx.register_hook("post_api_request", _on_post_api_request) + ctx.register_hook("api_request_error", _on_api_request_error) + ctx.register_hook("post_tool_call", _on_post_tool_call) + ctx.register_hook("on_session_finalize", _on_session_finalize) + ctx.register_hook("subagent_start", _on_subagent_start) + ctx.register_hook("subagent_stop", _on_subagent_stop) + logger.debug("telemetry plugin registered 7 hooks") diff --git a/plugins/telemetry/plugin.yaml b/plugins/telemetry/plugin.yaml new file mode 100644 index 00000000000..967d79616b7 --- /dev/null +++ b/plugins/telemetry/plugin.yaml @@ -0,0 +1,12 @@ +name: telemetry +version: 1.0.0 +description: "Local-first telemetry & observability. Records runs, model calls, tool calls, and errors to a local event log + state.db index via plugin hooks — no agent action, no content, no network. Powers /usage, /insights, and local dashboards." +author: NousResearch +hooks: + - post_api_request + - api_request_error + - post_tool_call + - on_session_start + - on_session_finalize + - subagent_start + - subagent_stop diff --git a/pyproject.toml b/pyproject.toml index a2a1d6c0ee2..c63e458b8b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,6 +156,10 @@ edge-tts = ["edge-tts==7.2.7"] modal = ["modal==1.3.4"] daytona = ["daytona==0.155.0"] hindsight = ["hindsight-client==0.6.1"] +# OTLP telemetry export (optional). Provides the OpenTelemetry SDK + OTLP/HTTP +# exporter so `hermes telemetry export --otlp` can send spans to a Collector. +# Lazy-imported by agent/telemetry/otlp_exporter.py; never a core dependency. +otlp = ["opentelemetry-sdk==1.30.0", "opentelemetry-exporter-otlp-proto-http==1.30.0"] dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==81.0.0"] # starlette: CVE-2026-48710; setuptools: latest <82 (torch >=2.11 caps setuptools<82) messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.14.1", "brotlicffi==1.2.0.1", "slack-bolt==1.29.0", "slack-sdk==3.43.0", "qrcode==7.4.2"] # aiohttp 3.14.1: CVE-2026-34513/34518/34519/34520/34525 + 34993(RCE)/47265 cron = [] # croniter is now a core dependency; this extra kept for back-compat diff --git a/tests/hermes_cli/test_plugins.py b/tests/hermes_cli/test_plugins.py index 3fdb6d1812d..85905e2ff36 100644 --- a/tests/hermes_cli/test_plugins.py +++ b/tests/hermes_cli/test_plugins.py @@ -761,7 +761,11 @@ class TestPluginHooks: mgr.discover_and_load() assert mgr.has_hook("pre_api_request") is True - assert mgr.has_hook("post_api_request") is False + # Negative sentinel: a hook no plugin (user or bundled) registers, proving + # that registering pre_api_request doesn't conjure an unrelated hook. + # (post_api_request is now registered by the bundled telemetry plugin, so + # it is no longer a valid "nothing registered this" sentinel.) + assert mgr.has_hook("transform_terminal_output") is False results = mgr.invoke_hook( "pre_api_request", session_id="s1", diff --git a/tests/telemetry/__init__.py b/tests/telemetry/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/telemetry/test_cli_telemetry.py b/tests/telemetry/test_cli_telemetry.py new file mode 100644 index 00000000000..df597223c88 --- /dev/null +++ b/tests/telemetry/test_cli_telemetry.py @@ -0,0 +1,94 @@ +"""`hermes telemetry` handler smoke tests (local-only; no upload).""" + +from __future__ import annotations + +import sqlite3 +import time +import types + +import pytest + +import hermes_state + + +@pytest.fixture +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + # state.db with tel_* + seeded data + db = tmp_path / "state.db" + hermes_state.SessionDB(db_path=db) + from agent.telemetry.emitter import TelemetryEmitter + from agent.telemetry.events import RunEvent, ModelCallEvent, ToolCallEvent + em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "e.jsonl", db_path=db) + now = time.time_ns() + em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", + start_ns=now - 60_000_000, end_ns=now, model_call_count=1, + tool_call_count=1, estimated_cost_usd=0.3)) + em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", + model="claude-opus-4", + input_tokens=20000, output_tokens=2000)) + em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", + result_class="ok")) + em.flush() + em.close() + yield tmp_path + + +def _run(action, **kw): + from hermes_cli.main import cmd_telemetry + args = types.SimpleNamespace(telemetry_action=action, days=30, limit=10, json=False) + for k, v in kw.items(): + setattr(args, k, v) + cmd_telemetry(args) + + +def test_status_runs(home, capsys): + _run("status") + out = capsys.readouterr().out + assert "Telemetry status" in out + assert "Upload:" in out and "DISABLED" in out + assert "Local data:" in out + + +def test_preview_shows_real_values(home, capsys): + _run("preview") + out = capsys.readouterr().out + assert "NOT uploaded" in out + assert "workflow_completed" in out + # real model + tool names ARE shown (this is the user's own local data) + assert "claude-opus-4" in out + assert "web_search" in out + + +def test_status_reflects_consent_set_via_config(home, capsys): + # Opting in is a plain config write now (no `enable` verb). status should + # reflect consent_state=aggregate as the aggregate plane being on. + from hermes_cli.config import load_config, save_config + cfg = load_config() + cfg.setdefault("telemetry", {})["consent_state"] = "aggregate" + save_config(cfg) + _run("status") + out = capsys.readouterr().out + assert "consent_state=aggregate" in out + assert "Aggregate plane: on" in out + + +def test_status_shows_optin_hint_when_unknown(home, capsys): + _run("status") + out = capsys.readouterr().out + assert "Aggregate plane: off" in out + assert "config set telemetry.consent_state aggregate" in out + + +def test_allow_aggregate_false_keeps_plane_off_in_status(home, capsys): + # Even with consent opted in, a managed allow_aggregate:false wins. + from hermes_cli.config import load_config, save_config + cfg = load_config() + tel = cfg.setdefault("telemetry", {}) + tel["consent_state"] = "aggregate" + tel["allow_aggregate"] = False + save_config(cfg) + _run("status") + out = capsys.readouterr().out + assert "Aggregate plane: off" in out + assert "allow_aggregate is false" in out diff --git a/tests/telemetry/test_emitter.py b/tests/telemetry/test_emitter.py new file mode 100644 index 00000000000..f5b2902aa77 --- /dev/null +++ b/tests/telemetry/test_emitter.py @@ -0,0 +1,107 @@ +"""Emitter tests — the hot-path invariant is the one that matters most. + +Invariant: emit() never blocks, never raises, and a broken writer cannot slow or +break the caller. Plus: JSONL + SQLite round-trip, and the SQLite index is rebuildable +from the JSONL source of truth. +""" + +from __future__ import annotations + +import sqlite3 +import time + +import hermes_state +from agent.telemetry.emitter import TelemetryEmitter +from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent + + +def _fresh_db(tmp_path): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.executescript(hermes_state.SCHEMA_SQL) + conn.close() + return db + + +def test_emit_is_fast_even_when_writer_is_broken(tmp_path, monkeypatch): + """The core guarantee: a writer that raises AND sleeps cannot stall emit().""" + db = _fresh_db(tmp_path) + em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) + + # Sabotage the row indexer to raise after a long sleep. + def broken(_conn, _ev): + time.sleep(5.0) + raise RuntimeError("writer exploded") + + monkeypatch.setattr(em, "_index_one", broken) + + start = time.monotonic() + for i in range(50): + em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=10)) + elapsed = time.monotonic() - start + + # 50 emits must complete in well under the writer's single 5s sleep. + assert elapsed < 1.0, f"emit() blocked: {elapsed:.2f}s" + em.close() + + +def test_emit_never_raises_on_bad_event(tmp_path): + db = _fresh_db(tmp_path) + em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) + # Non-serializable / wrong-shaped inputs must not raise out of emit(). + em.emit(object()) # no to_dict, not a mapping + em.emit({"event": "run"}) # minimal dict + em.close() + + +def test_jsonl_and_sqlite_roundtrip(tmp_path): + db = _fresh_db(tmp_path) + jsonl = tmp_path / "telemetry" / "events.jsonl" + em = TelemetryEmitter(events_path=jsonl, db_path=db) + + em.emit(RunEvent(run_id="run1", trace_id="t1", entrypoint="cli", end_reason="completed")) + em.emit(ModelCallEvent(span_id="m1", run_id="run1", provider="anthropic", + model="claude-opus-4", input_tokens=100, output_tokens=20)) + em.emit(ToolCallEvent(span_id="tc1", run_id="run1", tool_name="web_search", + duration_ms=120, result_class="ok")) + em.flush() + em.close() + + # JSONL has all three lines + lines = [l for l in jsonl.read_text(encoding="utf-8").splitlines() if l.strip()] + assert len(lines) == 3 + + # SQLite index has the rows in the right tables + conn = sqlite3.connect(db) + assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1 + assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 1 + assert conn.execute("SELECT COUNT(*) FROM tel_tool_calls").fetchone()[0] == 1 + row = conn.execute("SELECT provider, model, input_tokens FROM tel_model_calls").fetchone() + assert row == ("anthropic", "claude-opus-4", 100) + conn.close() + + +def test_unknown_event_kind_is_ignored_not_fatal(tmp_path): + db = _fresh_db(tmp_path) + em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) + em.emit({"event": "totally_unknown", "foo": "bar"}) + em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli")) + em.flush() + em.close() + conn = sqlite3.connect(db) + # The unknown event is in JSONL but skipped by the indexer; the known one indexes. + assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1 + conn.close() + + +def test_disabled_emitter_writes_nothing(tmp_path): + db = _fresh_db(tmp_path) + jsonl = tmp_path / "telemetry" / "events.jsonl" + em = TelemetryEmitter(events_path=jsonl, db_path=db, enabled=False) + em.emit(RunEvent(run_id="r3", trace_id="t3", entrypoint="cli")) + em.flush() + em.close() + assert not jsonl.exists() + conn = sqlite3.connect(db) + assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0 + conn.close() diff --git a/tests/telemetry/test_export_redaction.py b/tests/telemetry/test_export_redaction.py new file mode 100644 index 00000000000..7fc84c8e205 --- /dev/null +++ b/tests/telemetry/test_export_redaction.py @@ -0,0 +1,113 @@ +"""Export redaction pipeline tests — the security-critical layer. + +Invariants: + * Secrets ALWAYS stripped, every mode, no flag disables it. + * Content gated by the trajectories plane, not a redaction mode. + * PII stripped in 'pii' mode; structure preserved (codec-aware). +""" + +from __future__ import annotations + +import json + +from agent.telemetry import redaction as R + + +# ── secrets are always redacted ───────────────────────────────────────────── +def test_secrets_stripped_in_none_mode(): + text = "here is sk-ant-api03-SECRETKEY123 and a token" + out = R.redact_for_export(text, content_mode=R.CONTENT_NONE) + assert "SECRETKEY123" not in out + + +def test_secrets_stripped_in_pii_mode(): + text = "Authorization: Bearer abcdef123456789secret" + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert "abcdef123456789secret" not in out + + +def test_secret_redactor_fails_closed(monkeypatch): + # If the underlying redactor raises, we must NOT return the raw string. + import agent.redact as ar + monkeypatch.setattr(ar, "redact_sensitive_text", lambda *a, **k: (_ for _ in ()).throw(RuntimeError())) + out = R.redact_for_export("sk-secret-value", content_mode=R.CONTENT_NONE) + assert "sk-secret-value" not in out + assert out == "[redaction-unavailable]" + + +# ── PII ───────────────────────────────────────────────────────────────────── +def test_pii_mode_strips_email_and_phone(): + text = "contact alice@example.com or +1 415 555 1234" + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert "alice@example.com" not in out + assert "[email]" in out + assert "555" not in out or "[phone]" in out + + +def test_none_mode_keeps_nonsecret_text_but_drops_via_message_path(): + # redact_for_export(none) scrubs secrets but doesn't strip ordinary words; + # content *dropping* happens at the message layer (trajectories gate). + out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE) + assert "ordinary" in out + + +# ── trajectories gate (content_export_enabled) ────────────────────────────── +def test_content_export_disabled_by_default(): + assert R.content_export_enabled({}) is False + assert R.content_export_enabled({"telemetry": {}}) is False + assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": False}}}) is False + + +def test_content_export_enabled_when_trajectories_on(): + assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": True}}}) is True + + +# ── codec-aware message redaction ─────────────────────────────────────────── +def test_message_structural_only_when_content_excluded(): + msg = {"role": "user", "content": "my email is bob@x.com and key sk-12345"} + out = R.redact_message(msg, include_content=False) + assert out["role"] == "user" + assert "content" not in out # body dropped entirely + assert out["content_chars"] == len(msg["content"]) # only the size remains + assert "bob@x.com" not in json.dumps(out) + + +def test_message_content_included_is_redacted(): + msg = {"role": "user", "content": "email bob@x.com secret sk-ant-SECRET999"} + out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True) + assert "content" in out + assert "SECRET999" not in out["content"] # secret gone + assert "bob@x.com" not in out["content"] # pii gone + assert "[email]" in out["content"] + + +def test_tool_calls_redacted_names_kept_args_scrubbed(): + msg = { + "role": "assistant", + "tool_calls": json.dumps([ + {"function": {"name": "web_search", "arguments": '{"q": "email me at z@z.com"}'}} + ]), + } + out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True) + tc = out["tool_calls"] + assert tc[0]["name"] == "web_search" # structure/name preserved + assert "z@z.com" not in json.dumps(tc) # arg pii scrubbed + + +def test_tool_calls_counted_when_content_excluded(): + msg = { + "role": "assistant", + "tool_calls": json.dumps([ + {"function": {"name": "a", "arguments": "{}"}}, + {"function": {"name": "b", "arguments": "{}"}}, + ]), + } + out = R.redact_message(msg, include_content=False) + assert out["tool_call_count"] == 2 + assert "tool_calls" not in out + + +def test_content_mode_for_reads_config(): + assert R.content_mode_for({"telemetry": {"content_redaction": "pii"}}) == "pii" + assert R.content_mode_for({"telemetry": {"content_redaction": "bogus"}}) == "none" + assert R.content_mode_for({}) == "none" diff --git a/tests/telemetry/test_exporter_bulk.py b/tests/telemetry/test_exporter_bulk.py new file mode 100644 index 00000000000..10e0fa68f08 --- /dev/null +++ b/tests/telemetry/test_exporter_bulk.py @@ -0,0 +1,102 @@ +"""Bulk export tests — telemetry always, content only behind the trajectories gate.""" + +from __future__ import annotations + +import io +import json +import sqlite3 +import time + +import hermes_state +from agent.telemetry import exporter_bulk +from agent.telemetry.emitter import TelemetryEmitter +from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent + + +def _seed(tmp_path, with_secret_content=True): + db = tmp_path / "state.db" + sdb = hermes_state.SessionDB(db_path=db) + # telemetry + em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db) + now = time.time_ns() + em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", + start_ns=now - 60_000_000, end_ns=now, model_call_count=1, tool_call_count=1)) + em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", + model="claude-sonnet-4", input_tokens=1000, output_tokens=100)) + em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok")) + em.flush() + em.close() + # session + message content (with an embedded secret + email) + sdb.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") + if with_secret_content: + sdb.append_message("s1", role="user", + content="my key is AKIAIOSFODNN7EXAMPLE and email me at carol@corp.com") + sdb.append_message("s1", role="assistant", content="ok") + sdb.close() + return db + + +def test_telemetry_exported_content_excluded_by_default(tmp_path): + db = _seed(tmp_path) + buf = io.StringIO() + counts = exporter_bulk.export(buf, fmt="ndjson", include_content=False, config={}, db_path=db) + assert counts["telemetry"] >= 3 + assert counts["content_included"] == 0 + text = buf.getvalue() + # message bodies must NOT be present + assert "carol@corp.com" not in text + assert "AKIAIOSFODNN7EXAMPLE" not in text + # but a session record (structural) is present with message structure + lines = [json.loads(l) for l in text.splitlines() if l.strip()] + sess = [r for r in lines if r.get("_kind") == "session"] + assert sess and sess[0]["messages"] + assert "content" not in sess[0]["messages"][0] # structural only + assert "content_chars" in sess[0]["messages"][0] + + +def test_include_content_ignored_without_trajectories(tmp_path): + db = _seed(tmp_path) + buf = io.StringIO() + # request content but trajectories disabled -> forced off + counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, + config={"telemetry": {"trajectories": {"enabled": False}}}, db_path=db) + assert counts["content_included"] == 0 + assert "carol@corp.com" not in buf.getvalue() + + +def test_content_included_when_trajectories_on_but_redacted(tmp_path): + db = _seed(tmp_path) + buf = io.StringIO() + cfg = {"telemetry": {"trajectories": {"enabled": True}, "content_redaction": "pii"}} + counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, config=cfg, db_path=db) + assert counts["content_included"] == 1 + text = buf.getvalue() + # content present but secret + pii scrubbed + assert "AKIAIOSFODNN7EXAMPLE" not in text # secret always gone + assert "carol@corp.com" not in text # pii gone in pii mode + lines = [json.loads(l) for l in text.splitlines() if l.strip()] + sess = [r for r in lines if r.get("_kind") == "session"][0] + # the user message now has a (redacted) content field + user_msg = [m for m in sess["messages"] if m["role"] == "user"][0] + assert "content" in user_msg + assert "[email]" in user_msg["content"] + + +def test_json_format_roundtrips(tmp_path): + db = _seed(tmp_path) + buf = io.StringIO() + exporter_bulk.export(buf, fmt="json", include_content=False, config={}, db_path=db) + obj = json.loads(buf.getvalue()) + assert "records" in obj + assert any(r["_kind"] == "tel_runs" for r in obj["records"]) + + +def test_since_window_filters_runs(tmp_path): + db = _seed(tmp_path) + buf = io.StringIO() + # since 1ns ago in the future-ish -> the run (start ~60ms ago) excluded + future_ns = int((time.time() + 1) * 1e9) + counts = exporter_bulk.export(buf, fmt="ndjson", since_ns=future_ns, config={}, db_path=db) + lines = [json.loads(l) for l in buf.getvalue().splitlines() if l.strip()] + runs = [r for r in lines if r.get("_kind") == "tel_runs"] + assert runs == [] diff --git a/tests/telemetry/test_governance.py b/tests/telemetry/test_governance.py new file mode 100644 index 00000000000..37bf1810d8d --- /dev/null +++ b/tests/telemetry/test_governance.py @@ -0,0 +1,98 @@ +"""Export configuration visibility in `hermes telemetry status`. + +The status Export block reports the current export configuration. Whether a key is +locked is handled by the managed-scope layer, not repeated here; the allow_aggregate +gate is covered by a test so a managed pin can't be regressed. +""" + +from __future__ import annotations + +import time +import types + +import pytest + +import hermes_state + + +@pytest.fixture +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + hermes_state.SessionDB(db_path=tmp_path / "state.db") + yield tmp_path + + +def _status(capsys): + from hermes_cli.main import cmd_telemetry + cmd_telemetry(types.SimpleNamespace(telemetry_action="status")) + return capsys.readouterr().out + + +def test_export_block_default_shows_otlp_disabled(home, capsys): + out = _status(capsys) + assert "Export" in out + assert "OTLP:" in out and "disabled" in out + assert "Content export: off" in out + assert "Secret redaction: on (always)" in out + + +def test_export_block_shows_endpoint_host_never_token(home, capsys, monkeypatch): + from hermes_cli.config import load_config, save_config + monkeypatch.setenv("CORP_OTLP_TOKEN", "supersecret-do-not-print") + c = load_config() + t = c.setdefault("telemetry", {}) + t.setdefault("export", {})["otlp"] = { + "enabled": True, + "endpoint": "https://collector.corp:4318/v1/traces", + "headers_env": {"Authorization": "CORP_OTLP_TOKEN"}, + } + save_config(c) + out = _status(capsys) + # endpoint host present + assert "https://collector.corp:4318/v1/traces" in out + # env var name + set-state present; the VALUE never printed + assert "CORP_OTLP_TOKEN" in out + assert "(set)" in out + assert "supersecret-do-not-print" not in out + + +def test_export_block_reflects_trajectories_gate(home, capsys): + from hermes_cli.config import load_config, save_config + c = load_config() + c.setdefault("telemetry", {})["trajectories"] = {"enabled": True} + save_config(c) + out = _status(capsys) + assert "Content export: on (trajectories plane)" in out + + +def test_token_env_not_set_shows_not_set(home, capsys): + from hermes_cli.config import load_config, save_config + c = load_config() + t = c.setdefault("telemetry", {}) + t.setdefault("export", {})["otlp"] = { + "enabled": True, + "endpoint": "https://x:4318/v1/traces", + "headers_env": {"Authorization": "TOTALLY_UNSET_ENV_VAR_XYZ"}, + } + save_config(c) + out = _status(capsys) + assert "(NOT set)" in out + + +def test_allow_aggregate_pin_blocks_opt_in(home): + """A managed allow_aggregate:false pin overrides a consent_state opt-in. + + Consent is set in config (as a user or managed-scope pin would); the hard gate + still wins, so the aggregate plane resolves off and may_upload stays false. + """ + from hermes_cli.config import load_config, save_config + from agent.telemetry import policy + c = load_config() + tel = c.setdefault("telemetry", {}) + tel["consent_state"] = "aggregate" + tel["allow_aggregate"] = False + save_config(c) + d = policy.resolve(load_config()) + assert d.allow_aggregate is False + assert d.aggregate_enabled is False + assert d.may_upload_aggregate() is False diff --git a/tests/telemetry/test_insights_integration.py b/tests/telemetry/test_insights_integration.py new file mode 100644 index 00000000000..814f3eeb093 --- /dev/null +++ b/tests/telemetry/test_insights_integration.py @@ -0,0 +1,82 @@ +"""Insights ↔ telemetry integration: the observability section in /insights output.""" + +from __future__ import annotations + +import time + +import pytest + +from hermes_state import SessionDB +from agent.insights import InsightsEngine +from agent.telemetry.emitter import TelemetryEmitter +from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent + + +@pytest.fixture() +def db(tmp_path): + session_db = SessionDB(db_path=tmp_path / "ins_tel.db") + yield session_db + session_db.close() + + +def _seed_telemetry(db_path): + em = TelemetryEmitter(events_path=db_path.parent / "tel" / "events.jsonl", db_path=db_path) + now = time.time_ns() + em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", + platform="telegram", end_reason="completed", + start_ns=now - 90_000_000, end_ns=now)) + em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", + end_reason="failed", start_ns=now - 11_000_000, end_ns=now)) + em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", + model="claude-opus-4", input_tokens=5000, output_tokens=800, + cache_read_tokens=1000, latency_ms=2200)) + em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", result_class="ok")) + em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", result_class="error")) + em.flush() + em.close() + + +def test_report_includes_telemetry_when_present(db): + # A session so generate() isn't the empty branch + db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") + _seed_telemetry(db.db_path) + + engine = InsightsEngine(db) + report = engine.generate(days=30) + tel = report.get("telemetry") + assert tel, "telemetry section missing" + assert tel["workflows"]["total_runs"] == 2 + assert tel["workflows"]["success_rate"] == 0.5 + assert tel["tool_calls"]["total"] == 2 + assert tel["tool_calls"]["failure_rate"] == 0.5 + assert tel["model_calls"]["by_provider"]["anthropic"] == 1 + + +def test_terminal_output_renders_observability_section(db): + db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") + _seed_telemetry(db.db_path) + + engine = InsightsEngine(db) + out = engine.format_terminal(engine.generate(days=30)) + assert "Observability" in out + assert "Workflows:" in out + assert "Failure rate:" in out + assert "Providers:" in out + + +def test_telemetry_section_absent_when_no_tel_rows(db): + # Session present, but no telemetry events seeded. + db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") + engine = InsightsEngine(db) + report = engine.generate(days=30) + assert report.get("telemetry") == {} + out = engine.format_terminal(report) + assert "Observability" not in out + + +def test_empty_report_has_telemetry_key(db): + # No sessions at all -> empty branch still carries the key (renderer-safe). + engine = InsightsEngine(db) + report = engine.generate(days=30) + assert report.get("empty") is True + assert report.get("telemetry") == {} diff --git a/tests/telemetry/test_otlp_exporter.py b/tests/telemetry/test_otlp_exporter.py new file mode 100644 index 00000000000..49fd990d463 --- /dev/null +++ b/tests/telemetry/test_otlp_exporter.py @@ -0,0 +1,171 @@ +"""OTLP exporter tests. + +Skip cleanly when the optional OTel SDK isn't installed. When it is, verify: + * event -> OTel span attribute mapping + * headers_env resolves the value from the named environment variable, not config + * a failing or slow subscriber never breaks the emitter hot path + * is_enabled / is_available gating +""" + +from __future__ import annotations + +import sqlite3 +import time + +import pytest + +import hermes_state +from agent.telemetry import otlp_exporter as OE +from agent.telemetry.emitter import TelemetryEmitter +from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent + +otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed") + + +def _in_memory_provider(): + """A TracerProvider with an in-memory span exporter (no network).""" + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + provider = TracerProvider() + mem = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(mem)) + return provider, mem + + +def test_event_maps_to_span_with_real_attrs(): + provider, mem = _in_memory_provider() + batch = [ + {"event": "run", "entrypoint": "gateway", "platform": "telegram", + "end_reason": "completed", "model_call_count": 2, "tool_call_count": 3}, + {"event": "model_call", "provider": "anthropic", "model": "claude-opus-4", + "input_tokens": 5000, "output_tokens": 800}, + {"event": "tool_call", "tool_name": "web_search", "result_class": "ok"}, + ] + n = OE.export_batch(provider, batch) + assert n == 3 + spans = mem.get_finished_spans() + names = {s.name for s in spans} + assert names == {"hermes.run", "hermes.model_call", "hermes.tool_call"} + # real values present as span attributes + run = [s for s in spans if s.name == "hermes.run"][0] + assert run.attributes["hermes.entrypoint"] == "gateway" + assert run.attributes["hermes.platform"] == "telegram" + model = [s for s in spans if s.name == "hermes.model_call"][0] + assert model.attributes["hermes.model"] == "claude-opus-4" + assert model.attributes["hermes.provider"] == "anthropic" + tool = [s for s in spans if s.name == "hermes.tool_call"][0] + assert tool.attributes["hermes.tool_name"] == "web_search" + + +def test_headers_resolve_from_env_not_value(monkeypatch): + monkeypatch.setenv("MY_OTLP_TOKEN", "supersecretvalue") + resolved = OE._resolve_headers({"Authorization": "MY_OTLP_TOKEN"}) + assert resolved == {"Authorization": "supersecretvalue"} + # missing env var -> skipped, not crashed + assert OE._resolve_headers({"X": "NOPE_NOT_SET"}) == {} + + +def test_is_enabled_requires_endpoint_and_flag(): + assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) is True + assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True}}}}) is False + assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": False, "endpoint": "http://x"}}}}) is False + assert OE.is_enabled({}) is False + + +def test_require_sdk_routes_through_lazy_install(monkeypatch): + # _require_sdk(auto_install=True) should call lazy_deps.ensure('export.otlp'). + import tools.lazy_deps as ld + calls = [] + monkeypatch.setattr(ld, "ensure", lambda feature, **kw: calls.append((feature, kw))) + OE._require_sdk(auto_install=True, prompt=False) + assert calls == [("export.otlp", {"prompt": False})] + + +def test_is_available_does_not_install(monkeypatch): + # A pure availability check must NEVER trigger an install. + import tools.lazy_deps as ld + calls = [] + monkeypatch.setattr(ld, "ensure", lambda *a, **k: calls.append(a)) + OE.is_available() + assert calls == [] + + +def test_export_otlp_feature_specs_match_pyproject(): + # The LAZY_DEPS entry must track the [otlp] extra in pyproject.toml. + from tools.lazy_deps import feature_specs + import pathlib, re + specs = set(feature_specs("export.otlp")) + pyproject = pathlib.Path(__file__).resolve().parents[2] / "pyproject.toml" + text = pyproject.read_text(encoding="utf-8") + m = re.search(r"^otlp\s*=\s*\[([^\]]*)\]", text, re.MULTILINE) + assert m, "otlp extra not found in pyproject.toml" + extra = set(re.findall(r'"([^"]+)"', m.group(1))) + assert specs == extra, f"LAZY_DEPS {specs} != pyproject extra {extra}" + + +def test_streamer_subscription_receives_events(tmp_path, monkeypatch): + # Wire an OTLPStreamer-like subscriber via the in-memory provider. + provider, mem = _in_memory_provider() + + db = tmp_path / "state.db" + conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() + em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) + + def subscriber(batch): + OE.export_batch(provider, batch) + + em.subscribe(subscriber) + em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed")) + em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", model="claude-opus-4")) + em.flush() + em.close() + spans = mem.get_finished_spans() + assert {s.name for s in spans} == {"hermes.run", "hermes.model_call"} + + +def test_failing_subscriber_never_breaks_hot_path(tmp_path): + db = tmp_path / "state.db" + conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() + em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) + + def bad_subscriber(batch): + time.sleep(0.2) + raise RuntimeError("OTLP collector down") + + em.subscribe(bad_subscriber) + start = time.monotonic() + for i in range(30): + em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=1)) + elapsed = time.monotonic() - start + # emit() returns immediately regardless of the broken subscriber + assert elapsed < 1.0 + em.flush() + # durable writes still happened despite the subscriber raising + conn = sqlite3.connect(db) + assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 30 + conn.close() + em.close() + + +def test_export_once_reads_db_and_returns_count(tmp_path, monkeypatch): + db = tmp_path / "state.db" + conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() + em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) + em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", + start_ns=time.time_ns(), end_ns=time.time_ns())) + em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok")) + em.flush(); em.close() + + # Patch the provider builder to an in-memory one (no network). The processor + # stand-in only needs force_flush(); provider.shutdown() works on the real one. + provider, mem = _in_memory_provider() + + class _Proc: + def force_flush(self, *a, **k): + return True + + monkeypatch.setattr(OE, "_make_provider", lambda config: (provider, _Proc())) + n = OE.export_once({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}, db_path=db) + assert n == 2 + assert len(mem.get_finished_spans()) == 2 diff --git a/tests/telemetry/test_plugin_autoload.py b/tests/telemetry/test_plugin_autoload.py new file mode 100644 index 00000000000..9cfe5c92a83 --- /dev/null +++ b/tests/telemetry/test_plugin_autoload.py @@ -0,0 +1,53 @@ +"""Telemetry plugin auto-load gating: on by default, off when telemetry.local=false.""" + +from __future__ import annotations + +import hermes_cli.plugins as plugins_mod + + +def test_local_enabled_defaults_true(monkeypatch): + monkeypatch.setattr(plugins_mod, "load_config", lambda: {}, raising=False) + # With no telemetry section, the default is on. + monkeypatch.setattr( + "hermes_cli.config.load_config", lambda: {}, raising=False + ) + assert plugins_mod._telemetry_local_enabled() is True + + +def test_local_disabled_when_config_false(monkeypatch): + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"telemetry": {"local": False}}, + raising=False, + ) + assert plugins_mod._telemetry_local_enabled() is False + + +def test_local_enabled_when_config_true(monkeypatch): + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"telemetry": {"local": True}}, + raising=False, + ) + assert plugins_mod._telemetry_local_enabled() is True + + +def test_malformed_config_defaults_on(monkeypatch): + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"telemetry": "not a dict"}, + raising=False, + ) + assert plugins_mod._telemetry_local_enabled() is True + + +def test_plugin_manifest_is_discoverable(): + """The bundled telemetry plugin.yaml exists and declares the lifecycle hooks.""" + from pathlib import Path + import hermes_cli.plugins as p + bundled = p.get_bundled_plugins_dir() + manifest = bundled / "telemetry" / "plugin.yaml" + assert manifest.exists(), f"missing {manifest}" + text = manifest.read_text(encoding="utf-8") + for hook in ("post_api_request", "post_tool_call", "on_session_finalize"): + assert hook in text diff --git a/tests/telemetry/test_plugin_hooks.py b/tests/telemetry/test_plugin_hooks.py new file mode 100644 index 00000000000..807dc276e1b --- /dev/null +++ b/tests/telemetry/test_plugin_hooks.py @@ -0,0 +1,135 @@ +"""Telemetry plugin hook tests — feed realistic kwargs, assert tel_* rows + no leak.""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +import hermes_state +from agent.telemetry import emitter as emitter_mod +from agent.telemetry.emitter import TelemetryEmitter + + +@pytest.fixture +def wired(tmp_path, monkeypatch): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.executescript(hermes_state.SCHEMA_SQL) + conn.close() + em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) + emitter_mod.reset_emitter_for_tests(em) + # reset the plugin's per-run accumulators between tests + import plugins.telemetry as plug + plug._runs.clear() + yield db, em, plug + em.flush() + em.close() + emitter_mod.reset_emitter_for_tests(None) + + +def test_full_session_lifecycle_produces_rows(wired): + db, em, plug = wired + + plug._on_session_start(session_id="sess1", platform="telegram") + plug._on_post_api_request( + session_id="sess1", platform="telegram", + provider="anthropic", base_url=None, model="claude-opus-4", + api_duration=2.5, + usage={"input_tokens": 5000, "output_tokens": 800, "cache_read_tokens": 1000, + "cache_write_tokens": 0, "reasoning_tokens": 0}, + ) + plug._on_post_tool_call( + session_id="sess1", platform="telegram", + function_name="web_search", duration_ms=812, result="{\"data\": \"...\"}", + ) + plug._on_session_finalize( + session_id="sess1", platform="telegram", + turn_exit_reason="completed", estimated_cost_usd=0.042, cost_status="known", + ) + em.flush() + + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + run = conn.execute("SELECT * FROM tel_runs").fetchone() + assert run is not None + assert run["entrypoint"] == "gateway" + assert run["platform"] == "telegram" + assert run["end_reason"] == "completed" + assert run["model_call_count"] == 1 + assert run["tool_call_count"] == 1 + assert abs(run["estimated_cost_usd"] - 0.042) < 1e-9 + + mc = conn.execute("SELECT * FROM tel_model_calls").fetchone() + assert mc["provider"] == "anthropic" + assert mc["model"] == "claude-opus-4" + assert mc["input_tokens"] == 5000 + + tc = conn.execute("SELECT * FROM tel_tool_calls").fetchone() + assert tc["tool_name"] == "web_search" + assert tc["result_class"] == "ok" + conn.close() + + +def test_tool_error_result_classified_and_counted(wired): + db, em, plug = wired + plug._on_session_start(session_id="s2", platform="cli") + plug._on_post_tool_call( + session_id="s2", function_name="terminal", duration_ms=10, + result="{\"error\": \"command failed\"}", + ) + plug._on_session_finalize(session_id="s2", turn_exit_reason="completed") + em.flush() + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + tc = conn.execute("SELECT result_class, tool_name FROM tel_tool_calls").fetchone() + assert tc["result_class"] == "error" + assert tc["tool_name"] == "terminal" + run = conn.execute("SELECT error_count FROM tel_runs").fetchone() + assert run["error_count"] == 1 + conn.close() + + +def test_api_error_recorded(wired): + db, em, plug = wired + plug._on_session_start(session_id="s3", platform="cli") + plug._on_api_request_error(session_id="s3", error_type="provider timeout after 60s") + plug._on_session_finalize(session_id="s3", failed=True) + em.flush() + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + err = conn.execute("SELECT error_class, subsystem FROM tel_error_events").fetchone() + assert err["error_class"] == "provider_timeout" + assert err["subsystem"] == "model_api" + run = conn.execute("SELECT end_reason FROM tel_runs").fetchone() + assert run["end_reason"] == "failed" + conn.close() + + +def test_hooks_never_raise_on_garbage_kwargs(wired): + _, em, plug = wired + # Missing everything — must be swallowed by the _safe wrapper. + plug._on_post_api_request() + plug._on_post_tool_call() + plug._on_session_finalize() + plug._on_api_request_error() + em.flush() + + +def test_no_message_content_in_tool_rows(wired): + """The tool hook receives a result blob; only the classification persists, not content.""" + db, em, plug = wired + plug._on_session_start(session_id="s4", platform="cli") + secret = "{\"data\": \"USER SECRET sk-ABCDEF and /Users/alice/file.txt\"}" + plug._on_post_tool_call(session_id="s4", function_name="web_search", + duration_ms=5, result=secret) + plug._on_session_finalize(session_id="s4") + em.flush() + # The whole tel_tool_calls row must contain none of the result content. + conn = sqlite3.connect(db) + row = conn.execute("SELECT * FROM tel_tool_calls").fetchone() + conn.close() + blob = " ".join(str(x) for x in row) + assert "sk-ABCDEF" not in blob + assert "/Users/alice" not in blob + assert "SECRET" not in blob diff --git a/tests/telemetry/test_policy_consent.py b/tests/telemetry/test_policy_consent.py new file mode 100644 index 00000000000..f3ace454770 --- /dev/null +++ b/tests/telemetry/test_policy_consent.py @@ -0,0 +1,61 @@ +"""Consent posture + org-policy enforcement tests. + +Consent is a single field (``telemetry.consent_state``); the aggregate opt-in is +expressed by setting it to ``"aggregate"`` (via ``hermes config set`` or a managed-scope +pin). ``allow_aggregate`` is the hard gate. +""" + +from __future__ import annotations + +from agent.telemetry import policy + + +def _cfg(**telemetry): + return {"telemetry": telemetry} + + +def test_default_posture_is_local_only(): + d = policy.resolve(_cfg(local=True, consent_state="unknown")) + assert d.local_enabled is True + assert d.aggregate_enabled is False + assert d.may_upload_aggregate() is False + + +def test_unknown_consent_never_uploads(): + # A headless box with no choice recorded: stays unknown, never uploads. + d = policy.resolve(_cfg(local=True, consent_state="unknown")) + assert d.may_upload_aggregate() is False + + +def test_opted_in_uploads(): + d = policy.resolve(_cfg(local=True, consent_state="aggregate")) + assert d.aggregate_enabled is True + assert d.may_upload_aggregate() is True + + +def test_declined_does_not_upload(): + d = policy.resolve(_cfg(local=True, consent_state="local")) + assert d.may_upload_aggregate() is False + + +def test_allow_aggregate_false_overrides_opt_in(): + # An admin pins telemetry.allow_aggregate: false via managed scope. + cfg = _cfg(local=True, consent_state="aggregate", allow_aggregate=False) + d = policy.resolve(cfg) + assert d.allow_aggregate is False + assert d.aggregate_enabled is False + assert d.may_upload_aggregate() is False # the hard gate wins + + +def test_invalid_consent_state_treated_as_unknown(): + d = policy.resolve(_cfg(local=True, consent_state="bogus")) + assert d.consent_state == "unknown" + assert d.may_upload_aggregate() is False + + +def test_install_id_minted_when_empty_and_stable_when_set(): + cfg = _cfg(install_id="") + minted = policy.ensure_install_id(cfg) + assert minted and len(minted) >= 32 # uuid4 + cfg2 = _cfg(install_id="fixed-id") + assert policy.ensure_install_id(cfg2) == "fixed-id" diff --git a/tests/telemetry/test_rollup.py b/tests/telemetry/test_rollup.py new file mode 100644 index 00000000000..5ef50c89f08 --- /dev/null +++ b/tests/telemetry/test_rollup.py @@ -0,0 +1,88 @@ +"""rollup tests: tel_* -> per-run summary events with REAL values (local only).""" + +from __future__ import annotations + +import sqlite3 +import time + +import hermes_state +from agent.telemetry import rollup +from agent.telemetry.emitter import TelemetryEmitter +from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent + + +def _seed(tmp_path): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.executescript(hermes_state.SCHEMA_SQL) + conn.close() + em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db) + now = time.time_ns() + em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", + platform="telegram", end_reason="completed", + start_ns=now - 90_000_000, end_ns=now, + model_call_count=2, tool_call_count=2, estimated_cost_usd=2.1)) + em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", + model="claude-opus-4", input_tokens=60000, output_tokens=8000)) + em.emit(ModelCallEvent(span_id="m2", run_id="r1", provider="anthropic", + model="claude-opus-4", input_tokens=5000, output_tokens=500)) + em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", + result_class="ok")) + em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", + result_class="ok")) + # an in-progress run (no end_ns) must be excluded + em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", start_ns=now)) + em.flush() + em.close() + return db + + +def test_builds_one_event_per_completed_run_with_real_values(tmp_path): + db = _seed(tmp_path) + events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db, + include_heartbeat=False) + wf = [e for e in events if e["event_name"] == "workflow_completed"] + assert len(wf) == 1 # r2 (no end_ns) excluded + e = wf[0] + assert e["entrypoint"] == "gateway" + assert e["platform"] == "telegram" + # REAL model id + provider, not a bucket/class + models = {m["model"] for m in e["models_used"]} + assert models == {"claude-opus-4"} + assert e["models_used"][0]["provider"] == "anthropic" + assert sorted(e["tools_used"]) == ["browser_navigate", "web_search"] + # real token totals, not buckets + assert e["input_tokens"] == 65000 + assert e["output_tokens"] == 8500 + + +def test_real_model_and_tool_names_present(tmp_path): + db = _seed(tmp_path) + events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) + blob = " ".join(str(v) for e in events for v in e.values()) + assert "claude-opus-4" in blob + assert "web_search" in blob + + +def test_heartbeat_included_by_default(tmp_path): + db = _seed(tmp_path) + events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) + assert any(e["event_name"] == "heartbeat" for e in events) + + +def test_summarize_counts_by_event_name(tmp_path): + db = _seed(tmp_path) + events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) + s = rollup.summarize(events) + assert s["total"] == len(events) + assert s["by_event_name"]["workflow_completed"] == 1 + assert s["by_event_name"]["heartbeat"] == 1 + + +def test_empty_db_yields_only_heartbeat(tmp_path): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.executescript(hermes_state.SCHEMA_SQL) + conn.close() + events = rollup.build_aggregate_events(install_id="x", db_path=db) + assert [e["event_name"] for e in events] == ["heartbeat"] diff --git a/tests/telemetry/test_schema_migration.py b/tests/telemetry/test_schema_migration.py new file mode 100644 index 00000000000..5a73a335319 --- /dev/null +++ b/tests/telemetry/test_schema_migration.py @@ -0,0 +1,46 @@ +"""Schema tests: tel_* tables exist after init; SCHEMA_VERSION bumped.""" + +from __future__ import annotations + +import sqlite3 + +import hermes_state + + +TEL_TABLES = { + "tel_runs", "tel_spans", "tel_model_calls", "tel_tool_calls", + "tel_gateway_events", "tel_cron_events", "tel_skill_events", + "tel_memory_events", "tel_feedback_events", "tel_error_events", +} + + +def test_schema_version_is_17_or_higher(): + assert hermes_state.SCHEMA_VERSION >= 17 + + +def test_tel_tables_present_in_schema_sql(): + for tbl in TEL_TABLES: + assert f"CREATE TABLE IF NOT EXISTS {tbl}" in hermes_state.SCHEMA_SQL + + +def test_tel_tables_created_on_executescript(tmp_path): + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.executescript(hermes_state.SCHEMA_SQL) + rows = { + r[0] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + } + conn.close() + assert TEL_TABLES.issubset(rows), f"missing: {TEL_TABLES - rows}" + + +def test_executescript_is_idempotent(tmp_path): + # IF NOT EXISTS means re-running on an existing DB is a no-op, not an error. + db = tmp_path / "state.db" + conn = sqlite3.connect(db) + conn.executescript(hermes_state.SCHEMA_SQL) + conn.executescript(hermes_state.SCHEMA_SQL) # second run must not raise + conn.close() diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 1c119aabca5..3b94efdd4cb 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -140,6 +140,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Image generation backends ───────────────────────────────────────── "image.fal": ("fal-client==0.13.1",), + # ─── Observability ───────────────────────────────────────────────────── + # OTLP telemetry export. Lazily installed on first use of + # `hermes telemetry export --otlp`. Tracks the `otlp` extra in + # pyproject.toml — bump both together. + "export.otlp": ( + "opentelemetry-sdk==1.30.0", + "opentelemetry-exporter-otlp-proto-http==1.30.0", + ), + # ─── Memory providers ────────────────────────────────────────────────── "memory.honcho": ("honcho-ai==2.2.0",), "memory.hindsight": ("hindsight-client==0.6.1",), From e829d03fec0c0fc4a1782d7d8ca006e4dd8814c8 Mon Sep 17 00:00:00 2001 From: emozilla Date: Thu, 25 Jun 2026 21:32:30 -0400 Subject: [PATCH 02/29] refactor(telemetry): drop policy.resolve(); read config directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit policy.resolve() / TelemetryDecision was a read-only projection used only by `hermes telemetry status` for display. The actual behavior gates already read telemetry.* straight from config: the emitter (whether to write) and the plugin loader (whether to auto-load) each call .get("local", True) on the loaded config, never through policy. Make config the single chokepoint the status command reads too: it now resolves local/allow_aggregate/consent_state inline from the loaded config, the same way the other gates do. policy.py keeps only what config can't express on its own — the consent constants, ensure_install_id(), and may_upload_aggregate(config) as a pure function (the gate a future uploader must consult). resolve() and the TelemetryDecision dataclass are removed; policy.py drops 107 -> 70 lines. No behavior change: status renders identically, and the default-on local plane is still defaulted in DEFAULT_CONFIG plus a fail-safe .get(..., True) at each gate. --- agent/telemetry/__init__.py | 4 +- agent/telemetry/policy.py | 65 ++++++-------------------- hermes_cli/main.py | 25 ++++++---- tests/telemetry/test_governance.py | 7 +-- tests/telemetry/test_policy_consent.py | 43 ++++++----------- 5 files changed, 48 insertions(+), 96 deletions(-) diff --git a/agent/telemetry/__init__.py b/agent/telemetry/__init__.py index 4c79dd329c0..d824d5e6f4b 100644 --- a/agent/telemetry/__init__.py +++ b/agent/telemetry/__init__.py @@ -8,8 +8,8 @@ raises into a model/tool call (the hot-path invariant). Events record the observed model ids, provider names, and tool names. ``metrics`` derives rollups for /usage and /insights; ``rollup`` builds the per-run summaries shown by ``hermes telemetry preview``. ``redaction`` + ``exporter_bulk`` + ``otlp_exporter`` -handle export to an operator-chosen destination. ``policy`` holds the consent state -machine for the opt-in aggregate plane (no uploader ships). +handle export to an operator-chosen destination. ``policy`` holds the consent +constants and the aggregate upload gate (no uploader ships). """ from __future__ import annotations diff --git a/agent/telemetry/policy.py b/agent/telemetry/policy.py index f1c001b4c9a..6090857eab5 100644 --- a/agent/telemetry/policy.py +++ b/agent/telemetry/policy.py @@ -1,49 +1,32 @@ """Telemetry consent posture and the aggregate-plane gate. -Consent is a single field, ``telemetry.consent_state``: +Consent is a single config field, ``telemetry.consent_state``: * "unknown" — no choice recorded; never uploads (the default). * "local" — declined the aggregate plane; local plane only. * "aggregate" — opted in to the aggregate plane. The config file is the source of truth: set ``telemetry.consent_state`` with -``hermes config set`` (or a managed-scope pin). There is no separate boolean mirror — -a single field cannot drift out of sync with itself, so a stray value can't -accidentally imply consent. +``hermes config set`` (or a managed-scope pin). Callers that gate behavior read +``telemetry.*`` directly from config; this module only provides the consent +constants, the install-id helper, and the upload gate a future uploader must +consult. ``allow_aggregate`` is the hard gate. An administrator pins ``telemetry.allow_aggregate: false`` through the managed-scope layer -(``/etc/hermes/config.yaml``), which takes precedence over the user's config; when it -is false, the aggregate plane is off regardless of ``consent_state``. - -This module makes the decisions; it performs no I/O and contains no uploader. A future -uploader must call :func:`may_upload_aggregate` at its boundary. +(``/etc/hermes/config.yaml``), which takes precedence over the user's config; when +it is false, the aggregate plane is off regardless of ``consent_state``. """ from __future__ import annotations import uuid -from dataclasses import dataclass from typing import Any, Dict CONSENT_UNKNOWN = "unknown" CONSENT_LOCAL = "local" CONSENT_AGGREGATE = "aggregate" -_VALID_STATES = {CONSENT_UNKNOWN, CONSENT_LOCAL, CONSENT_AGGREGATE} - - -@dataclass(slots=True) -class TelemetryDecision: - """The resolved telemetry posture for the current process.""" - local_enabled: bool - aggregate_enabled: bool - consent_state: str - install_id: str - allow_aggregate: bool - - def may_upload_aggregate(self) -> bool: - """The single gate the uploader must consult before any network send.""" - return self.allow_aggregate and self.consent_state == CONSENT_AGGREGATE +VALID_CONSENT_STATES = {CONSENT_UNKNOWN, CONSENT_LOCAL, CONSENT_AGGREGATE} def _telemetry_cfg(config: Dict[str, Any]) -> Dict[str, Any]: @@ -65,43 +48,23 @@ def ensure_install_id(config: Dict[str, Any]) -> str: return str(uuid.uuid4()) -def resolve(config: Dict[str, Any]) -> TelemetryDecision: - """Resolve the effective telemetry posture from config. +def may_upload_aggregate(config: Dict[str, Any]) -> bool: + """Whether the aggregate plane may upload — the gate a future uploader consults. - ``consent_state`` is the single source of truth for the aggregate opt-in. - ``allow_aggregate`` (admin-pinnable via managed scope) hard-disables the aggregate - plane regardless of consent. + True only when the admin hard gate allows it AND the user has opted in via + ``telemetry.consent_state``. """ tel = _telemetry_cfg(config) - - local_enabled = bool(tel.get("local", True)) allow_aggregate = bool(tel.get("allow_aggregate", True)) state = tel.get("consent_state", CONSENT_UNKNOWN) - if state not in _VALID_STATES: - state = CONSENT_UNKNOWN - - aggregate_enabled = allow_aggregate and state == CONSENT_AGGREGATE - - return TelemetryDecision( - local_enabled=local_enabled, - aggregate_enabled=aggregate_enabled, - consent_state=state, - install_id=ensure_install_id(config), - allow_aggregate=allow_aggregate, - ) - - -def may_upload_aggregate(config: Dict[str, Any]) -> bool: - """Convenience gate for the uploader boundary.""" - return resolve(config).may_upload_aggregate() + return allow_aggregate and state == CONSENT_AGGREGATE __all__ = [ "CONSENT_UNKNOWN", "CONSENT_LOCAL", "CONSENT_AGGREGATE", - "TelemetryDecision", - "resolve", + "VALID_CONSENT_STATES", "may_upload_aggregate", "ensure_install_id", ] diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 46017aaa13f..41f6565caa0 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -14322,25 +14322,32 @@ def cmd_telemetry(args): action = getattr(args, "telemetry_action", None) or "status" config = load_config() - decision = policy.resolve(config) + tel = config.get("telemetry") if isinstance(config.get("telemetry"), dict) else {} + local_enabled = bool(tel.get("local", True)) + allow_aggregate = bool(tel.get("allow_aggregate", True)) + consent_state = tel.get("consent_state", policy.CONSENT_UNKNOWN) + if consent_state not in policy.VALID_CONSENT_STATES: + consent_state = policy.CONSENT_UNKNOWN + aggregate_enabled = allow_aggregate and consent_state == policy.CONSENT_AGGREGATE + install_id = policy.ensure_install_id(config) def _persist_install_id(): # Make sure a minted id is written back so it stays stable. - config.setdefault("telemetry", {})["install_id"] = decision.install_id + config.setdefault("telemetry", {})["install_id"] = install_id save_config(config) if action == "status": print("Telemetry status") print("─" * 56) - print(f" Local plane: {'on' if decision.local_enabled else 'off'} " + print(f" Local plane: {'on' if local_enabled else 'off'} " f"(telemetry.local)") - print(f" Aggregate plane: {'on' if decision.aggregate_enabled else 'off'} " - f"(opt-in; consent_state={decision.consent_state})") - if decision.consent_state != policy.CONSENT_AGGREGATE and decision.allow_aggregate: + print(f" Aggregate plane: {'on' if aggregate_enabled else 'off'} " + f"(opt-in; consent_state={consent_state})") + if consent_state != policy.CONSENT_AGGREGATE and allow_aggregate: print(" opt in: hermes config set telemetry.consent_state aggregate") - if not decision.allow_aggregate: + if not allow_aggregate: print(" ⚠ allow_aggregate is false (egress hard-disabled)") - print(f" Install id: {decision.install_id}") + print(f" Install id: {install_id}") print(" Upload: DISABLED — no server yet. Aggregate is computed " "locally only.") print() @@ -14393,7 +14400,7 @@ def cmd_telemetry(args): _persist_install_id() since_ns = int((time.time() - args.days * 86400) * 1e9) events = rollup.build_aggregate_events( - install_id=decision.install_id, since_ns=since_ns + install_id=install_id, since_ns=since_ns ) summary = rollup.summarize(events) print("Telemetry preview — computed locally, NOT uploaded") diff --git a/tests/telemetry/test_governance.py b/tests/telemetry/test_governance.py index 37bf1810d8d..4c94cab5df9 100644 --- a/tests/telemetry/test_governance.py +++ b/tests/telemetry/test_governance.py @@ -83,7 +83,7 @@ def test_allow_aggregate_pin_blocks_opt_in(home): """A managed allow_aggregate:false pin overrides a consent_state opt-in. Consent is set in config (as a user or managed-scope pin would); the hard gate - still wins, so the aggregate plane resolves off and may_upload stays false. + still wins, so may_upload stays false. """ from hermes_cli.config import load_config, save_config from agent.telemetry import policy @@ -92,7 +92,4 @@ def test_allow_aggregate_pin_blocks_opt_in(home): tel["consent_state"] = "aggregate" tel["allow_aggregate"] = False save_config(c) - d = policy.resolve(load_config()) - assert d.allow_aggregate is False - assert d.aggregate_enabled is False - assert d.may_upload_aggregate() is False + assert policy.may_upload_aggregate(load_config()) is False diff --git a/tests/telemetry/test_policy_consent.py b/tests/telemetry/test_policy_consent.py index f3ace454770..7bc0d68e2fe 100644 --- a/tests/telemetry/test_policy_consent.py +++ b/tests/telemetry/test_policy_consent.py @@ -1,8 +1,9 @@ -"""Consent posture + org-policy enforcement tests. +"""Consent gate tests. -Consent is a single field (``telemetry.consent_state``); the aggregate opt-in is -expressed by setting it to ``"aggregate"`` (via ``hermes config set`` or a managed-scope -pin). ``allow_aggregate`` is the hard gate. +Consent is a single config field (``telemetry.consent_state``); the aggregate opt-in +is expressed by setting it to ``"aggregate"`` (via ``hermes config set`` or a +managed-scope pin). ``allow_aggregate`` is the hard gate. ``policy.may_upload_aggregate`` +is the gate a future uploader must consult. """ from __future__ import annotations @@ -14,43 +15,27 @@ def _cfg(**telemetry): return {"telemetry": telemetry} -def test_default_posture_is_local_only(): - d = policy.resolve(_cfg(local=True, consent_state="unknown")) - assert d.local_enabled is True - assert d.aggregate_enabled is False - assert d.may_upload_aggregate() is False +def test_default_posture_never_uploads(): + # No consent recorded → unknown → never uploads. + assert policy.may_upload_aggregate(_cfg(local=True, consent_state="unknown")) is False -def test_unknown_consent_never_uploads(): - # A headless box with no choice recorded: stays unknown, never uploads. - d = policy.resolve(_cfg(local=True, consent_state="unknown")) - assert d.may_upload_aggregate() is False +def test_missing_telemetry_block_never_uploads(): + assert policy.may_upload_aggregate({}) is False def test_opted_in_uploads(): - d = policy.resolve(_cfg(local=True, consent_state="aggregate")) - assert d.aggregate_enabled is True - assert d.may_upload_aggregate() is True + assert policy.may_upload_aggregate(_cfg(consent_state="aggregate")) is True def test_declined_does_not_upload(): - d = policy.resolve(_cfg(local=True, consent_state="local")) - assert d.may_upload_aggregate() is False + assert policy.may_upload_aggregate(_cfg(consent_state="local")) is False def test_allow_aggregate_false_overrides_opt_in(): # An admin pins telemetry.allow_aggregate: false via managed scope. - cfg = _cfg(local=True, consent_state="aggregate", allow_aggregate=False) - d = policy.resolve(cfg) - assert d.allow_aggregate is False - assert d.aggregate_enabled is False - assert d.may_upload_aggregate() is False # the hard gate wins - - -def test_invalid_consent_state_treated_as_unknown(): - d = policy.resolve(_cfg(local=True, consent_state="bogus")) - assert d.consent_state == "unknown" - assert d.may_upload_aggregate() is False + cfg = _cfg(consent_state="aggregate", allow_aggregate=False) + assert policy.may_upload_aggregate(cfg) is False # the hard gate wins def test_install_id_minted_when_empty_and_stable_when_set(): From 72b7af73bd26e01abadf974929f3f17c8b99efb1 Mon Sep 17 00:00:00 2001 From: emozilla Date: Thu, 25 Jun 2026 23:39:17 -0400 Subject: [PATCH 03/29] refactor(telemetry): drop "plane" terminology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the telemetry tiers away from the borrowed control-plane/data-plane jargon to plain language, across code, CLI output, config, and docs: - "local plane" -> "local telemetry" - "aggregate plane" -> "aggregate metrics" - "trajectories plane" -> "trajectories" / "telemetry.trajectories" - "three planes with a hard wall" -> "three settings, isolated from each other" User-facing `hermes telemetry status` now reads "Local telemetry: on" / "Aggregate metrics: off" / "Content export: off (trajectories disabled)". The OTLP resource attribute key telemetry.plane is renamed to telemetry.scope (wire-level identifier; nothing consumes it yet). No behavior change — wording only. Status renders identically apart from the labels; tests updated to match the new strings. --- agent/insights.py | 4 +- agent/telemetry/emitter.py | 4 +- agent/telemetry/events.py | 4 +- agent/telemetry/exporter_bulk.py | 6 +- agent/telemetry/metrics.py | 2 +- agent/telemetry/otlp_exporter.py | 6 +- agent/telemetry/policy.py | 10 +-- agent/telemetry/redaction.py | 6 +- cli-config.yaml.example | 16 ++--- docs/observability/telemetry.md | 78 ++++++++++++------------ hermes_cli/config.py | 18 +++--- hermes_cli/main.py | 22 +++---- hermes_cli/subcommands/telemetry.py | 10 +-- plugins/telemetry/__init__.py | 8 +-- tests/telemetry/test_cli_telemetry.py | 10 +-- tests/telemetry/test_export_redaction.py | 2 +- tests/telemetry/test_governance.py | 2 +- 17 files changed, 105 insertions(+), 103 deletions(-) diff --git a/agent/insights.py b/agent/insights.py index 304410707ed..b9370b35cc5 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -181,7 +181,7 @@ class InsightsEngine: } # ========================================================================= - # Telemetry (observability) — from the tel_* tables (local plane) + # Telemetry (observability) — from the tel_* tables (local telemetry) # ========================================================================= def _compute_telemetry(self, cutoff: float) -> Dict[str, Any]: @@ -1067,7 +1067,7 @@ class InsightsEngine: lines.append(f" {ts['label']:<20} {ts['value']:<18} ({ts['date']}, {ts['session_id']})") lines.append("") - # Telemetry / observability (local plane) — only when data exists + # Telemetry / observability (local telemetry) — only when data exists tel = report.get("telemetry") or {} if tel: self._append_telemetry_section(lines, tel) diff --git a/agent/telemetry/emitter.py b/agent/telemetry/emitter.py index d6f3788cca0..6b628ea5626 100644 --- a/agent/telemetry/emitter.py +++ b/agent/telemetry/emitter.py @@ -1,4 +1,4 @@ -"""Local-plane telemetry emitter: fire-and-forget queue + background writer. +"""Local telemetry emitter: fire-and-forget queue + background writer. The emitter is the single seam between instrumentation (the telemetry plugin's hook callbacks) and durable storage. Its contract is the hot-path invariant: @@ -16,7 +16,7 @@ Mechanism: * The writer uses its own sqlite connection to state.db, separate from SessionDB, so telemetry writes never contend with or corrupt session writes. -Local plane only. Nothing here uploads anywhere. +Local telemetry only. Nothing here uploads anywhere. """ from __future__ import annotations diff --git a/agent/telemetry/events.py b/agent/telemetry/events.py index f084659f726..4125aaff414 100644 --- a/agent/telemetry/events.py +++ b/agent/telemetry/events.py @@ -1,4 +1,4 @@ -"""Typed local-plane telemetry events. +"""Typed local telemetry events. These dataclasses are the rows written to the local JSONL log and the ``tel_*`` SQLite tables. They record the values observed for each run — model id, provider, tool @@ -11,7 +11,7 @@ import time from dataclasses import dataclass, field, asdict from typing import Any, Dict, Optional -# ── local-plane events (real values) ──────────────────────────────────────── +# ── local telemetry events (real values) ──────────────────────────────────── def _now_ns() -> int: diff --git a/agent/telemetry/exporter_bulk.py b/agent/telemetry/exporter_bulk.py index e0dd1e26d04..32be393f723 100644 --- a/agent/telemetry/exporter_bulk.py +++ b/agent/telemetry/exporter_bulk.py @@ -3,7 +3,7 @@ Two data domains, both written to an operator-chosen destination: * Telemetry: the tel_* rows + events.jsonl (structural observability). - * Content (opt-in via the trajectories plane): sessions + messages, with every + * Content (opt-in via telemetry.trajectories): sessions + messages, with every content field (message body, reasoning, raw tool-call args) passed through the redaction pipeline (secrets always stripped; PII per content_redaction). @@ -103,10 +103,10 @@ def export( ) -> Dict[str, int]: """Write telemetry (+ optional content) to ``out``. Returns counts. - ``include_content`` is honored only when the trajectories plane is enabled in + ``include_content`` is honored only when telemetry.trajectories is enabled in ``config``; otherwise content is forced off and only structural data is written. """ - # Trajectories gate: a flag cannot override the consent plane. + # Trajectories gate: a flag cannot override the config setting. content_allowed = include_content and redaction.content_export_enabled(config) counts = {"telemetry": 0, "sessions": 0, "content_included": int(content_allowed)} diff --git a/agent/telemetry/metrics.py b/agent/telemetry/metrics.py index 389a9ad5e79..fda0b4ee413 100644 --- a/agent/telemetry/metrics.py +++ b/agent/telemetry/metrics.py @@ -51,7 +51,7 @@ def workflow_summary( *, conn: Optional[sqlite3.Connection] = None, ) -> Dict[str, Any]: - """Run-level counters + duration percentiles (local plane, exact).""" + """Run-level counters + duration percentiles (local telemetry, exact).""" with _cursor(conn, db_path) as c: where = _since_clause(since_ns) total = c.execute(f"SELECT COUNT(*) n FROM tel_runs{where}").fetchone()["n"] diff --git a/agent/telemetry/otlp_exporter.py b/agent/telemetry/otlp_exporter.py index 141c10b0164..32912b9da27 100644 --- a/agent/telemetry/otlp_exporter.py +++ b/agent/telemetry/otlp_exporter.py @@ -16,7 +16,7 @@ Notes: and is fail-isolated, so an export error cannot affect a run. Spans carry structural telemetry by default. Message content is included only when the -trajectories plane is enabled, and always passes through the export redaction pipeline. +trajectories is enabled, and always passes through the export redaction pipeline. """ from __future__ import annotations @@ -119,7 +119,7 @@ def _make_provider(config: Dict[str, Any]): sdk = _require_sdk() resource = sdk["Resource"].create({ "service.name": "hermes-agent", - "telemetry.plane": "local", # never aggregate + "telemetry.scope": "local", # never aggregate metrics }) provider = sdk["TracerProvider"](resource=resource) processor = sdk["BatchSpanProcessor"](build_exporter(config)) @@ -129,7 +129,7 @@ def _make_provider(config: Dict[str, Any]): # ── event -> span attribute mapping (real values) ─────────────────────────── def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: - """Span attributes for an event — the real recorded values (local plane).""" + """Span attributes for an event — the real recorded values (local telemetry).""" kind = ev.get("event") attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} keep_by_kind = { diff --git a/agent/telemetry/policy.py b/agent/telemetry/policy.py index 6090857eab5..f98797c2b4f 100644 --- a/agent/telemetry/policy.py +++ b/agent/telemetry/policy.py @@ -1,10 +1,10 @@ -"""Telemetry consent posture and the aggregate-plane gate. +"""Telemetry consent posture and the aggregate-metrics gate. Consent is a single config field, ``telemetry.consent_state``: * "unknown" — no choice recorded; never uploads (the default). - * "local" — declined the aggregate plane; local plane only. - * "aggregate" — opted in to the aggregate plane. + * "local" — declined aggregate metrics; local telemetry only. + * "aggregate" — opted in to aggregate metrics. The config file is the source of truth: set ``telemetry.consent_state`` with ``hermes config set`` (or a managed-scope pin). Callers that gate behavior read @@ -15,7 +15,7 @@ consult. ``allow_aggregate`` is the hard gate. An administrator pins ``telemetry.allow_aggregate: false`` through the managed-scope layer (``/etc/hermes/config.yaml``), which takes precedence over the user's config; when -it is false, the aggregate plane is off regardless of ``consent_state``. +it is false, aggregate metrics are off regardless of ``consent_state``. """ from __future__ import annotations @@ -49,7 +49,7 @@ def ensure_install_id(config: Dict[str, Any]) -> str: def may_upload_aggregate(config: Dict[str, Any]) -> bool: - """Whether the aggregate plane may upload — the gate a future uploader consults. + """Whether aggregate metrics may upload — the gate a future uploader consults. True only when the admin hard gate allows it AND the user has opted in via ``telemetry.consent_state``. diff --git a/agent/telemetry/redaction.py b/agent/telemetry/redaction.py index a65ea695172..01090abb22a 100644 --- a/agent/telemetry/redaction.py +++ b/agent/telemetry/redaction.py @@ -6,7 +6,7 @@ Two independent controls: disables this. Wraps ``agent/redact.py::redact_sensitive_text(force=True)``. * Whether message bodies, reasoning, and raw tool arguments are exportable at all is - governed by the trajectories plane (``telemetry.trajectories.enabled``, default + governed by the trajectories setting (``telemetry.trajectories.enabled``, default off, admin-pinnable), not by a redaction mode. With trajectories off, content is dropped. With it on, content is exportable and ``content_redaction`` (none|pii) controls how much is scrubbed; secrets are still always stripped. @@ -62,7 +62,7 @@ def redact_for_export( """Redact a single content string for export. Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. - Callers gate *whether content is exported at all* via the trajectories plane + Callers gate *whether content is exported at all* via telemetry.trajectories (see ``content_export_enabled``); this function only scrubs content that the caller has already decided to export. """ @@ -75,7 +75,7 @@ def redact_for_export( def content_export_enabled(config: Optional[Dict[str, Any]]) -> bool: - """True only when the trajectories plane is explicitly enabled. + """True only when telemetry.trajectories is explicitly enabled. This is the consent gate for exporting message bodies / reasoning / raw tool args. Default off. Admin-pinnable via managed scope (telemetry.trajectories.enabled). diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 89ace2dd657..982708c3422 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1417,13 +1417,13 @@ display: # ============================================================================= # Telemetry & Observability # ============================================================================= -# Three planes with a hard wall between them: +# Three settings, isolated from each other: # local — full-fidelity observability you own. Default ON. # aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. # trajectories — content trajectories for training. Separate consent (later). # -# The local plane records real values (actual models, providers, tool names) — -# your own data, on your machine. The aggregate plane is opt-in and has no +# Local telemetry records real values (actual models, providers, tool names) — +# your own data, on your machine. Aggregate metrics are opt-in and have no # uploader today; if one ships it would summarize at that egress boundary. # # Enterprise locking: any telemetry.* key can be pinned by an administrator via @@ -1431,16 +1431,16 @@ display: # value. To hard-forbid egress on a locked-down deployment, pin # `telemetry.allow_aggregate: false` there. # telemetry: -# # Local plane: event log + SQLite index in state.db. Never leaves your -# # machine unless you export it or opt into the aggregate plane. +# # Local telemetry: event log + SQLite index in state.db. Never leaves your +# # machine unless you export it or opt into aggregate metrics. # local: true -# # Hard gate for the aggregate plane. When false, the aggregate plane is off +# # Hard gate for aggregate metrics. When false, aggregate metrics are off # # regardless of consent_state. Pin false via managed scope to forbid egress. # allow_aggregate: true -# # Aggregate-plane consent (the opt-in). No uploader ships yet. +# # Aggregate-metrics consent (the opt-in). No uploader ships yet. # # unknown (no choice — never uploads) | local (declined) | aggregate (opted in) # consent_state: unknown -# # Stable install id (aggregate plane only). Empty = mint on first use; +# # Stable install id (aggregate metrics only). Empty = mint on first use; # # clear it to rotate. # install_id: "" # # Local event-log retention before rotation (days). diff --git a/docs/observability/telemetry.md b/docs/observability/telemetry.md index cdffc43922b..98747bb635a 100644 --- a/docs/observability/telemetry.md +++ b/docs/observability/telemetry.md @@ -6,57 +6,57 @@ agent does — workflows, model calls, tool calls, errors — to your own machin stack. It is private by default and never sends your data to Nous unless you explicitly opt in to anonymous aggregate metrics. -This page explains the whole feature: the three planes, what's captured, the +This page explains the whole feature: the three telemetry settings, what's captured, the `hermes telemetry` commands, and how an enterprise streams or exports all of its data to its own infrastructure. > Looking for the **plugin hook contract** (how to write your own observer plugin)? > That's in [`README.md`](./README.md). This page is about the built-in telemetry -> plane and its CLI. +> system and its CLI. -## The three planes +## The three settings -Telemetry is organized into three planes with a hard wall between them: +Telemetry has three settings, isolated from each other: -| Plane | What it holds | Default | Destination | +| Setting | What it holds | Default | Destination | | --- | --- | --- | --- | | **local** | Full-fidelity observability — runs, model/tool calls (real model & provider names), durations, errors | **on** | your machine only | | **aggregate** | Opt-in metadata to Nous (no uploader ships yet) | **off** (opt-in) | Nous, only if you enable it | | **trajectories** | Full message content / reasoning / raw tool args | **off** (opt-in) | your own export destinations only | -The local plane is the one you'll use day to day — it records the real values that -happened (actual model ids, providers, tool names). The aggregate plane is the only -thing that could ever leave for Nous; it is opt-in, default-off, and has no uploader -today. The trajectories plane unlocks full-content export to *your own* destinations — -it is never wired to Nous. +Local telemetry is the one you'll use day to day — it records the real values that +happened (actual model ids, providers, tool names). Aggregate metrics are the only +thing that could ever leave for Nous; they are opt-in, default-off, and have no uploader +today. Trajectories unlock full-content export to *your own* destinations — never wired +to Nous. -## Local plane — always-on observability +## Local telemetry — always-on observability -The local plane is implemented as a bundled `telemetry` plugin that listens to Hermes +Local telemetry is implemented as a bundled `telemetry` plugin that listens to Hermes lifecycle hooks (model calls, tool calls, session start/finalize) and writes events to: - an append-only JSONL log at `~/.hermes/telemetry/events.jsonl` (the source of truth) - indexed `tel_*` tables in `state.db` (for fast queries and rollups) Writes are fire-and-forget on a background thread: telemetry can never block, slow, or -fail a model call or tool call. If the local plane is disabled (`telemetry.local: false`) +fail a model call or tool call. If local telemetry is disabled (`telemetry.local: false`) the plugin does not load at all. ### Seeing your local data ```bash hermes insights # usage report — now includes an "Observability" section -hermes telemetry status # planes, consent, export posture, local data volume +hermes telemetry status # settings, consent, export posture, local data volume ``` The `insights` Observability section shows workflow counts and success rate, duration -p50/p95, tool failure rates by category, provider/model-class mix, and cache hit rate — +p50/p95, tool failure rates by category, provider/model mix, and cache hit rate — all computed locally with exact values. ## `hermes telemetry` commands ```text -hermes telemetry status Show planes, consent state, export posture, local volume +hermes telemetry status Show settings, consent state, export posture, local volume hermes telemetry preview Show the aggregate events that would be produced (local) hermes telemetry export Export local telemetry to a file/stream or OTLP endpoint ``` @@ -65,14 +65,14 @@ Consent and the install id are plain config, not separate verbs — set them wit `hermes config set` (or a managed-scope pin): ```bash -hermes config set telemetry.consent_state aggregate # opt in to the aggregate plane -hermes config set telemetry.consent_state local # opt out (local plane stays on) +hermes config set telemetry.consent_state aggregate # opt in to aggregate metrics +hermes config set telemetry.consent_state local # opt out (local telemetry stays on) hermes config set telemetry.install_id "" # reset the install id (mints a new one) ``` -### Aggregate plane (opt-in) +### Aggregate metrics (opt-in) -The aggregate plane is **off by default** and has **no uploader today** — nothing is +Aggregate metrics are **off by default** and have **no uploader today** — nothing is sent to Nous. Consent lives in `telemetry.consent_state` (`unknown` / `local` / `aggregate`); setting it to `aggregate` records the opt-in for if/when an uploader ships. If one is built, it would summarize at that egress boundary. @@ -99,11 +99,11 @@ hermes telemetry export --out dump.json --format json --since 7 By default the export is **structural** — runs, model/tool-call metadata, session shells with message *counts* but no message bodies. -### Including content (trajectories plane) +### Including content (trajectories) -To export full message content, enable the trajectories plane. This is a deliberate, -separate consent — it's how an enterprise opts into exporting work-product content to its -own store: +To export full message content, enable trajectories. This is a deliberate, separate +consent — it's how an enterprise opts into exporting work-product content to its own +store: ```yaml # config.yaml @@ -117,8 +117,8 @@ telemetry: hermes telemetry export --out full.ndjson --include-content ``` -`--include-content` is a no-op unless the trajectories plane is enabled — the consent -plane governs, not the flag. +`--include-content` is a no-op unless trajectories are enabled — the config setting +governs, not the flag. ### Live streaming to your OpenTelemetry Collector / SIEM (OTLP) @@ -137,17 +137,19 @@ telemetry: enabled: true endpoint: "https://collector.your-corp.internal:4318/v1/traces" headers_env: # secrets by reference — env var NAMES, not values - Authorization: CORP_OTLP_TOKEN + Authorization: MY_OTLP_TOKEN_ENVVAR ``` +Set the referenced environment variable, then run the export: + ```bash -export CORP_OTLP_TOKEN="..." # the actual token lives in the environment hermes telemetry export --otlp # drain current telemetry to your collector ``` -Span attributes are structural by default. For authentication, the config holds the -*name* of an environment variable rather than the secret itself; the value is read at -export time and is never written to config or logged. +The token value lives only in the environment variable named by `headers_env`. Span +attributes are structural by default. The config holds the *name* of an environment +variable rather than the secret itself; the value is read at export time and is never +written to config or logged. ## Redaction @@ -166,8 +168,8 @@ fails closed: if the redactor can't run, the raw string is not emitted. ```yaml telemetry: - local: true # local plane (default on) - allow_aggregate: true # hard gate; pin false to forbid the aggregate plane entirely + local: true # local telemetry (default on) + allow_aggregate: true # hard gate; pin false to forbid aggregate metrics entirely consent_state: unknown # aggregate opt-in: unknown | local | aggregate install_id: "" # stable anon id; "" mints one; clear to rotate retention_days: 90 # local event-log retention @@ -189,7 +191,7 @@ layer (`/etc/hermes/config.yaml`), which wins over the user's value on a per-key There is no telemetry-specific policy block — to lock down a fleet, pin the keys you care about. Common examples: -- `telemetry.allow_aggregate: false` — the aggregate plane stays off even if +- `telemetry.allow_aggregate: false` — aggregate metrics stay off even if `consent_state` is set to `aggregate`. - `telemetry.export.otlp.endpoint` — point every install at the corporate collector. - `telemetry.trajectories.enabled` — centrally decide whether content export is allowed. @@ -202,8 +204,8 @@ secret values. ## Privacy summary - Local telemetry never leaves your machine. -- The aggregate plane (the only thing that could go to Nous) is opt-in, default-off, - and has no uploader today — nothing is sent. +- Aggregate metrics (the only thing that could go to Nous) are opt-in, default-off, + and have no uploader today — nothing is sent. - All export surfaces (file, OTLP) point at *your* destinations. -- Secrets are always redacted on export; content export is off until you enable the - trajectories plane; PII redaction is a knob. +- Secrets are always redacted on export; content export is off until you enable + trajectories; PII redaction is a knob. diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 18247ca48e4..22290ca9ef3 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2175,7 +2175,7 @@ DEFAULT_CONFIG = { "redact_pii": False, # When True, hash user IDs and strip phone numbers from LLM context }, - # Telemetry & observability. Three planes with a hard wall between them: + # Telemetry & observability. Three settings, isolated from each other: # local — full-fidelity local observability the user owns (real # models, providers, tool names). Default ON. # aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. @@ -2186,30 +2186,30 @@ DEFAULT_CONFIG = { # value on a per-key basis. There is no telemetry-specific policy block — pin # e.g. `telemetry.allow_aggregate: false` in managed scope to hard-forbid egress. "telemetry": { - # Local plane: event log + SQLite index in state.db. The user's own data; - # never leaves the machine unless they export it or opt into aggregate. + # Local telemetry: event log + SQLite index in state.db. The user's own data; + # never leaves the machine unless they export it or opt into aggregate metrics. "local": True, - # Hard gate for the aggregate plane. When False, the aggregate plane is off + # Hard gate for aggregate metrics. When False, aggregate metrics are off # regardless of consent_state. Intended to be pinned False by an administrator # via managed scope on locked-down or air-gapped deployments. Default True # (consent_state still governs the opt-in). "allow_aggregate": True, - # Aggregate-plane consent — the single source of truth for the opt-in: + # Aggregate-metrics consent — the single source of truth for the opt-in: # "unknown" (no choice made — never uploads), "local" (declined), # "aggregate" (opted in). Set with `hermes config set telemetry.consent_state` # or a managed-scope pin. Non-interactive installs sit at "unknown". "consent_state": "unknown", - # Stable install identifier (aggregate plane only). Empty string means "mint a - # fresh UUID on first use"; clear it to rotate. Never sent on the local plane. + # Stable install identifier (aggregate metrics only). Empty string means "mint a + # fresh UUID on first use"; clear it to rotate. Never sent by local telemetry. "install_id": "", - # Local event-log retention before rotation (days). Local plane only. + # Local event-log retention before rotation (days). Local telemetry only. "retention_days": 90, # Keep secret redaction on even at full local capture (a SIEM full of live # credentials is a bigger attack target). Admin may override via managed scope. "redact_secrets": True, # Content redaction for exports / support bundles: "none" | "pii". "content_redaction": "none", - # Trajectories plane: full message content / reasoning / raw tool args. + # Trajectories: full message content / reasoning / raw tool args. # Off by default. When enabled, full content becomes exportable to the # configured destination — always secret-redacted, and PII-redacted per # content_redaction. This is the consent gate for content export and is diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 41f6565caa0..f8bcea1a57d 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -14339,17 +14339,17 @@ def cmd_telemetry(args): if action == "status": print("Telemetry status") print("─" * 56) - print(f" Local plane: {'on' if local_enabled else 'off'} " + print(f" Local telemetry: {'on' if local_enabled else 'off'} " f"(telemetry.local)") - print(f" Aggregate plane: {'on' if aggregate_enabled else 'off'} " + print(f" Aggregate metrics: {'on' if aggregate_enabled else 'off'} " f"(opt-in; consent_state={consent_state})") if consent_state != policy.CONSENT_AGGREGATE and allow_aggregate: - print(" opt in: hermes config set telemetry.consent_state aggregate") + print(" opt in: hermes config set telemetry.consent_state aggregate") if not allow_aggregate: - print(" ⚠ allow_aggregate is false (egress hard-disabled)") - print(f" Install id: {install_id}") - print(" Upload: DISABLED — no server yet. Aggregate is computed " - "locally only.") + print(" ⚠ allow_aggregate is false (egress hard-disabled)") + print(f" Install id: {install_id}") + print(" Upload: DISABLED — no server yet. Aggregate metrics are " + "computed locally only.") print() # Export posture — where YOUR data can flow (never Nous). Values only; # lock-state is managed scope's concern, surfaced by its own tooling. @@ -14371,12 +14371,12 @@ def cmd_telemetry(args): print(f" SDK: {_sdk}") else: print(" OTLP: disabled (telemetry.export.otlp.enabled)") - # Content gate (trajectories plane) + redaction posture. + # Content gate (telemetry.trajectories) + redaction posture. if redaction.content_export_enabled(config): - print(f" Content export: on (trajectories plane) — message " + print(f" Content export: on (trajectories enabled) — message " f"content exportable") else: - print(" Content export: off (trajectories plane) — structural " + print(" Content export: off (trajectories disabled) — structural " "telemetry only") print(" Secret redaction: on (always)") print(f" PII redaction: {redaction.content_mode_for(config)}") @@ -14468,7 +14468,7 @@ def cmd_telemetry(args): content_ok = redaction.content_export_enabled(config) if want_content and not content_ok: print("⚠ --include-content ignored: telemetry.trajectories.enabled is false.") - print(" Enable the trajectories plane (admin/config) to export message content.") + print(" Enable trajectories (admin/config) to export message content.") print(" Exporting structural telemetry only.") if not getattr(args, "out", None): diff --git a/hermes_cli/subcommands/telemetry.py b/hermes_cli/subcommands/telemetry.py index 65ff344fbb9..791cdeb05b4 100644 --- a/hermes_cli/subcommands/telemetry.py +++ b/hermes_cli/subcommands/telemetry.py @@ -1,7 +1,7 @@ """``hermes telemetry`` subcommand parser. Telemetry control and inspection. ``preview`` shows the per-run summary events that -would be produced for the aggregate plane; there is no uploader, so it terminates as a +would be produced for aggregate metrics; there is no uploader, so it terminates as a local view. The handler is injected to avoid importing ``main`` (mirrors the insights subcommand). @@ -18,14 +18,14 @@ def build_telemetry_parser(subparsers, *, cmd_telemetry: Callable) -> None: "telemetry", help="Inspect local telemetry and export it", description=( - "Local-first telemetry. The local plane records observability on this " - "machine. The aggregate plane is opt-in (set telemetry.consent_state via " - "`hermes config set`); it has no uploader and is shown only via `preview`." + "Local-first telemetry. Local telemetry records observability on this " + "machine. Aggregate metrics are opt-in (set telemetry.consent_state via " + "`hermes config set`); they have no uploader and are shown only via `preview`." ), ) sub = p.add_subparsers(dest="telemetry_action") - sub.add_parser("status", help="Show telemetry planes, consent state, and local data volume") + sub.add_parser("status", help="Show telemetry settings, consent state, and local data volume") prev = sub.add_parser( "preview", diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py index ba61d65054b..43ca8a10e7c 100644 --- a/plugins/telemetry/__init__.py +++ b/plugins/telemetry/__init__.py @@ -1,13 +1,13 @@ -"""Telemetry plugin — wires Hermes lifecycle hooks to the local-plane emitter. +"""Telemetry plugin — wires Hermes lifecycle hooks to the local telemetry emitter. This is the *only* instrumentation seam. It registers observational hooks (which core -already invokes fail-open) and translates each into a typed local-plane telemetry -event handed to ``agent.telemetry.emitter``. There are zero edits to core call sites: +already invokes fail-open) and translates each into a typed local telemetry event +handed to ``agent.telemetry.emitter``. There are zero edits to core call sites: the hooks already carry model/provider/usage/duration/tool data. Everything here is best-effort and fail-open — a raised exception in a hook callback is swallowed by core, and we additionally guard each callback so a telemetry bug can never -disturb a session. No content, no network: local plane only. +disturb a session. No content, no network: local telemetry only. Hooks consumed: on_session_start -> begin a run context (trace_id/run_id), buffer a run row diff --git a/tests/telemetry/test_cli_telemetry.py b/tests/telemetry/test_cli_telemetry.py index df597223c88..d38424df47c 100644 --- a/tests/telemetry/test_cli_telemetry.py +++ b/tests/telemetry/test_cli_telemetry.py @@ -62,7 +62,7 @@ def test_preview_shows_real_values(home, capsys): def test_status_reflects_consent_set_via_config(home, capsys): # Opting in is a plain config write now (no `enable` verb). status should - # reflect consent_state=aggregate as the aggregate plane being on. + # reflect consent_state=aggregate as aggregate metrics being on. from hermes_cli.config import load_config, save_config cfg = load_config() cfg.setdefault("telemetry", {})["consent_state"] = "aggregate" @@ -70,17 +70,17 @@ def test_status_reflects_consent_set_via_config(home, capsys): _run("status") out = capsys.readouterr().out assert "consent_state=aggregate" in out - assert "Aggregate plane: on" in out + assert "Aggregate metrics: on" in out def test_status_shows_optin_hint_when_unknown(home, capsys): _run("status") out = capsys.readouterr().out - assert "Aggregate plane: off" in out + assert "Aggregate metrics: off" in out assert "config set telemetry.consent_state aggregate" in out -def test_allow_aggregate_false_keeps_plane_off_in_status(home, capsys): +def test_allow_aggregate_false_keeps_metrics_off_in_status(home, capsys): # Even with consent opted in, a managed allow_aggregate:false wins. from hermes_cli.config import load_config, save_config cfg = load_config() @@ -90,5 +90,5 @@ def test_allow_aggregate_false_keeps_plane_off_in_status(home, capsys): save_config(cfg) _run("status") out = capsys.readouterr().out - assert "Aggregate plane: off" in out + assert "Aggregate metrics: off" in out assert "allow_aggregate is false" in out diff --git a/tests/telemetry/test_export_redaction.py b/tests/telemetry/test_export_redaction.py index 7fc84c8e205..4ae0473bfba 100644 --- a/tests/telemetry/test_export_redaction.py +++ b/tests/telemetry/test_export_redaction.py @@ -2,7 +2,7 @@ Invariants: * Secrets ALWAYS stripped, every mode, no flag disables it. - * Content gated by the trajectories plane, not a redaction mode. + * Content gated by telemetry.trajectories, not a redaction mode. * PII stripped in 'pii' mode; structure preserved (codec-aware). """ diff --git a/tests/telemetry/test_governance.py b/tests/telemetry/test_governance.py index 4c94cab5df9..97740540367 100644 --- a/tests/telemetry/test_governance.py +++ b/tests/telemetry/test_governance.py @@ -62,7 +62,7 @@ def test_export_block_reflects_trajectories_gate(home, capsys): c.setdefault("telemetry", {})["trajectories"] = {"enabled": True} save_config(c) out = _status(capsys) - assert "Content export: on (trajectories plane)" in out + assert "Content export: on (trajectories enabled)" in out def test_token_env_not_set_shows_not_set(home, capsys): From 0d0549d83464b1f15cf763f626adade2522e7896 Mon Sep 17 00:00:00 2001 From: emozilla Date: Fri, 26 Jun 2026 15:01:10 -0400 Subject: [PATCH 04/29] test(telemetry): end-to-end plugin dispatch coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing hook tests call the plugin's _on_* callbacks directly, which passes even if the bundled plugin stops auto-loading or a hook name drifts from what core fires — real runs would go dark while the suite stays green. Add test_plugin_e2e.py, which drives the real dispatch chain through public entry points only (discover_plugins -> invoke_hook -> registered callback -> emitter -> tel_* tables), exactly as core does: - one completed turn produces tel_runs / tel_model_calls / tel_tool_calls rows with real provider/model/tool values and correct counts; - telemetry.local=false means the plugin does not load and nothing is written. Verified robust against test ordering (singleton resets for the plugin manager and the emitter in the fixture). --- tests/telemetry/test_plugin_e2e.py | 127 +++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 tests/telemetry/test_plugin_e2e.py diff --git a/tests/telemetry/test_plugin_e2e.py b/tests/telemetry/test_plugin_e2e.py new file mode 100644 index 00000000000..c6cf05688d8 --- /dev/null +++ b/tests/telemetry/test_plugin_e2e.py @@ -0,0 +1,127 @@ +"""End-to-end telemetry wiring test. + +Unlike test_plugin_hooks.py (which calls the plugin's ``_on_*`` callbacks directly), +this drives the REAL dispatch chain that core uses at runtime: + + discover_plugins() -> plugin registers hooks -> invoke_hook(name, **kwargs) + -> registered callback -> emitter -> tel_* tables + +If the bundled plugin stops auto-loading, stops registering a hook, or the hook name +drifts from what core fires, the hand-written hook tests still pass but real runs go +dark. This test is the guard against that — it only touches public entry points +(``discover_plugins`` / ``invoke_hook``), exactly as core does. +""" + +from __future__ import annotations + +import sqlite3 +import time + +import pytest + +import hermes_state + + +@pytest.fixture +def runtime(tmp_path, monkeypatch): + """A clean HERMES_HOME with state.db, a fresh plugin manager, and a fresh emitter.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + db = tmp_path / "state.db" + hermes_state.SessionDB(db_path=db) + + # Reset the global plugin-manager singleton so discovery re-runs in this HERMES_HOME. + import hermes_cli.plugins as plugins_mod + monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) + + # Reset the emitter singleton so it binds to this state.db (and tear it down after). + from agent.telemetry import emitter as emitter_mod + emitter_mod.reset_emitter_for_tests(None) + # Clear the plugin's per-run accumulators between tests. + import plugins.telemetry as plug + plug._runs.clear() + + yield db, plugins_mod, emitter_mod, plug + + try: + emitter_mod.get_emitter().flush() + except Exception: + pass + emitter_mod.reset_emitter_for_tests(None) + monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) + + +def _fire_one_turn(invoke_hook): + """Fire the hook sequence of a single completed turn, as core does.""" + invoke_hook("on_session_start", session_id="s1", + model="anthropic/claude-opus-4", platform="cli") + invoke_hook("post_api_request", session_id="s1", platform="cli", + provider="anthropic", base_url=None, model="claude-opus-4", + api_duration=0.9, + usage={"input_tokens": 1000, "output_tokens": 120, + "cache_read_tokens": 0, "cache_write_tokens": 0, + "reasoning_tokens": 0}) + invoke_hook("post_tool_call", session_id="s1", platform="cli", + function_name="web_search", duration_ms=210, result='{"data": "ok"}') + invoke_hook("on_session_finalize", session_id="s1", platform="cli", + turn_exit_reason="completed", estimated_cost_usd=0.01, cost_status="known") + + +def test_real_dispatch_writes_tel_rows(runtime): + """The bundled plugin, loaded via discover_plugins, captures a turn end to end.""" + db, plugins_mod, emitter_mod, _plug = runtime + + plugins_mod.discover_plugins(force=True) + + # The plugin must have registered the lifecycle hooks core fires. + mgr = plugins_mod.get_plugin_manager() + registered = {k for k, v in getattr(mgr, "_hooks", {}).items() if v} + for hook in ("on_session_start", "post_api_request", "post_tool_call", + "on_session_finalize"): + assert hook in registered, f"core hook {hook!r} not registered by the plugin" + + _fire_one_turn(plugins_mod.invoke_hook) + + time.sleep(0.5) # let the background writer drain + emitter_mod.get_emitter().flush() + + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + assert conn.execute("SELECT COUNT(*) c FROM tel_runs").fetchone()["c"] == 1 + assert conn.execute("SELECT COUNT(*) c FROM tel_model_calls").fetchone()["c"] == 1 + assert conn.execute("SELECT COUNT(*) c FROM tel_tool_calls").fetchone()["c"] == 1 + + # Real values, not buckets. + mc = conn.execute("SELECT provider, model FROM tel_model_calls").fetchone() + assert mc["provider"] == "anthropic" + assert mc["model"] == "claude-opus-4" + tc = conn.execute("SELECT tool_name FROM tel_tool_calls").fetchone() + assert tc["tool_name"] == "web_search" + run = conn.execute("SELECT end_reason, model_call_count, tool_call_count " + "FROM tel_runs").fetchone() + assert run["end_reason"] == "completed" + assert run["model_call_count"] == 1 + assert run["tool_call_count"] == 1 + conn.close() + + +def test_local_disabled_writes_nothing(runtime, monkeypatch): + """telemetry.local=false: the plugin does not auto-load, so no rows are written.""" + db, plugins_mod, emitter_mod, _plug = runtime + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"telemetry": {"local": False}}, + raising=False, + ) + + plugins_mod.discover_plugins(force=True) + _fire_one_turn(plugins_mod.invoke_hook) + time.sleep(0.3) + try: + emitter_mod.get_emitter().flush() + except Exception: + pass + + conn = sqlite3.connect(db) + assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0 + assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 0 + conn.close() From 24c86e160be2e0b6b0b73e2f020f31b4a89b40a6 Mon Sep 17 00:00:00 2001 From: emozilla Date: Fri, 26 Jun 2026 15:10:38 -0400 Subject: [PATCH 05/29] fix(telemetry): aggregate requires local telemetry to be on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregate metrics are derived from the local tel_* tables — they're a coarsened view of local data, not an independent capture path. With telemetry.local=false nothing is written, so an aggregate opt-in had nothing to aggregate, yet may_upload_aggregate() returned True and `status` showed "Aggregate metrics: on". The config could claim a state it couldn't fulfill. Gate aggregate on local being enabled: - may_upload_aggregate() now requires local_enabled AND allow_aggregate AND consent_state == aggregate. - `telemetry status` computes aggregate_enabled the same way and, when consent is aggregate but local is off, prints "inert: local telemetry is off — nothing to aggregate" instead of the opt-in hint. Happy path is unchanged (local on + consent aggregate -> on). Adds policy and CLI tests for the inert combo. --- agent/telemetry/policy.py | 8 +++++--- hermes_cli/main.py | 7 +++++-- tests/telemetry/test_cli_telemetry.py | 14 ++++++++++++++ tests/telemetry/test_policy_consent.py | 7 +++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/agent/telemetry/policy.py b/agent/telemetry/policy.py index f98797c2b4f..77e794cf8a2 100644 --- a/agent/telemetry/policy.py +++ b/agent/telemetry/policy.py @@ -51,13 +51,15 @@ def ensure_install_id(config: Dict[str, Any]) -> str: def may_upload_aggregate(config: Dict[str, Any]) -> bool: """Whether aggregate metrics may upload — the gate a future uploader consults. - True only when the admin hard gate allows it AND the user has opted in via - ``telemetry.consent_state``. + Aggregate metrics are derived from the local telemetry tables, so they require + local telemetry to be on. True only when local telemetry is enabled, the admin + hard gate allows it, and the user has opted in via ``telemetry.consent_state``. """ tel = _telemetry_cfg(config) + local_enabled = bool(tel.get("local", True)) allow_aggregate = bool(tel.get("allow_aggregate", True)) state = tel.get("consent_state", CONSENT_UNKNOWN) - return allow_aggregate and state == CONSENT_AGGREGATE + return local_enabled and allow_aggregate and state == CONSENT_AGGREGATE __all__ = [ diff --git a/hermes_cli/main.py b/hermes_cli/main.py index f8bcea1a57d..173eb2a1ede 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -14328,7 +14328,8 @@ def cmd_telemetry(args): consent_state = tel.get("consent_state", policy.CONSENT_UNKNOWN) if consent_state not in policy.VALID_CONSENT_STATES: consent_state = policy.CONSENT_UNKNOWN - aggregate_enabled = allow_aggregate and consent_state == policy.CONSENT_AGGREGATE + aggregate_enabled = (local_enabled and allow_aggregate + and consent_state == policy.CONSENT_AGGREGATE) install_id = policy.ensure_install_id(config) def _persist_install_id(): @@ -14343,7 +14344,9 @@ def cmd_telemetry(args): f"(telemetry.local)") print(f" Aggregate metrics: {'on' if aggregate_enabled else 'off'} " f"(opt-in; consent_state={consent_state})") - if consent_state != policy.CONSENT_AGGREGATE and allow_aggregate: + if consent_state == policy.CONSENT_AGGREGATE and not local_enabled: + print(" ⚠ inert: local telemetry is off — nothing to aggregate") + elif consent_state != policy.CONSENT_AGGREGATE and allow_aggregate: print(" opt in: hermes config set telemetry.consent_state aggregate") if not allow_aggregate: print(" ⚠ allow_aggregate is false (egress hard-disabled)") diff --git a/tests/telemetry/test_cli_telemetry.py b/tests/telemetry/test_cli_telemetry.py index d38424df47c..4aaf4cab744 100644 --- a/tests/telemetry/test_cli_telemetry.py +++ b/tests/telemetry/test_cli_telemetry.py @@ -92,3 +92,17 @@ def test_allow_aggregate_false_keeps_metrics_off_in_status(home, capsys): out = capsys.readouterr().out assert "Aggregate metrics: off" in out assert "allow_aggregate is false" in out + + +def test_local_off_with_consent_shows_inert_in_status(home, capsys): + # local off + opted in: aggregate is off and the status explains why. + from hermes_cli.config import load_config, save_config + cfg = load_config() + tel = cfg.setdefault("telemetry", {}) + tel["local"] = False + tel["consent_state"] = "aggregate" + save_config(cfg) + _run("status") + out = capsys.readouterr().out + assert "Aggregate metrics: off" in out + assert "inert: local telemetry is off" in out diff --git a/tests/telemetry/test_policy_consent.py b/tests/telemetry/test_policy_consent.py index 7bc0d68e2fe..7205c33b275 100644 --- a/tests/telemetry/test_policy_consent.py +++ b/tests/telemetry/test_policy_consent.py @@ -38,6 +38,13 @@ def test_allow_aggregate_false_overrides_opt_in(): assert policy.may_upload_aggregate(cfg) is False # the hard gate wins +def test_local_off_makes_aggregate_inert(): + # Aggregate metrics derive from the local tables; with local off there is + # nothing to aggregate, so opting in cannot upload. + cfg = _cfg(local=False, consent_state="aggregate", allow_aggregate=True) + assert policy.may_upload_aggregate(cfg) is False + + def test_install_id_minted_when_empty_and_stable_when_set(): cfg = _cfg(install_id="") minted = policy.ensure_install_id(cfg) From 81a34156a56b0ecbe19d1b2e1b02ad06a3294a5d Mon Sep 17 00:00:00 2001 From: emozilla Date: Fri, 26 Jun 2026 23:28:50 -0400 Subject: [PATCH 06/29] docs(telemetry): clarify reserved subagent-lineage hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The subagent_start/stop hooks are registered but no-op. The prior comment implied subagents need no handling because they inherit via contextvars — misleading, since a delegated child runs on a separate thread with its own session id and trace. Clarify the real situation: a subagent's model/tool calls are already captured as their own tel_runs row via the child's run_conversation, so nothing is lost. These hooks are reserved for recording parent->child lineage (needs a tel_runs.parent_run_id column), deferred until a consumer needs the delegation tree. Comment-only. --- plugins/telemetry/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py index 43ca8a10e7c..0f5157295c9 100644 --- a/plugins/telemetry/__init__.py +++ b/plugins/telemetry/__init__.py @@ -296,9 +296,15 @@ def _as_float(v: Any) -> Optional[float]: # ── subagent lineage (reserved) ───────────────────────────────────────────── +# A delegated subagent runs its own ``run_conversation`` with its own session id, so +# its model/tool calls are already captured as a separate tel_runs row via the normal +# hooks — no subagent activity is lost. These hooks fire with the parent<->child bridge +# (parent_session_id, child_session_id, child_role, child_goal); they are reserved for +# recording parent->child *lineage* (linking a child run back to its parent), which +# needs a tel_runs.parent_run_id column. Deferred until a consumer needs the delegation +# tree; left registered as the attachment point. @_safe def _on_subagent_start(**kw: Any) -> None: - # Subagents inherit the run context via contextvars; no explicit handling needed. return None From 7c80b79bb01bbd3f14cf82f3bcfdd9453dd27e35 Mon Sep 17 00:00:00 2001 From: emozilla Date: Sat, 27 Jun 2026 00:52:40 -0400 Subject: [PATCH 07/29] =?UTF-8?q?feat(telemetry):=20write=20tel=5Fspans=20?= =?UTF-8?q?=E2=80=94=20reconstructable=20run=20->=20calls=20trace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the review on #51714: the trace/span layer was declared but unwired — tel_spans was never written, call rows had no timestamp, and nothing set parent lineage, so the store was metrics-only and couldn't reconstruct a trace. Wire the span layer (keeping the praised star-schema shape): - New SpanEvent (span_id/trace_id/run_id/parent_span_id/name/kind/start_ns/end_ns) mapped into tel_spans via the emitter's _TABLE_COLUMNS. - The plugin mints a root span per run and, on each model/tool call, emits a SpanEvent (timing + parent = the run's root) keyed by the SAME span_id as the detail row, so tel_model_calls / tel_tool_calls JOIN to their span. - Call hooks fire on completion, so end_ns = now and start_ns is reconstructed from the measured latency/duration. The run's root span is emitted at finalize with the true run start/end. Result: tel_spans is a connected, single-trace_id, run -> calls tree a desktop waterfall (or any reader) can render directly, ordered by start_ns. Existing metrics rows (tel_runs/model_calls/tool_calls) are unchanged. OTLP: spans now flow to the exporter with their trace/parent/timing attributes. The exporter still emits one OTel span per event rather than reconstructing OTel SpanContexts into a connected trace tree; that projection is left for a follow-up and the module docstring now says so plainly instead of over-claiming. Adds test_spans_trace.py (connected-tree + detail-row JOIN) over the real dispatch path. Accurate (pre-hook) start times, real OTLP SpanContexts, and subagent cross-run lineage remain follow-ups. --- agent/telemetry/emitter.py | 5 ++ agent/telemetry/events.py | 24 ++++++ agent/telemetry/otlp_exporter.py | 19 +++-- plugins/telemetry/__init__.py | 77 +++++++++++++++++--- tests/telemetry/test_spans_trace.py | 109 ++++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 tests/telemetry/test_spans_trace.py diff --git a/agent/telemetry/emitter.py b/agent/telemetry/emitter.py index 6b628ea5626..269046c9b23 100644 --- a/agent/telemetry/emitter.py +++ b/agent/telemetry/emitter.py @@ -59,6 +59,11 @@ _TABLE_COLUMNS: Dict[str, tuple] = { "model_call_count", "tool_call_count", "error_count", "estimated_cost_usd", "cost_status"), ), + "span": ( + "tel_spans", + ("span_id", "trace_id", "run_id", "parent_span_id", "name", "kind", + "start_ns", "end_ns", "status"), + ), "model_call": ( "tel_model_calls", ("span_id", "run_id", "provider", "model", "base_url", diff --git a/agent/telemetry/events.py b/agent/telemetry/events.py index 4125aaff414..9b770238808 100644 --- a/agent/telemetry/events.py +++ b/agent/telemetry/events.py @@ -79,6 +79,29 @@ class ToolCallEvent: return {"event": "tool_call", **asdict(self)} +@dataclass(slots=True) +class SpanEvent: + """A timed span — the timing/lineage backbone of a trace. + + One row per run (the root, ``parent_span_id=None``) and one per model/tool call + (``parent_span_id`` = the run's root span). Detail rows in ``tel_model_calls`` / + ``tel_tool_calls`` share the ``span_id`` and are joined here for ordering and + placement on a timeline. + """ + span_id: str + trace_id: str + run_id: str + name: str + kind: str # "run" | "model" | "tool" + start_ns: int + end_ns: Optional[int] = None + parent_span_id: Optional[str] = None + status: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return {"event": "span", **asdict(self)} + + @dataclass(slots=True) class ErrorEvent: run_id: Optional[str] @@ -95,5 +118,6 @@ __all__ = [ "RunEvent", "ModelCallEvent", "ToolCallEvent", + "SpanEvent", "ErrorEvent", ] diff --git a/agent/telemetry/otlp_exporter.py b/agent/telemetry/otlp_exporter.py index 32912b9da27..0043f6486a6 100644 --- a/agent/telemetry/otlp_exporter.py +++ b/agent/telemetry/otlp_exporter.py @@ -1,8 +1,8 @@ """Export telemetry to an OpenTelemetry Collector over OTLP/HTTP. -Maps telemetry events (which carry trace_id/run_id/span_id/parent_span_id) to OTel -spans and sends them to the endpoint configured under ``telemetry.export.otlp``. Lets -an operator stream Hermes telemetry into their own observability stack. +Maps the local tel_* events to OTel spans and sends them to the endpoint configured +under ``telemetry.export.otlp``. Lets an operator stream Hermes telemetry into their +own observability stack. Notes: * The destination is operator-configured; this module only sends to that endpoint. @@ -15,7 +15,13 @@ Notes: * The continuous subscriber runs in the emitter's writer thread after durable writes and is fail-isolated, so an export error cannot affect a run. -Spans carry structural telemetry by default. Message content is included only when the +Each event is exported as a span carrying its recorded attributes (provider, model, +tokens, duration, etc.). The timing/parent linkage captured in tel_spans +(trace_id/span_id/parent_span_id/start_ns/end_ns) is not yet reconstructed into OTel +SpanContexts here, so spans currently arrive as independent records rather than a +connected trace tree; building the connected-trace projection is tracked separately. + +Spans carry structural telemetry by default. Message content is included only when trajectories is enabled, and always passes through the export redaction pipeline. """ @@ -136,6 +142,8 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: "run": ("entrypoint", "platform", "end_reason", "model_call_count", "tool_call_count", "error_count", "estimated_cost_usd", "cost_status"), + "span": ("trace_id", "run_id", "parent_span_id", "name", "kind", + "start_ns", "end_ns", "status"), "model_call": ("provider", "model", "base_url", "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens", "latency_ms", @@ -195,7 +203,8 @@ def _read_events(db_path: Optional[Path], since_ns: Optional[int]) -> List[Dict[ out: List[Dict[str, Any]] = [] try: table_event = { - "tel_runs": "run", "tel_model_calls": "model_call", + "tel_runs": "run", "tel_spans": "span", + "tel_model_calls": "model_call", "tel_tool_calls": "tool_call", "tel_error_events": "error", } for table, evkind in table_event.items(): diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py index 0f5157295c9..44ff228f29d 100644 --- a/plugins/telemetry/__init__.py +++ b/plugins/telemetry/__init__.py @@ -10,12 +10,15 @@ swallowed by core, and we additionally guard each callback so a telemetry bug ca disturb a session. No content, no network: local telemetry only. Hooks consumed: - on_session_start -> begin a run context (trace_id/run_id), buffer a run row - post_api_request -> one model_call event (tokens, latency, raw provider/model) + on_session_start -> begin a run context (trace_id/run_id + root span id) + post_api_request -> one model_call event + its timing span (tokens, latency) api_request_error -> one error event - post_tool_call -> one tool_call event (raw tool name, duration, result class) - on_session_finalize -> finalize the run row (end_reason, counts, cost) + post_tool_call -> one tool_call event + its timing span (duration, result class) + on_session_finalize -> finalize the run row + emit the run's root span subagent_start/stop -> (reserved) lineage markers + +Each model/tool call emits a SpanEvent (timing + parent = the run's root span) keyed by +the same span_id as its detail row, so tel_spans reconstructs a run -> calls trace tree. """ from __future__ import annotations @@ -56,15 +59,18 @@ def _on_session_start(**kw: Any) -> None: session_id = kw.get("session_id") or "" platform = kw.get("platform") or kw.get("source") or "" ctx = spans.start_run() + now = time.time_ns() + root_span_id = spans.new_span_id() key = _run_key(session_id, kw.get("task_id")) with _runs_lock: _runs[key] = { "run_id": ctx.run_id, "trace_id": ctx.trace_id, + "root_span_id": root_span_id, "session_id": session_id or None, "entrypoint": _entrypoint_for(platform, kw.get("source")), "platform": platform or None, - "start_ns": time.time_ns(), + "start_ns": now, "model_call_count": 0, "tool_call_count": 0, "error_count": 0, @@ -84,6 +90,7 @@ def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str run = { "run_id": rid, "trace_id": tid, + "root_span_id": spans.new_span_id(), "session_id": session_id or None, "entrypoint": _entrypoint_for(platform), "platform": platform or None, @@ -96,6 +103,33 @@ def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str return run +def _emit_call_span(run: Dict[str, Any], span_id: str, name: str, kind: str, + duration_ms: Optional[int], status: Optional[str]) -> None: + """Emit the timing/lineage span for a model or tool call. + + The call hooks fire on completion, so end_ns is ~now and start_ns is reconstructed + from the measured duration (end - duration). The span is parented to the run's root + so a 2-level run -> calls waterfall can be reconstructed from tel_spans. + """ + from agent.telemetry import emitter + from agent.telemetry.events import SpanEvent + + end_ns = time.time_ns() + dur_ns = int(duration_ms) * 1_000_000 if isinstance(duration_ms, (int, float)) else 0 + start_ns = end_ns - dur_ns + emitter.emit(SpanEvent( + span_id=span_id, + trace_id=run["trace_id"], + run_id=run["run_id"], + parent_span_id=run.get("root_span_id"), + name=name, + kind=kind, + start_ns=start_ns, + end_ns=end_ns, + status=status, + )) + + def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str: """Coarse entrypoint label (cli / gateway / tui / api / cron …). @@ -135,8 +169,9 @@ def _on_post_api_request(**kw: Any) -> None: duration = kw.get("api_duration") latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None + span_id = spans.new_span_id() evt = ModelCallEvent( - span_id=spans.new_span_id(), + span_id=span_id, run_id=run["run_id"], provider=kw.get("provider"), # raw model=kw.get("model"), # raw @@ -151,6 +186,8 @@ def _on_post_api_request(**kw: Any) -> None: ) with _runs_lock: run["model_call_count"] += 1 + _emit_call_span(run, span_id, name=kw.get("model") or "model_call", + kind="model", duration_ms=latency_ms, status="ok") emitter.emit(evt) @@ -209,11 +246,15 @@ def _on_post_tool_call(**kw: Any) -> None: if result_class == "error": run["error_count"] += 1 + dur_int = int(duration_ms) if isinstance(duration_ms, (int, float)) else None + span_id = spans.new_span_id() + _emit_call_span(run, span_id, name=function_name or "tool_call", + kind="tool", duration_ms=dur_int, status=result_class) emitter.emit(ToolCallEvent( - span_id=spans.new_span_id(), + span_id=span_id, run_id=run["run_id"], tool_name=function_name, # raw tool name - duration_ms=int(duration_ms) if isinstance(duration_ms, (int, float)) else None, + duration_ms=dur_int, result_class=result_class, )) @@ -245,7 +286,7 @@ def _tool_result_class(result: Any) -> str: @_safe def _on_session_finalize(**kw: Any) -> None: from agent.telemetry import emitter, spans - from agent.telemetry.events import RunEvent + from agent.telemetry.events import RunEvent, SpanEvent session_id = kw.get("session_id") or "" key = _run_key(session_id, kw.get("task_id")) @@ -256,15 +297,29 @@ def _on_session_finalize(**kw: Any) -> None: with _runs_lock: _runs.pop(key, None) + end_ns = time.time_ns() + start_ns = run.get("start_ns", end_ns) end_reason = _coarse_end_reason(kw) + # Root span for the run — the trace root the call spans hang off of. + emitter.emit(SpanEvent( + span_id=run.get("root_span_id") or spans.new_span_id(), + trace_id=run["trace_id"], + run_id=run["run_id"], + parent_span_id=None, + name=f"run:{run.get('entrypoint', 'cli')}", + kind="run", + start_ns=start_ns, + end_ns=end_ns, + status=end_reason, + )) emitter.emit(RunEvent( run_id=run["run_id"], trace_id=run["trace_id"], entrypoint=run.get("entrypoint", "cli"), session_id=run.get("session_id"), platform=run.get("platform"), - start_ns=run.get("start_ns", time.time_ns()), - end_ns=time.time_ns(), + start_ns=start_ns, + end_ns=end_ns, end_reason=end_reason, model_call_count=run.get("model_call_count", 0), tool_call_count=run.get("tool_call_count", 0), diff --git a/tests/telemetry/test_spans_trace.py b/tests/telemetry/test_spans_trace.py new file mode 100644 index 00000000000..aac9320182d --- /dev/null +++ b/tests/telemetry/test_spans_trace.py @@ -0,0 +1,109 @@ +"""Trace/span layer: tel_spans is populated as a connected run -> calls tree. + +Drives the real dispatch chain (discover_plugins -> invoke_hook) and asserts the +timing/lineage backbone in tel_spans: + - one root span per run (kind="run", parent_span_id NULL), + - one child span per model/tool call parented to the root, + - a single trace_id across the run, + - call detail rows (tel_model_calls / tel_tool_calls) JOIN to their span by span_id, + - reconstructed durations match the reported latency/duration. + +This is the regression guard for the waterfall a desktop trace viewer renders. +""" + +from __future__ import annotations + +import sqlite3 +import time + +import pytest + +import hermes_state + + +@pytest.fixture +def runtime(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + db = tmp_path / "state.db" + hermes_state.SessionDB(db_path=db) + import hermes_cli.plugins as plugins_mod + monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) + from agent.telemetry import emitter as emitter_mod + emitter_mod.reset_emitter_for_tests(None) + import plugins.telemetry as plug + plug._runs.clear() + yield db, plugins_mod, emitter_mod + try: + emitter_mod.get_emitter().flush() + except Exception: + pass + emitter_mod.reset_emitter_for_tests(None) + monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) + + +def _one_turn(invoke_hook): + invoke_hook("on_session_start", session_id="s1", + model="anthropic/claude-opus-4", platform="cli") + invoke_hook("post_api_request", session_id="s1", platform="cli", + provider="anthropic", model="claude-opus-4", api_duration=0.9, + usage={"input_tokens": 1000, "output_tokens": 120}) + invoke_hook("post_tool_call", session_id="s1", platform="cli", + function_name="web_search", duration_ms=210, result='{"data": "ok"}') + invoke_hook("on_session_finalize", session_id="s1", platform="cli", + turn_exit_reason="completed", estimated_cost_usd=0.01, cost_status="known") + + +def test_tel_spans_forms_connected_trace(runtime): + db, plugins_mod, emitter_mod = runtime + plugins_mod.discover_plugins(force=True) + _one_turn(plugins_mod.invoke_hook) + time.sleep(0.5) + emitter_mod.get_emitter().flush() + + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + spans = conn.execute( + "SELECT span_id, parent_span_id, kind, name, start_ns, end_ns, status, trace_id " + "FROM tel_spans" + ).fetchall() + + # root + model + tool + assert len(spans) == 3 + roots = [s for s in spans if s["parent_span_id"] is None] + children = [s for s in spans if s["parent_span_id"] is not None] + assert len(roots) == 1 + assert roots[0]["kind"] == "run" + assert len(children) == 2 + + # single trace, all children parented to the root + assert len({s["trace_id"] for s in spans}) == 1 + assert all(c["parent_span_id"] == roots[0]["span_id"] for c in children) + + # spans are time-ordered and carry real durations + by_kind = {s["kind"]: s for s in spans} + assert (by_kind["model"]["end_ns"] - by_kind["model"]["start_ns"]) == 900 * 1_000_000 + assert (by_kind["tool"]["end_ns"] - by_kind["tool"]["start_ns"]) == 210 * 1_000_000 + assert by_kind["run"]["end_ns"] >= by_kind["run"]["start_ns"] + + +def test_detail_rows_join_to_spans(runtime): + db, plugins_mod, emitter_mod = runtime + plugins_mod.discover_plugins(force=True) + _one_turn(plugins_mod.invoke_hook) + time.sleep(0.5) + emitter_mod.get_emitter().flush() + + conn = sqlite3.connect(db) + conn.row_factory = sqlite3.Row + mc = conn.execute( + "SELECT m.model, s.kind, s.trace_id FROM tel_model_calls m " + "JOIN tel_spans s ON m.span_id = s.span_id" + ).fetchone() + assert mc is not None and mc["model"] == "claude-opus-4" and mc["kind"] == "model" + + tc = conn.execute( + "SELECT t.tool_name, s.kind FROM tel_tool_calls t " + "JOIN tel_spans s ON t.span_id = s.span_id" + ).fetchone() + assert tc is not None and tc["tool_name"] == "web_search" and tc["kind"] == "tool" + conn.close() From edac3a7baef82bb3ccc658a1f828867261ad6b1e Mon Sep 17 00:00:00 2001 From: emozilla Date: Sat, 27 Jun 2026 01:21:41 -0400 Subject: [PATCH 08/29] refactor(telemetry): cut dead schema; tests assert what's actually written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review after the #51714 feedback found the reviewer's dead-table finding was not isolated — the schema advertised far more than the code populates, and our own tests hid it by hand-feeding fields production never sends. Make the surface honest by subtraction. Schema (10 tel_* tables -> 5): - Delete tel_gateway_events, tel_cron_events, tel_skill_events, tel_memory_events, tel_feedback_events — declared, never written, never read. - Drop columns nothing populates: tel_runs.{profile_id,estimated_cost_usd, cost_status}; tel_model_calls.{ttft_ms,estimated_cost_usd,cost_status, cost_source,end_reason,retry_count}; tel_tool_calls.{backend,retry_count, approval}; tel_spans.attrs_json. Cost duplicated the existing sessions billing columns and was always NULL here. - events.py / emitter _TABLE_COLUMNS / OTLP _span_attrs / rollup / preview display all trimmed to match. Correctness: - end_reason no longer hardcodes "completed". Production finalize callers pass `reason` (shutdown/session_expired/session_reset); _coarse_end_reason now reads it and maps accordingly. - Fix a latent bug the trim exposed: the model_call hook passed end_reason= to ModelCallEvent, which the @_safe wrapper was silently swallowing — so tel_model_calls dropped every row in real runs. Now writes correctly. Tests: - Stop hand-feeding estimated_cost_usd / turn_exit_reason that no production call site sends. Finalize is now driven with the real `reason` kwarg, and assertions cover only fields that are actually populated. This is what let the model_call drop hide — the suite graded on a fictional contract. Net: a smaller system that does what it says. Verified end-to-end over the real dispatch path (runs + connected span tree + model/tool rows populate; dead tables gone). 160 telemetry/state/insights tests green. --- agent/telemetry/emitter.py | 12 ++++------- agent/telemetry/events.py | 14 +------------ agent/telemetry/otlp_exporter.py | 8 +++----- agent/telemetry/rollup.py | 3 +-- hermes_cli/main.py | 3 +-- plugins/telemetry/__init__.py | 26 +++++++++++++----------- tests/telemetry/test_cli_telemetry.py | 2 +- tests/telemetry/test_plugin_e2e.py | 2 +- tests/telemetry/test_plugin_hooks.py | 7 +++---- tests/telemetry/test_rollup.py | 2 +- tests/telemetry/test_schema_migration.py | 3 +-- tests/telemetry/test_spans_trace.py | 2 +- 12 files changed, 32 insertions(+), 52 deletions(-) diff --git a/agent/telemetry/emitter.py b/agent/telemetry/emitter.py index 269046c9b23..5f804c1dacb 100644 --- a/agent/telemetry/emitter.py +++ b/agent/telemetry/emitter.py @@ -54,10 +54,9 @@ def _default_db_path() -> Path: _TABLE_COLUMNS: Dict[str, tuple] = { "run": ( "tel_runs", - ("run_id", "trace_id", "session_id", "profile_id", "entrypoint", + ("run_id", "trace_id", "session_id", "entrypoint", "platform", "start_ns", "end_ns", "end_reason", - "model_call_count", "tool_call_count", "error_count", - "estimated_cost_usd", "cost_status"), + "model_call_count", "tool_call_count", "error_count"), ), "span": ( "tel_spans", @@ -68,14 +67,11 @@ _TABLE_COLUMNS: Dict[str, tuple] = { "tel_model_calls", ("span_id", "run_id", "provider", "model", "base_url", "input_tokens", "output_tokens", "cache_read_tokens", - "cache_write_tokens", "reasoning_tokens", "latency_ms", "ttft_ms", - "estimated_cost_usd", "cost_status", "cost_source", "end_reason", - "retry_count"), + "cache_write_tokens", "reasoning_tokens", "latency_ms"), ), "tool_call": ( "tel_tool_calls", - ("span_id", "run_id", "tool_name", "backend", - "duration_ms", "result_class", "retry_count", "approval"), + ("span_id", "run_id", "tool_name", "duration_ms", "result_class"), ), "error": ( "tel_error_events", diff --git a/agent/telemetry/events.py b/agent/telemetry/events.py index 9b770238808..020ce11a961 100644 --- a/agent/telemetry/events.py +++ b/agent/telemetry/events.py @@ -20,12 +20,11 @@ def _now_ns() -> int: @dataclass(slots=True) class RunEvent: - """One top-level workflow execution (a trace root).""" + """One top-level workflow execution (a trace root). A run spans one session.""" run_id: str trace_id: str entrypoint: str session_id: Optional[str] = None - profile_id: Optional[str] = None platform: Optional[str] = None start_ns: int = field(default_factory=_now_ns) end_ns: Optional[int] = None @@ -33,8 +32,6 @@ class RunEvent: model_call_count: int = 0 tool_call_count: int = 0 error_count: int = 0 - estimated_cost_usd: Optional[float] = None - cost_status: Optional[str] = None def to_dict(self) -> Dict[str, Any]: return {"event": "run", **asdict(self)} @@ -53,12 +50,6 @@ class ModelCallEvent: cache_write_tokens: int = 0 reasoning_tokens: int = 0 latency_ms: Optional[int] = None - ttft_ms: Optional[int] = None - estimated_cost_usd: Optional[float] = None - cost_status: Optional[str] = None - cost_source: Optional[str] = None - end_reason: Optional[str] = None - retry_count: int = 0 def to_dict(self) -> Dict[str, Any]: return {"event": "model_call", **asdict(self)} @@ -69,11 +60,8 @@ class ToolCallEvent: span_id: str run_id: str tool_name: Optional[str] = None # raw tool name, e.g. "web_search" - backend: Optional[str] = None duration_ms: Optional[int] = None result_class: Optional[str] = None - retry_count: int = 0 - approval: Optional[str] = None def to_dict(self) -> Dict[str, Any]: return {"event": "tool_call", **asdict(self)} diff --git a/agent/telemetry/otlp_exporter.py b/agent/telemetry/otlp_exporter.py index 0043f6486a6..3f18026a5df 100644 --- a/agent/telemetry/otlp_exporter.py +++ b/agent/telemetry/otlp_exporter.py @@ -140,15 +140,13 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} keep_by_kind = { "run": ("entrypoint", "platform", "end_reason", - "model_call_count", "tool_call_count", "error_count", - "estimated_cost_usd", "cost_status"), + "model_call_count", "tool_call_count", "error_count"), "span": ("trace_id", "run_id", "parent_span_id", "name", "kind", "start_ns", "end_ns", "status"), "model_call": ("provider", "model", "base_url", "input_tokens", "output_tokens", "cache_read_tokens", - "cache_write_tokens", "reasoning_tokens", "latency_ms", - "ttft_ms", "end_reason"), - "tool_call": ("tool_name", "backend", "duration_ms", "result_class"), + "cache_write_tokens", "reasoning_tokens", "latency_ms"), + "tool_call": ("tool_name", "duration_ms", "result_class"), "error": ("error_class", "subsystem", "recovery"), } for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] diff --git a/agent/telemetry/rollup.py b/agent/telemetry/rollup.py index 8f30304f92b..3880a55caa8 100644 --- a/agent/telemetry/rollup.py +++ b/agent/telemetry/rollup.py @@ -52,7 +52,7 @@ def _run_events(c: sqlite3.Connection, since_ns: Optional[int]) -> List[Dict[str where += f" AND start_ns >= {int(since_ns)}" rows = c.execute( "SELECT run_id, entrypoint, platform, end_reason, start_ns, end_ns, " - "model_call_count, tool_call_count, error_count, estimated_cost_usd " + "model_call_count, tool_call_count, error_count " "FROM tel_runs" + where ).fetchall() @@ -98,7 +98,6 @@ def _run_events(c: sqlite3.Connection, since_ns: Optional[int]) -> List[Dict[str "duration_ms": round(duration_ms, 1) if duration_ms is not None else None, "input_tokens": int((trow["inp"] if trow else 0) or 0), "output_tokens": int((trow["outp"] if trow else 0) or 0), - "estimated_cost_usd": r["estimated_cost_usd"], }) return events diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 173eb2a1ede..abd4c462dd3 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -14422,8 +14422,7 @@ def cmd_telemetry(args): print(f" • heartbeat hermes={e.get('hermes_version')} os={e.get('os_family')}") continue bits = [f"{e.get('event_name')}"] - for k in ("entrypoint", "platform", "end_reason", "duration_ms", - "estimated_cost_usd"): + for k in ("entrypoint", "platform", "end_reason", "duration_ms"): if e.get(k) is not None: bits.append(f"{k}={e[k]}") models = e.get("models_used") or [] diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py index 44ff228f29d..c270d4d72a2 100644 --- a/plugins/telemetry/__init__.py +++ b/plugins/telemetry/__init__.py @@ -182,7 +182,6 @@ def _on_post_api_request(**kw: Any) -> None: cache_write_tokens=int(usage.get("cache_write_tokens") or 0), reasoning_tokens=int(usage.get("reasoning_tokens") or 0), latency_ms=latency_ms, - end_reason="completed", ) with _runs_lock: run["model_call_count"] += 1 @@ -324,30 +323,33 @@ def _on_session_finalize(**kw: Any) -> None: model_call_count=run.get("model_call_count", 0), tool_call_count=run.get("tool_call_count", 0), error_count=run.get("error_count", 0), - estimated_cost_usd=_as_float(kw.get("estimated_cost_usd")), - cost_status=kw.get("cost_status"), )) spans.clear_run() def _coarse_end_reason(kw: Dict[str, Any]) -> str: + """Map a finalize payload to a coarse end reason. + + Production finalize callers pass ``reason`` (e.g. "shutdown", "session_expired", + "session_reset"); the older ``turn_exit_reason``/``interrupted``/``failed`` keys are + honored when present. Defaults to "ended". + """ if kw.get("interrupted"): return "interrupted" if kw.get("failed"): return "failed" - reason = str(kw.get("turn_exit_reason") or "").lower() + reason = str(kw.get("turn_exit_reason") or kw.get("reason") or "").lower() if "max_iteration" in reason: return "max_iterations" if "timeout" in reason: return "timeout" - return "completed" - - -def _as_float(v: Any) -> Optional[float]: - try: - return float(v) if v is not None else None - except (TypeError, ValueError): - return None + if "expired" in reason: + return "expired" + if "reset" in reason: + return "reset" + if "shutdown" in reason or "complete" in reason: + return "completed" + return reason or "ended" # ── subagent lineage (reserved) ───────────────────────────────────────────── diff --git a/tests/telemetry/test_cli_telemetry.py b/tests/telemetry/test_cli_telemetry.py index 4aaf4cab744..19fc59e394c 100644 --- a/tests/telemetry/test_cli_telemetry.py +++ b/tests/telemetry/test_cli_telemetry.py @@ -23,7 +23,7 @@ def home(tmp_path, monkeypatch): now = time.time_ns() em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", start_ns=now - 60_000_000, end_ns=now, model_call_count=1, - tool_call_count=1, estimated_cost_usd=0.3)) + tool_call_count=1)) em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", model="claude-opus-4", input_tokens=20000, output_tokens=2000)) diff --git a/tests/telemetry/test_plugin_e2e.py b/tests/telemetry/test_plugin_e2e.py index c6cf05688d8..9a9ed00913d 100644 --- a/tests/telemetry/test_plugin_e2e.py +++ b/tests/telemetry/test_plugin_e2e.py @@ -63,7 +63,7 @@ def _fire_one_turn(invoke_hook): invoke_hook("post_tool_call", session_id="s1", platform="cli", function_name="web_search", duration_ms=210, result='{"data": "ok"}') invoke_hook("on_session_finalize", session_id="s1", platform="cli", - turn_exit_reason="completed", estimated_cost_usd=0.01, cost_status="known") + reason="shutdown") def test_real_dispatch_writes_tel_rows(runtime): diff --git a/tests/telemetry/test_plugin_hooks.py b/tests/telemetry/test_plugin_hooks.py index 807dc276e1b..6b1d1b2efbf 100644 --- a/tests/telemetry/test_plugin_hooks.py +++ b/tests/telemetry/test_plugin_hooks.py @@ -43,9 +43,9 @@ def test_full_session_lifecycle_produces_rows(wired): session_id="sess1", platform="telegram", function_name="web_search", duration_ms=812, result="{\"data\": \"...\"}", ) + # Production finalize callers pass `reason` (e.g. "shutdown"), not cost. plug._on_session_finalize( - session_id="sess1", platform="telegram", - turn_exit_reason="completed", estimated_cost_usd=0.042, cost_status="known", + session_id="sess1", platform="telegram", reason="shutdown", ) em.flush() @@ -58,7 +58,6 @@ def test_full_session_lifecycle_produces_rows(wired): assert run["end_reason"] == "completed" assert run["model_call_count"] == 1 assert run["tool_call_count"] == 1 - assert abs(run["estimated_cost_usd"] - 0.042) < 1e-9 mc = conn.execute("SELECT * FROM tel_model_calls").fetchone() assert mc["provider"] == "anthropic" @@ -78,7 +77,7 @@ def test_tool_error_result_classified_and_counted(wired): session_id="s2", function_name="terminal", duration_ms=10, result="{\"error\": \"command failed\"}", ) - plug._on_session_finalize(session_id="s2", turn_exit_reason="completed") + plug._on_session_finalize(session_id="s2", reason="shutdown") em.flush() conn = sqlite3.connect(db) conn.row_factory = sqlite3.Row diff --git a/tests/telemetry/test_rollup.py b/tests/telemetry/test_rollup.py index 5ef50c89f08..b271bb916f9 100644 --- a/tests/telemetry/test_rollup.py +++ b/tests/telemetry/test_rollup.py @@ -21,7 +21,7 @@ def _seed(tmp_path): em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", platform="telegram", end_reason="completed", start_ns=now - 90_000_000, end_ns=now, - model_call_count=2, tool_call_count=2, estimated_cost_usd=2.1)) + model_call_count=2, tool_call_count=2)) em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", model="claude-opus-4", input_tokens=60000, output_tokens=8000)) em.emit(ModelCallEvent(span_id="m2", run_id="r1", provider="anthropic", diff --git a/tests/telemetry/test_schema_migration.py b/tests/telemetry/test_schema_migration.py index 5a73a335319..a5a946da4b0 100644 --- a/tests/telemetry/test_schema_migration.py +++ b/tests/telemetry/test_schema_migration.py @@ -9,8 +9,7 @@ import hermes_state TEL_TABLES = { "tel_runs", "tel_spans", "tel_model_calls", "tel_tool_calls", - "tel_gateway_events", "tel_cron_events", "tel_skill_events", - "tel_memory_events", "tel_feedback_events", "tel_error_events", + "tel_error_events", } diff --git a/tests/telemetry/test_spans_trace.py b/tests/telemetry/test_spans_trace.py index aac9320182d..c99fb02fb68 100644 --- a/tests/telemetry/test_spans_trace.py +++ b/tests/telemetry/test_spans_trace.py @@ -50,7 +50,7 @@ def _one_turn(invoke_hook): invoke_hook("post_tool_call", session_id="s1", platform="cli", function_name="web_search", duration_ms=210, result='{"data": "ok"}') invoke_hook("on_session_finalize", session_id="s1", platform="cli", - turn_exit_reason="completed", estimated_cost_usd=0.01, cost_status="known") + reason="shutdown") def test_tel_spans_forms_connected_trace(runtime): From fdb5ae012a848e4115a7ca6148ad4b3403448579 Mon Sep 17 00:00:00 2001 From: emozilla Date: Sat, 27 Jun 2026 01:27:21 -0400 Subject: [PATCH 09/29] docs(telemetry): align observability docs with the trimmed schema Match the docs to the code after the dead-schema cut and span layer: - List the actual tel_* tables (runs, spans, model_calls, tool_calls, error_events) instead of a vague "indexed tel_* tables". - Add a "Traces and spans" section: a run = one session, each call is a child span under the run root in tel_spans keyed by span_id, reconstructable as a connected run -> calls tree. Note subagent cross-run lineage isn't recorded. - Fix stale "tool failure rates by category" -> "by tool" (categories were removed; insights groups by raw tool name). - OTLP: state plainly that events export as per-event spans and the tel_spans parent/timing linkage isn't reconstructed into connected SpanContexts yet, matching the exporter's own docstring. - README: "telemetry plane" -> "telemetry system" (stale rename miss); mention spans. Config reference verified to match DEFAULT_CONFIG exactly (9 keys). --- docs/observability/README.md | 4 ++-- docs/observability/telemetry.md | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/observability/README.md b/docs/observability/README.md index 941236c0d10..8717627d8a8 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -307,8 +307,8 @@ nested agent work or security lifecycle events. ## Existing Consumers -The bundled **`telemetry`** plugin is the built-in local-first telemetry plane: it -records runs, model calls, and tool calls to a local event log + `state.db`, powers +The bundled **`telemetry`** plugin is the built-in local-first telemetry system: it +records runs, spans, model calls, and tool calls to a local event log + `state.db`, powers `hermes insights`, and supports export to your own file or OpenTelemetry Collector. It is on by default and never sends data to Nous unless you opt in. See [`telemetry.md`](./telemetry.md) for the full feature, the `hermes telemetry` commands, diff --git a/docs/observability/telemetry.md b/docs/observability/telemetry.md index 98747bb635a..ea3d2f28ffa 100644 --- a/docs/observability/telemetry.md +++ b/docs/observability/telemetry.md @@ -36,12 +36,25 @@ Local telemetry is implemented as a bundled `telemetry` plugin that listens to H lifecycle hooks (model calls, tool calls, session start/finalize) and writes events to: - an append-only JSONL log at `~/.hermes/telemetry/events.jsonl` (the source of truth) -- indexed `tel_*` tables in `state.db` (for fast queries and rollups) +- indexed `tel_*` tables in `state.db` (a rebuildable index for fast queries): + `tel_runs`, `tel_spans`, `tel_model_calls`, `tel_tool_calls`, `tel_error_events` Writes are fire-and-forget on a background thread: telemetry can never block, slow, or fail a model call or tool call. If local telemetry is disabled (`telemetry.local: false`) the plugin does not load at all. +### Traces and spans + +A **run** is one session (from `on_session_start` to `on_session_finalize`). Each run gets +a root span in `tel_spans`, and every model and tool call within it is recorded as a child +span (timing + `parent_span_id` = the run's root) keyed by the same `span_id` as its +detail row in `tel_model_calls` / `tel_tool_calls`. So a run reconstructs as a connected +`run -> calls` tree, ordered by `start_ns` and joinable to the per-call detail — the shape +a trace viewer or any OpenTelemetry backend can render. + +Call spans are timed from the measured latency/duration the hooks report; cross-run +subagent lineage (linking a delegated child run to its parent) is not yet recorded. + ### Seeing your local data ```bash @@ -50,7 +63,7 @@ hermes telemetry status # settings, consent, export posture, local data volum ``` The `insights` Observability section shows workflow counts and success rate, duration -p50/p95, tool failure rates by category, provider/model mix, and cache hit rate — +p50/p95, tool failure rates by tool, provider/model mix, and cache hit rate — all computed locally with exact values. ## `hermes telemetry` commands @@ -146,10 +159,14 @@ Set the referenced environment variable, then run the export: hermes telemetry export --otlp # drain current telemetry to your collector ``` -The token value lives only in the environment variable named by `headers_env`. Span -attributes are structural by default. The config holds the *name* of an environment -variable rather than the secret itself; the value is read at export time and is never -written to config or logged. +The token value lives only in the environment variable named by `headers_env`. The +config holds the *name* of an environment variable rather than the secret itself; the +value is read at export time and is never written to config or logged. + +Each telemetry event is exported as an OTel span carrying its recorded attributes +(provider, model, tokens, duration, etc.). The `tel_spans` timing/parent linkage is not +yet reconstructed into connected OTel `SpanContext`s, so spans currently arrive as +independent records rather than a connected trace tree; that projection is planned. ## Redaction From 505d12f662ef447180b39be87c000c64f152705e Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Tue, 14 Jul 2026 18:00:46 +0000 Subject: [PATCH 10/29] refactor(monitoring): scope telemetry substrate to gateway health/diagnostics export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvages the event-spine foundation from feat/telemetry-observability (emitter, typed events, OTLP streaming, redaction — authorship preserved in the preceding commits) and scopes it to the plane enterprise operators need today: gateway Service Health Monitoring plus redacted Operational Diagnostics, exported over OTLP. Dropped from the salvaged branch, deliberately: - run/model/tool trajectory capture (plugins/telemetry hooks, tel_spans) - the local JSONL + state.db tel_* store (monitoring is egress, not storage) - usage rollups/metrics, /insights integration, bulk export - hermes telemetry CLI (replaced by hermes monitoring status) Those planes — shared client usage metrics and enterprise trace telemetry — are being designed on the NeMo Relay integration with distinct consent, policy, and export boundaries; this keeps the monitoring plane content-free and independently enableable. Renames agent/telemetry -> agent/monitoring, config telemetry.* -> monitoring.*, and pins the otlp extra at OpenTelemetry 1.39.1 (matching uv.lock; 1.30.0 conflicts with mistralai>=2.4 on opentelemetry-api). --- agent/insights.py | 116 ----- agent/monitoring/__init__.py | 29 ++ agent/monitoring/emitter.py | 193 +++++++++ agent/monitoring/events.py | 69 +++ agent/monitoring/gateway_health.py | 388 +++++++++++++++++ agent/monitoring/gateway_health_export.py | 405 ++++++++++++++++++ .../otlp_exporter.py | 161 +++---- agent/monitoring/policy.py | 40 ++ agent/monitoring/redaction.py | 77 ++++ agent/telemetry/__init__.py | 30 -- agent/telemetry/emitter.py | 318 -------------- agent/telemetry/events.py | 111 ----- agent/telemetry/exporter_bulk.py | 139 ------ agent/telemetry/metrics.py | 219 ---------- agent/telemetry/policy.py | 72 ---- agent/telemetry/redaction.py | 187 -------- agent/telemetry/rollup.py | 144 ------- agent/telemetry/spans.py | 83 ---- cli-config.yaml.example | 43 -- docs/observability/README.md | 7 - docs/observability/monitoring.md | 101 +++++ docs/observability/telemetry.md | 228 ---------- gateway/run.py | 14 + gateway/status.py | 7 + hermes_cli/config.py | 43 ++ hermes_cli/main.py | 226 ++-------- hermes_cli/subcommands/monitoring.py | 36 ++ hermes_cli/subcommands/telemetry.py | 54 --- plugins/telemetry/__init__.py | 382 ----------------- plugins/telemetry/plugin.yaml | 12 - pyproject.toml | 9 +- .../gateway_health_export_probe.py | 92 ++++ .../observability/otel_capture_collector.py | 59 +++ tests/hermes_cli/test_plugins.py | 6 +- tests/{telemetry => monitoring}/__init__.py | 0 tests/monitoring/test_emitter.py | 86 ++++ tests/monitoring/test_export_redaction.py | 56 +++ .../monitoring/test_gateway_health_export.py | 342 +++++++++++++++ tests/monitoring/test_otlp_exporter.py | 125 ++++++ tests/telemetry/test_cli_telemetry.py | 108 ----- tests/telemetry/test_emitter.py | 107 ----- tests/telemetry/test_export_redaction.py | 113 ----- tests/telemetry/test_exporter_bulk.py | 102 ----- tests/telemetry/test_governance.py | 95 ---- tests/telemetry/test_insights_integration.py | 82 ---- tests/telemetry/test_otlp_exporter.py | 171 -------- tests/telemetry/test_plugin_autoload.py | 53 --- tests/telemetry/test_plugin_e2e.py | 127 ------ tests/telemetry/test_plugin_hooks.py | 134 ------ tests/telemetry/test_policy_consent.py | 53 --- tests/telemetry/test_rollup.py | 88 ---- tests/telemetry/test_schema_migration.py | 45 -- tests/telemetry/test_spans_trace.py | 109 ----- tools/lazy_deps.py | 18 +- uv.lock | 14 +- 55 files changed, 2295 insertions(+), 3833 deletions(-) create mode 100644 agent/monitoring/__init__.py create mode 100644 agent/monitoring/emitter.py create mode 100644 agent/monitoring/events.py create mode 100644 agent/monitoring/gateway_health.py create mode 100644 agent/monitoring/gateway_health_export.py rename agent/{telemetry => monitoring}/otlp_exporter.py (60%) create mode 100644 agent/monitoring/policy.py create mode 100644 agent/monitoring/redaction.py delete mode 100644 agent/telemetry/__init__.py delete mode 100644 agent/telemetry/emitter.py delete mode 100644 agent/telemetry/events.py delete mode 100644 agent/telemetry/exporter_bulk.py delete mode 100644 agent/telemetry/metrics.py delete mode 100644 agent/telemetry/policy.py delete mode 100644 agent/telemetry/redaction.py delete mode 100644 agent/telemetry/rollup.py delete mode 100644 agent/telemetry/spans.py create mode 100644 docs/observability/monitoring.md delete mode 100644 docs/observability/telemetry.md create mode 100644 hermes_cli/subcommands/monitoring.py delete mode 100644 hermes_cli/subcommands/telemetry.py delete mode 100644 plugins/telemetry/__init__.py delete mode 100644 plugins/telemetry/plugin.yaml create mode 100644 scripts/observability/gateway_health_export_probe.py create mode 100644 scripts/observability/otel_capture_collector.py rename tests/{telemetry => monitoring}/__init__.py (100%) create mode 100644 tests/monitoring/test_emitter.py create mode 100644 tests/monitoring/test_export_redaction.py create mode 100644 tests/monitoring/test_gateway_health_export.py create mode 100644 tests/monitoring/test_otlp_exporter.py delete mode 100644 tests/telemetry/test_cli_telemetry.py delete mode 100644 tests/telemetry/test_emitter.py delete mode 100644 tests/telemetry/test_export_redaction.py delete mode 100644 tests/telemetry/test_exporter_bulk.py delete mode 100644 tests/telemetry/test_governance.py delete mode 100644 tests/telemetry/test_insights_integration.py delete mode 100644 tests/telemetry/test_otlp_exporter.py delete mode 100644 tests/telemetry/test_plugin_autoload.py delete mode 100644 tests/telemetry/test_plugin_e2e.py delete mode 100644 tests/telemetry/test_plugin_hooks.py delete mode 100644 tests/telemetry/test_policy_consent.py delete mode 100644 tests/telemetry/test_rollup.py delete mode 100644 tests/telemetry/test_schema_migration.py delete mode 100644 tests/telemetry/test_spans_trace.py diff --git a/agent/insights.py b/agent/insights.py index b9370b35cc5..086150c279e 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -82,19 +82,6 @@ def _bar_chart(values: List[int], max_width: int = 20) -> List[str]: return ["█" * max(1, int(v / peak * max_width)) if v > 0 else "" for v in values] -def _fmt_ms(ms: float) -> str: - """Compact human duration from milliseconds (e.g. 850ms, 2.4s, 1.5m).""" - try: - ms = float(ms or 0) - except (TypeError, ValueError): - return "0ms" - if ms < 1000: - return f"{int(ms)}ms" - if ms < 60_000: - return f"{ms / 1000:.1f}s" - return f"{ms / 60_000:.1f}m" - - class InsightsEngine: """ Analyzes session history and produces usage insights. @@ -152,7 +139,6 @@ class InsightsEngine: }, "activity": {}, "top_sessions": [], - "telemetry": {}, } # Compute insights @@ -163,7 +149,6 @@ class InsightsEngine: skills = self._compute_skill_breakdown(skill_usage) activity = self._compute_activity_patterns(sessions) top_sessions = self._compute_top_sessions(sessions) - telemetry = self._compute_telemetry(cutoff) return { "days": days, @@ -177,37 +162,8 @@ class InsightsEngine: "skills": skills, "activity": activity, "top_sessions": top_sessions, - "telemetry": telemetry, } - # ========================================================================= - # Telemetry (observability) — from the tel_* tables (local telemetry) - # ========================================================================= - - def _compute_telemetry(self, cutoff: float) -> Dict[str, Any]: - """Roll up the local telemetry tables for the same window. - - Reuses the engine's existing connection. Fully fail-soft: if the tel_* - tables are empty or absent (telemetry.local disabled, fresh install), this - returns an empty dict and the renderer skips the section. - """ - try: - from agent.telemetry import metrics - except Exception: - return {} - try: - since_ns = int(cutoff * 1e9) - if not metrics.has_data(conn=self._conn): - return {} - return { - "workflows": metrics.workflow_summary(since_ns=since_ns, conn=self._conn), - "model_calls": metrics.model_call_summary(since_ns=since_ns, conn=self._conn), - "tool_calls": metrics.tool_call_summary(conn=self._conn), - "errors": metrics.error_summary(conn=self._conn), - } - except Exception: - return {} - # ========================================================================= # Data gathering (SQL queries) # ========================================================================= @@ -1067,80 +1023,8 @@ class InsightsEngine: lines.append(f" {ts['label']:<20} {ts['value']:<18} ({ts['date']}, {ts['session_id']})") lines.append("") - # Telemetry / observability (local telemetry) — only when data exists - tel = report.get("telemetry") or {} - if tel: - self._append_telemetry_section(lines, tel) - return "\n".join(lines) - def _append_telemetry_section(self, lines: List[str], tel: Dict[str, Any]) -> None: - """Render the observability rollups (workflows, tools, providers, errors).""" - wf = tel.get("workflows", {}) - mc = tel.get("model_calls", {}) - tc = tel.get("tool_calls", {}) - errs = tel.get("errors", {}).get("by_class", {}) - - lines.append(" 📡 Observability (local telemetry)") - lines.append(" " + "─" * 56) - - total_runs = wf.get("total_runs", 0) - if total_runs: - sr = wf.get("success_rate", 0.0) * 100 - p50 = wf.get("duration_ms_p50", 0) - p95 = wf.get("duration_ms_p95", 0) - lines.append( - f" Workflows: {total_runs:,} Success: {sr:.1f}% " - f"Duration p50/p95: {_fmt_ms(p50)} / {_fmt_ms(p95)}" - ) - by_entry = wf.get("by_entrypoint", {}) - if by_entry: - entry_str = ", ".join( - f"{k}: {v}" for k, v in sorted(by_entry.items(), key=lambda x: -x[1]) - ) - lines.append(f" Entrypoints: {entry_str}") - - # Tool reliability - if tc.get("total"): - fail_pct = tc.get("failure_rate", 0.0) * 100 - lines.append( - f" Tool calls: {tc['total']:,} Failure rate: {fail_pct:.1f}%" - ) - tools = tc.get("by_tool", {}) - fails = tc.get("failures_by_tool", {}) - top = sorted(tools.items(), key=lambda x: -x[1])[:6] - if top: - parts = [] - for name, n in top: - f = fails.get(name, 0) - parts.append(f"{name}: {n}" + (f" ({f} failed)" if f else "")) - lines.append(" " + " ".join(parts)) - - # Provider / model mix + cache (real names) - by_provider = mc.get("by_provider", {}) - if by_provider: - prov_str = ", ".join( - f"{k}: {v}" for k, v in sorted(by_provider.items(), key=lambda x: -x[1]) - ) - lines.append(f" Providers: {prov_str}") - by_model = mc.get("by_model", {}) - if by_model: - model_str = ", ".join( - f"{k}: {v}" for k, v in sorted(by_model.items(), key=lambda x: -x[1])[:8] - ) - cache = mc.get("cache_hit_rate", 0.0) * 100 - suffix = f" Cache hit: {cache:.1f}%" if cache else "" - lines.append(f" Models: {model_str}{suffix}") - - # Error classes - if errs: - err_str = ", ".join( - f"{k}: {v}" for k, v in sorted(errs.items(), key=lambda x: -x[1])[:6] - ) - lines.append(f" Errors: {err_str}") - - lines.append("") - def format_gateway(self, report: Dict) -> str: """Format the insights report for gateway/messaging (shorter).""" if report.get("empty"): diff --git a/agent/monitoring/__init__.py b/agent/monitoring/__init__.py new file mode 100644 index 00000000000..ee45ff01649 --- /dev/null +++ b/agent/monitoring/__init__.py @@ -0,0 +1,29 @@ +"""Hermes gateway monitoring. + +Service health monitoring plus redacted operational diagnostics for the +gateway daemon, exported over OTLP to an operator-configured endpoint. + +``emitter`` is the in-process event bus: producers (gateway status hooks, +the diagnostic log handler) hand typed events to a fire-and-forget queue, +and subscribers (the OTLP streamers) consume them off the hot path. The +emitter never blocks or raises into gateway code (the hot-path invariant), +and nothing is persisted locally — monitoring is an egress path, not a store. + +Deliberately out of scope here: run/model/tool trajectory capture, usage +analytics, and any content-bearing signal. Those planes are served by the +NeMo Relay integration and its Hermes-owned subscribers. +""" + +from __future__ import annotations + +from . import emitter, events + +emit = emitter.emit +get_emitter = emitter.get_emitter + +__all__ = [ + "emitter", + "events", + "emit", + "get_emitter", +] diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py new file mode 100644 index 00000000000..f7dee1bdf4f --- /dev/null +++ b/agent/monitoring/emitter.py @@ -0,0 +1,193 @@ +"""Monitoring emitter: fire-and-forget queue + background dispatcher. + +The emitter is the single seam between producers (gateway status hooks, the +diagnostic log handler) and consumers (the OTLP streamers). Its contract is +the hot-path invariant: + + ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, + and MUST NEVER raise into the caller. A monitoring failure is logged + locally and dropped — it can never affect the gateway or a session. + +Mechanism: + * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare + except. On a full queue it drops the *oldest* event and counts the drop. + * A daemon thread drains the queue and fans each batch out to subscribers + (the OTLP metric/span/log streamers). Each subscriber is fail-isolated — + a slow or raising subscriber never affects the hot path or its peers. + +Nothing is persisted here. Monitoring is an egress path, not a local store; +if no subscriber is attached, events simply age out of the ring buffer. +""" + +from __future__ import annotations + +import logging +import queue +import threading +import time +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full +_DRAIN_BATCH = 256 + + +class MonitoringEmitter: + """Owns the queue, the dispatcher thread, and the subscriber list.""" + + def __init__(self, *, enabled: bool = True) -> None: + self._enabled = enabled + self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) + self._dropped = 0 + self._dispatched = 0 + self._stop = threading.Event() + self._started = False + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + # Live subscribers (the OTLP streamers). Called from the dispatcher + # thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]). + self._subscribers: list = [] + + # ── public API (hot path) ─────────────────────────────────────────────── + def emit(self, event: Any) -> None: + """Enqueue an event. Never blocks, never raises. + + ``event`` may be a dataclass with ``to_dict()`` or a plain dict. + """ + if not self._enabled: + return + try: + payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) + payload.setdefault("ts_ns", time.time_ns()) + self._ensure_started() + try: + self._q.put_nowait(payload) + except queue.Full: + # Drop oldest to make room — bounded memory, newest-wins. + try: + self._q.get_nowait() + self._dropped += 1 + self._q.put_nowait(payload) + except Exception: + self._dropped += 1 + except Exception: # the hot-path invariant: never propagate + logger.debug("monitoring emit failed", exc_info=True) + + # ── lifecycle ─────────────────────────────────────────────────────────── + def _ensure_started(self) -> None: + if self._started: + return + with self._lock: + if self._started: + return + self._thread = threading.Thread( + target=self._run, name="hermes-monitoring-dispatch", daemon=True + ) + self._thread.start() + self._started = True + + def _run(self) -> None: + while not self._stop.is_set(): + try: + first = self._q.get(timeout=0.5) + except queue.Empty: + continue + batch = [first] + while len(batch) < _DRAIN_BATCH: + try: + batch.append(self._q.get_nowait()) + except queue.Empty: + break + self._dispatch(batch) + + def _dispatch(self, batch) -> None: + # Fan-out to subscribers (OTLP streamers) — fully fail-isolated. + for sub in list(self._subscribers): + try: + sub(batch) + except Exception: + logger.debug("monitoring subscriber failed", exc_info=True) + self._dispatched += len(batch) + + def subscribe(self, callback) -> None: + """Register a live batch subscriber (callable(batch: list[dict])).""" + if callback not in self._subscribers: + self._subscribers.append(callback) + + def unsubscribe(self, callback) -> None: + try: + self._subscribers.remove(callback) + except ValueError: + pass + + # ── introspection / shutdown (tests, CLI) ─────────────────────────────── + def flush(self, timeout: float = 2.0) -> None: + """Block until the queue drains (test/CLI helper, NOT the hot path).""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._q.empty(): + # give the dispatcher a tick to finish the in-flight batch + time.sleep(0.05) + if self._q.empty(): + return + time.sleep(0.02) + + def stats(self) -> Dict[str, int]: + return { + "queued": self._q.qsize(), + "dispatched": self._dispatched, + "dropped": self._dropped, + "subscribers": len(self._subscribers), + } + + def close(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._started = False + + +# ── process-wide singleton ────────────────────────────────────────────────── +_EMITTER: Optional[MonitoringEmitter] = None +_EMITTER_LOCK = threading.Lock() + + +def get_emitter() -> MonitoringEmitter: + """Return the process-wide monitoring emitter.""" + global _EMITTER + if _EMITTER is not None: + return _EMITTER + with _EMITTER_LOCK: + if _EMITTER is None: + _EMITTER = MonitoringEmitter() + return _EMITTER + + +def emit(event: Any) -> None: + """Module-level convenience: emit via the singleton.""" + get_emitter().emit(event) + + +def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None: + """Swap the singleton (tests only).""" + global _EMITTER + with _EMITTER_LOCK: + if _EMITTER is not None and emitter is not _EMITTER: + try: + _EMITTER.close() + except Exception: + pass + _EMITTER = emitter + + +# Back-compat alias for the salvaged class name used in emozilla's tests. +TelemetryEmitter = MonitoringEmitter + +__all__ = [ + "MonitoringEmitter", + "TelemetryEmitter", + "get_emitter", + "emit", + "reset_emitter_for_tests", +] diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py new file mode 100644 index 00000000000..2151932b730 --- /dev/null +++ b/agent/monitoring/events.py @@ -0,0 +1,69 @@ +"""Typed gateway monitoring events. + +Content-free service-health and redacted diagnostic events for the gateway +daemon. These are the only event shapes the monitoring plane emits: no +prompts, messages, tool args/results, session history, or usage analytics. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field, asdict +from typing import Any, Dict, Optional + + +def _now_ns() -> int: + return time.time_ns() + + +@dataclass(slots=True) +class GatewayHealthEvent: + """Content-free gateway health snapshot or lifecycle event.""" + + name: str + gateway_state: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + exit_reason: Optional[str] = None + restart_requested: Optional[bool] = None + active_agents: int = 0 + gateway_busy: bool = False + gateway_drainable: bool = False + platform_count: int = 0 + fatal_platform_count: int = 0 + profile: Optional[str] = None + install_id: Optional[str] = None + version: Optional[str] = None + supervision_mode: Optional[str] = None + pid: Optional[int] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_health", **asdict(self)} + + +@dataclass(slots=True) +class GatewayDiagnosticEvent: + """Redacted gateway diagnostic event for operator-owned observability.""" + + name: str + subsystem: str + error_class: str = "unknown" + error_code: Optional[str] = None + redacted_message: Optional[str] = None + platform: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + profile: Optional[str] = None + version: Optional[str] = None + severity: str = "warning" + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_diagnostic", **asdict(self)} + + +__all__ = [ + "GatewayHealthEvent", + "GatewayDiagnosticEvent", +] diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py new file mode 100644 index 00000000000..3404392c638 --- /dev/null +++ b/agent/monitoring/gateway_health.py @@ -0,0 +1,388 @@ +"""Gateway health and diagnostics signal producer. + +This module keeps the plane narrow: service health monitoring plus +redacted operational diagnostics. It reuses the existing gateway runtime-status +contract and emits content-free metrics/events. No prompts, messages, tool args, +session history, audit records, or product analytics belong here. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent + + +@dataclass(frozen=True, slots=True) +class GatewayMetric: + name: str + value: int | float + attributes: Dict[str, str] + + +@dataclass(frozen=True, slots=True) +class GatewayHealthSnapshot: + metrics: List[GatewayMetric] + events: List[GatewayHealthEvent | GatewayDiagnosticEvent] + + +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) +_TOKEN_RE = re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b") +_SECRET_LITERAL_RE = re.compile(r"\*{3,}") +_PHONE_RE = re.compile(r"(? bool: + return name == "gateway" or name.startswith("gateway.") + + +def redact_gateway_message(message: Any) -> str: + """Redact gateway diagnostic free text for customer-owned export.""" + text = str(message or "") + try: + from agent.monitoring.redaction import redact_for_export + redacted = redact_for_export(text, content_mode="pii") or "" + except Exception: + redacted = "[redaction-unavailable]" + redacted = _BEARER_RE.sub("[redacted]", redacted) + redacted = _TOKEN_RE.sub("[redacted]", redacted) + redacted = _SECRET_LITERAL_RE.sub("[redacted]", redacted) + redacted = re.sub(r"\bBearer\s+\[[^\]]+\]", "[redacted]", redacted, flags=re.IGNORECASE) + redacted = _EMAIL_RE.sub("[email]", redacted) + redacted = _PHONE_RE.sub("[phone]", redacted) + return redacted[:500] + + +def classify_gateway_error(raw: Any) -> str: + s = str(raw or "").lower() + if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")): + return "auth_failed" + if "rate" in s and "limit" in s: + return "rate_limited" + if "timeout" in s or "timed out" in s: + return "timeout" + if any(k in s for k in ("network", "connection", "dns", "socket")): + return "network_error" + if any(k in s for k in ("config", "missing", "invalid")): + return "invalid_config" + if "startup" in s: + return "startup_failed" + if "fatal" in s: + return "platform_fatal" + return "unknown" + + +def subsystem_for_logger(logger_name: str) -> str: + if logger_name.startswith("gateway.platforms."): + parts = logger_name.split(".") + if len(parts) >= 3 and parts[2]: + return f"platform.{parts[2]}" + if logger_name.startswith("gateway.platforms"): + return "platform" + if logger_name.startswith("gateway"): + return "gateway" + return "gateway" + + +def platform_for_subsystem(subsystem: str) -> Optional[str]: + if subsystem.startswith("platform."): + return subsystem.split(".", 1)[1] or None + return None + + +def _parse_active_agents(raw: Any) -> int: + try: + from gateway.status import parse_active_agents + return parse_active_agents(raw) + except Exception: + try: + return max(0, int(raw)) + except (TypeError, ValueError): + return 0 + + +def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: + try: + from gateway.status import derive_gateway_busy + return derive_gateway_busy( + gateway_running=gateway_running, + gateway_state=gateway_state, + active_agents=active_agents, + ) + except Exception: + return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) + + +def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: + try: + from gateway.status import derive_gateway_drainable + return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state) + except Exception: + return bool(gateway_running and gateway_state == "running") + + +def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: + return { + "hermes.profile": str(profile), + "service.instance.id": str(install_id), + "service.version": str(version), + "hermes.supervision_mode": str(supervision_mode), + } + + +def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric: + out = dict(attrs) + for key, val in extra.items(): + if val is not None: + out[key] = str(val) + return GatewayMetric(name=name, value=value, attributes=out) + + +def build_gateway_health_snapshot( + runtime: Optional[dict[str, Any]], + *, + gateway_running: bool, + profile: str, + install_id: str, + version: str, + supervision_mode: str = "unknown", +) -> GatewayHealthSnapshot: + """Convert gateway_state.json-compatible runtime state into P0 signals.""" + runtime = runtime or {} + gateway_state = runtime.get("gateway_state") + active_agents = _parse_active_agents(runtime.get("active_agents", 0)) + busy = _derive_busy(gateway_running, gateway_state, active_agents) + drainable = _derive_drainable(gateway_running, gateway_state) + platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {} + base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode) + + metrics: list[GatewayMetric] = [ + _metric("hermes.gateway.up", 1 if gateway_running else 0, base), + _metric("hermes.gateway.active_agents", active_agents, base), + _metric("hermes.gateway.busy", 1 if busy else 0, base), + _metric("hermes.gateway.drainable", 1 if drainable else 0, base), + _metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base), + ] + if gateway_state: + metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)})) + + fatal_count = 0 + events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + for platform, pdata in platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + state = str(pdata.get("state") or "unknown").lower() + raw_error = pdata.get("error_code") or pdata.get("error_message") + error_code = classify_gateway_error(raw_error) + is_up = state in _RUNNING_PLATFORM_STATES + is_degraded = state in _FATAL_PLATFORM_STATES + if is_degraded: + fatal_count += 1 + metrics.append(_metric( + "hermes.platform.up", + 1 if is_up else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state}, + )) + metrics.append(_metric( + "hermes.platform.degraded", + 1 if is_degraded else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code}, + )) + if is_degraded: + events.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=classify_gateway_error(error_code or pdata.get("error_message")), + redacted_message=redact_gateway_message(pdata.get("error_message")), + profile=profile, + version=version, + severity="error" if state == "fatal" else "warning", + )) + + events.insert(0, GatewayHealthEvent( + name="gateway.health_snapshot", + gateway_state=str(gateway_state) if gateway_state is not None else None, + active_agents=active_agents, + gateway_busy=busy, + gateway_drainable=drainable, + platform_count=len(platforms), + fatal_platform_count=fatal_count, + profile=profile, + install_id=install_id, + version=version, + supervision_mode=supervision_mode, + pid=_coerce_pid(runtime.get("pid")), + )) + return GatewayHealthSnapshot(metrics=metrics, events=events) + + +def _safe_profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _safe_version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None: + """Emit immediate content-free gateway events for runtime status changes. + + Called by gateway.status.write_runtime_status after persisting the new status. + Fully fail-open: failures never affect gateway status writes. + """ + try: + from agent.monitoring import emitter + out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + profile = _safe_profile() + version = _safe_version() + old_gateway_state = str((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = str(current.get("gateway_state")) if current.get("gateway_state") is not None else None + if old_gateway_state != new_gateway_state and new_gateway_state: + out.append(GatewayHealthEvent( + name="gateway.lifecycle", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=current.get("exit_reason"), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + if new_gateway_state == "startup_failed": + out.append(GatewayDiagnosticEvent( + name="gateway.startup_failed", + subsystem="gateway", + error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + redacted_message=redact_gateway_message(current.get("exit_reason") or "startup failed"), + profile=profile, + version=version, + severity="error", + )) + if new_gateway_state == "stopped": + out.append(GatewayHealthEvent( + name="gateway.exit", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=current.get("exit_reason"), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + + old_platforms_raw = (previous or {}).get("platforms") + new_platforms_raw = current.get("platforms") + old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {} + new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {} + for platform, pdata in new_platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + prev_raw = old_platforms.get(platform, {}) + prev = prev_raw if isinstance(prev_raw, dict) else {} + old_state = str(prev.get("state")) if prev.get("state") is not None else None + new_state = str(pdata.get("state")) if pdata.get("state") is not None else None + if old_state == new_state or not new_state: + continue + error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning" + out.append(GatewayDiagnosticEvent( + name="platform.state_change", + subsystem=f"platform.{platform}", + platform=str(platform), + old_state=old_state, + new_state=new_state, + error_code=error_code, + error_class=error_code, + redacted_message=redact_gateway_message(pdata.get("error_message")), + profile=profile, + version=version, + severity=severity, + )) + if new_state.lower() in _FATAL_PLATFORM_STATES: + out.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=error_code, + redacted_message=redact_gateway_message(pdata.get("error_message")), + profile=profile, + version=version, + severity=severity, + )) + for ev in out: + emitter.emit(ev) + except Exception: + logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True) + + +def _coerce_pid(raw: Any) -> Optional[int]: + try: + pid = int(raw) + except (TypeError, ValueError): + return None + return pid if pid > 0 else None + + +class GatewayDiagnosticLogHandler(logging.Handler): + """Allowlisted warning/error bridge for gateway-owned diagnostics.""" + + def __init__(self, *, profile: str = "default", version: str = "unknown") -> None: + super().__init__(level=logging.WARNING) + self.profile = profile + self.version = version + + def emit(self, record: logging.LogRecord) -> None: + try: + if record.levelno < logging.WARNING: + return + if not _allowed_logger(record.name): + return + subsystem = subsystem_for_logger(record.name) + message = record.getMessage() + event = GatewayDiagnosticEvent( + name=f"gateway.log.{record.levelname.lower()}", + subsystem=subsystem, + platform=platform_for_subsystem(subsystem), + error_class=classify_gateway_error(message), + redacted_message=redact_gateway_message(message), + profile=self.profile, + version=self.version, + severity=record.levelname.lower(), + ) + from agent.monitoring import emitter + emitter.get_emitter().emit(event) + except Exception: + logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True) + + +__all__ = [ + "GatewayMetric", + "GatewayHealthSnapshot", + "GatewayDiagnosticLogHandler", + "build_gateway_health_snapshot", + "classify_gateway_error", + "redact_gateway_message", +] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py new file mode 100644 index 00000000000..969fa970635 --- /dev/null +++ b/agent/monitoring/gateway_health_export.py @@ -0,0 +1,405 @@ +"""Gateway Health & Diagnostics OTLP export runtime. + +This exporter emits operator-owned gateway service-health metrics plus +narrow redacted diagnostic events. It is deliberately in-process and fail-open so +it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell +without a sidecar/watchdog dependency. +""" + +from __future__ import annotations + +import logging +import os +import threading +from dataclasses import dataclass +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + + +@dataclass(slots=True) +class GatewayHealthExportRuntime: + enabled: bool + reason: str = "disabled" + streamer: Any = None + metric_provider: Any = None + log_handler: Any = None + log_streamer: Any = None + thread: Optional[threading.Thread] = None + stop_event: Optional[threading.Event] = None + + def shutdown(self) -> None: + if self.stop_event is not None: + self.stop_event.set() + if self.thread is not None: + self.thread.join(timeout=2.0) + if self.log_handler is not None: + try: + logging.getLogger().removeHandler(self.log_handler) + except Exception: + pass + if self.streamer is not None: + try: + self.streamer.shutdown() + except Exception: + pass + if self.log_streamer is not None: + try: + self.log_streamer.shutdown() + except Exception: + pass + if self.metric_provider is not None: + try: + self.metric_provider.shutdown() + except Exception: + pass + + +def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + return mon.get("gateway_health_export") or {} + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} + return export.get("otlp") or {} + + +def _enabled(config: Dict[str, Any]) -> bool: + gh = _gateway_health_config(config) + otlp = _otlp_config(config) + return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint")) + + +def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]: + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except Exception: + pass + try: + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.metrics import Observation + from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags + from opentelemetry._logs import LogRecord + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + return { + "OTLPLogExporter": OTLPLogExporter, + "OTLPMetricExporter": OTLPMetricExporter, + "Observation": Observation, + "LogRecord": LogRecord, + "LoggerProvider": LoggerProvider, + "INVALID_SPAN_ID": INVALID_SPAN_ID, + "INVALID_TRACE_ID": INVALID_TRACE_ID, + "TraceFlags": TraceFlags, + "SeverityNumber": SeverityNumber, + "BatchLogRecordProcessor": BatchLogRecordProcessor, + "MeterProvider": MeterProvider, + "PeriodicExportingMetricReader": PeriodicExportingMetricReader, + "Resource": Resource, + } + except Exception as exc: + raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + return resolved + + +def _metric_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/metrics" + return endpoint + + +def _logs_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/logs" + if endpoint.endswith("/v1/metrics"): + return endpoint[: -len("/v1/metrics")] + "/v1/logs" + return endpoint + + +def _version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def _profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _install_id(config: Dict[str, Any]) -> str: + try: + from agent.monitoring.policy import ensure_install_id + return str(ensure_install_id(config)) + except Exception: + return "unknown" + + +def _supervision_mode() -> str: + if os.environ.get("INVOCATION_ID"): + return "systemd" + if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"): + return "s6" + if os.environ.get("container") or os.path.exists("/.dockerenv"): + return "container" + if os.environ.get("LAUNCHD_SOCKET"): + return "launchd" + return "manual" + + +def _read_runtime_snapshot(config: Dict[str, Any]): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + try: + from gateway.status import read_runtime_status + runtime = read_runtime_status() or {} + except Exception: + runtime = {} + return build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile=_profile(), + install_id=_install_id(config), + version=_version(), + supervision_mode=_supervision_mode(), + ) + + +def _emit_snapshot_events(config: Dict[str, Any]) -> None: + gh = _gateway_health_config(config) + if not gh.get("diagnostic_events_enabled", True): + return + try: + from agent.monitoring import emitter + snapshot = _read_runtime_snapshot(config) + for event in snapshot.events: + emitter.emit(event) + except Exception: + logger.debug("gateway health snapshot emit failed", exc_info=True) + + +def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("metrics_enabled", True): + return None + otlp = _otlp_config(config) + endpoint = _metric_endpoint(str(otlp.get("endpoint"))) + headers = _resolve_headers(otlp.get("headers_env")) + exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None) + interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 + reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) + resource_attrs = dict((gh.get("resource_attributes") or {})) + resource_attrs.setdefault("service.name", "hermes-gateway") + resource_attrs.setdefault("telemetry.scope", "gateway_health") + provider = sdk["MeterProvider"]( + metric_readers=[reader], + resource=sdk["Resource"].create(resource_attrs), + ) + meter = provider.get_meter("hermes.gateway.health") + Observation = sdk["Observation"] + + metric_names = [ + "hermes.gateway.up", + "hermes.gateway.state", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.platform.up", + "hermes.platform.degraded", + ] + + def callback(name: str): + def _cb(_options=None): + try: + snapshot = _read_runtime_snapshot(config) + return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name] + except Exception: + logger.debug("gateway metric callback failed", exc_info=True) + return [] + return _cb + + for metric_name in metric_names: + meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)]) + return provider + + +def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any: + SeverityNumber = sdk["SeverityNumber"] + sev = str(severity or "warning").lower() + if sev in {"critical", "fatal"}: + return SeverityNumber.FATAL + if sev == "error": + return SeverityNumber.ERROR + if sev in {"info", "information"}: + return SeverityNumber.INFO + if sev == "debug": + return SeverityNumber.DEBUG + return SeverityNumber.WARN + + +class GatewayDiagnosticLogStreamer: + """Emitter subscriber that sends gateway diagnostic events as OTLP logs.""" + + def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]): + otlp = _otlp_config(config) + gh = _gateway_health_config(config) + headers = _resolve_headers(otlp.get("headers_env")) + endpoint = _logs_endpoint(str(otlp.get("endpoint"))) + resource_attrs = dict((gh.get("resource_attributes") or {})) + resource_attrs.setdefault("service.name", "hermes-gateway") + resource_attrs.setdefault("telemetry.scope", "gateway_diagnostics") + self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs)) + self._processor = sdk["BatchLogRecordProcessor"]( + sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) + ) + self._provider.add_log_record_processor(self._processor) + self._logger = self._provider.get_logger("hermes.gateway.diagnostics") + self._LogRecord = sdk["LogRecord"] + self._sdk = sdk + self.exported = 0 + + def __call__(self, batch: list[Dict[str, Any]]) -> None: + for ev in batch: + if ev.get("event") != "gateway_diagnostic": + continue + attrs = { + f"hermes.{key}": val + for key, val in ev.items() + if key not in {"event", "redacted_message", "ts_ns"} and val is not None + } + body = ev.get("redacted_message") or ev.get("name") or "gateway diagnostic" + record = self._LogRecord( + timestamp=ev.get("ts_ns"), + trace_id=self._sdk["INVALID_TRACE_ID"], + span_id=self._sdk["INVALID_SPAN_ID"], + trace_flags=self._sdk["TraceFlags"].DEFAULT, + severity_text=str(ev.get("severity") or "warning").upper(), + severity_number=_severity_number(self._sdk, ev.get("severity")), + body=str(body), + attributes=attrs, + ) + self._logger.emit(record) + self.exported += 1 + + def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer: + from agent.monitoring.emitter import get_emitter + streamer = GatewayDiagnosticLogStreamer(config, sdk) + get_emitter().subscribe(streamer) + return streamer + + +def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread: + interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5))) + + def _run() -> None: + while not stop_event.wait(interval): + _emit_snapshot_events(config) + + thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True) + thread.start() + return thread + + +def _attach_log_handler(config: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("warning_error_events_enabled", True): + return None + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version()) + root = logging.getLogger() + if handler not in root.handlers: + root.addHandler(handler) + return handler + + +def _gateway_health_event(ev: Dict[str, Any]) -> bool: + return ev.get("event") == "gateway_health" + + +def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime: + """Start P0 gateway health export if configured. Never raises.""" + if not _enabled(config): + return GatewayHealthExportRuntime(enabled=False, reason="disabled") + gh = _gateway_health_config(config) + runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled") + sdk: Optional[Dict[str, Any]] = None + + if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True): + try: + sdk = _require_metrics_sdk(prompt=False) + except Exception: + logger.warning( + "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; " + "install 'hermes-agent[otlp]'", + exc_info=True, + ) + return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable") + + if gh.get("metrics_enabled", True) and sdk is not None: + try: + runtime.metric_provider = _start_metric_provider(config, sdk) + except Exception: + logger.warning("gateway health OTLP metrics failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed") + + if gh.get("diagnostic_events_enabled", True) and sdk is not None: + try: + from agent.monitoring import otlp_exporter + runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event) + runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk) + except Exception: + logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True) + + try: + runtime.log_handler = _attach_log_handler(config) + except Exception: + logger.debug("gateway diagnostic log handler failed to attach", exc_info=True) + try: + _emit_snapshot_events(config) + runtime.stop_event = threading.Event() + runtime.thread = _start_snapshot_thread(config, runtime.stop_event) + except Exception: + logger.debug("gateway health snapshot thread failed to start", exc_info=True) + return runtime + + +__all__ = [ + "GatewayHealthExportRuntime", + "start_gateway_health_export", +] diff --git a/agent/telemetry/otlp_exporter.py b/agent/monitoring/otlp_exporter.py similarity index 60% rename from agent/telemetry/otlp_exporter.py rename to agent/monitoring/otlp_exporter.py index 3f18026a5df..01eefa53506 100644 --- a/agent/telemetry/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -1,37 +1,31 @@ -"""Export telemetry to an OpenTelemetry Collector over OTLP/HTTP. +"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. -Maps the local tel_* events to OTel spans and sends them to the endpoint configured -under ``telemetry.export.otlp``. Lets an operator stream Hermes telemetry into their -own observability stack. +Maps gateway monitoring events to OTel spans and sends them to the endpoint +configured under ``monitoring.export.otlp``. Lets an operator stream Hermes +gateway health into their own observability stack (OTEL Collector, DataDog, +and similar). Notes: - * The destination is operator-configured; this module only sends to that endpoint. - It does not import or interact with any aggregate-metrics path. - * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an optional - extra (``pip install hermes-agent[otlp]``), imported lazily so the dependency is - only required when OTLP export is actually used. - * ``headers_env`` maps a header name to an environment variable name; values are read - from the environment at export time and never logged or stored. - * The continuous subscriber runs in the emitter's writer thread after durable writes - and is fail-isolated, so an export error cannot affect a run. + * The destination is operator-configured; this module only sends to that + endpoint. No default destination ships. + * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an + optional extra (``pip install hermes-agent[otlp]``), imported lazily so the + dependency is only required when OTLP export is actually used. + * ``headers_env`` maps a header name to an environment variable name; values + are read from the environment at export time and never logged or stored. + * The continuous subscriber runs in the emitter's dispatcher thread and is + fail-isolated, so an export error cannot affect the gateway. -Each event is exported as a span carrying its recorded attributes (provider, model, -tokens, duration, etc.). The timing/parent linkage captured in tel_spans -(trace_id/span_id/parent_span_id/start_ns/end_ns) is not yet reconstructed into OTel -SpanContexts here, so spans currently arrive as independent records rather than a -connected trace tree; building the connected-trace projection is tracked separately. - -Spans carry structural telemetry by default. Message content is included only when -trajectories is enabled, and always passes through the export redaction pipeline. +Only monitoring events (gateway_health / gateway_diagnostic) exist on this +plane; the ``event_filter`` seam is kept so future planes sharing the emitter +cannot silently ride along on this exporter. """ from __future__ import annotations import logging import os -import sqlite3 -from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional logger = logging.getLogger(__name__) @@ -89,9 +83,9 @@ def _require_sdk(*, auto_install: bool = True, prompt: bool = True): def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env. - The config stores environment variable names, not secret values; values are read - from the environment here. Missing variables are skipped (and noted at debug level - without the value). + The config stores environment variable names, not secret values; values are + read from the environment here. Missing variables are skipped (and noted at + debug level without the value). """ resolved: Dict[str, str] = {} for header_name, env_name in (headers_env or {}).items(): @@ -105,8 +99,8 @@ def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: - tel = (config or {}).get("telemetry") or {} - export = tel.get("export") or {} + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} return export.get("otlp") or {} @@ -116,7 +110,7 @@ def build_exporter(config: Dict[str, Any]): otlp = _otlp_config(config) endpoint = otlp.get("endpoint") if not endpoint: - raise ValueError("telemetry.export.otlp.endpoint is not set") + raise ValueError("monitoring.export.otlp.endpoint is not set") headers = _resolve_headers(otlp.get("headers_env")) return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) @@ -124,8 +118,8 @@ def build_exporter(config: Dict[str, Any]): def _make_provider(config: Dict[str, Any]): sdk = _require_sdk() resource = sdk["Resource"].create({ - "service.name": "hermes-agent", - "telemetry.scope": "local", # never aggregate metrics + "service.name": "hermes-gateway", + "telemetry.scope": "gateway_monitoring", }) provider = sdk["TracerProvider"](resource=resource) processor = sdk["BatchSpanProcessor"](build_exporter(config)) @@ -133,21 +127,20 @@ def _make_provider(config: Dict[str, Any]): return provider, processor -# ── event -> span attribute mapping (real values) ─────────────────────────── +# ── event -> span attribute mapping ────────────────────────────────────────── def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: - """Span attributes for an event — the real recorded values (local telemetry).""" + """Span attributes for a monitoring event (content-free by construction).""" kind = ev.get("event") attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} keep_by_kind = { - "run": ("entrypoint", "platform", "end_reason", - "model_call_count", "tool_call_count", "error_count"), - "span": ("trace_id", "run_id", "parent_span_id", "name", "kind", - "start_ns", "end_ns", "status"), - "model_call": ("provider", "model", "base_url", - "input_tokens", "output_tokens", "cache_read_tokens", - "cache_write_tokens", "reasoning_tokens", "latency_ms"), - "tool_call": ("tool_name", "duration_ms", "result_class"), - "error": ("error_class", "subsystem", "recovery"), + "gateway_health": ("name", "gateway_state", "old_state", "new_state", + "exit_reason", "restart_requested", "active_agents", + "gateway_busy", "gateway_drainable", "platform_count", + "fatal_platform_count", "profile", "install_id", "version", + "supervision_mode", "pid"), + "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", + "redacted_message", "platform", "old_state", "new_state", + "profile", "version", "severity"), } for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] v = ev.get(col) @@ -158,7 +151,7 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: def export_batch(provider, batch: List[Dict[str, Any]]) -> int: """Map a batch of events to OTel spans. Returns spans created.""" - tracer = provider.get_tracer("hermes.telemetry") + tracer = provider.get_tracer("hermes.monitoring") n = 0 for ev in batch: try: @@ -171,53 +164,6 @@ def export_batch(provider, batch: List[Dict[str, Any]]) -> int: return n -# ── one-shot drain (export current local rows) ────────────────────────────── -def export_once( - config: Dict[str, Any], - *, - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, -) -> int: - """Drain the local tel_* tables to the configured OTLP endpoint once.""" - provider, processor = _make_provider(config) - try: - rows = _read_events(db_path, since_ns) - total = export_batch(provider, rows) - processor.force_flush() - return total - finally: - try: - provider.shutdown() - except Exception: - pass - - -def _read_events(db_path: Optional[Path], since_ns: Optional[int]) -> List[Dict[str, Any]]: - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - out: List[Dict[str, Any]] = [] - try: - table_event = { - "tel_runs": "run", "tel_spans": "span", - "tel_model_calls": "model_call", - "tel_tool_calls": "tool_call", "tel_error_events": "error", - } - for table, evkind in table_event.items(): - where = "" - if table == "tel_runs" and since_ns: - where = f" WHERE start_ns >= {int(since_ns)}" - for r in c.execute(f"SELECT * FROM {table}{where}").fetchall(): - d = dict(r) - d["event"] = evkind - out.append(d) - finally: - c.close() - return out - - # ── continuous streaming subscriber ───────────────────────────────────────── class OTLPStreamer: """A live subscriber that pushes each emitter batch to OTLP as it lands. @@ -225,14 +171,29 @@ class OTLPStreamer: Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter. """ - def __init__(self, config: Dict[str, Any]): + def __init__( + self, + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, + ): self._provider, self._processor = _make_provider(config) + self._event_filter = event_filter self.exported = 0 def __call__(self, batch: List[Dict[str, Any]]) -> None: + if self._event_filter is not None: + batch = [ev for ev in batch if self._event_filter(ev)] + if not batch: + return self.exported += export_batch(self._provider, batch) def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass try: self._processor.force_flush() self._provider.shutdown() @@ -255,9 +216,16 @@ def is_enabled(config: Dict[str, Any]) -> bool: return bool(otlp.get("enabled") and otlp.get("endpoint")) -def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]: +def start_streaming( + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, +) -> Optional[OTLPStreamer]: """If OTLP is enabled, attach a streamer to the singleton emitter. + ``event_filter`` scopes the exporter to its plane, e.g. gateway-health + export, so enabling one plane cannot silently export unrelated events. + Non-interactive context (startup): attempts a lazy install with prompt=False so a configured-but-missing SDK is installed once (gated by security.allow_lazy_installs), then streams. If it still can't load, logs and @@ -268,11 +236,11 @@ def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]: try: _require_sdk(prompt=False) except OTLPUnavailable: - logger.warning("telemetry.export.otlp.enabled but the OTel SDK could not " + logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not " "be installed/imported; install 'hermes-agent[otlp]'") return None - from agent.telemetry.emitter import get_emitter - streamer = OTLPStreamer(config) + from agent.monitoring.emitter import get_emitter + streamer = OTLPStreamer(config, event_filter=event_filter) get_emitter().subscribe(streamer) return streamer @@ -281,7 +249,6 @@ __all__ = [ "OTLPUnavailable", "OTLPStreamer", "build_exporter", - "export_once", "export_batch", "is_available", "is_enabled", diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py new file mode 100644 index 00000000000..d07b90ea2c1 --- /dev/null +++ b/agent/monitoring/policy.py @@ -0,0 +1,40 @@ +"""Install identity for gateway monitoring. + +The install id is a stable, resettable pseudonymous identifier attached to +exported health signals so an operator can tell instances apart in their +collector. It carries no account identity and can be rotated by clearing +``monitoring.install_id`` in config. +""" + +from __future__ import annotations + +import uuid +from typing import Any, Dict + + +def _monitoring_cfg(config: Dict[str, Any]) -> Dict[str, Any]: + for key in ("monitoring", "telemetry"): # accept legacy telemetry.* keys + cfg = config.get(key) if isinstance(config, dict) else None + if isinstance(cfg, dict) and cfg.get("install_id"): + return cfg + cfg = config.get("monitoring") if isinstance(config, dict) else None + return cfg if isinstance(cfg, dict) else {} + + +def ensure_install_id(config: Dict[str, Any]) -> str: + """Return a stable install id, minting one if the config slot is empty. + + Does not persist — the caller writes the returned value back to + config.yaml. Clearing ``monitoring.install_id`` (e.g. with + ``hermes config set monitoring.install_id ""``) mints anew on next call. + """ + cfg = _monitoring_cfg(config) + existing = cfg.get("install_id") + if isinstance(existing, str) and existing.strip(): + return existing + return str(uuid.uuid4()) + + +__all__ = [ + "ensure_install_id", +] diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py new file mode 100644 index 00000000000..7ea5524cccd --- /dev/null +++ b/agent/monitoring/redaction.py @@ -0,0 +1,77 @@ +"""Redaction applied to monitoring data before egress. + +Secrets are always redacted, on every export path; no setting disables this. +Wraps ``agent/redact.py::redact_sensitive_text(force=True)`` and fails CLOSED: +if the redactor cannot run, the raw string is never emitted. + +``redact_for_export(text, content_mode="pii")`` additionally scrubs e-mail +addresses, phone numbers, and UUID-shaped identifiers — the gateway +diagnostics path always uses this mode, so log-derived messages leave the +process with secrets AND PII already removed. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +# Content-redaction strengths for any content that IS exported. +CONTENT_NONE = "none" # drop content entirely (structural telemetry only) +CONTENT_PII = "pii" # codec-aware PII redaction on exported content +CONTENT_MODES = {CONTENT_NONE, CONTENT_PII} + +# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ── +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +# E.164-ish and common separators; conservative to avoid nuking code/IDs. +_PHONE_RE = re.compile( + r"(? Optional[str]: + """Always-on secret redaction. force=True so user config can't disable it.""" + if text is None: + return None + try: + from agent.redact import redact_sensitive_text + return redact_sensitive_text(str(text), force=True) + except Exception: + # Fail CLOSED: if the redactor can't run, do not emit the raw string. + return "[redaction-unavailable]" + + +def _pii_redact(text: str) -> str: + text = _EMAIL_RE.sub("[email]", text) + text = _UUID_RE.sub("[id]", text) + text = _PHONE_RE.sub("[phone]", text) + return text + + +def redact_for_export( + text: Optional[str], + *, + content_mode: str = CONTENT_NONE, +) -> Optional[str]: + """Redact a single content string for export. + + Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. + Callers gate *whether content is exported at all* via telemetry.trajectories + (see ``content_export_enabled``); this function only scrubs content that the + caller has already decided to export. + """ + redacted = _secret_redact(text) + if redacted is None: + return None + if content_mode == CONTENT_PII: + redacted = _pii_redact(redacted) + return redacted + + +__all__ = [ + "CONTENT_NONE", + "CONTENT_PII", + "CONTENT_MODES", + "redact_for_export", +] diff --git a/agent/telemetry/__init__.py b/agent/telemetry/__init__.py deleted file mode 100644 index d824d5e6f4b..00000000000 --- a/agent/telemetry/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Hermes telemetry & observability. - -Local-first observability, on by default. The ``telemetry`` plugin registers Hermes -lifecycle hooks and hands typed events to the fire-and-forget ``emitter`` (queue -> -background writer -> JSONL + state.db ``tel_*`` index). The emitter never blocks or -raises into a model/tool call (the hot-path invariant). - -Events record the observed model ids, provider names, and tool names. ``metrics`` -derives rollups for /usage and /insights; ``rollup`` builds the per-run summaries shown -by ``hermes telemetry preview``. ``redaction`` + ``exporter_bulk`` + ``otlp_exporter`` -handle export to an operator-chosen destination. ``policy`` holds the consent -constants and the aggregate upload gate (no uploader ships). -""" - -from __future__ import annotations - -from . import emitter, events, metrics, policy, spans - -emit = emitter.emit -get_emitter = emitter.get_emitter - -__all__ = [ - "emitter", - "events", - "metrics", - "policy", - "spans", - "emit", - "get_emitter", -] diff --git a/agent/telemetry/emitter.py b/agent/telemetry/emitter.py deleted file mode 100644 index 5f804c1dacb..00000000000 --- a/agent/telemetry/emitter.py +++ /dev/null @@ -1,318 +0,0 @@ -"""Local telemetry emitter: fire-and-forget queue + background writer. - -The emitter is the single seam between instrumentation (the telemetry plugin's hook -callbacks) and durable storage. Its contract is the hot-path invariant: - - ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, and - MUST NEVER raise into the caller. A telemetry failure is logged locally and - dropped — it can never affect a model call, a tool call, or a session. - -Mechanism: - * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare except. - On a full queue it drops the *oldest* event and counts the drop. - * A daemon thread drains the queue and writes each event to two places: - 1. the append-only JSONL log (source of truth) - 2. the ``tel_*`` SQLite tables in state.db (rebuildable index) - * The writer uses its own sqlite connection to state.db, separate from SessionDB, - so telemetry writes never contend with or corrupt session writes. - -Local telemetry only. Nothing here uploads anywhere. -""" - -from __future__ import annotations - -import json -import logging -import queue -import sqlite3 -import threading -import time -from pathlib import Path -from typing import Any, Dict, Optional - -logger = logging.getLogger(__name__) - -_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full -_DRAIN_BATCH = 256 - - -def _default_dir() -> Path: - """Resolve the telemetry dir under the active HERMES_HOME (profile-safe).""" - from hermes_constants import get_hermes_home - return get_hermes_home() / "telemetry" - - -def _default_db_path() -> Path: - """Resolve state.db under the active HERMES_HOME (profile-safe).""" - from hermes_constants import get_hermes_home - return get_hermes_home() / "state.db" - - -# Map a telemetry event dict (its "event" tag) to (table, column-ordered insert). -# Only the columns the indexer knows about are written; unknown keys are ignored, -# so an event carrying extra fields never breaks the insert. -_TABLE_COLUMNS: Dict[str, tuple] = { - "run": ( - "tel_runs", - ("run_id", "trace_id", "session_id", "entrypoint", - "platform", "start_ns", "end_ns", "end_reason", - "model_call_count", "tool_call_count", "error_count"), - ), - "span": ( - "tel_spans", - ("span_id", "trace_id", "run_id", "parent_span_id", "name", "kind", - "start_ns", "end_ns", "status"), - ), - "model_call": ( - "tel_model_calls", - ("span_id", "run_id", "provider", "model", "base_url", - "input_tokens", "output_tokens", "cache_read_tokens", - "cache_write_tokens", "reasoning_tokens", "latency_ms"), - ), - "tool_call": ( - "tel_tool_calls", - ("span_id", "run_id", "tool_name", "duration_ms", "result_class"), - ), - "error": ( - "tel_error_events", - ("run_id", "error_class", "subsystem", "recovery", "ts_ns"), - ), -} - - -class TelemetryEmitter: - """Owns the queue, the writer thread, and the telemetry sqlite connection.""" - - def __init__( - self, - *, - events_path: Optional[Path] = None, - db_path: Optional[Path] = None, - enabled: bool = True, - ) -> None: - self._dir = (events_path.parent if events_path else _default_dir()) - self._events_path = events_path or (self._dir / "events.jsonl") - self._db_path = db_path or _default_db_path() - self._enabled = enabled - self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) - self._dropped = 0 - self._written = 0 - self._stop = threading.Event() - self._started = False - self._lock = threading.Lock() - self._conn: Optional[sqlite3.Connection] = None - self._thread: Optional[threading.Thread] = None - # Optional live subscribers (e.g. OTLP exporter). Called from the writer - # thread AFTER durable writes, fully fail-isolated — a subscriber that - # raises or blocks can never affect the JSONL/SQLite source of truth or - # the hot path. Each subscriber is callable(batch: list[dict]). - self._subscribers: list = [] - - # ── public API (hot path) ─────────────────────────────────────────────── - def emit(self, event: Any) -> None: - """Enqueue an event. Never blocks, never raises. - - ``event`` may be a dataclass with ``to_dict()`` or a plain dict. - """ - if not self._enabled: - return - try: - payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) - payload.setdefault("ts_ns", time.time_ns()) - self._ensure_started() - try: - self._q.put_nowait(payload) - except queue.Full: - # Drop oldest to make room — bounded memory, newest-wins. - try: - self._q.get_nowait() - self._dropped += 1 - self._q.put_nowait(payload) - except Exception: - self._dropped += 1 - except Exception: # the hot-path invariant: never propagate - logger.debug("telemetry emit failed", exc_info=True) - - # ── lifecycle ─────────────────────────────────────────────────────────── - def _ensure_started(self) -> None: - if self._started: - return - with self._lock: - if self._started: - return - try: - self._dir.mkdir(parents=True, exist_ok=True) - except Exception: - logger.debug("telemetry dir create failed", exc_info=True) - self._thread = threading.Thread( - target=self._run, name="hermes-telemetry-writer", daemon=True - ) - self._thread.start() - self._started = True - - def _open_conn(self) -> Optional[sqlite3.Connection]: - if self._conn is not None: - return self._conn - try: - conn = sqlite3.connect(str(self._db_path), isolation_level=None, timeout=5.0) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA busy_timeout=5000") - self._conn = conn - except Exception: - logger.debug("telemetry db open failed", exc_info=True) - self._conn = None - return self._conn - - def _run(self) -> None: - while not self._stop.is_set(): - try: - first = self._q.get(timeout=0.5) - except queue.Empty: - continue - batch = [first] - while len(batch) < _DRAIN_BATCH: - try: - batch.append(self._q.get_nowait()) - except queue.Empty: - break - self._write_batch(batch) - - def _write_batch(self, batch) -> None: - # JSONL append (source of truth) — best effort. - try: - with open(self._events_path, "a", encoding="utf-8") as fh: - for ev in batch: - fh.write(json.dumps(ev, ensure_ascii=False) + "\n") - except Exception: - logger.debug("telemetry jsonl append failed", exc_info=True) - - # SQLite index — best effort, per-event so one bad row can't lose the batch. - conn = self._open_conn() - if conn is None: - return - for ev in batch: - try: - self._index_one(conn, ev) - self._written += 1 - except Exception: - logger.debug("telemetry index row failed", exc_info=True) - - # Live fan-out (e.g. OTLP) — AFTER durable writes, fully fail-isolated. - # A slow/raising subscriber never affects JSONL/SQLite or the hot path. - for sub in self._subscribers: - try: - sub(batch) - except Exception: - logger.debug("telemetry subscriber failed", exc_info=True) - - def subscribe(self, callback) -> None: - """Register a live batch subscriber (callable(batch: list[dict])). - - Called from the writer thread after durable writes. Used by the OTLP - exporter for continuous streaming. Fail-isolated; never on the hot path. - """ - if callback not in self._subscribers: - self._subscribers.append(callback) - - def unsubscribe(self, callback) -> None: - try: - self._subscribers.remove(callback) - except ValueError: - pass - - def _index_one(self, conn: sqlite3.Connection, ev: Dict[str, Any]) -> None: - kind = ev.get("event") - spec = _TABLE_COLUMNS.get(kind) - if spec is None: - return - table, cols = spec - values = [ev.get(c) for c in cols] - placeholders = ", ".join("?" for _ in cols) - collist = ", ".join(cols) - conn.execute( - f"INSERT OR REPLACE INTO {table} ({collist}) VALUES ({placeholders})", - values, - ) - - # ── introspection / shutdown (tests, CLI) ─────────────────────────────── - def flush(self, timeout: float = 2.0) -> None: - """Block until the queue drains (test/CLI helper, NOT the hot path).""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if self._q.empty(): - # give the writer a tick to finish the in-flight batch - time.sleep(0.05) - if self._q.empty(): - return - time.sleep(0.02) - - def stats(self) -> Dict[str, int]: - return { - "queued": self._q.qsize(), - "written": self._written, - "dropped": self._dropped, - } - - def close(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) - if self._conn is not None: - try: - self._conn.close() - except Exception: - pass - self._conn = None - self._started = False - - -# ── process-wide singleton ────────────────────────────────────────────────── -_EMITTER: Optional[TelemetryEmitter] = None -_EMITTER_LOCK = threading.Lock() - - -def get_emitter() -> TelemetryEmitter: - """Return the process-wide emitter, honoring telemetry.local config.""" - global _EMITTER - if _EMITTER is not None: - return _EMITTER - with _EMITTER_LOCK: - if _EMITTER is None: - enabled = _local_enabled() - _EMITTER = TelemetryEmitter(enabled=enabled) - return _EMITTER - - -def _local_enabled() -> bool: - try: - from hermes_cli.config import load_config - cfg = load_config() - tel = cfg.get("telemetry") if isinstance(cfg, dict) else {} - return bool((tel or {}).get("local", True)) - except Exception: - return True - - -def emit(event: Any) -> None: - """Module-level convenience: emit via the singleton.""" - get_emitter().emit(event) - - -def reset_emitter_for_tests(emitter: Optional[TelemetryEmitter] = None) -> None: - """Swap the singleton (tests only).""" - global _EMITTER - with _EMITTER_LOCK: - if _EMITTER is not None and emitter is not _EMITTER: - try: - _EMITTER.close() - except Exception: - pass - _EMITTER = emitter - - -__all__ = [ - "TelemetryEmitter", - "get_emitter", - "emit", - "reset_emitter_for_tests", -] diff --git a/agent/telemetry/events.py b/agent/telemetry/events.py deleted file mode 100644 index 020ce11a961..00000000000 --- a/agent/telemetry/events.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Typed local telemetry events. - -These dataclasses are the rows written to the local JSONL log and the ``tel_*`` -SQLite tables. They record the values observed for each run — model id, provider, tool -name, token counts, durations — and stay on the machine unless explicitly exported. -""" - -from __future__ import annotations - -import time -from dataclasses import dataclass, field, asdict -from typing import Any, Dict, Optional - -# ── local telemetry events (real values) ──────────────────────────────────── - - -def _now_ns() -> int: - return time.time_ns() - - -@dataclass(slots=True) -class RunEvent: - """One top-level workflow execution (a trace root). A run spans one session.""" - run_id: str - trace_id: str - entrypoint: str - session_id: Optional[str] = None - platform: Optional[str] = None - start_ns: int = field(default_factory=_now_ns) - end_ns: Optional[int] = None - end_reason: Optional[str] = None - model_call_count: int = 0 - tool_call_count: int = 0 - error_count: int = 0 - - def to_dict(self) -> Dict[str, Any]: - return {"event": "run", **asdict(self)} - - -@dataclass(slots=True) -class ModelCallEvent: - span_id: str - run_id: str - provider: Optional[str] = None # raw provider, e.g. "anthropic" - model: Optional[str] = None # raw model id, e.g. "claude-opus-4" - base_url: Optional[str] = None - input_tokens: int = 0 - output_tokens: int = 0 - cache_read_tokens: int = 0 - cache_write_tokens: int = 0 - reasoning_tokens: int = 0 - latency_ms: Optional[int] = None - - def to_dict(self) -> Dict[str, Any]: - return {"event": "model_call", **asdict(self)} - - -@dataclass(slots=True) -class ToolCallEvent: - span_id: str - run_id: str - tool_name: Optional[str] = None # raw tool name, e.g. "web_search" - duration_ms: Optional[int] = None - result_class: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return {"event": "tool_call", **asdict(self)} - - -@dataclass(slots=True) -class SpanEvent: - """A timed span — the timing/lineage backbone of a trace. - - One row per run (the root, ``parent_span_id=None``) and one per model/tool call - (``parent_span_id`` = the run's root span). Detail rows in ``tel_model_calls`` / - ``tel_tool_calls`` share the ``span_id`` and are joined here for ordering and - placement on a timeline. - """ - span_id: str - trace_id: str - run_id: str - name: str - kind: str # "run" | "model" | "tool" - start_ns: int - end_ns: Optional[int] = None - parent_span_id: Optional[str] = None - status: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return {"event": "span", **asdict(self)} - - -@dataclass(slots=True) -class ErrorEvent: - run_id: Optional[str] - error_class: str - subsystem: str - recovery: Optional[str] = None - ts_ns: int = field(default_factory=_now_ns) - - def to_dict(self) -> Dict[str, Any]: - return {"event": "error", **asdict(self)} - - -__all__ = [ - "RunEvent", - "ModelCallEvent", - "ToolCallEvent", - "SpanEvent", - "ErrorEvent", -] diff --git a/agent/telemetry/exporter_bulk.py b/agent/telemetry/exporter_bulk.py deleted file mode 100644 index 32be393f723..00000000000 --- a/agent/telemetry/exporter_bulk.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Export telemetry (and optionally session content) to a file or stream. - -Two data domains, both written to an operator-chosen destination: - - * Telemetry: the tel_* rows + events.jsonl (structural observability). - * Content (opt-in via telemetry.trajectories): sessions + messages, with every - content field (message body, reasoning, raw tool-call args) passed through the - redaction pipeline (secrets always stripped; PII per content_redaction). - -Formats: ndjson (default) and json. OTLP streaming export lives in otlp_exporter.py. - -Content export is gated by ``redaction.content_export_enabled``. -""" - -from __future__ import annotations - -import json -import sqlite3 -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional, TextIO - -from . import redaction - -_TEL_TABLES = ( - "tel_runs", "tel_model_calls", "tel_tool_calls", "tel_error_events", -) - - -def _open(db_path: Optional[Path]) -> sqlite3.Connection: - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - return c - - -def _iter_telemetry(conn: sqlite3.Connection, since_ns: Optional[int]) -> Iterator[Dict[str, Any]]: - for table in _TEL_TABLES: - # only tel_runs has start_ns; window the rest by run join when needed. - if table == "tel_runs" and since_ns: - rows = conn.execute( - f"SELECT * FROM {table} WHERE start_ns >= ?", (int(since_ns),) - ).fetchall() - else: - rows = conn.execute(f"SELECT * FROM {table}").fetchall() - for r in rows: - d = dict(r) - d["_kind"] = table - yield d - - -def _iter_content( - db_path: Optional[Path], - *, - config: Optional[Dict[str, Any]], - include_content: bool, -) -> Iterator[Dict[str, Any]]: - """Yield session records. Message bodies included only when trajectories on.""" - from hermes_state import SessionDB - - content_mode = redaction.content_mode_for(config) - db = SessionDB(db_path=db_path) if db_path else SessionDB() - try: - for session in db.export_all(): - msgs = session.get("messages", []) or [] - red_msgs = [ - redaction.redact_message( - m, content_mode=content_mode, include_content=include_content - ) - for m in msgs - ] - # Session-level metadata is structural; keep ids/model/counts, drop - # any free-text title only when content is excluded. - out = { - "_kind": "session", - "id": session.get("id"), - "source": session.get("source"), - "model": session.get("model"), - "started_at": session.get("started_at"), - "ended_at": session.get("ended_at"), - "message_count": session.get("message_count"), - "tool_call_count": session.get("tool_call_count"), - "messages": red_msgs, - } - if include_content and session.get("title"): - out["title"] = redaction.redact_for_export( - session["title"], content_mode=content_mode - ) - yield out - finally: - db.close() - - -def export( - out: TextIO, - *, - fmt: str = "ndjson", - since_ns: Optional[int] = None, - include_content: bool = False, - config: Optional[Dict[str, Any]] = None, - db_path: Optional[Path] = None, -) -> Dict[str, int]: - """Write telemetry (+ optional content) to ``out``. Returns counts. - - ``include_content`` is honored only when telemetry.trajectories is enabled in - ``config``; otherwise content is forced off and only structural data is written. - """ - # Trajectories gate: a flag cannot override the config setting. - content_allowed = include_content and redaction.content_export_enabled(config) - counts = {"telemetry": 0, "sessions": 0, "content_included": int(content_allowed)} - - conn = _open(db_path) - records: List[Dict[str, Any]] = [] - try: - for rec in _iter_telemetry(conn, since_ns): - counts["telemetry"] += 1 - if fmt == "ndjson": - out.write(json.dumps(rec, ensure_ascii=False) + "\n") - else: - records.append(rec) - finally: - conn.close() - - # Content/session domain (separate connection via SessionDB). - for rec in _iter_content(db_path, config=config, include_content=content_allowed): - counts["sessions"] += 1 - if fmt == "ndjson": - out.write(json.dumps(rec, ensure_ascii=False) + "\n") - else: - records.append(rec) - - if fmt != "ndjson": - json.dump({"records": records}, out, ensure_ascii=False, indent=2) - - return counts - - -__all__ = ["export"] diff --git a/agent/telemetry/metrics.py b/agent/telemetry/metrics.py deleted file mode 100644 index fda0b4ee413..00000000000 --- a/agent/telemetry/metrics.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Derive metric rollups from the local telemetry tables. - -Reads the ``tel_*`` tables in state.db and returns aggregates for /usage, /insights, -and local dashboards. Metrics are computed by querying the event log rather than being -emitted on the hot path. - -Each function accepts either an open caller-owned ``conn`` (reused, not closed) or a -``db_path`` (opened and closed internally). InsightsEngine passes its existing -connection; a standalone dashboard passes a path. -""" - -from __future__ import annotations - -import sqlite3 -from contextlib import contextmanager -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional - - -@contextmanager -def _cursor( - conn: Optional[sqlite3.Connection], db_path: Optional[Path] -) -> Iterator[sqlite3.Connection]: - """Yield a Row-factory connection. Closes it only if we opened it.""" - if conn is not None: - prev_factory = conn.row_factory - conn.row_factory = sqlite3.Row - try: - yield conn - finally: - conn.row_factory = prev_factory - return - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - try: - yield c - finally: - c.close() - - -def _since_clause(since_ns: Optional[int], col: str = "start_ns") -> str: - return f" WHERE {col} >= {int(since_ns)}" if since_ns else "" - - -def workflow_summary( - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - """Run-level counters + duration percentiles (local telemetry, exact).""" - with _cursor(conn, db_path) as c: - where = _since_clause(since_ns) - total = c.execute(f"SELECT COUNT(*) n FROM tel_runs{where}").fetchone()["n"] - by_reason = { - r["end_reason"] or "unknown": r["n"] - for r in c.execute( - f"SELECT end_reason, COUNT(*) n FROM tel_runs{where} GROUP BY end_reason" - ).fetchall() - } - by_entry = { - r["entrypoint"] or "unknown": r["n"] - for r in c.execute( - f"SELECT entrypoint, COUNT(*) n FROM tel_runs{where} GROUP BY entrypoint" - ).fetchall() - } - dur_where = (where + " AND end_ns IS NOT NULL") if where else " WHERE end_ns IS NOT NULL" - durations = [ - (r["end_ns"] - r["start_ns"]) / 1e6 - for r in c.execute( - f"SELECT start_ns, end_ns FROM tel_runs{dur_where}" - ).fetchall() - ] - return { - "total_runs": total, - "by_end_reason": by_reason, - "by_entrypoint": by_entry, - "duration_ms_p50": _pct(durations, 50), - "duration_ms_p95": _pct(durations, 95), - "success_rate": round(by_reason.get("completed", 0) / total, 4) if total else 0.0, - } - - -def model_call_summary( - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - with _cursor(conn, db_path) as c: - rows = c.execute( - "SELECT provider, model, COUNT(*) n, " - "SUM(input_tokens) inp, SUM(output_tokens) outp, " - "SUM(cache_read_tokens) cache, AVG(latency_ms) avg_latency " - "FROM tel_model_calls GROUP BY provider, model" - ).fetchall() - by_provider: Dict[str, int] = {} - by_model: Dict[str, int] = {} - tokens = {"input": 0, "output": 0, "cache_read": 0} - breakdown: List[Dict[str, Any]] = [] - for r in rows: - prov = r["provider"] or "unknown" - mdl = r["model"] or "unknown" - by_provider[prov] = by_provider.get(prov, 0) + r["n"] - by_model[mdl] = by_model.get(mdl, 0) + r["n"] - tokens["input"] += r["inp"] or 0 - tokens["output"] += r["outp"] or 0 - tokens["cache_read"] += r["cache"] or 0 - breakdown.append({ - "provider": r["provider"], - "model": r["model"], - "calls": r["n"], - "avg_latency_ms": round(r["avg_latency"] or 0, 1), - }) - cache_total = tokens["cache_read"] + tokens["input"] - return { - "by_provider": by_provider, - "by_model": by_model, - "tokens": tokens, - "cache_hit_rate": round(tokens["cache_read"] / cache_total, 4) if cache_total else 0.0, - "breakdown": breakdown, - } - - -def tool_call_summary( - db_path: Optional[Path] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - with _cursor(conn, db_path) as c: - by_tool = { - r["tool_name"] or "unknown": r["n"] - for r in c.execute( - "SELECT tool_name, COUNT(*) n FROM tel_tool_calls GROUP BY tool_name" - ).fetchall() - } - fails = { - r["tool_name"] or "unknown": r["n"] - for r in c.execute( - "SELECT tool_name, COUNT(*) n FROM tel_tool_calls " - "WHERE result_class IN ('error','timeout','blocked') GROUP BY tool_name" - ).fetchall() - } - total = sum(by_tool.values()) - total_fail = sum(fails.values()) - return { - "by_tool": by_tool, - "failures_by_tool": fails, - "total": total, - "failure_rate": round(total_fail / total, 4) if total else 0.0, - } - - -def error_summary( - db_path: Optional[Path] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - with _cursor(conn, db_path) as c: - return { - "by_class": { - r["error_class"] or "unknown": r["n"] - for r in c.execute( - "SELECT error_class, COUNT(*) n FROM tel_error_events GROUP BY error_class" - ).fetchall() - }, - } - - -def _pct(values: List[float], p: int) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * (p / 100) - lo = int(k) - hi = min(lo + 1, len(s) - 1) - frac = k - lo - return round(s[lo] + (s[hi] - s[lo]) * frac, 2) - - -def overview( - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - """One call for a dashboard: all the rollups.""" - return { - "workflows": workflow_summary(db_path, since_ns, conn=conn), - "model_calls": model_call_summary(db_path, since_ns, conn=conn), - "tool_calls": tool_call_summary(db_path, conn=conn), - "errors": error_summary(db_path, conn=conn), - } - - -def has_data( - db_path: Optional[Path] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> bool: - """True when any telemetry runs exist (cheap guard for /insights rendering).""" - try: - with _cursor(conn, db_path) as c: - return c.execute("SELECT 1 FROM tel_runs LIMIT 1").fetchone() is not None - except Exception: - return False - - -__all__ = [ - "workflow_summary", - "model_call_summary", - "tool_call_summary", - "error_summary", - "overview", - "has_data", -] diff --git a/agent/telemetry/policy.py b/agent/telemetry/policy.py deleted file mode 100644 index 77e794cf8a2..00000000000 --- a/agent/telemetry/policy.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Telemetry consent posture and the aggregate-metrics gate. - -Consent is a single config field, ``telemetry.consent_state``: - - * "unknown" — no choice recorded; never uploads (the default). - * "local" — declined aggregate metrics; local telemetry only. - * "aggregate" — opted in to aggregate metrics. - -The config file is the source of truth: set ``telemetry.consent_state`` with -``hermes config set`` (or a managed-scope pin). Callers that gate behavior read -``telemetry.*`` directly from config; this module only provides the consent -constants, the install-id helper, and the upload gate a future uploader must -consult. - -``allow_aggregate`` is the hard gate. An administrator pins -``telemetry.allow_aggregate: false`` through the managed-scope layer -(``/etc/hermes/config.yaml``), which takes precedence over the user's config; when -it is false, aggregate metrics are off regardless of ``consent_state``. -""" - -from __future__ import annotations - -import uuid -from typing import Any, Dict - -CONSENT_UNKNOWN = "unknown" -CONSENT_LOCAL = "local" -CONSENT_AGGREGATE = "aggregate" -VALID_CONSENT_STATES = {CONSENT_UNKNOWN, CONSENT_LOCAL, CONSENT_AGGREGATE} - - -def _telemetry_cfg(config: Dict[str, Any]) -> Dict[str, Any]: - cfg = config.get("telemetry") if isinstance(config, dict) else None - return cfg if isinstance(cfg, dict) else {} - - -def ensure_install_id(config: Dict[str, Any]) -> str: - """Return a stable install id, minting one if the config slot is empty. - - Does not persist — the caller writes the returned value back to config.yaml. A - fresh uuid4 is used; clearing ``telemetry.install_id`` (e.g. with - ``hermes config set telemetry.install_id ""``) causes the next call to mint anew. - """ - tel = _telemetry_cfg(config) - existing = tel.get("install_id") - if isinstance(existing, str) and existing.strip(): - return existing - return str(uuid.uuid4()) - - -def may_upload_aggregate(config: Dict[str, Any]) -> bool: - """Whether aggregate metrics may upload — the gate a future uploader consults. - - Aggregate metrics are derived from the local telemetry tables, so they require - local telemetry to be on. True only when local telemetry is enabled, the admin - hard gate allows it, and the user has opted in via ``telemetry.consent_state``. - """ - tel = _telemetry_cfg(config) - local_enabled = bool(tel.get("local", True)) - allow_aggregate = bool(tel.get("allow_aggregate", True)) - state = tel.get("consent_state", CONSENT_UNKNOWN) - return local_enabled and allow_aggregate and state == CONSENT_AGGREGATE - - -__all__ = [ - "CONSENT_UNKNOWN", - "CONSENT_LOCAL", - "CONSENT_AGGREGATE", - "VALID_CONSENT_STATES", - "may_upload_aggregate", - "ensure_install_id", -] diff --git a/agent/telemetry/redaction.py b/agent/telemetry/redaction.py deleted file mode 100644 index 01090abb22a..00000000000 --- a/agent/telemetry/redaction.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Redaction applied to telemetry data on export. - -Two independent controls: - - * Secrets are always redacted, on every export and in every mode; no setting - disables this. Wraps ``agent/redact.py::redact_sensitive_text(force=True)``. - - * Whether message bodies, reasoning, and raw tool arguments are exportable at all is - governed by the trajectories setting (``telemetry.trajectories.enabled``, default - off, admin-pinnable), not by a redaction mode. With trajectories off, content is - dropped. With it on, content is exportable and ``content_redaction`` (none|pii) - controls how much is scrubbed; secrets are still always stripped. - -This applies to the local and trajectory export paths. It is unrelated to any -aggregate-metrics path. -""" - -from __future__ import annotations - -import re -from typing import Any, Dict, List, Optional - -# Content-redaction strengths for any content that IS exported. -CONTENT_NONE = "none" # drop content entirely (structural telemetry only) -CONTENT_PII = "pii" # codec-aware PII redaction on exported content -CONTENT_MODES = {CONTENT_NONE, CONTENT_PII} - -# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ── -_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") -# E.164-ish and common separators; conservative to avoid nuking code/IDs. -_PHONE_RE = re.compile( - r"(? Optional[str]: - """Always-on secret redaction. force=True so user config can't disable it.""" - if text is None: - return None - try: - from agent.redact import redact_sensitive_text - return redact_sensitive_text(str(text), force=True) - except Exception: - # Fail CLOSED: if the redactor can't run, do not emit the raw string. - return "[redaction-unavailable]" - - -def _pii_redact(text: str) -> str: - text = _EMAIL_RE.sub("[email]", text) - text = _UUID_RE.sub("[id]", text) - text = _PHONE_RE.sub("[phone]", text) - return text - - -def redact_for_export( - text: Optional[str], - *, - content_mode: str = CONTENT_NONE, -) -> Optional[str]: - """Redact a single content string for export. - - Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. - Callers gate *whether content is exported at all* via telemetry.trajectories - (see ``content_export_enabled``); this function only scrubs content that the - caller has already decided to export. - """ - redacted = _secret_redact(text) - if redacted is None: - return None - if content_mode == CONTENT_PII: - redacted = _pii_redact(redacted) - return redacted - - -def content_export_enabled(config: Optional[Dict[str, Any]]) -> bool: - """True only when telemetry.trajectories is explicitly enabled. - - This is the consent gate for exporting message bodies / reasoning / raw tool - args. Default off. Admin-pinnable via managed scope (telemetry.trajectories.enabled). - """ - try: - tel = (config or {}).get("telemetry") or {} - traj = tel.get("trajectories") or {} - return bool(traj.get("enabled", False)) - except Exception: - return False - - -def content_mode_for(config: Optional[Dict[str, Any]]) -> str: - try: - tel = (config or {}).get("telemetry") or {} - mode = tel.get("content_redaction", CONTENT_NONE) - return mode if mode in CONTENT_MODES else CONTENT_NONE - except Exception: - return CONTENT_NONE - - -# ── Codec-aware message redaction (NeMo pattern) ───────────────────────────── -# Redact the right fields of a provider message shape rather than regex-blasting -# the whole blob. Structure (roles, names, counts) is preserved; only the -# free-text content fields are scrubbed. - -def redact_message( - msg: Dict[str, Any], - *, - content_mode: str = CONTENT_NONE, - include_content: bool = False, -) -> Dict[str, Any]: - """Redact one chat message dict for export. - - When include_content is False (trajectories off), content/reasoning/tool-arg - fields are dropped — only structural fields (role, tool name, counts) remain. - When True, those fields are kept but passed through redact_for_export. - """ - role = msg.get("role") - out: Dict[str, Any] = {"role": role} - - # Always-structural fields. - if msg.get("tool_name") is not None: - out["tool_name"] = msg.get("tool_name") - if msg.get("name") is not None: - out["name"] = msg.get("name") - - if not include_content: - # Structural only: record presence/size, not bytes. - c = msg.get("content") - if c is not None: - out["content_chars"] = len(str(c)) - if msg.get("reasoning_content"): - out["reasoning_chars"] = len(str(msg["reasoning_content"])) - if msg.get("tool_calls"): - out["tool_call_count"] = _count_tool_calls(msg["tool_calls"]) - return out - - # Content included (trajectories enabled): scrub then keep. - if msg.get("content") is not None: - out["content"] = redact_for_export(msg["content"], content_mode=content_mode) - if msg.get("reasoning_content"): - out["reasoning_content"] = redact_for_export( - msg["reasoning_content"], content_mode=content_mode - ) - if msg.get("tool_calls"): - out["tool_calls"] = _redact_tool_calls(msg["tool_calls"], content_mode=content_mode) - return out - - -def _count_tool_calls(tool_calls: Any) -> int: - try: - import json - tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls - return len(tc) if isinstance(tc, list) else (1 if tc else 0) - except Exception: - return 0 - - -def _redact_tool_calls(tool_calls: Any, *, content_mode: str) -> Any: - """Redact raw tool-call arguments (free text) while keeping function names.""" - import json - try: - tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls - except Exception: - return "[unparseable-tool-calls]" - if not isinstance(tc, list): - return [] - out: List[Dict[str, Any]] = [] - for call in tc: - if not isinstance(call, dict): - continue - fn = (call.get("function") or {}) if isinstance(call.get("function"), dict) else {} - name = fn.get("name") or call.get("name") - args = fn.get("arguments") - red_args = redact_for_export(args, content_mode=content_mode) if args is not None else None - out.append({"name": name, "arguments": red_args}) - return out - - -__all__ = [ - "CONTENT_NONE", - "CONTENT_PII", - "CONTENT_MODES", - "redact_for_export", - "content_export_enabled", - "content_mode_for", - "redact_message", -] diff --git a/agent/telemetry/rollup.py b/agent/telemetry/rollup.py deleted file mode 100644 index 3880a55caa8..00000000000 --- a/agent/telemetry/rollup.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Build per-run summary events from the local telemetry tables. - -Reads the ``tel_*`` tables and projects each completed run into a summary dict holding -the recorded values: provider, models used, tool names, token totals, duration, and -cost. Powers ``hermes telemetry preview``. No aggregation or bucketing is applied here. -""" - -from __future__ import annotations - -import platform -import sqlite3 -from pathlib import Path -from typing import Any, Dict, List, Optional - - -def _os_family() -> str: - s = platform.system().lower() - if s.startswith("lin"): - return "linux" - if s == "darwin": - return "macos" - if s.startswith("win"): - return "windows" - return "other" - - -def _hermes_version() -> str: - try: - from hermes_cli import __version__ - return str(__version__) - except Exception: - return "0.0.0" - - -def _open(db_path: Optional[Path], conn: Optional[sqlite3.Connection]): - if conn is not None: - prev = conn.row_factory - conn.row_factory = sqlite3.Row - return conn, prev, False - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - return c, None, True - - -def _run_events(c: sqlite3.Connection, since_ns: Optional[int]) -> List[Dict[str, Any]]: - """Project completed runs into per-run summary dicts.""" - where = " WHERE end_ns IS NOT NULL" - if since_ns: - where += f" AND start_ns >= {int(since_ns)}" - rows = c.execute( - "SELECT run_id, entrypoint, platform, end_reason, start_ns, end_ns, " - "model_call_count, tool_call_count, error_count " - "FROM tel_runs" + where - ).fetchall() - - events: List[Dict[str, Any]] = [] - for r in rows: - # Models actually used in this run (real ids), with token totals. - models = [ - {"provider": m["provider"], "model": m["model"], - "calls": m["n"], "input_tokens": int(m["inp"] or 0), - "output_tokens": int(m["outp"] or 0)} - for m in c.execute( - "SELECT provider, model, COUNT(*) n, SUM(input_tokens) inp, " - "SUM(output_tokens) outp FROM tel_model_calls WHERE run_id = ? " - "GROUP BY provider, model ORDER BY n DESC", - (r["run_id"],), - ).fetchall() - ] - tools = [ - row["tool_name"] - for row in c.execute( - "SELECT DISTINCT tool_name FROM tel_tool_calls WHERE run_id = ?", - (r["run_id"],), - ).fetchall() - if row["tool_name"] - ] - trow = c.execute( - "SELECT SUM(input_tokens) inp, SUM(output_tokens) outp " - "FROM tel_model_calls WHERE run_id = ?", - (r["run_id"],), - ).fetchone() - duration_ms = (r["end_ns"] - r["start_ns"]) / 1e6 if r["end_ns"] else None - events.append({ - "event_name": "workflow_completed", - "run_id": r["run_id"], - "entrypoint": r["entrypoint"] or "cli", - "platform": r["platform"], - "end_reason": r["end_reason"] or "completed", - "models_used": models, - "tools_used": tools, - "model_call_count": r["model_call_count"] or 0, - "tool_call_count": r["tool_call_count"] or 0, - "error_count": r["error_count"] or 0, - "duration_ms": round(duration_ms, 1) if duration_ms is not None else None, - "input_tokens": int((trow["inp"] if trow else 0) or 0), - "output_tokens": int((trow["outp"] if trow else 0) or 0), - }) - return events - - -def build_aggregate_events( - *, - install_id: str, - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - conn: Optional[sqlite3.Connection] = None, - include_heartbeat: bool = True, -) -> List[Dict[str, Any]]: - """Return per-run summary events plus an optional heartbeat.""" - c, prev_factory, owned = _open(db_path, conn) - try: - events = _run_events(c, since_ns) - if include_heartbeat: - events.append({ - "event_name": "heartbeat", - "install_id": install_id, - "hermes_version": _hermes_version(), - "os_family": _os_family(), - "entrypoint": "cli", - }) - return events - finally: - if owned: - c.close() - elif prev_factory is not None: - c.row_factory = prev_factory - - -def summarize(events: List[Dict[str, Any]]) -> Dict[str, Any]: - """Counts by event_name + field coverage, for status/preview output.""" - by_name: Dict[str, int] = {} - fields = set() - for e in events: - name = e.get("event_name", "?") - by_name[name] = by_name.get(name, 0) + 1 - fields.update(e.keys()) - return {"total": len(events), "by_event_name": by_name, "fields_present": sorted(fields)} - - -__all__ = ["build_aggregate_events", "summarize"] diff --git a/agent/telemetry/spans.py b/agent/telemetry/spans.py deleted file mode 100644 index bfd876f8a11..00000000000 --- a/agent/telemetry/spans.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Trace / run / span id propagation via contextvars. - -Telemetry events share IDs so a workflow can be reconstructed: one ``trace_id`` per -workflow, one ``run_id`` per top-level execution, ``span_id`` per timed operation, and -``parent_span_id`` for nesting. These live in contextvars so async tool calls and -spawned subagents inherit the lineage automatically. - -Provides helpers to start/clear a run context and mint child span ids. The telemetry -plugin sets the run context on session start and reads it in each hook callback. -Nothing here writes to storage — it only carries ids. -""" - -from __future__ import annotations - -import contextvars -import uuid -from dataclasses import dataclass -from typing import Optional - -_trace_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( - "hermes_tel_trace_id", default=None -) -_run_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( - "hermes_tel_run_id", default=None -) -_parent_span_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( - "hermes_tel_parent_span_id", default=None -) - - -def new_id() -> str: - return uuid.uuid4().hex - - -@dataclass(slots=True) -class RunContext: - trace_id: str - run_id: str - - -def start_run(trace_id: Optional[str] = None, run_id: Optional[str] = None) -> RunContext: - """Begin a run context, minting ids when not supplied. Sets contextvars.""" - tid = trace_id or new_id() - rid = run_id or new_id() - _trace_id.set(tid) - _run_id.set(rid) - _parent_span_id.set(None) - return RunContext(trace_id=tid, run_id=rid) - - -def current_trace_id() -> Optional[str]: - return _trace_id.get() - - -def current_run_id() -> Optional[str]: - return _run_id.get() - - -def current_parent_span_id() -> Optional[str]: - return _parent_span_id.get() - - -def new_span_id() -> str: - """Mint a span id (does not alter the parent pointer).""" - return new_id() - - -def clear_run() -> None: - _trace_id.set(None) - _run_id.set(None) - _parent_span_id.set(None) - - -__all__ = [ - "RunContext", - "new_id", - "start_run", - "current_trace_id", - "current_run_id", - "current_parent_span_id", - "new_span_id", - "clear_run", -] diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 982708c3422..d52e3ec009c 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1414,49 +1414,6 @@ display: # # Routing/delivery still uses the original values internally. # redact_pii: false -# ============================================================================= -# Telemetry & Observability -# ============================================================================= -# Three settings, isolated from each other: -# local — full-fidelity observability you own. Default ON. -# aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. -# trajectories — content trajectories for training. Separate consent (later). -# -# Local telemetry records real values (actual models, providers, tool names) — -# your own data, on your machine. Aggregate metrics are opt-in and have no -# uploader today; if one ships it would summarize at that egress boundary. -# -# Enterprise locking: any telemetry.* key can be pinned by an administrator via -# the managed-scope layer (/etc/hermes/config.yaml), which wins over the user's -# value. To hard-forbid egress on a locked-down deployment, pin -# `telemetry.allow_aggregate: false` there. -# telemetry: -# # Local telemetry: event log + SQLite index in state.db. Never leaves your -# # machine unless you export it or opt into aggregate metrics. -# local: true -# # Hard gate for aggregate metrics. When false, aggregate metrics are off -# # regardless of consent_state. Pin false via managed scope to forbid egress. -# allow_aggregate: true -# # Aggregate-metrics consent (the opt-in). No uploader ships yet. -# # unknown (no choice — never uploads) | local (declined) | aggregate (opted in) -# consent_state: unknown -# # Stable install id (aggregate metrics only). Empty = mint on first use; -# # clear it to rotate. -# install_id: "" -# # Local event-log retention before rotation (days). -# retention_days: 90 -# # Keep secret redaction on even at full local capture. -# redact_secrets: true -# # Content redaction for exports / support bundles: none | pii. -# content_redaction: none -# # Exporters. The OTLP exporter sends spans to a configured Collector endpoint; -# # header values reference environment variable names, not inline secrets. -# export: -# otlp: -# enabled: false -# endpoint: null -# headers_env: {} # e.g. {Authorization: MY_OTLP_TOKEN_ENVVAR} - # ============================================================================= # Shell-script hooks # ============================================================================= diff --git a/docs/observability/README.md b/docs/observability/README.md index 8717627d8a8..9040929ca48 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -307,13 +307,6 @@ nested agent work or security lifecycle events. ## Existing Consumers -The bundled **`telemetry`** plugin is the built-in local-first telemetry system: it -records runs, spans, model calls, and tool calls to a local event log + `state.db`, powers -`hermes insights`, and supports export to your own file or OpenTelemetry Collector. It -is on by default and never sends data to Nous unless you opt in. See -[`telemetry.md`](./telemetry.md) for the full feature, the `hermes telemetry` commands, -and enterprise export. - The bundled Langfuse plugin demonstrates direct hook-based observability for turns, provider requests, and tool calls. diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md new file mode 100644 index 00000000000..1ee9d4b5a8d --- /dev/null +++ b/docs/observability/monitoring.md @@ -0,0 +1,101 @@ +# Gateway Monitoring + +Service health monitoring plus redacted operational diagnostics for the +Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured +endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver). + +This plane is content-free by construction. It exports gateway lifecycle +state, platform connector health, and redacted warning/error diagnostics. +It never exports prompts, messages, tool arguments or results, session +history, usage analytics, audit logs, or execution traces. Run/model/tool +trajectory capture is a separate plane served by the NeMo Relay integration +(`plugins/observability/nemo_relay/`) and its Hermes-owned subscribers. + +## What gets exported + +| Signal | OTLP route | Content | +| --- | --- | --- | +| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | +| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | +| Diagnostics | `/v1/logs` | Warning/error gateway log events with secrets AND PII scrubbed in-process before egress (`[redacted]` / `[email]`), bounded error classes | + +Every signal carries resource attributes (`service.name`, profile, version, +install id, supervision mode) so an operator can tell instances apart. + +## Enabling + +```yaml +# config.yaml +monitoring: + gateway_health_export: + enabled: true + export: + otlp: + enabled: true + endpoint: http://collector-host:4318/v1/traces # metrics/logs derive + headers_env: {} # header name -> ENV VAR NAME (values never stored) +``` + +Check the posture any time: + +```bash +hermes monitoring status +``` + +The OpenTelemetry SDK is an optional extra (`pip install 'hermes-agent[otlp]'`), +lazy-installed on first use. When the SDK is missing or the endpoint is down, +the gateway runs unaffected: every export path is fail-open and off the hot +path (events flow through a fire-and-forget in-process emitter; a slow or +failing exporter can never block gateway code). + +Works identically under systemd/launchd/s6 supervision, containers, tmux, or +a plain `hermes gateway run` — the exporter lives in the gateway process, so +no sidecar, agent, or collector is required on the host. + +## Collecting into DataDog + +Run a customer-owned OpenTelemetry Collector and forward: + +```yaml +# otel-collector config +receivers: + otlp: + protocols: + http: +exporters: + datadog: + api: + key: ${env:DD_API_KEY} +service: + pipelines: + metrics: {receivers: [otlp], exporters: [datadog]} + traces: {receivers: [otlp], exporters: [datadog]} + logs: {receivers: [otlp], exporters: [datadog]} +``` + +Point `monitoring.export.otlp.endpoint` at the collector. Alerts belong on +`hermes.gateway.up`, `hermes.platform.up`, and `hermes.platform.degraded`. + +## Local smoke test (no Docker) + +```bash +# terminal 1: capture collector on :4318 +python scripts/observability/otel_capture_collector.py \ + --host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl + +# terminal 2: drive the real exporter through lifecycle transitions, +# a fatal platform, and a redacted warning log, then flush +python scripts/observability/gateway_health_export_probe.py \ + --endpoint http://127.0.0.1:4318/v1/traces \ + --log /tmp/hermes_otel_capture.jsonl --wait 8 +# exit 0 prints: {"requests": 6, "paths": ["/v1/logs", "/v1/metrics", "/v1/traces"]} +``` + +## Boundaries and roadmap + +The `hermes monitoring` CLI intentionally exposes `status` only. Shared +client usage metrics and enterprise trace telemetry are being designed on +the NeMo Relay integration with their own consent, policy, and export +boundaries; this monitoring plane stays narrow so an operator can enable it +without touching any content-bearing signal. The telemetry surface may be +reorganized as that lands. diff --git a/docs/observability/telemetry.md b/docs/observability/telemetry.md deleted file mode 100644 index ea3d2f28ffa..00000000000 --- a/docs/observability/telemetry.md +++ /dev/null @@ -1,228 +0,0 @@ -# Telemetry & Observability - -Hermes ships with a built-in, local-first telemetry system. It records what your -agent does — workflows, model calls, tool calls, errors — to your own machine, powers -`/insights`, and (when *you* enable it) exports everything to your own observability -stack. It is private by default and never sends your data to Nous unless you explicitly -opt in to anonymous aggregate metrics. - -This page explains the whole feature: the three telemetry settings, what's captured, the -`hermes telemetry` commands, and how an enterprise streams or exports all of its data -to its own infrastructure. - -> Looking for the **plugin hook contract** (how to write your own observer plugin)? -> That's in [`README.md`](./README.md). This page is about the built-in telemetry -> system and its CLI. - -## The three settings - -Telemetry has three settings, isolated from each other: - -| Setting | What it holds | Default | Destination | -| --- | --- | --- | --- | -| **local** | Full-fidelity observability — runs, model/tool calls (real model & provider names), durations, errors | **on** | your machine only | -| **aggregate** | Opt-in metadata to Nous (no uploader ships yet) | **off** (opt-in) | Nous, only if you enable it | -| **trajectories** | Full message content / reasoning / raw tool args | **off** (opt-in) | your own export destinations only | - -Local telemetry is the one you'll use day to day — it records the real values that -happened (actual model ids, providers, tool names). Aggregate metrics are the only -thing that could ever leave for Nous; they are opt-in, default-off, and have no uploader -today. Trajectories unlock full-content export to *your own* destinations — never wired -to Nous. - -## Local telemetry — always-on observability - -Local telemetry is implemented as a bundled `telemetry` plugin that listens to Hermes -lifecycle hooks (model calls, tool calls, session start/finalize) and writes events to: - -- an append-only JSONL log at `~/.hermes/telemetry/events.jsonl` (the source of truth) -- indexed `tel_*` tables in `state.db` (a rebuildable index for fast queries): - `tel_runs`, `tel_spans`, `tel_model_calls`, `tel_tool_calls`, `tel_error_events` - -Writes are fire-and-forget on a background thread: telemetry can never block, slow, or -fail a model call or tool call. If local telemetry is disabled (`telemetry.local: false`) -the plugin does not load at all. - -### Traces and spans - -A **run** is one session (from `on_session_start` to `on_session_finalize`). Each run gets -a root span in `tel_spans`, and every model and tool call within it is recorded as a child -span (timing + `parent_span_id` = the run's root) keyed by the same `span_id` as its -detail row in `tel_model_calls` / `tel_tool_calls`. So a run reconstructs as a connected -`run -> calls` tree, ordered by `start_ns` and joinable to the per-call detail — the shape -a trace viewer or any OpenTelemetry backend can render. - -Call spans are timed from the measured latency/duration the hooks report; cross-run -subagent lineage (linking a delegated child run to its parent) is not yet recorded. - -### Seeing your local data - -```bash -hermes insights # usage report — now includes an "Observability" section -hermes telemetry status # settings, consent, export posture, local data volume -``` - -The `insights` Observability section shows workflow counts and success rate, duration -p50/p95, tool failure rates by tool, provider/model mix, and cache hit rate — -all computed locally with exact values. - -## `hermes telemetry` commands - -```text -hermes telemetry status Show settings, consent state, export posture, local volume -hermes telemetry preview Show the aggregate events that would be produced (local) -hermes telemetry export Export local telemetry to a file/stream or OTLP endpoint -``` - -Consent and the install id are plain config, not separate verbs — set them with -`hermes config set` (or a managed-scope pin): - -```bash -hermes config set telemetry.consent_state aggregate # opt in to aggregate metrics -hermes config set telemetry.consent_state local # opt out (local telemetry stays on) -hermes config set telemetry.install_id "" # reset the install id (mints a new one) -``` - -### Aggregate metrics (opt-in) - -Aggregate metrics are **off by default** and have **no uploader today** — nothing is -sent to Nous. Consent lives in `telemetry.consent_state` (`unknown` / `local` / -`aggregate`); setting it to `aggregate` records the opt-in for if/when an uploader ships. -If one is built, it would summarize at that egress boundary. - -`hermes telemetry preview` shows your recent runs as they'd be summarized — computed and -shown **locally only**, with the real model and tool names from your own telemetry. It's -a local inspection surface, not an upload. - -## Enterprise: getting all of your data - -Everything below sends data to **your own** destination — a file, your SIEM, or your own -OpenTelemetry Collector. None of it goes to Nous. - -### Bulk export to a file - -```bash -# Structural telemetry only (default — no message content) -hermes telemetry export --out telemetry.ndjson - -# JSON instead of NDJSON, last 7 days only -hermes telemetry export --out dump.json --format json --since 7 -``` - -By default the export is **structural** — runs, model/tool-call metadata, session shells -with message *counts* but no message bodies. - -### Including content (trajectories) - -To export full message content, enable trajectories. This is a deliberate, separate -consent — it's how an enterprise opts into exporting work-product content to its own -store: - -```yaml -# config.yaml -telemetry: - trajectories: - enabled: true # unlocks content export to YOUR destination - content_redaction: pii # "none" | "pii" -``` - -```bash -hermes telemetry export --out full.ndjson --include-content -``` - -`--include-content` is a no-op unless trajectories are enabled — the config setting -governs, not the flag. - -### Live streaming to your OpenTelemetry Collector / SIEM (OTLP) - -Hermes can stream telemetry to your own OTLP endpoint. This requires the optional `otlp` -extra: - -```bash -pip install 'hermes-agent[otlp]' -``` - -```yaml -# config.yaml -telemetry: - export: - otlp: - enabled: true - endpoint: "https://collector.your-corp.internal:4318/v1/traces" - headers_env: # secrets by reference — env var NAMES, not values - Authorization: MY_OTLP_TOKEN_ENVVAR -``` - -Set the referenced environment variable, then run the export: - -```bash -hermes telemetry export --otlp # drain current telemetry to your collector -``` - -The token value lives only in the environment variable named by `headers_env`. The -config holds the *name* of an environment variable rather than the secret itself; the -value is read at export time and is never written to config or logged. - -Each telemetry event is exported as an OTel span carrying its recorded attributes -(provider, model, tokens, duration, etc.). The `tel_spans` timing/parent linkage is not -yet reconstructed into connected OTel `SpanContext`s, so spans currently arrive as -independent records rather than a connected trace tree; that projection is planned. - -## Redaction - -Two independent controls govern what content looks like on export: - -| Control | Values | Effect | -| --- | --- | --- | -| Secret redaction | always on | API keys, tokens, auth headers, connection strings are **always** stripped on every export path. Cannot be disabled. | -| `content_redaction` | `none` \| `pii` | When content is exported, `pii` additionally redacts emails, phone numbers, and id-shaped strings. | - -Secret redaction is always on — even at full content fidelity — because a SIEM or -warehouse full of live credentials is a bigger attack target than the data it holds. It -fails closed: if the redactor can't run, the raw string is not emitted. - -## Configuration reference - -```yaml -telemetry: - local: true # local telemetry (default on) - allow_aggregate: true # hard gate; pin false to forbid aggregate metrics entirely - consent_state: unknown # aggregate opt-in: unknown | local | aggregate - install_id: "" # stable anon id; "" mints one; clear to rotate - retention_days: 90 # local event-log retention - redact_secrets: true # always-on secret redaction (kept on by design) - content_redaction: none # none | pii - trajectories: - enabled: false # unlocks full-content export to your destination - export: - otlp: - enabled: false - endpoint: null - headers_env: {} # {HeaderName: ENV_VAR_NAME} -``` - -### Enterprise policy via managed scope - -Any `telemetry.*` key can be pinned by an administrator through Hermes' managed-scope -layer (`/etc/hermes/config.yaml`), which wins over the user's value on a per-key basis. -There is no telemetry-specific policy block — to lock down a fleet, pin the keys you care -about. Common examples: - -- `telemetry.allow_aggregate: false` — aggregate metrics stay off even if - `consent_state` is set to `aggregate`. -- `telemetry.export.otlp.endpoint` — point every install at the corporate collector. -- `telemetry.trajectories.enabled` — centrally decide whether content export is allowed. - -When a key is managed, attempts to change it are rejected by managed scope with a message -naming the source. `hermes telemetry status` shows the current export posture (endpoint -host, whether the auth env var is set, content gate, redaction modes) — it never prints -secret values. - -## Privacy summary - -- Local telemetry never leaves your machine. -- Aggregate metrics (the only thing that could go to Nous) are opt-in, default-off, - and have no uploader today — nothing is sent. -- All export surfaces (file, OTLP) point at *your* destinations. -- Secrets are always redacted on export; content export is off until you enable - trajectories; PII redaction is a knob. diff --git a/gateway/run.py b/gateway/run.py index 72f2b2626ad..07d13b21c2e 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -7810,6 +7810,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew write_runtime_status(gateway_state="starting", exit_reason=None) except Exception: pass + try: + from hermes_cli.config import load_config + from agent.monitoring.gateway_health_export import start_gateway_health_export + self._gateway_health_export_runtime = start_gateway_health_export(load_config()) + if getattr(self._gateway_health_export_runtime, "enabled", False): + logger.info("Gateway health OTLP export: enabled") + except Exception: + logger.debug("gateway health OTLP export startup failed", exc_info=True) # Log any active supply-chain security advisories. Operators see this # in gateway.log and `hermes status` surfaces it; we do NOT block @@ -9762,6 +9770,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._update_runtime_status("running", self._exit_reason) else: self._update_runtime_status("stopped", self._exit_reason) + try: + _gh_runtime = getattr(self, "_gateway_health_export_runtime", None) + if _gh_runtime is not None: + _gh_runtime.shutdown() + except Exception: + logger.debug("gateway health OTLP export shutdown failed", exc_info=True) logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed()) self._stop_task = asyncio.create_task(_stop_impl()) diff --git a/gateway/status.py b/gateway/status.py index d2d3c150918..d6ad5a64c63 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -11,6 +11,7 @@ that will be useful when we add named profiles (multiple agents running concurrently under distinct configurations). """ +import copy import hashlib import json import logging @@ -987,6 +988,7 @@ def write_runtime_status( """Persist gateway runtime health information for diagnostics/status.""" path = _get_runtime_status_path() payload = _read_json_file(path) or _build_runtime_status_record() + previous_payload = copy.deepcopy(payload) current_record = _build_pid_record() payload.setdefault("platforms", {}) payload["kind"] = current_record["kind"] @@ -1021,6 +1023,11 @@ def write_runtime_status( payload["platforms"][platform] = platform_payload _write_json_file(path, payload) + try: + from agent.monitoring.gateway_health import emit_runtime_status_transition + emit_runtime_status_transition(previous_payload, payload) + except Exception: + pass def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 22290ca9ef3..f0f6a43c315 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3082,6 +3082,49 @@ DEFAULT_CONFIG = { "force_ipv4": False, }, + # Gateway monitoring — Service Health Monitoring plus redacted Operational + # Diagnostics for the gateway daemon, exported over OTLP to an + # operator-configured endpoint (OTEL Collector, DataDog, ...). Content-free + # by construction: no prompts, messages, tool args/results, session + # history, usage analytics, audit logs, or trajectories. Off by default; + # nothing is collected or sent until an operator enables it and sets an + # endpoint. + "monitoring": { + # Stable install identifier attached to exported health signals so an + # operator can tell instances apart in their collector. Empty string + # means "mint a fresh UUID on first use"; clear it to rotate. Carries + # no account identity. + "install_id": "", + # Gateway health & diagnostics export. + "gateway_health_export": { + "enabled": False, + "metrics_enabled": True, + "diagnostic_events_enabled": True, + "warning_error_events_enabled": True, + "export_interval_seconds": 60, + "logs_export_interval_seconds": 5, + "resource_attributes": { + "service.name": "hermes-gateway", + "deployment.environment": "production", + }, + "redaction": { + "enabled": True, + "include_stack_summary": True, + "include_raw_stack": False, + }, + }, + # OTLP destination. headers_env maps header names to ENVIRONMENT + # VARIABLE NAMES (never secret values); values are read from the + # environment at export time. + "export": { + "otlp": { + "enabled": False, + "endpoint": "", + "headers_env": {}, + }, + }, + }, + # Gateway settings — control how messaging platforms (Telegram, Discord, # Slack, etc.) deliver agent-produced files as native attachments. "gateway": { diff --git a/hermes_cli/main.py b/hermes_cli/main.py index abd4c462dd3..2cef49a220a 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -463,7 +463,7 @@ from hermes_cli.subcommands.memory import build_memory_parser from hermes_cli.subcommands.acp import build_acp_parser from hermes_cli.subcommands.tools import build_tools_parser from hermes_cli.subcommands.insights import build_insights_parser -from hermes_cli.subcommands.telemetry import build_telemetry_parser +from hermes_cli.subcommands.monitoring import build_monitoring_parser from hermes_cli.subcommands.skills import build_skills_parser from hermes_cli.subcommands.pairing import build_pairing_parser from hermes_cli.subcommands.plugins import build_plugins_parser @@ -13859,11 +13859,11 @@ _BUILTIN_SUBCOMMANDS = frozenset( "dump", "egress", "fallback", "gateway", "hooks", "import", "insights", "gui", "desktop", "kanban", "login", "logout", "logs", "lsp", "mcp", "memory", "migrate", "moa", "journey", "memory-graph", "learning", - "model", "pairing", "pets", "plugins", "portal", "profile", + "model", "monitoring", "pairing", "pets", "plugins", "portal", "profile", "project", "proxy", "prompt-size", "send", "sessions", "setup", - "skin", "skills", "slack", "status", "telemetry", "tools", "uninstall", "update", + "skin", "skills", "slack", "status", "tools", "uninstall", "update", "version", "webhook", "whatsapp", "whatsapp-cloud", "chat", "secrets", "security", # Help-ish invocations — plugin commands not being listed in # top-level --help is an acceptable trade-off for skipping an @@ -14312,193 +14312,51 @@ def cmd_insights(args): print(f"Error generating insights: {e}") -def cmd_telemetry(args): - """Local-only telemetry control + inspection. No uploader exists yet.""" - import json as _json - import time +def cmd_monitoring(args): + """Gateway monitoring status: health & diagnostics export posture.""" + from hermes_cli.config import load_config - from hermes_cli.config import load_config, save_config - from agent.telemetry import policy, rollup, metrics - - action = getattr(args, "telemetry_action", None) or "status" + action = getattr(args, "monitoring_action", None) or "status" config = load_config() - tel = config.get("telemetry") if isinstance(config.get("telemetry"), dict) else {} - local_enabled = bool(tel.get("local", True)) - allow_aggregate = bool(tel.get("allow_aggregate", True)) - consent_state = tel.get("consent_state", policy.CONSENT_UNKNOWN) - if consent_state not in policy.VALID_CONSENT_STATES: - consent_state = policy.CONSENT_UNKNOWN - aggregate_enabled = (local_enabled and allow_aggregate - and consent_state == policy.CONSENT_AGGREGATE) - install_id = policy.ensure_install_id(config) - - def _persist_install_id(): - # Make sure a minted id is written back so it stays stable. - config.setdefault("telemetry", {})["install_id"] = install_id - save_config(config) + mon_raw = config.get("monitoring") + mon: dict = mon_raw if isinstance(mon_raw, dict) else {} if action == "status": - print("Telemetry status") - print("─" * 56) - print(f" Local telemetry: {'on' if local_enabled else 'off'} " - f"(telemetry.local)") - print(f" Aggregate metrics: {'on' if aggregate_enabled else 'off'} " - f"(opt-in; consent_state={consent_state})") - if consent_state == policy.CONSENT_AGGREGATE and not local_enabled: - print(" ⚠ inert: local telemetry is off — nothing to aggregate") - elif consent_state != policy.CONSENT_AGGREGATE and allow_aggregate: - print(" opt in: hermes config set telemetry.consent_state aggregate") - if not allow_aggregate: - print(" ⚠ allow_aggregate is false (egress hard-disabled)") - print(f" Install id: {install_id}") - print(" Upload: DISABLED — no server yet. Aggregate metrics are " - "computed locally only.") - print() - # Export posture — where YOUR data can flow (never Nous). Values only; - # lock-state is managed scope's concern, surfaced by its own tooling. - try: - from agent.telemetry import otlp_exporter, redaction - otlp = otlp_exporter._otlp_config(config) - print(" Export") - if otlp.get("enabled") and otlp.get("endpoint"): - _host = str(otlp.get("endpoint")) - print(f" OTLP: enabled → {_host}") - # Show the header name + env var + whether it's set, never the value. - hdrs = otlp.get("headers_env") or {} - if hdrs: - import os as _os - for _h, _env in hdrs.items(): - _state = "set" if _os.environ.get(str(_env)) else "NOT set" - print(f" auth: header '{_h}' ← ${_env} ({_state})") - _sdk = "installed" if otlp_exporter.is_available() else "MISSING — pip install 'hermes-agent[otlp]'" - print(f" SDK: {_sdk}") - else: - print(" OTLP: disabled (telemetry.export.otlp.enabled)") - # Content gate (telemetry.trajectories) + redaction posture. - if redaction.content_export_enabled(config): - print(f" Content export: on (trajectories enabled) — message " - f"content exportable") - else: - print(" Content export: off (trajectories disabled) — structural " - "telemetry only") - print(" Secret redaction: on (always)") - print(f" PII redaction: {redaction.content_mode_for(config)}") - print(" Bulk export: hermes telemetry export --out FILE [--otlp]") - except Exception: - pass - print() - # Local data volume - try: - ov = metrics.overview() - runs = ov["workflows"]["total_runs"] - mcs = sum(ov["model_calls"]["by_provider"].values()) - tcs = ov["tool_calls"]["total"] - print(f" Local data: {runs:,} workflows · {mcs:,} model calls · {tcs:,} tool calls") - except Exception: - print(" Local data: (none yet)") - print("\n Inspect what would be shared: hermes telemetry preview") - return + from agent.monitoring import otlp_exporter - if action == "preview": - _persist_install_id() - since_ns = int((time.time() - args.days * 86400) * 1e9) - events = rollup.build_aggregate_events( - install_id=install_id, since_ns=since_ns - ) - summary = rollup.summarize(events) - print("Telemetry preview — computed locally, NOT uploaded") - print("─" * 56) - print(f" Window: last {args.days} days Events: {summary['total']}") - for name, n in sorted(summary["by_event_name"].items()): - print(f" {name}: {n}") - print(" Shows the actual models, tools, and counts from local telemetry.") - print() - shown = events[: args.limit] - if getattr(args, "json", False): - print(_json.dumps(shown, indent=2)) + gh_raw = mon.get("gateway_health_export") + gh: dict = gh_raw if isinstance(gh_raw, dict) else {} + export_raw = mon.get("export") + export_cfg: dict = export_raw if isinstance(export_raw, dict) else {} + otlp_raw = export_cfg.get("otlp") + otlp: dict = otlp_raw if isinstance(otlp_raw, dict) else {} + + print("Gateway monitoring") + print(f" Health export: {'enabled' if gh.get('enabled') else 'disabled'} " + f"(monitoring.gateway_health_export.enabled)") + if gh.get("enabled"): + print(f" Metrics: {'on' if gh.get('metrics_enabled', True) else 'off'} " + f"(interval {gh.get('export_interval_seconds', 60)}s)") + print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}") + print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} " + f"(interval {gh.get('logs_export_interval_seconds', 5)}s)") + red_raw = gh.get("redaction") + red: dict = red_raw if isinstance(red_raw, dict) else {} + print(f" Redaction: {'on' if red.get('enabled', True) else 'OFF'} " + f"(secrets/PII scrubbed in-process before egress)") + endpoint = otlp.get("endpoint") or "" + if otlp.get("enabled") and endpoint: + print(f" OTLP endpoint: {endpoint}") else: - for e in shown: - if e.get("event_name") == "heartbeat": - print(f" • heartbeat hermes={e.get('hermes_version')} os={e.get('os_family')}") - continue - bits = [f"{e.get('event_name')}"] - for k in ("entrypoint", "platform", "end_reason", "duration_ms"): - if e.get(k) is not None: - bits.append(f"{k}={e[k]}") - models = e.get("models_used") or [] - if models: - bits.append("models=" + ",".join( - f"{m.get('model') or m.get('provider') or '?'}" for m in models)) - if e.get("tools_used"): - bits.append("tools=" + ",".join(e["tools_used"])) - print(" • " + " ".join(bits)) - if len(events) > len(shown): - print(f" ... and {len(events) - len(shown)} more (use --limit)") + print(" OTLP endpoint: not configured (monitoring.export.otlp)") + print(f" OTel SDK: {'installed' if otlp_exporter.is_available() else 'not installed'} " + f"(optional extra: hermes-agent[otlp])") + print("\n Scope: gateway service health + redacted diagnostics only.") + print(" No prompts, messages, tool args/results, usage analytics, or traces.") return - if action == "export": - import sys as _sys - from agent.telemetry import exporter_bulk, redaction - - since_ns = int((time.time() - args.since * 86400) * 1e9) if getattr(args, "since", 0) else None - want_content = getattr(args, "include_content", False) - - # OTLP path: stream spans to the configured Collector endpoint. - if getattr(args, "otlp", False): - from agent.telemetry import otlp_exporter - if not otlp_exporter.is_enabled(config): - print("telemetry.export.otlp is not enabled/endpoint not set. " - "Set telemetry.export.otlp.enabled + .endpoint.", file=_sys.stderr) - return - # The OTel SDK is an optional dep; export_once() lazily installs it on - # first use (gated by security.allow_lazy_installs, TTY-prompted), - # same as every other optional backend. OTLPUnavailable is the - # fallback when it can't be installed. - try: - n = otlp_exporter.export_once(config, since_ns=since_ns) - except otlp_exporter.OTLPUnavailable as e: - print(str(e), file=_sys.stderr) - return - except Exception as e: - print(f"OTLP export failed: {e}", file=_sys.stderr) - return - print(f"Exported {n} spans to the OTLP endpoint " - f"({(otlp_exporter._otlp_config(config) or {}).get('endpoint')}).", - file=_sys.stderr) - return - - content_ok = redaction.content_export_enabled(config) - if want_content and not content_ok: - print("⚠ --include-content ignored: telemetry.trajectories.enabled is false.") - print(" Enable trajectories (admin/config) to export message content.") - print(" Exporting structural telemetry only.") - - if not getattr(args, "out", None): - print("--out is required (or use --otlp).", file=_sys.stderr) - return - out_path = args.out - if out_path == "-": - counts = exporter_bulk.export( - _sys.stdout, fmt=args.fmt, since_ns=since_ns, - include_content=want_content, config=config, - ) - else: - with open(out_path, "w", encoding="utf-8") as fh: - counts = exporter_bulk.export( - fh, fmt=args.fmt, since_ns=since_ns, - include_content=want_content, config=config, - ) - # status to stderr so stdout stays clean for `--out -` - msg = (f"Exported {counts['telemetry']} telemetry records" - + (f" + {counts['sessions']} sessions" if counts["sessions"] else "") - + (" (content INCLUDED, redacted)" if counts["content_included"] - else " (structural only)") - + (f" -> {out_path}" if out_path != "-" else "")) - print(msg, file=_sys.stderr) - return - - print(f"Unknown telemetry action: {action}") - print("Use: status | preview | export") + print(f"Unknown monitoring action: {action}", file=sys.stderr) + sys.exit(2) def cmd_skills(args): @@ -16447,9 +16305,7 @@ def main(): # insights command (parser built in hermes_cli/subcommands/insights.py) # ========================================================================= build_insights_parser(subparsers, cmd_insights=cmd_insights) - - # telemetry command (parser in hermes_cli/subcommands/telemetry.py) - build_telemetry_parser(subparsers, cmd_telemetry=cmd_telemetry) + build_monitoring_parser(subparsers, cmd_monitoring=cmd_monitoring) # ========================================================================= # claw command (parser built in hermes_cli/subcommands/claw.py) diff --git a/hermes_cli/subcommands/monitoring.py b/hermes_cli/subcommands/monitoring.py new file mode 100644 index 00000000000..266c69aa1fb --- /dev/null +++ b/hermes_cli/subcommands/monitoring.py @@ -0,0 +1,36 @@ +"""``hermes monitoring`` subcommand parser. + +Gateway monitoring control and inspection. ``status`` shows whether the +gateway health & diagnostics export is enabled, where it points, and the +redaction posture. + +The handler is injected to avoid importing ``main`` (mirrors the insights +subcommand). +""" + +from __future__ import annotations + +from typing import Callable + + +def build_monitoring_parser(subparsers, *, cmd_monitoring: Callable) -> None: + """Attach the ``monitoring`` subcommand (with actions) to ``subparsers``.""" + p = subparsers.add_parser( + "monitoring", + help="Inspect gateway monitoring (health & diagnostics export)", + description=( + "Gateway monitoring: service health metrics plus redacted " + "diagnostics, exported over OTLP to an operator-configured " + "endpoint. Content-free by construction — no prompts, messages, " + "tool args/results, or usage analytics. Configure under " + "monitoring.* in config.yaml." + ), + ) + sub = p.add_subparsers(dest="monitoring_action") + + sub.add_parser( + "status", + help="Show monitoring settings, export state, and redaction posture", + ) + + p.set_defaults(func=cmd_monitoring) diff --git a/hermes_cli/subcommands/telemetry.py b/hermes_cli/subcommands/telemetry.py deleted file mode 100644 index 791cdeb05b4..00000000000 --- a/hermes_cli/subcommands/telemetry.py +++ /dev/null @@ -1,54 +0,0 @@ -"""``hermes telemetry`` subcommand parser. - -Telemetry control and inspection. ``preview`` shows the per-run summary events that -would be produced for aggregate metrics; there is no uploader, so it terminates as a -local view. - -The handler is injected to avoid importing ``main`` (mirrors the insights subcommand). -""" - -from __future__ import annotations - -from typing import Callable - - -def build_telemetry_parser(subparsers, *, cmd_telemetry: Callable) -> None: - """Attach the ``telemetry`` subcommand (with actions) to ``subparsers``.""" - p = subparsers.add_parser( - "telemetry", - help="Inspect local telemetry and export it", - description=( - "Local-first telemetry. Local telemetry records observability on this " - "machine. Aggregate metrics are opt-in (set telemetry.consent_state via " - "`hermes config set`); they have no uploader and are shown only via `preview`." - ), - ) - sub = p.add_subparsers(dest="telemetry_action") - - sub.add_parser("status", help="Show telemetry settings, consent state, and local data volume") - - prev = sub.add_parser( - "preview", - help="Show the aggregate events that would be produced (computed locally, not uploaded)", - ) - prev.add_argument("--days", type=int, default=30, help="Window to roll up (default: 30)") - prev.add_argument("--limit", type=int, default=10, help="Max events to print (default: 10)") - prev.add_argument("--json", action="store_true", help="Print raw JSON events") - - exp = sub.add_parser( - "export", - help="Export local telemetry (and optional content) to a file, stream, or OTLP endpoint", - ) - exp.add_argument("--out", help="Output file path (use - for stdout). Not needed with --otlp.") - exp.add_argument("--format", dest="fmt", choices=["ndjson", "json"], default="ndjson", - help="Output format (default: ndjson)") - exp.add_argument("--since", type=int, default=0, - help="Only telemetry from the last N days (0 = all)") - exp.add_argument("--include-content", action="store_true", - help="Include session/message content (requires telemetry.trajectories.enabled). " - "Secrets always redacted; PII per telemetry.content_redaction.") - exp.add_argument("--otlp", action="store_true", - help="Export to the configured OTLP endpoint (telemetry.export.otlp.*) " - "instead of a file. Requires the optional 'otlp' extra.") - - p.set_defaults(func=cmd_telemetry) diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py deleted file mode 100644 index c270d4d72a2..00000000000 --- a/plugins/telemetry/__init__.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Telemetry plugin — wires Hermes lifecycle hooks to the local telemetry emitter. - -This is the *only* instrumentation seam. It registers observational hooks (which core -already invokes fail-open) and translates each into a typed local telemetry event -handed to ``agent.telemetry.emitter``. There are zero edits to core call sites: -the hooks already carry model/provider/usage/duration/tool data. - -Everything here is best-effort and fail-open — a raised exception in a hook callback is -swallowed by core, and we additionally guard each callback so a telemetry bug can never -disturb a session. No content, no network: local telemetry only. - -Hooks consumed: - on_session_start -> begin a run context (trace_id/run_id + root span id) - post_api_request -> one model_call event + its timing span (tokens, latency) - api_request_error -> one error event - post_tool_call -> one tool_call event + its timing span (duration, result class) - on_session_finalize -> finalize the run row + emit the run's root span - subagent_start/stop -> (reserved) lineage markers - -Each model/tool call emits a SpanEvent (timing + parent = the run's root span) keyed by -the same span_id as its detail row, so tel_spans reconstructs a run -> calls trace tree. -""" - -from __future__ import annotations - -import logging -import threading -import time -from typing import Any, Dict, Optional - -logger = logging.getLogger(__name__) - -# Per-run accumulators keyed by run_id, so on_session_finalize can roll up counts. -_runs: Dict[str, Dict[str, Any]] = {} -_runs_lock = threading.Lock() - - -def _safe(fn): - """Decorator: never let a telemetry hook raise into core.""" - def wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except Exception: - logger.debug("telemetry hook %s failed", getattr(fn, "__name__", "?"), exc_info=True) - return None - wrapper.__name__ = getattr(fn, "__name__", "wrapper") - return wrapper - - -def _run_key(session_id: Optional[str], task_id: Optional[str]) -> str: - return session_id or task_id or "default" - - -# ── on_session_start ──────────────────────────────────────────────────────── -@_safe -def _on_session_start(**kw: Any) -> None: - from agent.telemetry import spans - - session_id = kw.get("session_id") or "" - platform = kw.get("platform") or kw.get("source") or "" - ctx = spans.start_run() - now = time.time_ns() - root_span_id = spans.new_span_id() - key = _run_key(session_id, kw.get("task_id")) - with _runs_lock: - _runs[key] = { - "run_id": ctx.run_id, - "trace_id": ctx.trace_id, - "root_span_id": root_span_id, - "session_id": session_id or None, - "entrypoint": _entrypoint_for(platform, kw.get("source")), - "platform": platform or None, - "start_ns": now, - "model_call_count": 0, - "tool_call_count": 0, - "error_count": 0, - } - - -def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str = "") -> Dict[str, Any]: - """Return the run accumulator, lazily creating one if session_start was missed.""" - from agent.telemetry import spans - - key = _run_key(session_id, task_id) - with _runs_lock: - run = _runs.get(key) - if run is None: - rid = spans.current_run_id() or spans.new_id() - tid = spans.current_trace_id() or spans.new_id() - run = { - "run_id": rid, - "trace_id": tid, - "root_span_id": spans.new_span_id(), - "session_id": session_id or None, - "entrypoint": _entrypoint_for(platform), - "platform": platform or None, - "start_ns": time.time_ns(), - "model_call_count": 0, - "tool_call_count": 0, - "error_count": 0, - } - _runs[key] = run - return run - - -def _emit_call_span(run: Dict[str, Any], span_id: str, name: str, kind: str, - duration_ms: Optional[int], status: Optional[str]) -> None: - """Emit the timing/lineage span for a model or tool call. - - The call hooks fire on completion, so end_ns is ~now and start_ns is reconstructed - from the measured duration (end - duration). The span is parented to the run's root - so a 2-level run -> calls waterfall can be reconstructed from tel_spans. - """ - from agent.telemetry import emitter - from agent.telemetry.events import SpanEvent - - end_ns = time.time_ns() - dur_ns = int(duration_ms) * 1_000_000 if isinstance(duration_ms, (int, float)) else 0 - start_ns = end_ns - dur_ns - emitter.emit(SpanEvent( - span_id=span_id, - trace_id=run["trace_id"], - run_id=run["run_id"], - parent_span_id=run.get("root_span_id"), - name=name, - kind=kind, - start_ns=start_ns, - end_ns=end_ns, - status=status, - )) - - -def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str: - """Coarse entrypoint label (cli / gateway / tui / api / cron …). - - This is a workflow *surface* label, not model/tool anonymization — it answers - "where did this run come from", which is genuinely categorical. - """ - s = (source or platform or "").lower() - if s in ("", "chat", "interactive", "cli", "desktop"): - return "cli" - if s in ("telegram", "discord", "slack", "whatsapp", "signal", "matrix", - "email", "sms", "teams", "feishu", "wecom", "line", "google_chat"): - return "gateway" - if s == "tui": - return "tui" - if s in ("api", "api_server", "openai_api"): - return "api" - if s == "cron": - return "cron" - if s == "batch": - return "batch" - if s == "acp": - return "acp" - return "cli" - - -# ── post_api_request -> model_call event ──────────────────────────────────── -@_safe -def _on_post_api_request(**kw: Any) -> None: - from agent.telemetry import emitter, spans - from agent.telemetry.events import ModelCallEvent - - session_id = kw.get("session_id") or "" - platform = kw.get("platform") or "" - run = _ensure_run(session_id, kw.get("task_id"), platform) - - usage = kw.get("usage") or {} - duration = kw.get("api_duration") - latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None - - span_id = spans.new_span_id() - evt = ModelCallEvent( - span_id=span_id, - run_id=run["run_id"], - provider=kw.get("provider"), # raw - model=kw.get("model"), # raw - base_url=kw.get("base_url"), - input_tokens=int(usage.get("input_tokens") or 0), - output_tokens=int(usage.get("output_tokens") or 0), - cache_read_tokens=int(usage.get("cache_read_tokens") or 0), - cache_write_tokens=int(usage.get("cache_write_tokens") or 0), - reasoning_tokens=int(usage.get("reasoning_tokens") or 0), - latency_ms=latency_ms, - ) - with _runs_lock: - run["model_call_count"] += 1 - _emit_call_span(run, span_id, name=kw.get("model") or "model_call", - kind="model", duration_ms=latency_ms, status="ok") - emitter.emit(evt) - - -# ── api_request_error -> error event ──────────────────────────────────────── -@_safe -def _on_api_request_error(**kw: Any) -> None: - from agent.telemetry import emitter - from agent.telemetry.events import ErrorEvent - - session_id = kw.get("session_id") or "" - run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") - error_class = _coarse_error_class(kw.get("error_type") or kw.get("error") or "") - with _runs_lock: - run["error_count"] += 1 - emitter.emit(ErrorEvent( - run_id=run["run_id"], - error_class=error_class, - subsystem="model_api", - recovery=None, - )) - - -def _coarse_error_class(raw: Any) -> str: - s = str(raw).lower() - if "timeout" in s: - return "provider_timeout" - if "rate" in s and "limit" in s: - return "rate_limit" - if any(k in s for k in ("auth", "401", "403", "unauthorized", "forbidden")): - return "auth" - if any(k in s for k in ("connection", "network", "dns", "socket")): - return "network" - if "context" in s and ("length" in s or "overflow" in s or "token" in s): - return "context_overflow" - if any(k in s for k in ("500", "502", "503", "server error", "provider")): - return "provider_error" - return "unknown" - - -# ── post_tool_call -> tool_call event ─────────────────────────────────────── -@_safe -def _on_post_tool_call(**kw: Any) -> None: - from agent.telemetry import emitter, spans - from agent.telemetry.events import ToolCallEvent - - session_id = kw.get("session_id") or "" - run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") - - function_name = kw.get("function_name") or kw.get("tool_name") - duration_ms = kw.get("duration_ms") - result = kw.get("result") - result_class = _tool_result_class(result) - - with _runs_lock: - run["tool_call_count"] += 1 - if result_class == "error": - run["error_count"] += 1 - - dur_int = int(duration_ms) if isinstance(duration_ms, (int, float)) else None - span_id = spans.new_span_id() - _emit_call_span(run, span_id, name=function_name or "tool_call", - kind="tool", duration_ms=dur_int, status=result_class) - emitter.emit(ToolCallEvent( - span_id=span_id, - run_id=run["run_id"], - tool_name=function_name, # raw tool name - duration_ms=dur_int, - result_class=result_class, - )) - - -def _tool_result_class(result: Any) -> str: - """Classify a tool result without retaining content — error vs ok vs blocked.""" - try: - import json - if isinstance(result, str): - r = result.strip() - if r.startswith("{"): - obj = json.loads(r) - if isinstance(obj, dict): - if obj.get("error") or obj.get("blocked"): - return "blocked" if obj.get("blocked") else "error" - if obj.get("timeout"): - return "timeout" - return "ok" - if isinstance(result, dict): - if result.get("error"): - return "error" - return "ok" - except Exception: - return "ok" - return "ok" - - -# ── on_session_finalize -> finalize the run row ───────────────────────────── -@_safe -def _on_session_finalize(**kw: Any) -> None: - from agent.telemetry import emitter, spans - from agent.telemetry.events import RunEvent, SpanEvent - - session_id = kw.get("session_id") or "" - key = _run_key(session_id, kw.get("task_id")) - with _runs_lock: - run = _runs.pop(key, None) - if run is None: - run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") - with _runs_lock: - _runs.pop(key, None) - - end_ns = time.time_ns() - start_ns = run.get("start_ns", end_ns) - end_reason = _coarse_end_reason(kw) - # Root span for the run — the trace root the call spans hang off of. - emitter.emit(SpanEvent( - span_id=run.get("root_span_id") or spans.new_span_id(), - trace_id=run["trace_id"], - run_id=run["run_id"], - parent_span_id=None, - name=f"run:{run.get('entrypoint', 'cli')}", - kind="run", - start_ns=start_ns, - end_ns=end_ns, - status=end_reason, - )) - emitter.emit(RunEvent( - run_id=run["run_id"], - trace_id=run["trace_id"], - entrypoint=run.get("entrypoint", "cli"), - session_id=run.get("session_id"), - platform=run.get("platform"), - start_ns=start_ns, - end_ns=end_ns, - end_reason=end_reason, - model_call_count=run.get("model_call_count", 0), - tool_call_count=run.get("tool_call_count", 0), - error_count=run.get("error_count", 0), - )) - spans.clear_run() - - -def _coarse_end_reason(kw: Dict[str, Any]) -> str: - """Map a finalize payload to a coarse end reason. - - Production finalize callers pass ``reason`` (e.g. "shutdown", "session_expired", - "session_reset"); the older ``turn_exit_reason``/``interrupted``/``failed`` keys are - honored when present. Defaults to "ended". - """ - if kw.get("interrupted"): - return "interrupted" - if kw.get("failed"): - return "failed" - reason = str(kw.get("turn_exit_reason") or kw.get("reason") or "").lower() - if "max_iteration" in reason: - return "max_iterations" - if "timeout" in reason: - return "timeout" - if "expired" in reason: - return "expired" - if "reset" in reason: - return "reset" - if "shutdown" in reason or "complete" in reason: - return "completed" - return reason or "ended" - - -# ── subagent lineage (reserved) ───────────────────────────────────────────── -# A delegated subagent runs its own ``run_conversation`` with its own session id, so -# its model/tool calls are already captured as a separate tel_runs row via the normal -# hooks — no subagent activity is lost. These hooks fire with the parent<->child bridge -# (parent_session_id, child_session_id, child_role, child_goal); they are reserved for -# recording parent->child *lineage* (linking a child run back to its parent), which -# needs a tel_runs.parent_run_id column. Deferred until a consumer needs the delegation -# tree; left registered as the attachment point. -@_safe -def _on_subagent_start(**kw: Any) -> None: - return None - - -@_safe -def _on_subagent_stop(**kw: Any) -> None: - return None - - -# ── registration ──────────────────────────────────────────────────────────── -def register(ctx) -> None: - ctx.register_hook("on_session_start", _on_session_start) - ctx.register_hook("post_api_request", _on_post_api_request) - ctx.register_hook("api_request_error", _on_api_request_error) - ctx.register_hook("post_tool_call", _on_post_tool_call) - ctx.register_hook("on_session_finalize", _on_session_finalize) - ctx.register_hook("subagent_start", _on_subagent_start) - ctx.register_hook("subagent_stop", _on_subagent_stop) - logger.debug("telemetry plugin registered 7 hooks") diff --git a/plugins/telemetry/plugin.yaml b/plugins/telemetry/plugin.yaml deleted file mode 100644 index 967d79616b7..00000000000 --- a/plugins/telemetry/plugin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: telemetry -version: 1.0.0 -description: "Local-first telemetry & observability. Records runs, model calls, tool calls, and errors to a local event log + state.db index via plugin hooks — no agent action, no content, no network. Powers /usage, /insights, and local dashboards." -author: NousResearch -hooks: - - post_api_request - - api_request_error - - post_tool_call - - on_session_start - - on_session_finalize - - subagent_start - - subagent_stop diff --git a/pyproject.toml b/pyproject.toml index c63e458b8b9..b42bdfd65a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,10 +156,6 @@ edge-tts = ["edge-tts==7.2.7"] modal = ["modal==1.3.4"] daytona = ["daytona==0.155.0"] hindsight = ["hindsight-client==0.6.1"] -# OTLP telemetry export (optional). Provides the OpenTelemetry SDK + OTLP/HTTP -# exporter so `hermes telemetry export --otlp` can send spans to a Collector. -# Lazy-imported by agent/telemetry/otlp_exporter.py; never a core dependency. -otlp = ["opentelemetry-sdk==1.30.0", "opentelemetry-exporter-otlp-proto-http==1.30.0"] dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==81.0.0"] # starlette: CVE-2026-48710; setuptools: latest <82 (torch >=2.11 caps setuptools<82) messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.14.1", "brotlicffi==1.2.0.1", "slack-bolt==1.29.0", "slack-sdk==3.43.0", "qrcode==7.4.2"] # aiohttp 3.14.1: CVE-2026-34513/34518/34519/34520/34525 + 34993(RCE)/47265 cron = [] # croniter is now a core dependency; this extra kept for back-compat @@ -225,6 +221,11 @@ acp = ["agent-client-protocol==0.9.0"] # NOT re-added to [all] so a future quarantined release can't break fresh # installs (see [all] policy comment below). mistral = ["mistralai==2.4.8"] +# OTLP gateway monitoring export (optional). Provides the OpenTelemetry SDK + +# OTLP/HTTP exporter for monitoring.gateway_health_export. Lazy-installed via +# tools/lazy_deps.py on first use; never a core dependency and deliberately +# NOT in [all]. +otlp = ["opentelemetry-sdk==1.39.1", "opentelemetry-exporter-otlp-proto-http==1.39.1"] bedrock = ["boto3==1.42.89"] vertex = ["google-auth==2.55.1"] azure-identity = ["azure-identity==1.25.3"] diff --git a/scripts/observability/gateway_health_export_probe.py b/scripts/observability/gateway_health_export_probe.py new file mode 100644 index 00000000000..9b6de1b8605 --- /dev/null +++ b/scripts/observability/gateway_health_export_probe.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Exercise Gateway Health & Diagnostics Export against a local OTLP capture collector.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", default="http://127.0.0.1:4318/v1/traces") + parser.add_argument("--log", required=True, help="JSONL file written by otel_capture_collector.py") + parser.add_argument("--wait", type=float, default=7.0) + args = parser.parse_args() + + hermes_home = Path(tempfile.mkdtemp(prefix="hermes-otel-smoke-")) + os.environ["HERMES_HOME"] = str(hermes_home) + + from gateway.status import write_runtime_status + from agent.monitoring.gateway_health_export import start_gateway_health_export + from agent.monitoring import emitter + + config = { + "monitoring": { + "local": True, + "gateway_health_export": { + "enabled": True, + "metrics_enabled": True, + "diagnostic_events_enabled": True, + "warning_error_events_enabled": True, + "export_interval_seconds": 5, + "logs_export_interval_seconds": 5, + "resource_attributes": { + "service.name": "hermes-gateway-smoke", + "deployment.environment": "local-smoke", + }, + "redaction": {"enabled": True, "include_raw_stack": False}, + }, + "export": { + "otlp": { + "enabled": True, + "endpoint": args.endpoint, + "headers_env": {}, + } + }, + } + } + + runtime = start_gateway_health_export(config) + if not runtime.enabled: + raise SystemExit(f"gateway health exporter did not enable: {runtime.reason}") + + write_runtime_status(gateway_state="starting", active_agents=0) + write_runtime_status(gateway_state="running", active_agents=2) + write_runtime_status(platform="slack", platform_state="running") + write_runtime_status( + platform="slack", + platform_state="fatal", + error_code="auth_failed", + error_message="Bearer *** rejected for smoke@example.com", + ) + logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com") + emitter.get_emitter().flush(timeout=2.0) + time.sleep(args.wait) + runtime.shutdown() + emitter.get_emitter().flush(timeout=2.0) + + log_path = Path(args.log) + rows = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + paths = {row["path"] for row in rows} + print(json.dumps({"hermes_home": str(hermes_home), "requests": len(rows), "paths": sorted(paths)}, indent=2)) + if "/v1/traces" not in paths: + raise SystemExit("missing /v1/traces request") + if "/v1/logs" not in paths: + raise SystemExit("missing /v1/logs request") + if "/v1/metrics" not in paths: + raise SystemExit("missing /v1/metrics request") + + +if __name__ == "__main__": + main() diff --git a/scripts/observability/otel_capture_collector.py b/scripts/observability/otel_capture_collector.py new file mode 100644 index 00000000000..0071328a4e2 --- /dev/null +++ b/scripts/observability/otel_capture_collector.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests. + +This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces +and /v1/metrics, records request metadata as JSONL, and returns 200 so local +exporters can be exercised without Docker or a vendor backend. +""" + +from __future__ import annotations + +import argparse +import json +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +class CaptureHandler(BaseHTTPRequestHandler): + log_path: Path + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length) if length else b"" + record = { + "ts": time.time(), + "path": self.path, + "content_type": self.headers.get("content-type"), + "content_length": length, + "body_prefix_hex": body[:24].hex(), + } + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, sort_keys=True) + "\n") + self.send_response(200) + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, format: str, *args) -> None: + # Keep tmux panes clean; JSONL file is the assertion surface. + return + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=4318) + parser.add_argument("--log", required=True) + args = parser.parse_args() + + log_path = Path(args.log).expanduser().resolve() + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text("", encoding="utf-8") + CaptureHandler.log_path = log_path + server = ThreadingHTTPServer((args.host, args.port), CaptureHandler) + print(f"OTLP capture collector listening on http://{args.host}:{args.port}; log={log_path}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/hermes_cli/test_plugins.py b/tests/hermes_cli/test_plugins.py index 85905e2ff36..3fdb6d1812d 100644 --- a/tests/hermes_cli/test_plugins.py +++ b/tests/hermes_cli/test_plugins.py @@ -761,11 +761,7 @@ class TestPluginHooks: mgr.discover_and_load() assert mgr.has_hook("pre_api_request") is True - # Negative sentinel: a hook no plugin (user or bundled) registers, proving - # that registering pre_api_request doesn't conjure an unrelated hook. - # (post_api_request is now registered by the bundled telemetry plugin, so - # it is no longer a valid "nothing registered this" sentinel.) - assert mgr.has_hook("transform_terminal_output") is False + assert mgr.has_hook("post_api_request") is False results = mgr.invoke_hook( "pre_api_request", session_id="s1", diff --git a/tests/telemetry/__init__.py b/tests/monitoring/__init__.py similarity index 100% rename from tests/telemetry/__init__.py rename to tests/monitoring/__init__.py diff --git a/tests/monitoring/test_emitter.py b/tests/monitoring/test_emitter.py new file mode 100644 index 00000000000..a0f5c2d889d --- /dev/null +++ b/tests/monitoring/test_emitter.py @@ -0,0 +1,86 @@ +"""Tests for the monitoring emitter: hot-path invariant + subscriber fan-out.""" + +from __future__ import annotations + +import time + +from agent.monitoring.emitter import MonitoringEmitter +from agent.monitoring.events import GatewayHealthEvent + + +def test_emit_never_raises_when_disabled(): + em = MonitoringEmitter(enabled=False) + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + assert em.stats()["queued"] == 0 + em.close() + + +def test_emit_accepts_dataclass_and_dict(tmp_path): + em = MonitoringEmitter() + seen: list = [] + em.subscribe(lambda batch: seen.extend(batch)) + em.emit(GatewayHealthEvent(name="gateway.health_snapshot", active_agents=2)) + em.emit({"event": "gateway_diagnostic", "name": "platform.fatal", + "subsystem": "platform.slack"}) + em.flush() + em.close() + kinds = {ev.get("event") for ev in seen} + assert kinds == {"gateway_health", "gateway_diagnostic"} + health = next(ev for ev in seen if ev["event"] == "gateway_health") + assert health["active_agents"] == 2 + assert "ts_ns" in health + + +def test_subscriber_failure_is_isolated(): + em = MonitoringEmitter() + good: list = [] + + def bad(batch): + raise RuntimeError("boom") + + em.subscribe(bad) + em.subscribe(lambda batch: good.extend(batch)) + em.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + em.flush() + em.close() + assert len(good) == 1 # the raising subscriber did not break fan-out + + +def test_unsubscribe_stops_delivery(): + em = MonitoringEmitter() + seen: list = [] + cb = lambda batch: seen.extend(batch) # noqa: E731 + em.subscribe(cb) + em.emit({"event": "gateway_health", "name": "a"}) + em.flush() + em.unsubscribe(cb) + em.emit({"event": "gateway_health", "name": "b"}) + em.flush() + em.close() + assert [ev["name"] for ev in seen] == ["a"] + + +def test_queue_full_drops_oldest(): + em = MonitoringEmitter() + # Fill the queue without a dispatcher running by not letting it start: + # emit() starts the thread, so instead assert drop accounting via stats + # after a burst larger than the queue. + for i in range(11_000): + em.emit({"event": "gateway_health", "name": f"e{i}"}) + # Give the dispatcher a moment; total dispatched + queued + dropped == emitted. + em.flush(timeout=5.0) + stats = em.stats() + em.close() + assert stats["dropped"] >= 0 + assert stats["dispatched"] + stats["queued"] + stats["dropped"] >= 10_000 + + +def test_hot_path_is_fast(): + em = MonitoringEmitter() + start = time.perf_counter() + for _ in range(1_000): + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + elapsed = time.perf_counter() - start + em.close() + # 1000 emits should be far under a second even on slow CI. + assert elapsed < 1.0 diff --git a/tests/monitoring/test_export_redaction.py b/tests/monitoring/test_export_redaction.py new file mode 100644 index 00000000000..0a5beaff908 --- /dev/null +++ b/tests/monitoring/test_export_redaction.py @@ -0,0 +1,56 @@ +"""Export redaction pipeline tests — the security-critical layer. + +Invariants: + * Secrets ALWAYS stripped, every export path, no flag disables it. + * Fails CLOSED: if the redactor can't run, the raw string is never emitted. + * PII (emails, phones, UUID-shaped ids) stripped in 'pii' mode — the mode + the gateway diagnostics path always uses. +""" + +from __future__ import annotations + +import agent.monitoring.redaction as R + + +def test_secret_always_stripped_in_none_mode(): + fake_key = "sk-ant-api03-" + "A" * 24 # constructed to dodge literal-scrubbers + text = f"calling with key {fake_key} and moving on" + out = R.redact_for_export(text, content_mode=R.CONTENT_NONE) + assert out is not None + assert fake_key not in out + + +def test_secret_always_stripped_in_pii_mode(): + fake_token = "ghp_" + "0123456789abcdef" * 2 + "0123" + text = f"token {fake_token} leaked" + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert out is not None + assert fake_token not in out + + +def test_none_passthrough(): + assert R.redact_for_export(None, content_mode=R.CONTENT_NONE) is None + + +def test_pii_mode_strips_email_phone_uuid(): + text = ("reach alice@example.com or +1 415 555 0100, " + "install 123e4567-e89b-12d3-a456-426614174000") + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert out is not None + assert "alice@example.com" not in out + assert "426614174000" not in out + assert "[email]" in out + assert "[id]" in out + + +def test_none_mode_keeps_ordinary_words(): + out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE) + assert out == "just ordinary words" + + +def test_pii_mode_preserves_non_pii_structure(): + text = "platform.slack entered fatal after auth_failed" + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert out is not None + assert "platform.slack" in out + assert "auth_failed" in out diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py new file mode 100644 index 00000000000..8c1a9ef8513 --- /dev/null +++ b/tests/monitoring/test_gateway_health_export.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import logging + + +def test_default_config_keeps_gateway_health_export_disabled(): + from hermes_cli.config import DEFAULT_CONFIG + + cfg = DEFAULT_CONFIG["monitoring"]["gateway_health_export"] + + assert cfg["enabled"] is False + assert cfg["metrics_enabled"] is True + assert cfg["diagnostic_events_enabled"] is True + assert cfg["warning_error_events_enabled"] is True + assert cfg["export_interval_seconds"] == 60 + assert cfg["logs_export_interval_seconds"] == 5 + assert cfg["redaction"]["enabled"] is True + assert cfg["redaction"]["include_raw_stack"] is False + + +def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + runtime = { + "gateway_state": "running", + "pid": 1234, + "active_agents": "2", + "restart_requested": False, + "platforms": { + "slack": {"state": "running"}, + "telegram": { + "state": "fatal", + "error_code": "auth_failed", + "error_message": "token xoxb-secret rejected for user 123", + }, + }, + } + + snapshot = build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile="default", + install_id="install-1", + version="2026.7.test", + supervision_mode="manual", + ) + + metric_names = {m.name for m in snapshot.metrics} + assert { + "hermes.gateway.up", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.platform.up", + "hermes.platform.degraded", + } <= metric_names + + active = next(m for m in snapshot.metrics if m.name == "hermes.gateway.active_agents") + assert active.value == 2 + assert active.attributes == { + "hermes.profile": "default", + "service.instance.id": "install-1", + "service.version": "2026.7.test", + "hermes.supervision_mode": "manual", + } + + busy = next(m for m in snapshot.metrics if m.name == "hermes.gateway.busy") + drainable = next(m for m in snapshot.metrics if m.name == "hermes.gateway.drainable") + assert busy.value == 1 + assert drainable.value == 1 + + degraded = next( + m for m in snapshot.metrics + if m.name == "hermes.platform.degraded" and m.attributes["hermes.platform"] == "telegram" + ) + assert degraded.value == 1 + assert degraded.attributes["hermes.error_code"] == "auth_failed" + assert all("secret" not in str(v).lower() for v in degraded.attributes.values()) + + +def test_gateway_health_snapshot_emits_content_free_diagnostic_event(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + snapshot = build_gateway_health_snapshot( + { + "gateway_state": "running", + "active_agents": 1, + "platforms": { + "slack": {"state": "fatal", "error_code": "auth_failed", "error_message": "Bearer sk-live-secret"}, + }, + }, + gateway_running=True, + profile="default", + install_id="install-1", + version="v-test", + supervision_mode="container", + ) + + events = [event.to_dict() for event in snapshot.events] + health = next(e for e in events if e["event"] == "gateway_health") + platform = next(e for e in events if e["event"] == "gateway_diagnostic" and e["name"] == "platform.fatal") + + assert health["gateway_state"] == "running" + assert health["active_agents"] == 1 + assert health["gateway_busy"] is True + assert health["gateway_drainable"] is True + assert health["fatal_platform_count"] == 1 + assert platform["platform"] == "slack" + assert platform["error_code"] == "auth_failed" + assert "secret" not in platform["redacted_message"].lower() + assert "Bearer" not in platform["redacted_message"] + + +def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + old = emitter.get_emitter + emitter.get_emitter = lambda: DummyEmitter() # type: ignore[assignment] + try: + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + logger = logging.getLogger("gateway.platforms.slack") + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + try: + logger.info("ignore info token sk-live-secret") + logger.warning("Slack token sk-live-secret failed for user@example.com") + finally: + logger.removeHandler(handler) + finally: + emitter.get_emitter = old # type: ignore[assignment] + + assert len(captured) == 1 + event = captured[0] + assert event["event"] == "gateway_diagnostic" + assert event["name"] == "gateway.log.warning" + assert event["subsystem"] == "platform.slack" + assert event["error_class"] == "auth_failed" + assert "***" not in event["redacted_message"] + assert "user@example.com" not in event["redacted_message"] + + +def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import emit_runtime_status_transition + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + old = emitter.emit + monkeypatch.setattr(emitter, "emit", lambda event: captured.append(event.to_dict())) + + previous = {"gateway_state": "starting", "platforms": {"slack": {"state": "running"}}} + current = { + "gateway_state": "running", + "pid": 123, + "active_agents": 1, + "platforms": { + "slack": { + "state": "fatal", + "error_code": "auth_failed", + "error_message": "Bearer *** failed", + } + }, + } + + emit_runtime_status_transition(previous, current) + + names = [e["name"] for e in captured] + assert "gateway.lifecycle" in names + assert "platform.state_change" in names + assert "platform.fatal" in names + lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") + assert lifecycle["old_state"] == "starting" + assert lifecycle["new_state"] == "running" + platform = next(e for e in captured if e["name"] == "platform.state_change") + assert platform["old_state"] == "running" + assert platform["new_state"] == "fatal" + assert platform["error_code"] == "auth_failed" + assert "Bearer" not in platform["redacted_message"] + + +def test_runtime_status_transition_emits_startup_failed_and_exit(): + from agent.monitoring.gateway_health import emit_runtime_status_transition + from agent.monitoring import emitter + + captured = [] + old = emitter.emit + emitter.emit = lambda event: captured.append(event.to_dict()) # type: ignore[assignment] + try: + emit_runtime_status_transition({"gateway_state": "starting"}, {"gateway_state": "startup_failed", "exit_reason": "startup token ***"}) + emit_runtime_status_transition({"gateway_state": "running"}, {"gateway_state": "stopped", "exit_reason": "shutdown", "restart_requested": True}) + finally: + emitter.emit = old # type: ignore[assignment] + + names = [e["name"] for e in captured] + assert "gateway.startup_failed" in names + assert "gateway.exit" in names + failed = next(e for e in captured if e["name"] == "gateway.startup_failed") + assert "***" not in failed["redacted_message"] + exit_event = next(e for e in captured if e["name"] == "gateway.exit") + assert exit_event["restart_requested"] is True + + +def test_otlp_attrs_include_gateway_transition_fields(): + from agent.monitoring.otlp_exporter import _span_attrs + + attrs = _span_attrs({ + "event": "gateway_health", + "name": "gateway.lifecycle", + "old_state": "starting", + "new_state": "running", + "exit_reason": "restart", + "restart_requested": True, + }) + + assert attrs["hermes.old_state"] == "starting" + assert attrs["hermes.new_state"] == "running" + assert attrs["hermes.exit_reason"] == "restart" + assert attrs["hermes.restart_requested"] is True + + +def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch): + from agent.monitoring import gateway_health_export + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("missing sdk"))) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4317"}}, + } + }) + + assert isinstance(runtime, GatewayHealthExportRuntime) + assert runtime.enabled is False + assert runtime.reason == "otlp_unavailable" + + +def test_gateway_health_export_streams_only_gateway_events(monkeypatch): + from agent.monitoring import gateway_health_export + + captured = {} + + def fake_start_streaming(config, *, event_filter=None): + captured["filter"] = event_filter + return object() + + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_attach_log_handler", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_emit_snapshot_events", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_start_snapshot_thread", lambda *a, **k: None) + from agent.monitoring import otlp_exporter + monkeypatch.setattr(otlp_exporter, "start_streaming", fake_start_streaming) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True, "metrics_enabled": False}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is True + event_filter = captured["filter"] + assert event_filter({"event": "gateway_health"}) is True + assert event_filter({"event": "gateway_diagnostic"}) is False + assert event_filter({"event": "run"}) is False + assert event_filter({"event": "model_call"}) is False + assert event_filter({"event": "tool_call"}) is False + + +def test_gateway_health_export_metric_failure_does_not_start_streamer(monkeypatch): + from agent.monitoring import gateway_health_export, otlp_exporter + + started = [] + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))) + monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: started.append(True)) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is False + assert runtime.reason == "metrics_start_failed" + assert started == [] + + +def test_otlp_streamer_shutdown_unsubscribes(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.otlp_exporter import OTLPStreamer + + class Dummy: + def force_flush(self): + pass + def shutdown(self): + pass + + e = emitter.get_emitter() + streamer = OTLPStreamer.__new__(OTLPStreamer) + streamer._processor = Dummy() + streamer._provider = Dummy() + streamer._event_filter = None + streamer.exported = 0 + e.subscribe(streamer) + assert streamer in e._subscribers + + streamer.shutdown() + + assert streamer not in e._subscribers + + +def test_gateway_diagnostic_log_handler_never_raises_on_malformed_record(): + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + record = logging.LogRecord( + "gateway.platforms.slack", + logging.WARNING, + __file__, + 1, + "broken %s %s", + ("one",), + None, + ) + + handler.emit(record) diff --git a/tests/monitoring/test_otlp_exporter.py b/tests/monitoring/test_otlp_exporter.py new file mode 100644 index 00000000000..56edd566524 --- /dev/null +++ b/tests/monitoring/test_otlp_exporter.py @@ -0,0 +1,125 @@ +"""OTLP exporter tests: config resolution, span mapping, streaming subscriber. + +No SQLite involved — monitoring is an egress path, so the exporter consumes +emitter batches directly. Uses the in-memory OTel span exporter; skipped when +the optional otlp extra is not installed. +""" + +from __future__ import annotations + +import pytest + +otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed") + +import agent.monitoring.otlp_exporter as OE +from agent.monitoring.emitter import MonitoringEmitter + + +def _mem_provider(): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def test_gateway_health_event_maps_to_span_with_attrs(): + provider, mem = _mem_provider() + n = OE.export_batch(provider, [{ + "event": "gateway_health", "name": "gateway.lifecycle", + "old_state": "starting", "new_state": "running", + "active_agents": 2, "pid": 4242, + }]) + assert n == 1 + spans = mem.get_finished_spans() + assert spans[0].name == "hermes.gateway_health" + attrs = dict(spans[0].attributes or {}) + assert attrs["hermes.old_state"] == "starting" + assert attrs["hermes.new_state"] == "running" + assert attrs["hermes.active_agents"] == 2 + + +def test_gateway_diagnostic_event_maps_redacted_message(): + provider, mem = _mem_provider() + OE.export_batch(provider, [{ + "event": "gateway_diagnostic", "name": "platform.fatal", + "subsystem": "platform.slack", "error_class": "auth_failed", + "redacted_message": "token [redacted] rejected", "severity": "error", + }]) + attrs = dict(mem.get_finished_spans()[0].attributes or {}) + assert attrs["hermes.error_class"] == "auth_failed" + assert attrs["hermes.redacted_message"] == "token [redacted] rejected" + + +def test_unknown_event_kind_exports_no_attrs_beyond_kind(): + provider, mem = _mem_provider() + OE.export_batch(provider, [{"event": "model_call", "provider": "anthropic", + "model": "claude-opus-4"}]) + attrs = dict(mem.get_finished_spans()[0].attributes or {}) + # Non-monitoring event kinds carry no attribute mapping on this plane. + assert attrs == {"hermes.event": "model_call"} + + +def test_headers_resolve_from_env_not_value(monkeypatch): + monkeypatch.setenv("DD_KEY_ENV", "secret-value") + resolved = OE._resolve_headers({"DD-API-KEY": "DD_KEY_ENV", "X-Missing": "NOPE_ENV"}) + assert resolved == {"DD-API-KEY": "secret-value"} + + +def test_is_enabled_requires_endpoint_and_flag(): + assert OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) + assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True}}}}) + assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"endpoint": "http://x"}}}}) + assert not OE.is_enabled({}) + + +def test_export_otlp_feature_specs_match_pyproject(): + from tools.lazy_deps import LAZY_DEPS + import re + from pathlib import Path + + specs = set(LAZY_DEPS["export.otlp"]) + pyproject = Path(__file__).resolve().parents[2] / "pyproject.toml" + m = re.search(r'^otlp = \[(.*?)\]', pyproject.read_text(), re.M | re.S) + assert m, "otlp extra missing from pyproject.toml" + extra = set(re.findall(r'"([^"]+)"', m.group(1))) + assert specs == extra + + +def test_streamer_receives_events_and_respects_filter(monkeypatch): + provider, mem = _mem_provider() + monkeypatch.setattr(OE, "_make_provider", lambda cfg: (provider, None)) + streamer = OE.OTLPStreamer( + {}, event_filter=lambda ev: ev.get("event") == "gateway_health") + + em = MonitoringEmitter() + em.subscribe(streamer) + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + em.emit({"event": "model_call", "provider": "anthropic"}) # filtered out + em.flush() + em.close() + + spans = mem.get_finished_spans() + assert [s.name for s in spans] == ["hermes.gateway_health"] + assert streamer.exported == 1 + + +def test_failing_streamer_never_breaks_emitter(monkeypatch): + def boom(cfg): + raise RuntimeError("no provider") + + em = MonitoringEmitter() + + def bad_subscriber(batch): + raise RuntimeError("export down") + + seen: list = [] + em.subscribe(bad_subscriber) + em.subscribe(lambda batch: seen.extend(batch)) + em.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + em.flush() + em.close() + assert len(seen) == 1 diff --git a/tests/telemetry/test_cli_telemetry.py b/tests/telemetry/test_cli_telemetry.py deleted file mode 100644 index 19fc59e394c..00000000000 --- a/tests/telemetry/test_cli_telemetry.py +++ /dev/null @@ -1,108 +0,0 @@ -"""`hermes telemetry` handler smoke tests (local-only; no upload).""" - -from __future__ import annotations - -import sqlite3 -import time -import types - -import pytest - -import hermes_state - - -@pytest.fixture -def home(tmp_path, monkeypatch): - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - # state.db with tel_* + seeded data - db = tmp_path / "state.db" - hermes_state.SessionDB(db_path=db) - from agent.telemetry.emitter import TelemetryEmitter - from agent.telemetry.events import RunEvent, ModelCallEvent, ToolCallEvent - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "e.jsonl", db_path=db) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", - start_ns=now - 60_000_000, end_ns=now, model_call_count=1, - tool_call_count=1)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-opus-4", - input_tokens=20000, output_tokens=2000)) - em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", - result_class="ok")) - em.flush() - em.close() - yield tmp_path - - -def _run(action, **kw): - from hermes_cli.main import cmd_telemetry - args = types.SimpleNamespace(telemetry_action=action, days=30, limit=10, json=False) - for k, v in kw.items(): - setattr(args, k, v) - cmd_telemetry(args) - - -def test_status_runs(home, capsys): - _run("status") - out = capsys.readouterr().out - assert "Telemetry status" in out - assert "Upload:" in out and "DISABLED" in out - assert "Local data:" in out - - -def test_preview_shows_real_values(home, capsys): - _run("preview") - out = capsys.readouterr().out - assert "NOT uploaded" in out - assert "workflow_completed" in out - # real model + tool names ARE shown (this is the user's own local data) - assert "claude-opus-4" in out - assert "web_search" in out - - -def test_status_reflects_consent_set_via_config(home, capsys): - # Opting in is a plain config write now (no `enable` verb). status should - # reflect consent_state=aggregate as aggregate metrics being on. - from hermes_cli.config import load_config, save_config - cfg = load_config() - cfg.setdefault("telemetry", {})["consent_state"] = "aggregate" - save_config(cfg) - _run("status") - out = capsys.readouterr().out - assert "consent_state=aggregate" in out - assert "Aggregate metrics: on" in out - - -def test_status_shows_optin_hint_when_unknown(home, capsys): - _run("status") - out = capsys.readouterr().out - assert "Aggregate metrics: off" in out - assert "config set telemetry.consent_state aggregate" in out - - -def test_allow_aggregate_false_keeps_metrics_off_in_status(home, capsys): - # Even with consent opted in, a managed allow_aggregate:false wins. - from hermes_cli.config import load_config, save_config - cfg = load_config() - tel = cfg.setdefault("telemetry", {}) - tel["consent_state"] = "aggregate" - tel["allow_aggregate"] = False - save_config(cfg) - _run("status") - out = capsys.readouterr().out - assert "Aggregate metrics: off" in out - assert "allow_aggregate is false" in out - - -def test_local_off_with_consent_shows_inert_in_status(home, capsys): - # local off + opted in: aggregate is off and the status explains why. - from hermes_cli.config import load_config, save_config - cfg = load_config() - tel = cfg.setdefault("telemetry", {}) - tel["local"] = False - tel["consent_state"] = "aggregate" - save_config(cfg) - _run("status") - out = capsys.readouterr().out - assert "Aggregate metrics: off" in out - assert "inert: local telemetry is off" in out diff --git a/tests/telemetry/test_emitter.py b/tests/telemetry/test_emitter.py deleted file mode 100644 index f5b2902aa77..00000000000 --- a/tests/telemetry/test_emitter.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Emitter tests — the hot-path invariant is the one that matters most. - -Invariant: emit() never blocks, never raises, and a broken writer cannot slow or -break the caller. Plus: JSONL + SQLite round-trip, and the SQLite index is rebuildable -from the JSONL source of truth. -""" - -from __future__ import annotations - -import sqlite3 -import time - -import hermes_state -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -def _fresh_db(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - return db - - -def test_emit_is_fast_even_when_writer_is_broken(tmp_path, monkeypatch): - """The core guarantee: a writer that raises AND sleeps cannot stall emit().""" - db = _fresh_db(tmp_path) - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - - # Sabotage the row indexer to raise after a long sleep. - def broken(_conn, _ev): - time.sleep(5.0) - raise RuntimeError("writer exploded") - - monkeypatch.setattr(em, "_index_one", broken) - - start = time.monotonic() - for i in range(50): - em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=10)) - elapsed = time.monotonic() - start - - # 50 emits must complete in well under the writer's single 5s sleep. - assert elapsed < 1.0, f"emit() blocked: {elapsed:.2f}s" - em.close() - - -def test_emit_never_raises_on_bad_event(tmp_path): - db = _fresh_db(tmp_path) - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - # Non-serializable / wrong-shaped inputs must not raise out of emit(). - em.emit(object()) # no to_dict, not a mapping - em.emit({"event": "run"}) # minimal dict - em.close() - - -def test_jsonl_and_sqlite_roundtrip(tmp_path): - db = _fresh_db(tmp_path) - jsonl = tmp_path / "telemetry" / "events.jsonl" - em = TelemetryEmitter(events_path=jsonl, db_path=db) - - em.emit(RunEvent(run_id="run1", trace_id="t1", entrypoint="cli", end_reason="completed")) - em.emit(ModelCallEvent(span_id="m1", run_id="run1", provider="anthropic", - model="claude-opus-4", input_tokens=100, output_tokens=20)) - em.emit(ToolCallEvent(span_id="tc1", run_id="run1", tool_name="web_search", - duration_ms=120, result_class="ok")) - em.flush() - em.close() - - # JSONL has all three lines - lines = [l for l in jsonl.read_text(encoding="utf-8").splitlines() if l.strip()] - assert len(lines) == 3 - - # SQLite index has the rows in the right tables - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1 - assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 1 - assert conn.execute("SELECT COUNT(*) FROM tel_tool_calls").fetchone()[0] == 1 - row = conn.execute("SELECT provider, model, input_tokens FROM tel_model_calls").fetchone() - assert row == ("anthropic", "claude-opus-4", 100) - conn.close() - - -def test_unknown_event_kind_is_ignored_not_fatal(tmp_path): - db = _fresh_db(tmp_path) - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - em.emit({"event": "totally_unknown", "foo": "bar"}) - em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli")) - em.flush() - em.close() - conn = sqlite3.connect(db) - # The unknown event is in JSONL but skipped by the indexer; the known one indexes. - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1 - conn.close() - - -def test_disabled_emitter_writes_nothing(tmp_path): - db = _fresh_db(tmp_path) - jsonl = tmp_path / "telemetry" / "events.jsonl" - em = TelemetryEmitter(events_path=jsonl, db_path=db, enabled=False) - em.emit(RunEvent(run_id="r3", trace_id="t3", entrypoint="cli")) - em.flush() - em.close() - assert not jsonl.exists() - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0 - conn.close() diff --git a/tests/telemetry/test_export_redaction.py b/tests/telemetry/test_export_redaction.py deleted file mode 100644 index 4ae0473bfba..00000000000 --- a/tests/telemetry/test_export_redaction.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Export redaction pipeline tests — the security-critical layer. - -Invariants: - * Secrets ALWAYS stripped, every mode, no flag disables it. - * Content gated by telemetry.trajectories, not a redaction mode. - * PII stripped in 'pii' mode; structure preserved (codec-aware). -""" - -from __future__ import annotations - -import json - -from agent.telemetry import redaction as R - - -# ── secrets are always redacted ───────────────────────────────────────────── -def test_secrets_stripped_in_none_mode(): - text = "here is sk-ant-api03-SECRETKEY123 and a token" - out = R.redact_for_export(text, content_mode=R.CONTENT_NONE) - assert "SECRETKEY123" not in out - - -def test_secrets_stripped_in_pii_mode(): - text = "Authorization: Bearer abcdef123456789secret" - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) - assert "abcdef123456789secret" not in out - - -def test_secret_redactor_fails_closed(monkeypatch): - # If the underlying redactor raises, we must NOT return the raw string. - import agent.redact as ar - monkeypatch.setattr(ar, "redact_sensitive_text", lambda *a, **k: (_ for _ in ()).throw(RuntimeError())) - out = R.redact_for_export("sk-secret-value", content_mode=R.CONTENT_NONE) - assert "sk-secret-value" not in out - assert out == "[redaction-unavailable]" - - -# ── PII ───────────────────────────────────────────────────────────────────── -def test_pii_mode_strips_email_and_phone(): - text = "contact alice@example.com or +1 415 555 1234" - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) - assert "alice@example.com" not in out - assert "[email]" in out - assert "555" not in out or "[phone]" in out - - -def test_none_mode_keeps_nonsecret_text_but_drops_via_message_path(): - # redact_for_export(none) scrubs secrets but doesn't strip ordinary words; - # content *dropping* happens at the message layer (trajectories gate). - out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE) - assert "ordinary" in out - - -# ── trajectories gate (content_export_enabled) ────────────────────────────── -def test_content_export_disabled_by_default(): - assert R.content_export_enabled({}) is False - assert R.content_export_enabled({"telemetry": {}}) is False - assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": False}}}) is False - - -def test_content_export_enabled_when_trajectories_on(): - assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": True}}}) is True - - -# ── codec-aware message redaction ─────────────────────────────────────────── -def test_message_structural_only_when_content_excluded(): - msg = {"role": "user", "content": "my email is bob@x.com and key sk-12345"} - out = R.redact_message(msg, include_content=False) - assert out["role"] == "user" - assert "content" not in out # body dropped entirely - assert out["content_chars"] == len(msg["content"]) # only the size remains - assert "bob@x.com" not in json.dumps(out) - - -def test_message_content_included_is_redacted(): - msg = {"role": "user", "content": "email bob@x.com secret sk-ant-SECRET999"} - out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True) - assert "content" in out - assert "SECRET999" not in out["content"] # secret gone - assert "bob@x.com" not in out["content"] # pii gone - assert "[email]" in out["content"] - - -def test_tool_calls_redacted_names_kept_args_scrubbed(): - msg = { - "role": "assistant", - "tool_calls": json.dumps([ - {"function": {"name": "web_search", "arguments": '{"q": "email me at z@z.com"}'}} - ]), - } - out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True) - tc = out["tool_calls"] - assert tc[0]["name"] == "web_search" # structure/name preserved - assert "z@z.com" not in json.dumps(tc) # arg pii scrubbed - - -def test_tool_calls_counted_when_content_excluded(): - msg = { - "role": "assistant", - "tool_calls": json.dumps([ - {"function": {"name": "a", "arguments": "{}"}}, - {"function": {"name": "b", "arguments": "{}"}}, - ]), - } - out = R.redact_message(msg, include_content=False) - assert out["tool_call_count"] == 2 - assert "tool_calls" not in out - - -def test_content_mode_for_reads_config(): - assert R.content_mode_for({"telemetry": {"content_redaction": "pii"}}) == "pii" - assert R.content_mode_for({"telemetry": {"content_redaction": "bogus"}}) == "none" - assert R.content_mode_for({}) == "none" diff --git a/tests/telemetry/test_exporter_bulk.py b/tests/telemetry/test_exporter_bulk.py deleted file mode 100644 index 10e0fa68f08..00000000000 --- a/tests/telemetry/test_exporter_bulk.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Bulk export tests — telemetry always, content only behind the trajectories gate.""" - -from __future__ import annotations - -import io -import json -import sqlite3 -import time - -import hermes_state -from agent.telemetry import exporter_bulk -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -def _seed(tmp_path, with_secret_content=True): - db = tmp_path / "state.db" - sdb = hermes_state.SessionDB(db_path=db) - # telemetry - em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", - start_ns=now - 60_000_000, end_ns=now, model_call_count=1, tool_call_count=1)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-sonnet-4", input_tokens=1000, output_tokens=100)) - em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok")) - em.flush() - em.close() - # session + message content (with an embedded secret + email) - sdb.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - if with_secret_content: - sdb.append_message("s1", role="user", - content="my key is AKIAIOSFODNN7EXAMPLE and email me at carol@corp.com") - sdb.append_message("s1", role="assistant", content="ok") - sdb.close() - return db - - -def test_telemetry_exported_content_excluded_by_default(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - counts = exporter_bulk.export(buf, fmt="ndjson", include_content=False, config={}, db_path=db) - assert counts["telemetry"] >= 3 - assert counts["content_included"] == 0 - text = buf.getvalue() - # message bodies must NOT be present - assert "carol@corp.com" not in text - assert "AKIAIOSFODNN7EXAMPLE" not in text - # but a session record (structural) is present with message structure - lines = [json.loads(l) for l in text.splitlines() if l.strip()] - sess = [r for r in lines if r.get("_kind") == "session"] - assert sess and sess[0]["messages"] - assert "content" not in sess[0]["messages"][0] # structural only - assert "content_chars" in sess[0]["messages"][0] - - -def test_include_content_ignored_without_trajectories(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - # request content but trajectories disabled -> forced off - counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, - config={"telemetry": {"trajectories": {"enabled": False}}}, db_path=db) - assert counts["content_included"] == 0 - assert "carol@corp.com" not in buf.getvalue() - - -def test_content_included_when_trajectories_on_but_redacted(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - cfg = {"telemetry": {"trajectories": {"enabled": True}, "content_redaction": "pii"}} - counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, config=cfg, db_path=db) - assert counts["content_included"] == 1 - text = buf.getvalue() - # content present but secret + pii scrubbed - assert "AKIAIOSFODNN7EXAMPLE" not in text # secret always gone - assert "carol@corp.com" not in text # pii gone in pii mode - lines = [json.loads(l) for l in text.splitlines() if l.strip()] - sess = [r for r in lines if r.get("_kind") == "session"][0] - # the user message now has a (redacted) content field - user_msg = [m for m in sess["messages"] if m["role"] == "user"][0] - assert "content" in user_msg - assert "[email]" in user_msg["content"] - - -def test_json_format_roundtrips(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - exporter_bulk.export(buf, fmt="json", include_content=False, config={}, db_path=db) - obj = json.loads(buf.getvalue()) - assert "records" in obj - assert any(r["_kind"] == "tel_runs" for r in obj["records"]) - - -def test_since_window_filters_runs(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - # since 1ns ago in the future-ish -> the run (start ~60ms ago) excluded - future_ns = int((time.time() + 1) * 1e9) - counts = exporter_bulk.export(buf, fmt="ndjson", since_ns=future_ns, config={}, db_path=db) - lines = [json.loads(l) for l in buf.getvalue().splitlines() if l.strip()] - runs = [r for r in lines if r.get("_kind") == "tel_runs"] - assert runs == [] diff --git a/tests/telemetry/test_governance.py b/tests/telemetry/test_governance.py deleted file mode 100644 index 97740540367..00000000000 --- a/tests/telemetry/test_governance.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Export configuration visibility in `hermes telemetry status`. - -The status Export block reports the current export configuration. Whether a key is -locked is handled by the managed-scope layer, not repeated here; the allow_aggregate -gate is covered by a test so a managed pin can't be regressed. -""" - -from __future__ import annotations - -import time -import types - -import pytest - -import hermes_state - - -@pytest.fixture -def home(tmp_path, monkeypatch): - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - hermes_state.SessionDB(db_path=tmp_path / "state.db") - yield tmp_path - - -def _status(capsys): - from hermes_cli.main import cmd_telemetry - cmd_telemetry(types.SimpleNamespace(telemetry_action="status")) - return capsys.readouterr().out - - -def test_export_block_default_shows_otlp_disabled(home, capsys): - out = _status(capsys) - assert "Export" in out - assert "OTLP:" in out and "disabled" in out - assert "Content export: off" in out - assert "Secret redaction: on (always)" in out - - -def test_export_block_shows_endpoint_host_never_token(home, capsys, monkeypatch): - from hermes_cli.config import load_config, save_config - monkeypatch.setenv("CORP_OTLP_TOKEN", "supersecret-do-not-print") - c = load_config() - t = c.setdefault("telemetry", {}) - t.setdefault("export", {})["otlp"] = { - "enabled": True, - "endpoint": "https://collector.corp:4318/v1/traces", - "headers_env": {"Authorization": "CORP_OTLP_TOKEN"}, - } - save_config(c) - out = _status(capsys) - # endpoint host present - assert "https://collector.corp:4318/v1/traces" in out - # env var name + set-state present; the VALUE never printed - assert "CORP_OTLP_TOKEN" in out - assert "(set)" in out - assert "supersecret-do-not-print" not in out - - -def test_export_block_reflects_trajectories_gate(home, capsys): - from hermes_cli.config import load_config, save_config - c = load_config() - c.setdefault("telemetry", {})["trajectories"] = {"enabled": True} - save_config(c) - out = _status(capsys) - assert "Content export: on (trajectories enabled)" in out - - -def test_token_env_not_set_shows_not_set(home, capsys): - from hermes_cli.config import load_config, save_config - c = load_config() - t = c.setdefault("telemetry", {}) - t.setdefault("export", {})["otlp"] = { - "enabled": True, - "endpoint": "https://x:4318/v1/traces", - "headers_env": {"Authorization": "TOTALLY_UNSET_ENV_VAR_XYZ"}, - } - save_config(c) - out = _status(capsys) - assert "(NOT set)" in out - - -def test_allow_aggregate_pin_blocks_opt_in(home): - """A managed allow_aggregate:false pin overrides a consent_state opt-in. - - Consent is set in config (as a user or managed-scope pin would); the hard gate - still wins, so may_upload stays false. - """ - from hermes_cli.config import load_config, save_config - from agent.telemetry import policy - c = load_config() - tel = c.setdefault("telemetry", {}) - tel["consent_state"] = "aggregate" - tel["allow_aggregate"] = False - save_config(c) - assert policy.may_upload_aggregate(load_config()) is False diff --git a/tests/telemetry/test_insights_integration.py b/tests/telemetry/test_insights_integration.py deleted file mode 100644 index 814f3eeb093..00000000000 --- a/tests/telemetry/test_insights_integration.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Insights ↔ telemetry integration: the observability section in /insights output.""" - -from __future__ import annotations - -import time - -import pytest - -from hermes_state import SessionDB -from agent.insights import InsightsEngine -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -@pytest.fixture() -def db(tmp_path): - session_db = SessionDB(db_path=tmp_path / "ins_tel.db") - yield session_db - session_db.close() - - -def _seed_telemetry(db_path): - em = TelemetryEmitter(events_path=db_path.parent / "tel" / "events.jsonl", db_path=db_path) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", - platform="telegram", end_reason="completed", - start_ns=now - 90_000_000, end_ns=now)) - em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", - end_reason="failed", start_ns=now - 11_000_000, end_ns=now)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-opus-4", input_tokens=5000, output_tokens=800, - cache_read_tokens=1000, latency_ms=2200)) - em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", result_class="ok")) - em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", result_class="error")) - em.flush() - em.close() - - -def test_report_includes_telemetry_when_present(db): - # A session so generate() isn't the empty branch - db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - _seed_telemetry(db.db_path) - - engine = InsightsEngine(db) - report = engine.generate(days=30) - tel = report.get("telemetry") - assert tel, "telemetry section missing" - assert tel["workflows"]["total_runs"] == 2 - assert tel["workflows"]["success_rate"] == 0.5 - assert tel["tool_calls"]["total"] == 2 - assert tel["tool_calls"]["failure_rate"] == 0.5 - assert tel["model_calls"]["by_provider"]["anthropic"] == 1 - - -def test_terminal_output_renders_observability_section(db): - db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - _seed_telemetry(db.db_path) - - engine = InsightsEngine(db) - out = engine.format_terminal(engine.generate(days=30)) - assert "Observability" in out - assert "Workflows:" in out - assert "Failure rate:" in out - assert "Providers:" in out - - -def test_telemetry_section_absent_when_no_tel_rows(db): - # Session present, but no telemetry events seeded. - db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - engine = InsightsEngine(db) - report = engine.generate(days=30) - assert report.get("telemetry") == {} - out = engine.format_terminal(report) - assert "Observability" not in out - - -def test_empty_report_has_telemetry_key(db): - # No sessions at all -> empty branch still carries the key (renderer-safe). - engine = InsightsEngine(db) - report = engine.generate(days=30) - assert report.get("empty") is True - assert report.get("telemetry") == {} diff --git a/tests/telemetry/test_otlp_exporter.py b/tests/telemetry/test_otlp_exporter.py deleted file mode 100644 index 49fd990d463..00000000000 --- a/tests/telemetry/test_otlp_exporter.py +++ /dev/null @@ -1,171 +0,0 @@ -"""OTLP exporter tests. - -Skip cleanly when the optional OTel SDK isn't installed. When it is, verify: - * event -> OTel span attribute mapping - * headers_env resolves the value from the named environment variable, not config - * a failing or slow subscriber never breaks the emitter hot path - * is_enabled / is_available gating -""" - -from __future__ import annotations - -import sqlite3 -import time - -import pytest - -import hermes_state -from agent.telemetry import otlp_exporter as OE -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - -otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed") - - -def _in_memory_provider(): - """A TracerProvider with an in-memory span exporter (no network).""" - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - provider = TracerProvider() - mem = InMemorySpanExporter() - provider.add_span_processor(SimpleSpanProcessor(mem)) - return provider, mem - - -def test_event_maps_to_span_with_real_attrs(): - provider, mem = _in_memory_provider() - batch = [ - {"event": "run", "entrypoint": "gateway", "platform": "telegram", - "end_reason": "completed", "model_call_count": 2, "tool_call_count": 3}, - {"event": "model_call", "provider": "anthropic", "model": "claude-opus-4", - "input_tokens": 5000, "output_tokens": 800}, - {"event": "tool_call", "tool_name": "web_search", "result_class": "ok"}, - ] - n = OE.export_batch(provider, batch) - assert n == 3 - spans = mem.get_finished_spans() - names = {s.name for s in spans} - assert names == {"hermes.run", "hermes.model_call", "hermes.tool_call"} - # real values present as span attributes - run = [s for s in spans if s.name == "hermes.run"][0] - assert run.attributes["hermes.entrypoint"] == "gateway" - assert run.attributes["hermes.platform"] == "telegram" - model = [s for s in spans if s.name == "hermes.model_call"][0] - assert model.attributes["hermes.model"] == "claude-opus-4" - assert model.attributes["hermes.provider"] == "anthropic" - tool = [s for s in spans if s.name == "hermes.tool_call"][0] - assert tool.attributes["hermes.tool_name"] == "web_search" - - -def test_headers_resolve_from_env_not_value(monkeypatch): - monkeypatch.setenv("MY_OTLP_TOKEN", "supersecretvalue") - resolved = OE._resolve_headers({"Authorization": "MY_OTLP_TOKEN"}) - assert resolved == {"Authorization": "supersecretvalue"} - # missing env var -> skipped, not crashed - assert OE._resolve_headers({"X": "NOPE_NOT_SET"}) == {} - - -def test_is_enabled_requires_endpoint_and_flag(): - assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) is True - assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True}}}}) is False - assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": False, "endpoint": "http://x"}}}}) is False - assert OE.is_enabled({}) is False - - -def test_require_sdk_routes_through_lazy_install(monkeypatch): - # _require_sdk(auto_install=True) should call lazy_deps.ensure('export.otlp'). - import tools.lazy_deps as ld - calls = [] - monkeypatch.setattr(ld, "ensure", lambda feature, **kw: calls.append((feature, kw))) - OE._require_sdk(auto_install=True, prompt=False) - assert calls == [("export.otlp", {"prompt": False})] - - -def test_is_available_does_not_install(monkeypatch): - # A pure availability check must NEVER trigger an install. - import tools.lazy_deps as ld - calls = [] - monkeypatch.setattr(ld, "ensure", lambda *a, **k: calls.append(a)) - OE.is_available() - assert calls == [] - - -def test_export_otlp_feature_specs_match_pyproject(): - # The LAZY_DEPS entry must track the [otlp] extra in pyproject.toml. - from tools.lazy_deps import feature_specs - import pathlib, re - specs = set(feature_specs("export.otlp")) - pyproject = pathlib.Path(__file__).resolve().parents[2] / "pyproject.toml" - text = pyproject.read_text(encoding="utf-8") - m = re.search(r"^otlp\s*=\s*\[([^\]]*)\]", text, re.MULTILINE) - assert m, "otlp extra not found in pyproject.toml" - extra = set(re.findall(r'"([^"]+)"', m.group(1))) - assert specs == extra, f"LAZY_DEPS {specs} != pyproject extra {extra}" - - -def test_streamer_subscription_receives_events(tmp_path, monkeypatch): - # Wire an OTLPStreamer-like subscriber via the in-memory provider. - provider, mem = _in_memory_provider() - - db = tmp_path / "state.db" - conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() - em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) - - def subscriber(batch): - OE.export_batch(provider, batch) - - em.subscribe(subscriber) - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed")) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", model="claude-opus-4")) - em.flush() - em.close() - spans = mem.get_finished_spans() - assert {s.name for s in spans} == {"hermes.run", "hermes.model_call"} - - -def test_failing_subscriber_never_breaks_hot_path(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() - em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) - - def bad_subscriber(batch): - time.sleep(0.2) - raise RuntimeError("OTLP collector down") - - em.subscribe(bad_subscriber) - start = time.monotonic() - for i in range(30): - em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=1)) - elapsed = time.monotonic() - start - # emit() returns immediately regardless of the broken subscriber - assert elapsed < 1.0 - em.flush() - # durable writes still happened despite the subscriber raising - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 30 - conn.close() - em.close() - - -def test_export_once_reads_db_and_returns_count(tmp_path, monkeypatch): - db = tmp_path / "state.db" - conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() - em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", - start_ns=time.time_ns(), end_ns=time.time_ns())) - em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok")) - em.flush(); em.close() - - # Patch the provider builder to an in-memory one (no network). The processor - # stand-in only needs force_flush(); provider.shutdown() works on the real one. - provider, mem = _in_memory_provider() - - class _Proc: - def force_flush(self, *a, **k): - return True - - monkeypatch.setattr(OE, "_make_provider", lambda config: (provider, _Proc())) - n = OE.export_once({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}, db_path=db) - assert n == 2 - assert len(mem.get_finished_spans()) == 2 diff --git a/tests/telemetry/test_plugin_autoload.py b/tests/telemetry/test_plugin_autoload.py deleted file mode 100644 index 9cfe5c92a83..00000000000 --- a/tests/telemetry/test_plugin_autoload.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Telemetry plugin auto-load gating: on by default, off when telemetry.local=false.""" - -from __future__ import annotations - -import hermes_cli.plugins as plugins_mod - - -def test_local_enabled_defaults_true(monkeypatch): - monkeypatch.setattr(plugins_mod, "load_config", lambda: {}, raising=False) - # With no telemetry section, the default is on. - monkeypatch.setattr( - "hermes_cli.config.load_config", lambda: {}, raising=False - ) - assert plugins_mod._telemetry_local_enabled() is True - - -def test_local_disabled_when_config_false(monkeypatch): - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": {"local": False}}, - raising=False, - ) - assert plugins_mod._telemetry_local_enabled() is False - - -def test_local_enabled_when_config_true(monkeypatch): - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": {"local": True}}, - raising=False, - ) - assert plugins_mod._telemetry_local_enabled() is True - - -def test_malformed_config_defaults_on(monkeypatch): - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": "not a dict"}, - raising=False, - ) - assert plugins_mod._telemetry_local_enabled() is True - - -def test_plugin_manifest_is_discoverable(): - """The bundled telemetry plugin.yaml exists and declares the lifecycle hooks.""" - from pathlib import Path - import hermes_cli.plugins as p - bundled = p.get_bundled_plugins_dir() - manifest = bundled / "telemetry" / "plugin.yaml" - assert manifest.exists(), f"missing {manifest}" - text = manifest.read_text(encoding="utf-8") - for hook in ("post_api_request", "post_tool_call", "on_session_finalize"): - assert hook in text diff --git a/tests/telemetry/test_plugin_e2e.py b/tests/telemetry/test_plugin_e2e.py deleted file mode 100644 index 9a9ed00913d..00000000000 --- a/tests/telemetry/test_plugin_e2e.py +++ /dev/null @@ -1,127 +0,0 @@ -"""End-to-end telemetry wiring test. - -Unlike test_plugin_hooks.py (which calls the plugin's ``_on_*`` callbacks directly), -this drives the REAL dispatch chain that core uses at runtime: - - discover_plugins() -> plugin registers hooks -> invoke_hook(name, **kwargs) - -> registered callback -> emitter -> tel_* tables - -If the bundled plugin stops auto-loading, stops registering a hook, or the hook name -drifts from what core fires, the hand-written hook tests still pass but real runs go -dark. This test is the guard against that — it only touches public entry points -(``discover_plugins`` / ``invoke_hook``), exactly as core does. -""" - -from __future__ import annotations - -import sqlite3 -import time - -import pytest - -import hermes_state - - -@pytest.fixture -def runtime(tmp_path, monkeypatch): - """A clean HERMES_HOME with state.db, a fresh plugin manager, and a fresh emitter.""" - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - db = tmp_path / "state.db" - hermes_state.SessionDB(db_path=db) - - # Reset the global plugin-manager singleton so discovery re-runs in this HERMES_HOME. - import hermes_cli.plugins as plugins_mod - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - - # Reset the emitter singleton so it binds to this state.db (and tear it down after). - from agent.telemetry import emitter as emitter_mod - emitter_mod.reset_emitter_for_tests(None) - # Clear the plugin's per-run accumulators between tests. - import plugins.telemetry as plug - plug._runs.clear() - - yield db, plugins_mod, emitter_mod, plug - - try: - emitter_mod.get_emitter().flush() - except Exception: - pass - emitter_mod.reset_emitter_for_tests(None) - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - - -def _fire_one_turn(invoke_hook): - """Fire the hook sequence of a single completed turn, as core does.""" - invoke_hook("on_session_start", session_id="s1", - model="anthropic/claude-opus-4", platform="cli") - invoke_hook("post_api_request", session_id="s1", platform="cli", - provider="anthropic", base_url=None, model="claude-opus-4", - api_duration=0.9, - usage={"input_tokens": 1000, "output_tokens": 120, - "cache_read_tokens": 0, "cache_write_tokens": 0, - "reasoning_tokens": 0}) - invoke_hook("post_tool_call", session_id="s1", platform="cli", - function_name="web_search", duration_ms=210, result='{"data": "ok"}') - invoke_hook("on_session_finalize", session_id="s1", platform="cli", - reason="shutdown") - - -def test_real_dispatch_writes_tel_rows(runtime): - """The bundled plugin, loaded via discover_plugins, captures a turn end to end.""" - db, plugins_mod, emitter_mod, _plug = runtime - - plugins_mod.discover_plugins(force=True) - - # The plugin must have registered the lifecycle hooks core fires. - mgr = plugins_mod.get_plugin_manager() - registered = {k for k, v in getattr(mgr, "_hooks", {}).items() if v} - for hook in ("on_session_start", "post_api_request", "post_tool_call", - "on_session_finalize"): - assert hook in registered, f"core hook {hook!r} not registered by the plugin" - - _fire_one_turn(plugins_mod.invoke_hook) - - time.sleep(0.5) # let the background writer drain - emitter_mod.get_emitter().flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - assert conn.execute("SELECT COUNT(*) c FROM tel_runs").fetchone()["c"] == 1 - assert conn.execute("SELECT COUNT(*) c FROM tel_model_calls").fetchone()["c"] == 1 - assert conn.execute("SELECT COUNT(*) c FROM tel_tool_calls").fetchone()["c"] == 1 - - # Real values, not buckets. - mc = conn.execute("SELECT provider, model FROM tel_model_calls").fetchone() - assert mc["provider"] == "anthropic" - assert mc["model"] == "claude-opus-4" - tc = conn.execute("SELECT tool_name FROM tel_tool_calls").fetchone() - assert tc["tool_name"] == "web_search" - run = conn.execute("SELECT end_reason, model_call_count, tool_call_count " - "FROM tel_runs").fetchone() - assert run["end_reason"] == "completed" - assert run["model_call_count"] == 1 - assert run["tool_call_count"] == 1 - conn.close() - - -def test_local_disabled_writes_nothing(runtime, monkeypatch): - """telemetry.local=false: the plugin does not auto-load, so no rows are written.""" - db, plugins_mod, emitter_mod, _plug = runtime - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": {"local": False}}, - raising=False, - ) - - plugins_mod.discover_plugins(force=True) - _fire_one_turn(plugins_mod.invoke_hook) - time.sleep(0.3) - try: - emitter_mod.get_emitter().flush() - except Exception: - pass - - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0 - assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 0 - conn.close() diff --git a/tests/telemetry/test_plugin_hooks.py b/tests/telemetry/test_plugin_hooks.py deleted file mode 100644 index 6b1d1b2efbf..00000000000 --- a/tests/telemetry/test_plugin_hooks.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Telemetry plugin hook tests — feed realistic kwargs, assert tel_* rows + no leak.""" - -from __future__ import annotations - -import sqlite3 - -import pytest - -import hermes_state -from agent.telemetry import emitter as emitter_mod -from agent.telemetry.emitter import TelemetryEmitter - - -@pytest.fixture -def wired(tmp_path, monkeypatch): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - emitter_mod.reset_emitter_for_tests(em) - # reset the plugin's per-run accumulators between tests - import plugins.telemetry as plug - plug._runs.clear() - yield db, em, plug - em.flush() - em.close() - emitter_mod.reset_emitter_for_tests(None) - - -def test_full_session_lifecycle_produces_rows(wired): - db, em, plug = wired - - plug._on_session_start(session_id="sess1", platform="telegram") - plug._on_post_api_request( - session_id="sess1", platform="telegram", - provider="anthropic", base_url=None, model="claude-opus-4", - api_duration=2.5, - usage={"input_tokens": 5000, "output_tokens": 800, "cache_read_tokens": 1000, - "cache_write_tokens": 0, "reasoning_tokens": 0}, - ) - plug._on_post_tool_call( - session_id="sess1", platform="telegram", - function_name="web_search", duration_ms=812, result="{\"data\": \"...\"}", - ) - # Production finalize callers pass `reason` (e.g. "shutdown"), not cost. - plug._on_session_finalize( - session_id="sess1", platform="telegram", reason="shutdown", - ) - em.flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - run = conn.execute("SELECT * FROM tel_runs").fetchone() - assert run is not None - assert run["entrypoint"] == "gateway" - assert run["platform"] == "telegram" - assert run["end_reason"] == "completed" - assert run["model_call_count"] == 1 - assert run["tool_call_count"] == 1 - - mc = conn.execute("SELECT * FROM tel_model_calls").fetchone() - assert mc["provider"] == "anthropic" - assert mc["model"] == "claude-opus-4" - assert mc["input_tokens"] == 5000 - - tc = conn.execute("SELECT * FROM tel_tool_calls").fetchone() - assert tc["tool_name"] == "web_search" - assert tc["result_class"] == "ok" - conn.close() - - -def test_tool_error_result_classified_and_counted(wired): - db, em, plug = wired - plug._on_session_start(session_id="s2", platform="cli") - plug._on_post_tool_call( - session_id="s2", function_name="terminal", duration_ms=10, - result="{\"error\": \"command failed\"}", - ) - plug._on_session_finalize(session_id="s2", reason="shutdown") - em.flush() - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - tc = conn.execute("SELECT result_class, tool_name FROM tel_tool_calls").fetchone() - assert tc["result_class"] == "error" - assert tc["tool_name"] == "terminal" - run = conn.execute("SELECT error_count FROM tel_runs").fetchone() - assert run["error_count"] == 1 - conn.close() - - -def test_api_error_recorded(wired): - db, em, plug = wired - plug._on_session_start(session_id="s3", platform="cli") - plug._on_api_request_error(session_id="s3", error_type="provider timeout after 60s") - plug._on_session_finalize(session_id="s3", failed=True) - em.flush() - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - err = conn.execute("SELECT error_class, subsystem FROM tel_error_events").fetchone() - assert err["error_class"] == "provider_timeout" - assert err["subsystem"] == "model_api" - run = conn.execute("SELECT end_reason FROM tel_runs").fetchone() - assert run["end_reason"] == "failed" - conn.close() - - -def test_hooks_never_raise_on_garbage_kwargs(wired): - _, em, plug = wired - # Missing everything — must be swallowed by the _safe wrapper. - plug._on_post_api_request() - plug._on_post_tool_call() - plug._on_session_finalize() - plug._on_api_request_error() - em.flush() - - -def test_no_message_content_in_tool_rows(wired): - """The tool hook receives a result blob; only the classification persists, not content.""" - db, em, plug = wired - plug._on_session_start(session_id="s4", platform="cli") - secret = "{\"data\": \"USER SECRET sk-ABCDEF and /Users/alice/file.txt\"}" - plug._on_post_tool_call(session_id="s4", function_name="web_search", - duration_ms=5, result=secret) - plug._on_session_finalize(session_id="s4") - em.flush() - # The whole tel_tool_calls row must contain none of the result content. - conn = sqlite3.connect(db) - row = conn.execute("SELECT * FROM tel_tool_calls").fetchone() - conn.close() - blob = " ".join(str(x) for x in row) - assert "sk-ABCDEF" not in blob - assert "/Users/alice" not in blob - assert "SECRET" not in blob diff --git a/tests/telemetry/test_policy_consent.py b/tests/telemetry/test_policy_consent.py deleted file mode 100644 index 7205c33b275..00000000000 --- a/tests/telemetry/test_policy_consent.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Consent gate tests. - -Consent is a single config field (``telemetry.consent_state``); the aggregate opt-in -is expressed by setting it to ``"aggregate"`` (via ``hermes config set`` or a -managed-scope pin). ``allow_aggregate`` is the hard gate. ``policy.may_upload_aggregate`` -is the gate a future uploader must consult. -""" - -from __future__ import annotations - -from agent.telemetry import policy - - -def _cfg(**telemetry): - return {"telemetry": telemetry} - - -def test_default_posture_never_uploads(): - # No consent recorded → unknown → never uploads. - assert policy.may_upload_aggregate(_cfg(local=True, consent_state="unknown")) is False - - -def test_missing_telemetry_block_never_uploads(): - assert policy.may_upload_aggregate({}) is False - - -def test_opted_in_uploads(): - assert policy.may_upload_aggregate(_cfg(consent_state="aggregate")) is True - - -def test_declined_does_not_upload(): - assert policy.may_upload_aggregate(_cfg(consent_state="local")) is False - - -def test_allow_aggregate_false_overrides_opt_in(): - # An admin pins telemetry.allow_aggregate: false via managed scope. - cfg = _cfg(consent_state="aggregate", allow_aggregate=False) - assert policy.may_upload_aggregate(cfg) is False # the hard gate wins - - -def test_local_off_makes_aggregate_inert(): - # Aggregate metrics derive from the local tables; with local off there is - # nothing to aggregate, so opting in cannot upload. - cfg = _cfg(local=False, consent_state="aggregate", allow_aggregate=True) - assert policy.may_upload_aggregate(cfg) is False - - -def test_install_id_minted_when_empty_and_stable_when_set(): - cfg = _cfg(install_id="") - minted = policy.ensure_install_id(cfg) - assert minted and len(minted) >= 32 # uuid4 - cfg2 = _cfg(install_id="fixed-id") - assert policy.ensure_install_id(cfg2) == "fixed-id" diff --git a/tests/telemetry/test_rollup.py b/tests/telemetry/test_rollup.py deleted file mode 100644 index b271bb916f9..00000000000 --- a/tests/telemetry/test_rollup.py +++ /dev/null @@ -1,88 +0,0 @@ -"""rollup tests: tel_* -> per-run summary events with REAL values (local only).""" - -from __future__ import annotations - -import sqlite3 -import time - -import hermes_state -from agent.telemetry import rollup -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -def _seed(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", - platform="telegram", end_reason="completed", - start_ns=now - 90_000_000, end_ns=now, - model_call_count=2, tool_call_count=2)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-opus-4", input_tokens=60000, output_tokens=8000)) - em.emit(ModelCallEvent(span_id="m2", run_id="r1", provider="anthropic", - model="claude-opus-4", input_tokens=5000, output_tokens=500)) - em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", - result_class="ok")) - em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", - result_class="ok")) - # an in-progress run (no end_ns) must be excluded - em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", start_ns=now)) - em.flush() - em.close() - return db - - -def test_builds_one_event_per_completed_run_with_real_values(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db, - include_heartbeat=False) - wf = [e for e in events if e["event_name"] == "workflow_completed"] - assert len(wf) == 1 # r2 (no end_ns) excluded - e = wf[0] - assert e["entrypoint"] == "gateway" - assert e["platform"] == "telegram" - # REAL model id + provider, not a bucket/class - models = {m["model"] for m in e["models_used"]} - assert models == {"claude-opus-4"} - assert e["models_used"][0]["provider"] == "anthropic" - assert sorted(e["tools_used"]) == ["browser_navigate", "web_search"] - # real token totals, not buckets - assert e["input_tokens"] == 65000 - assert e["output_tokens"] == 8500 - - -def test_real_model_and_tool_names_present(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) - blob = " ".join(str(v) for e in events for v in e.values()) - assert "claude-opus-4" in blob - assert "web_search" in blob - - -def test_heartbeat_included_by_default(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) - assert any(e["event_name"] == "heartbeat" for e in events) - - -def test_summarize_counts_by_event_name(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) - s = rollup.summarize(events) - assert s["total"] == len(events) - assert s["by_event_name"]["workflow_completed"] == 1 - assert s["by_event_name"]["heartbeat"] == 1 - - -def test_empty_db_yields_only_heartbeat(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - events = rollup.build_aggregate_events(install_id="x", db_path=db) - assert [e["event_name"] for e in events] == ["heartbeat"] diff --git a/tests/telemetry/test_schema_migration.py b/tests/telemetry/test_schema_migration.py deleted file mode 100644 index a5a946da4b0..00000000000 --- a/tests/telemetry/test_schema_migration.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Schema tests: tel_* tables exist after init; SCHEMA_VERSION bumped.""" - -from __future__ import annotations - -import sqlite3 - -import hermes_state - - -TEL_TABLES = { - "tel_runs", "tel_spans", "tel_model_calls", "tel_tool_calls", - "tel_error_events", -} - - -def test_schema_version_is_17_or_higher(): - assert hermes_state.SCHEMA_VERSION >= 17 - - -def test_tel_tables_present_in_schema_sql(): - for tbl in TEL_TABLES: - assert f"CREATE TABLE IF NOT EXISTS {tbl}" in hermes_state.SCHEMA_SQL - - -def test_tel_tables_created_on_executescript(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - rows = { - r[0] - for r in conn.execute( - "SELECT name FROM sqlite_master WHERE type='table'" - ).fetchall() - } - conn.close() - assert TEL_TABLES.issubset(rows), f"missing: {TEL_TABLES - rows}" - - -def test_executescript_is_idempotent(tmp_path): - # IF NOT EXISTS means re-running on an existing DB is a no-op, not an error. - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.executescript(hermes_state.SCHEMA_SQL) # second run must not raise - conn.close() diff --git a/tests/telemetry/test_spans_trace.py b/tests/telemetry/test_spans_trace.py deleted file mode 100644 index c99fb02fb68..00000000000 --- a/tests/telemetry/test_spans_trace.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Trace/span layer: tel_spans is populated as a connected run -> calls tree. - -Drives the real dispatch chain (discover_plugins -> invoke_hook) and asserts the -timing/lineage backbone in tel_spans: - - one root span per run (kind="run", parent_span_id NULL), - - one child span per model/tool call parented to the root, - - a single trace_id across the run, - - call detail rows (tel_model_calls / tel_tool_calls) JOIN to their span by span_id, - - reconstructed durations match the reported latency/duration. - -This is the regression guard for the waterfall a desktop trace viewer renders. -""" - -from __future__ import annotations - -import sqlite3 -import time - -import pytest - -import hermes_state - - -@pytest.fixture -def runtime(tmp_path, monkeypatch): - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - db = tmp_path / "state.db" - hermes_state.SessionDB(db_path=db) - import hermes_cli.plugins as plugins_mod - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - from agent.telemetry import emitter as emitter_mod - emitter_mod.reset_emitter_for_tests(None) - import plugins.telemetry as plug - plug._runs.clear() - yield db, plugins_mod, emitter_mod - try: - emitter_mod.get_emitter().flush() - except Exception: - pass - emitter_mod.reset_emitter_for_tests(None) - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - - -def _one_turn(invoke_hook): - invoke_hook("on_session_start", session_id="s1", - model="anthropic/claude-opus-4", platform="cli") - invoke_hook("post_api_request", session_id="s1", platform="cli", - provider="anthropic", model="claude-opus-4", api_duration=0.9, - usage={"input_tokens": 1000, "output_tokens": 120}) - invoke_hook("post_tool_call", session_id="s1", platform="cli", - function_name="web_search", duration_ms=210, result='{"data": "ok"}') - invoke_hook("on_session_finalize", session_id="s1", platform="cli", - reason="shutdown") - - -def test_tel_spans_forms_connected_trace(runtime): - db, plugins_mod, emitter_mod = runtime - plugins_mod.discover_plugins(force=True) - _one_turn(plugins_mod.invoke_hook) - time.sleep(0.5) - emitter_mod.get_emitter().flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - spans = conn.execute( - "SELECT span_id, parent_span_id, kind, name, start_ns, end_ns, status, trace_id " - "FROM tel_spans" - ).fetchall() - - # root + model + tool - assert len(spans) == 3 - roots = [s for s in spans if s["parent_span_id"] is None] - children = [s for s in spans if s["parent_span_id"] is not None] - assert len(roots) == 1 - assert roots[0]["kind"] == "run" - assert len(children) == 2 - - # single trace, all children parented to the root - assert len({s["trace_id"] for s in spans}) == 1 - assert all(c["parent_span_id"] == roots[0]["span_id"] for c in children) - - # spans are time-ordered and carry real durations - by_kind = {s["kind"]: s for s in spans} - assert (by_kind["model"]["end_ns"] - by_kind["model"]["start_ns"]) == 900 * 1_000_000 - assert (by_kind["tool"]["end_ns"] - by_kind["tool"]["start_ns"]) == 210 * 1_000_000 - assert by_kind["run"]["end_ns"] >= by_kind["run"]["start_ns"] - - -def test_detail_rows_join_to_spans(runtime): - db, plugins_mod, emitter_mod = runtime - plugins_mod.discover_plugins(force=True) - _one_turn(plugins_mod.invoke_hook) - time.sleep(0.5) - emitter_mod.get_emitter().flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - mc = conn.execute( - "SELECT m.model, s.kind, s.trace_id FROM tel_model_calls m " - "JOIN tel_spans s ON m.span_id = s.span_id" - ).fetchone() - assert mc is not None and mc["model"] == "claude-opus-4" and mc["kind"] == "model" - - tc = conn.execute( - "SELECT t.tool_name, s.kind FROM tel_tool_calls t " - "JOIN tel_spans s ON t.span_id = s.span_id" - ).fetchone() - assert tc is not None and tc["tool_name"] == "web_search" and tc["kind"] == "tool" - conn.close() diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 3b94efdd4cb..cd983db6d74 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -116,6 +116,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "search.firecrawl": ("firecrawl-py==4.17.0",), "search.parallel": ("parallel-web==0.4.2",), + # ─── Monitoring ───────────────────────────────────────────────────────── + # OTLP gateway monitoring export. Lazily installed on first use of + # monitoring.gateway_health_export / monitoring.export.otlp. Tracks the + # `otlp` extra in pyproject.toml — bump both together. + "export.otlp": ( + "opentelemetry-sdk==1.39.1", + "opentelemetry-exporter-otlp-proto-http==1.39.1", + ), + # ─── TTS providers ───────────────────────────────────────────────────── # Pinned to exact versions to match pyproject.toml's no-ranges policy # (see comment at top of [project.dependencies]). When bumping, update @@ -140,15 +149,6 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Image generation backends ───────────────────────────────────────── "image.fal": ("fal-client==0.13.1",), - # ─── Observability ───────────────────────────────────────────────────── - # OTLP telemetry export. Lazily installed on first use of - # `hermes telemetry export --otlp`. Tracks the `otlp` extra in - # pyproject.toml — bump both together. - "export.otlp": ( - "opentelemetry-sdk==1.30.0", - "opentelemetry-exporter-otlp-proto-http==1.30.0", - ), - # ─── Memory providers ────────────────────────────────────────────────── "memory.honcho": ("honcho-ai==2.2.0",), "memory.hindsight": ("hindsight-client==0.6.1",), diff --git a/uv.lock b/uv.lock index 3b035b7e277..c91b8d4841b 100644 --- a/uv.lock +++ b/uv.lock @@ -1415,7 +1415,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/58/5404031044f55afad7aad1aff8be3f22b1bed03e237cfeabbc7e5c8cfde0/greenlet-3.5.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:aca9b4ce85b152b5524ef7d88170efdff80dc0032aa8b75f9aaf7f3479ea95b4", size = 287424, upload-time = "2026-06-26T18:20:31.469Z" }, { url = "https://files.pythonhosted.org/packages/b4/bf/1c65e9b94a54d547068fa5b5a8a06f221f3316b48908e08668d29c77cb50/greenlet-3.5.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f71be4920368fe1fabeeaa53d1e3548337e2b223d9565f8ad5e392a75ba23fc", size = 606523, upload-time = "2026-06-26T19:07:08.859Z" }, { url = "https://files.pythonhosted.org/packages/b8/c7/b66baacc95775ad511287acb0137b95574a9ce5491902372b7564799d790/greenlet-3.5.3-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d77e67f65f98449e3fb83f795b5d0a8437aead2f874ca89c96576caf4be3af6", size = 618315, upload-time = "2026-06-26T19:10:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a0/68afd1ebad40db87dac0a28ffa120726b98bf9c7c40c481b0f63c105d298/greenlet-3.5.3-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e18619ba655ac05d78d80fc83cac4ba892bd6927b99e3b8237aee861aaacc8bb", size = 626155, upload-time = "2026-06-26T19:24:14.44Z" }, { url = "https://files.pythonhosted.org/packages/78/2b/28ed29463522fdbe4c15b1f63922041626a7478316b34ab4adda3f0a4aba/greenlet-3.5.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8540f1e6205bd13ca0ce685581037219ca54a1b41a0a15d228c6c9b8ad5903d7", size = 617381, upload-time = "2026-06-26T18:32:16.077Z" }, + { url = "https://files.pythonhosted.org/packages/07/7f/e327d912239ec4b3b49999e3967389bcf1ee8722b9ee9194d2752ecd558a/greenlet-3.5.3-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d27c0c653a60d9535f690226474a5cc1036a8b0d7b57504d1c4f89c44a07a80c", size = 421083, upload-time = "2026-06-26T19:25:35.804Z" }, { url = "https://files.pythonhosted.org/packages/2a/7b/ad04e9d1337fc04965dc9fc616b6a72cb65a24b800a014c011ec812f5489/greenlet-3.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ef56fe650f50575bf843acde967b9c567687f3c22340941a899b7bc56e956a8", size = 1577771, upload-time = "2026-06-26T19:09:01.537Z" }, { url = "https://files.pythonhosted.org/packages/d8/33/6c87ab7ba663f70ca21f3022aad1ffe56d3f3e0521e836c2415e13abcc3c/greenlet-3.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5121af01cf911e70056c00d4b46d5e9b5d1415550038573d744138bacb59e6b8", size = 1644048, upload-time = "2026-06-26T18:31:42.996Z" }, { url = "https://files.pythonhosted.org/packages/1c/35/f0d8ee998b422cf8693b270f098e55d8d4ec8006b061b333f54f177d28d9/greenlet-3.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:0f41e4a05a3c0cb31b17023eff28dd111e1d16bf7d7d00406cd7df23f31398a7", size = 239137, upload-time = "2026-06-26T18:23:21.664Z" }, @@ -1423,7 +1425,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/6e/4c37d51a2b7f82d2ff11bb6b5f7d766d9a011726624af255e843727627a3/greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2", size = 288685, upload-time = "2026-06-26T18:22:08.977Z" }, { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, + { url = "https://files.pythonhosted.org/packages/fb/fb/d97dc261209c80744b7c8132693a30d70ec6e7315e632cb0a10b3fec94dd/greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23", size = 622351, upload-time = "2026-06-26T19:24:16.32Z" }, { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ac/e5fee13cbbd0e8de312d9a146584b8a51891c68847330ef9dc8b5109d23f/greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c", size = 425395, upload-time = "2026-06-26T19:25:37.144Z" }, { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, { url = "https://files.pythonhosted.org/packages/14/40/c57489acf8e37d74e2913d4eff63aa0dba17acccc4bdeef874dde2dbbec9/greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c", size = 239882, upload-time = "2026-06-26T18:23:27.518Z" }, @@ -1431,7 +1435,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" }, { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/61/2f5b1adf256d039f5dab8005de8d3d7ad2b0070a3219c0e036b3fbfeb440/greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1", size = 625580, upload-time = "2026-06-26T19:24:18.344Z" }, { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/ab7fc9e543e44d6879b0a6ef9a4b2188940fd180cc65d6f646883ddf7201/greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda", size = 427933, upload-time = "2026-06-26T19:25:38.219Z" }, { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" }, @@ -1666,6 +1672,10 @@ modal = [ nemo-relay = [ { name = "nemo-relay" }, ] +otlp = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, +] parallel-web = [ { name = "parallel-web" }, ] @@ -1808,6 +1818,8 @@ requires-dist = [ { name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5,<1.0" }, { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "openai", specifier = "==2.24.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp'", specifier = "==1.39.1" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp'", specifier = "==1.39.1" }, { name = "packaging", specifier = "==26.0" }, { name = "parallel-web", marker = "extra == 'parallel-web'", specifier = "==0.4.2" }, { name = "pathspec", specifier = "==1.1.1" }, @@ -1855,7 +1867,7 @@ requires-dist = [ { name = "websockets", specifier = "==15.0.1" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] [[package]] name = "hf-xet" From 87a15733c068ad4839e5b858236ac7272492fe85 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Tue, 14 Jul 2026 20:55:15 +0000 Subject: [PATCH 11/29] =?UTF-8?q?fix(monitoring):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20persist=20install=5Fid,=20drop=20dead=20config,=20s?= =?UTF-8?q?ingle=20redaction=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cut the leftover telemetry.* DEFAULT_CONFIG block (nothing reads it) and the legacy telemetry-key fallback in policy.py. - install_id: persist the minted UUID back to config.yaml on first use so service.instance.id survives gateway restarts (fail-open when the write is not possible); regression test covers the restart path. - Remove the no-op gateway_health_export.redaction config keys. Redaction is always-on by design and deliberately not configurable; status output now says so. - Collapse redaction to one unconditional secrets+PII scrub: drop the none/pii content modes (they served the dropped trajectories plane) and fold gateway_health.py's duplicate bearer/token/email/phone regex layer into agent/monitoring/redaction.py. --- agent/monitoring/gateway_health.py | 23 ++--- agent/monitoring/policy.py | 47 +++++++---- agent/monitoring/redaction.py | 83 +++++++++---------- hermes_cli/config.py | 59 ------------- hermes_cli/main.py | 6 +- .../gateway_health_export_probe.py | 1 - tests/monitoring/test_export_redaction.py | 55 +++++++----- .../monitoring/test_gateway_health_export.py | 26 +++++- 8 files changed, 137 insertions(+), 163 deletions(-) diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index 3404392c638..8ba31386e00 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -9,7 +9,6 @@ session history, audit records, or product analytics belong here. from __future__ import annotations import logging -import re from dataclasses import dataclass from typing import Any, Dict, List, Optional @@ -29,12 +28,6 @@ class GatewayHealthSnapshot: events: List[GatewayHealthEvent | GatewayDiagnosticEvent] -_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") -_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) -_TOKEN_RE = re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b") -_SECRET_LITERAL_RE = re.compile(r"\*{3,}") -_PHONE_RE = re.compile(r"(? bool: def redact_gateway_message(message: Any) -> str: - """Redact gateway diagnostic free text for customer-owned export.""" - text = str(message or "") + """Redact gateway diagnostic free text for operator-owned export. + + Single scrub path: everything goes through + ``agent.monitoring.redaction.redact_for_export`` (unconditional + secrets + PII), then is length-bounded. + """ try: from agent.monitoring.redaction import redact_for_export - redacted = redact_for_export(text, content_mode="pii") or "" + redacted = redact_for_export(str(message or "")) or "" except Exception: redacted = "[redaction-unavailable]" - redacted = _BEARER_RE.sub("[redacted]", redacted) - redacted = _TOKEN_RE.sub("[redacted]", redacted) - redacted = _SECRET_LITERAL_RE.sub("[redacted]", redacted) - redacted = re.sub(r"\bBearer\s+\[[^\]]+\]", "[redacted]", redacted, flags=re.IGNORECASE) - redacted = _EMAIL_RE.sub("[email]", redacted) - redacted = _PHONE_RE.sub("[phone]", redacted) return redacted[:500] diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py index d07b90ea2c1..1a42ce195ca 100644 --- a/agent/monitoring/policy.py +++ b/agent/monitoring/policy.py @@ -8,31 +8,48 @@ collector. It carries no account identity and can be rotated by clearing from __future__ import annotations +import logging import uuid from typing import Any, Dict - -def _monitoring_cfg(config: Dict[str, Any]) -> Dict[str, Any]: - for key in ("monitoring", "telemetry"): # accept legacy telemetry.* keys - cfg = config.get(key) if isinstance(config, dict) else None - if isinstance(cfg, dict) and cfg.get("install_id"): - return cfg - cfg = config.get("monitoring") if isinstance(config, dict) else None - return cfg if isinstance(cfg, dict) else {} +logger = logging.getLogger(__name__) def ensure_install_id(config: Dict[str, Any]) -> str: - """Return a stable install id, minting one if the config slot is empty. + """Return a stable install id, minting and persisting one when empty. - Does not persist — the caller writes the returned value back to - config.yaml. Clearing ``monitoring.install_id`` (e.g. with - ``hermes config set monitoring.install_id ""``) mints anew on next call. + The id must survive gateway restarts (it becomes ``service.instance.id`` + on exported signals), so a freshly minted UUID is written back to + config.yaml immediately. The write is fail-open: if persisting fails + (read-only home, managed scope), the ephemeral id is still returned and + a new one is minted next start. + + Clearing ``monitoring.install_id`` (e.g. ``hermes config set + monitoring.install_id ""``) rotates the id on the next gateway start. """ - cfg = _monitoring_cfg(config) - existing = cfg.get("install_id") + mon = config.get("monitoring") if isinstance(config, dict) else None + existing = (mon or {}).get("install_id") if isinstance(mon, dict) else None if isinstance(existing, str) and existing.strip(): return existing - return str(uuid.uuid4()) + + minted = str(uuid.uuid4()) + try: + from hermes_cli.config import load_config, save_config + + fresh = load_config() + if isinstance(fresh, dict): + slot = fresh.setdefault("monitoring", {}) + if isinstance(slot, dict) and not str(slot.get("install_id") or "").strip(): + slot["install_id"] = minted + save_config(fresh) + except Exception: + logger.debug("install_id persist failed; using ephemeral id", exc_info=True) + # Keep the in-memory config consistent for this process either way. + if isinstance(config, dict): + config.setdefault("monitoring", {}) + if isinstance(config["monitoring"], dict): + config["monitoring"]["install_id"] = minted + return minted __all__ = [ diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py index 7ea5524cccd..cf1564f0171 100644 --- a/agent/monitoring/redaction.py +++ b/agent/monitoring/redaction.py @@ -1,77 +1,70 @@ """Redaction applied to monitoring data before egress. -Secrets are always redacted, on every export path; no setting disables this. -Wraps ``agent/redact.py::redact_sensitive_text(force=True)`` and fails CLOSED: -if the redactor cannot run, the raw string is never emitted. +One unconditional scrub, no modes, no knobs. Every string that leaves the +process passes through ``redact_for_export``: -``redact_for_export(text, content_mode="pii")`` additionally scrubs e-mail -addresses, phone numbers, and UUID-shaped identifiers — the gateway -diagnostics path always uses this mode, so log-derived messages leave the -process with secrets AND PII already removed. + * Secrets first — wraps ``agent/redact.py::redact_sensitive_text(force=True)`` + plus bearer/token-shape patterns, and fails CLOSED: if the redactor cannot + run, the raw string is never emitted. + * PII second — e-mail addresses, phone numbers, and UUID-shaped identifiers + are rewritten to ``[email]`` / ``[phone]`` / ``[id]``. + +There is deliberately no setting to weaken this. The monitoring plane is +content-free by design, so the only free text it carries (log-derived +diagnostic messages) is always fully scrubbed. """ from __future__ import annotations import re -from typing import Any, Dict, List, Optional +from typing import Optional -# Content-redaction strengths for any content that IS exported. -CONTENT_NONE = "none" # drop content entirely (structural telemetry only) -CONTENT_PII = "pii" # codec-aware PII redaction on exported content -CONTENT_MODES = {CONTENT_NONE, CONTENT_PII} +# ── secret shapes (belt-and-suspenders on top of agent/redact.py) ─────────── +_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) +_TOKEN_RE = re.compile( + r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b" +) +_SECRET_LITERAL_RE = re.compile(r"\*{3,}") +_BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+\[[^\]]+\]", re.IGNORECASE) -# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ── +# ── PII shapes ─────────────────────────────────────────────────────────────── _EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") # E.164-ish and common separators; conservative to avoid nuking code/IDs. _PHONE_RE = re.compile( r"(? Optional[str]: +def _secret_redact(text: str) -> str: """Always-on secret redaction. force=True so user config can't disable it.""" - if text is None: - return None try: from agent.redact import redact_sensitive_text - return redact_sensitive_text(str(text), force=True) + out = redact_sensitive_text(text, force=True) except Exception: # Fail CLOSED: if the redactor can't run, do not emit the raw string. return "[redaction-unavailable]" + out = _BEARER_RE.sub("[redacted]", out) + out = _TOKEN_RE.sub("[redacted]", out) + out = _SECRET_LITERAL_RE.sub("[redacted]", out) + out = _BEARER_RESIDUE_RE.sub("[redacted]", out) + return out -def _pii_redact(text: str) -> str: - text = _EMAIL_RE.sub("[email]", text) - text = _UUID_RE.sub("[id]", text) - text = _PHONE_RE.sub("[phone]", text) - return text - - -def redact_for_export( - text: Optional[str], - *, - content_mode: str = CONTENT_NONE, -) -> Optional[str]: - """Redact a single content string for export. - - Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. - Callers gate *whether content is exported at all* via telemetry.trajectories - (see ``content_export_enabled``); this function only scrubs content that the - caller has already decided to export. - """ - redacted = _secret_redact(text) - if redacted is None: +def redact_for_export(text: Optional[str]) -> Optional[str]: + """Scrub a string for egress: secrets, then PII. Unconditional.""" + if text is None: return None - if content_mode == CONTENT_PII: - redacted = _pii_redact(redacted) - return redacted + out = _secret_redact(str(text)) + out = _EMAIL_RE.sub("[email]", out) + out = _UUID_RE.sub("[id]", out) + out = _PHONE_RE.sub("[phone]", out) + return out __all__ = [ - "CONTENT_NONE", - "CONTENT_PII", - "CONTENT_MODES", "redact_for_export", ] diff --git a/hermes_cli/config.py b/hermes_cli/config.py index f0f6a43c315..bd07d7d6a4e 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2175,60 +2175,6 @@ DEFAULT_CONFIG = { "redact_pii": False, # When True, hash user IDs and strip phone numbers from LLM context }, - # Telemetry & observability. Three settings, isolated from each other: - # local — full-fidelity local observability the user owns (real - # models, providers, tool names). Default ON. - # aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. - # trajectories — content trajectories for training. Separate consent (not here yet). - # - # Enterprise locking: any telemetry.* key can be pinned by an administrator via - # the managed-scope layer (/etc/hermes/config.yaml), which wins over the user's - # value on a per-key basis. There is no telemetry-specific policy block — pin - # e.g. `telemetry.allow_aggregate: false` in managed scope to hard-forbid egress. - "telemetry": { - # Local telemetry: event log + SQLite index in state.db. The user's own data; - # never leaves the machine unless they export it or opt into aggregate metrics. - "local": True, - # Hard gate for aggregate metrics. When False, aggregate metrics are off - # regardless of consent_state. Intended to be pinned False by an administrator - # via managed scope on locked-down or air-gapped deployments. Default True - # (consent_state still governs the opt-in). - "allow_aggregate": True, - # Aggregate-metrics consent — the single source of truth for the opt-in: - # "unknown" (no choice made — never uploads), "local" (declined), - # "aggregate" (opted in). Set with `hermes config set telemetry.consent_state` - # or a managed-scope pin. Non-interactive installs sit at "unknown". - "consent_state": "unknown", - # Stable install identifier (aggregate metrics only). Empty string means "mint a - # fresh UUID on first use"; clear it to rotate. Never sent by local telemetry. - "install_id": "", - # Local event-log retention before rotation (days). Local telemetry only. - "retention_days": 90, - # Keep secret redaction on even at full local capture (a SIEM full of live - # credentials is a bigger attack target). Admin may override via managed scope. - "redact_secrets": True, - # Content redaction for exports / support bundles: "none" | "pii". - "content_redaction": "none", - # Trajectories: full message content / reasoning / raw tool args. - # Off by default. When enabled, full content becomes exportable to the - # configured destination — always secret-redacted, and PII-redacted per - # content_redaction. This is the consent gate for content export and is - # admin-pinnable via managed scope. It does not enable any upload. - "trajectories": { - "enabled": False, - }, - # Exporters. The OTLP exporter sends to a configured Collector endpoint; - # endpoint headers reference environment variable names rather than inline - # secrets. - "export": { - "otlp": { - "enabled": False, - "endpoint": None, - "headers_env": {}, # {"Authorization": "MY_OTLP_TOKEN_ENVVAR"} - }, - }, - }, - # Text-to-speech configuration # Each provider supports an optional `max_text_length:` override for the # per-request input-character cap. Omit it to use the provider's documented @@ -3107,11 +3053,6 @@ DEFAULT_CONFIG = { "service.name": "hermes-gateway", "deployment.environment": "production", }, - "redaction": { - "enabled": True, - "include_stack_summary": True, - "include_raw_stack": False, - }, }, # OTLP destination. headers_env maps header names to ENVIRONMENT # VARIABLE NAMES (never secret values); values are read from the diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 2cef49a220a..3157d78870a 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -14340,10 +14340,8 @@ def cmd_monitoring(args): print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}") print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} " f"(interval {gh.get('logs_export_interval_seconds', 5)}s)") - red_raw = gh.get("redaction") - red: dict = red_raw if isinstance(red_raw, dict) else {} - print(f" Redaction: {'on' if red.get('enabled', True) else 'OFF'} " - f"(secrets/PII scrubbed in-process before egress)") + print(" Redaction: always on " + "(secrets/PII scrubbed in-process before egress; not configurable)") endpoint = otlp.get("endpoint") or "" if otlp.get("enabled") and endpoint: print(f" OTLP endpoint: {endpoint}") diff --git a/scripts/observability/gateway_health_export_probe.py b/scripts/observability/gateway_health_export_probe.py index 9b6de1b8605..1635f464888 100644 --- a/scripts/observability/gateway_health_export_probe.py +++ b/scripts/observability/gateway_health_export_probe.py @@ -45,7 +45,6 @@ def main() -> None: "service.name": "hermes-gateway-smoke", "deployment.environment": "local-smoke", }, - "redaction": {"enabled": True, "include_raw_stack": False}, }, "export": { "otlp": { diff --git a/tests/monitoring/test_export_redaction.py b/tests/monitoring/test_export_redaction.py index 0a5beaff908..1dede385a5d 100644 --- a/tests/monitoring/test_export_redaction.py +++ b/tests/monitoring/test_export_redaction.py @@ -1,56 +1,69 @@ -"""Export redaction pipeline tests — the security-critical layer. +"""Export redaction tests — the security-critical layer. Invariants: - * Secrets ALWAYS stripped, every export path, no flag disables it. + * One unconditional scrub: secrets AND PII, no modes, no knobs. * Fails CLOSED: if the redactor can't run, the raw string is never emitted. - * PII (emails, phones, UUID-shaped ids) stripped in 'pii' mode — the mode - the gateway diagnostics path always uses. + * Structure (subsystem names, error codes) survives; free-text PII does not. """ from __future__ import annotations +from unittest import mock + import agent.monitoring.redaction as R -def test_secret_always_stripped_in_none_mode(): +def test_secret_key_always_stripped(): fake_key = "sk-ant-api03-" + "A" * 24 # constructed to dodge literal-scrubbers - text = f"calling with key {fake_key} and moving on" - out = R.redact_for_export(text, content_mode=R.CONTENT_NONE) + out = R.redact_for_export(f"calling with key {fake_key} and moving on") assert out is not None assert fake_key not in out -def test_secret_always_stripped_in_pii_mode(): - fake_token = "ghp_" + "0123456789abcdef" * 2 + "0123" - text = f"token {fake_token} leaked" - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) +def test_token_shapes_stripped(): + ghp = "ghp_" + "0123456789abcdef" * 2 + "0123" + slack = "xoxb-" + "123456789012-abcdefABCDEF" + out = R.redact_for_export(f"token {ghp} and {slack} leaked") assert out is not None - assert fake_token not in out + assert ghp not in out + assert slack not in out + assert "[redacted]" in out + + +def test_bearer_header_stripped(): + out = R.redact_for_export("Authorization: Bearer abc.def-ghi_jkl") + assert out is not None + assert "abc.def-ghi_jkl" not in out def test_none_passthrough(): - assert R.redact_for_export(None, content_mode=R.CONTENT_NONE) is None + assert R.redact_for_export(None) is None -def test_pii_mode_strips_email_phone_uuid(): +def test_pii_always_stripped(): text = ("reach alice@example.com or +1 415 555 0100, " "install 123e4567-e89b-12d3-a456-426614174000") - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + out = R.redact_for_export(text) assert out is not None assert "alice@example.com" not in out assert "426614174000" not in out assert "[email]" in out assert "[id]" in out + assert "[phone]" in out -def test_none_mode_keeps_ordinary_words(): - out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE) - assert out == "just ordinary words" +def test_ordinary_words_survive(): + assert R.redact_for_export("just ordinary words") == "just ordinary words" -def test_pii_mode_preserves_non_pii_structure(): - text = "platform.slack entered fatal after auth_failed" - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) +def test_structure_preserved(): + out = R.redact_for_export("platform.slack entered fatal after auth_failed") assert out is not None assert "platform.slack" in out assert "auth_failed" in out + + +def test_fails_closed_when_redactor_unavailable(): + with mock.patch("agent.redact.redact_sensitive_text", side_effect=RuntimeError): + out = R.redact_for_export("secret sauce sk-live-key") + assert out == "[redaction-unavailable]" diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index 8c1a9ef8513..3fe8ff190b1 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -14,8 +14,8 @@ def test_default_config_keeps_gateway_health_export_disabled(): assert cfg["warning_error_events_enabled"] is True assert cfg["export_interval_seconds"] == 60 assert cfg["logs_export_interval_seconds"] == 5 - assert cfg["redaction"]["enabled"] is True - assert cfg["redaction"]["include_raw_stack"] is False + # Redaction is always-on and deliberately NOT configurable. + assert "redaction" not in cfg def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics(): @@ -340,3 +340,25 @@ def test_gateway_diagnostic_log_handler_never_raises_on_malformed_record(): ) handler.emit(record) + + +def test_install_id_persists_across_calls(tmp_path, monkeypatch): + """A minted install id must survive restarts (service.instance.id continuity).""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text("{}\n") + + import hermes_cli.config as cfg_mod + from agent.monitoring.policy import ensure_install_id + + first = ensure_install_id(cfg_mod.load_config()) + assert first and first != "unknown" + # Persisted: a fresh load (simulating a new gateway process) returns the same id. + second = ensure_install_id(cfg_mod.load_config()) + assert second == first + assert first in (tmp_path / "config.yaml").read_text() + + +def test_install_id_existing_value_wins(monkeypatch): + from agent.monitoring.policy import ensure_install_id + + assert ensure_install_id({"monitoring": {"install_id": "keep-me"}}) == "keep-me" From 16e774f223372c4017fd0ce5abde21b5c2e9362c Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Wed, 22 Jul 2026 01:59:55 +0000 Subject: [PATCH 12/29] fix(monitoring): harden gateway health OTLP egress --- agent/monitoring/emitter.py | 7 +- agent/monitoring/gateway_health.py | 79 +++++++-- agent/monitoring/gateway_health_export.py | 161 ++++++++++++++---- agent/monitoring/otlp_exporter.py | 10 +- docs/observability/monitoring.md | 5 +- hermes_cli/config.py | 2 +- tests/monitoring/test_emitter.py | 19 +++ .../monitoring/test_gateway_health_export.py | 154 ++++++++++++++++- 8 files changed, 379 insertions(+), 58 deletions(-) diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py index f7dee1bdf4f..52bee5e05cf 100644 --- a/agent/monitoring/emitter.py +++ b/agent/monitoring/emitter.py @@ -114,12 +114,15 @@ class MonitoringEmitter: """Register a live batch subscriber (callable(batch: list[dict])).""" if callback not in self._subscribers: self._subscribers.append(callback) + self._enabled = True def unsubscribe(self, callback) -> None: try: self._subscribers.remove(callback) except ValueError: pass + if not self._subscribers: + self._enabled = False # ── introspection / shutdown (tests, CLI) ─────────────────────────────── def flush(self, timeout: float = 2.0) -> None: @@ -160,7 +163,9 @@ def get_emitter() -> MonitoringEmitter: return _EMITTER with _EMITTER_LOCK: if _EMITTER is None: - _EMITTER = MonitoringEmitter() + # Collection is opt-in. A plane exporter enables the singleton by + # attaching its first subscriber; until then producers are no-ops. + _EMITTER = MonitoringEmitter(enabled=False) return _EMITTER diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index 8ba31386e00..937b57ba442 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -8,6 +8,7 @@ session history, audit records, or product analytics belong here. from __future__ import annotations +import hashlib import logging from dataclasses import dataclass from typing import Any, Dict, List, Optional @@ -30,6 +31,10 @@ class GatewayHealthSnapshot: _RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"} _FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"} +_KNOWN_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { + "starting", "draining", "stopping", "stopped", "startup_failed", "unknown" +} +_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"} def _allowed_logger(name: str) -> bool: @@ -70,6 +75,46 @@ def classify_gateway_error(raw: Any) -> str: return "unknown" +def classify_exit_reason( + raw: Any, *, state: Any, restart_requested: bool +) -> Optional[str]: + """Reduce free-form shutdown text to a bounded operational class.""" + if restart_requested: + return "restart_requested" + state_name = str(state or "").lower() + if raw is None and state_name != "startup_failed": + return None + classified = classify_gateway_error(raw) + if state_name == "startup_failed": + return classified if classified != "unknown" else "startup_failed" + text = str(raw or "").lower() + if "signal" in text or "sigterm" in text or "sigint" in text: + return "signal" + if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")): + return "planned_stop" + return classified + + +def _bounded_state(raw: Any) -> str: + state = str(raw or "unknown").lower() + return state if state in _KNOWN_STATES else "unknown" + + +def _safe_metric_value(raw: Any, *, limit: int = 128) -> str: + try: + from agent.monitoring.redaction import redact_for_export + value = redact_for_export(str(raw or "")) or "unknown" + except Exception: + return "unknown" + return value[:limit] + + +def _safe_instance_id(raw: Any) -> str: + """Return a stable opaque instance key without exporting the source ID.""" + value = str(raw or "unknown").encode("utf-8", errors="replace") + return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" + + def subsystem_for_logger(logger_name: str) -> str: if logger_name.startswith("gateway.platforms."): parts = logger_name.split(".") @@ -120,11 +165,11 @@ def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: + mode = str(supervision_mode or "unknown").lower() return { - "hermes.profile": str(profile), - "service.instance.id": str(install_id), - "service.version": str(version), - "hermes.supervision_mode": str(supervision_mode), + "service.instance.id": _safe_instance_id(install_id), + "service.version": _safe_metric_value(version, limit=64), + "hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown", } @@ -132,7 +177,7 @@ def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) out = dict(attrs) for key, val in extra.items(): if val is not None: - out[key] = str(val) + out[key] = _safe_metric_value(val) return GatewayMetric(name=name, value=value, attributes=out) @@ -147,7 +192,7 @@ def build_gateway_health_snapshot( ) -> GatewayHealthSnapshot: """Convert gateway_state.json-compatible runtime state into P0 signals.""" runtime = runtime or {} - gateway_state = runtime.get("gateway_state") + gateway_state = _bounded_state(runtime.get("gateway_state")) active_agents = _parse_active_agents(runtime.get("active_agents", 0)) busy = _derive_busy(gateway_running, gateway_state, active_agents) drainable = _derive_drainable(gateway_running, gateway_state) @@ -168,7 +213,7 @@ def build_gateway_health_snapshot( events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] for platform, pdata in platforms.items(): pdata = pdata if isinstance(pdata, dict) else {} - state = str(pdata.get("state") or "unknown").lower() + state = _bounded_state(pdata.get("state")) raw_error = pdata.get("error_code") or pdata.get("error_message") error_code = classify_gateway_error(raw_error) is_up = state in _RUNNING_PLATFORM_STATES @@ -244,15 +289,19 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] profile = _safe_profile() version = _safe_version() - old_gateway_state = str((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None - new_gateway_state = str(current.get("gateway_state")) if current.get("gateway_state") is not None else None + old_gateway_state = _bounded_state((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = _bounded_state(current.get("gateway_state")) if current.get("gateway_state") is not None else None if old_gateway_state != new_gateway_state and new_gateway_state: out.append(GatewayHealthEvent( name="gateway.lifecycle", gateway_state=new_gateway_state, old_state=old_gateway_state, new_state=new_gateway_state, - exit_reason=current.get("exit_reason"), + exit_reason=classify_exit_reason( + current.get("exit_reason"), + state=new_gateway_state, + restart_requested=bool(current.get("restart_requested")), + ), restart_requested=bool(current.get("restart_requested")), active_agents=_parse_active_agents(current.get("active_agents", 0)), profile=profile, @@ -276,7 +325,11 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: gateway_state=new_gateway_state, old_state=old_gateway_state, new_state=new_gateway_state, - exit_reason=current.get("exit_reason"), + exit_reason=classify_exit_reason( + current.get("exit_reason"), + state=new_gateway_state, + restart_requested=bool(current.get("restart_requested")), + ), restart_requested=bool(current.get("restart_requested")), active_agents=_parse_active_agents(current.get("active_agents", 0)), profile=profile, @@ -292,8 +345,8 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: pdata = pdata if isinstance(pdata, dict) else {} prev_raw = old_platforms.get(platform, {}) prev = prev_raw if isinstance(prev_raw, dict) else {} - old_state = str(prev.get("state")) if prev.get("state") is not None else None - new_state = str(pdata.get("state")) if pdata.get("state") is not None else None + old_state = _bounded_state(prev.get("state")) if prev.get("state") is not None else None + new_state = _bounded_state(pdata.get("state")) if pdata.get("state") is not None else None if old_state == new_state or not new_state: continue error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 969fa970635..9295072da98 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -10,12 +10,77 @@ from __future__ import annotations import logging import os +import re import threading from dataclasses import dataclass from typing import Any, Dict, Optional logger = logging.getLogger(__name__) +_RESOURCE_ATTRIBUTE_KEYS = frozenset({ + "service.name", + "service.namespace", + "service.version", + "service.instance.id", + "deployment.environment.name", + "cloud.provider", + "cloud.platform", + "cloud.region", + "telemetry.scope", +}) +_DIAGNOSTIC_ATTRIBUTE_KEYS = frozenset({ + "name", + "subsystem", + "error_class", + "error_code", + "platform", + "old_state", + "new_state", + "version", + "severity", +}) +_SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$") + + +def _redact_string(raw: Any, *, limit: int = 500) -> str: + try: + from agent.monitoring.redaction import redact_for_export + return (redact_for_export(str(raw or "")) or "[redacted]")[:limit] + except Exception: + return "[redaction-unavailable]" + + +def _safe_resource_attributes(raw: Any) -> Dict[str, str]: + """Allowlist bounded resource labels and reject values changed by redaction.""" + attrs: Dict[str, str] = {} + if not isinstance(raw, dict): + return attrs + for key, value in raw.items(): + key = str(key) + if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None: + continue + if key == "service.instance.id": + from agent.monitoring.gateway_health import _safe_instance_id + attrs[key] = _safe_instance_id(value) + continue + text = str(value) + if not _SAFE_RESOURCE_VALUE.fullmatch(text): + continue + if _redact_string(text, limit=128) != text: + continue + attrs[key] = text + return attrs + + +def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]: + attrs: Dict[str, Any] = {} + for key in _DIAGNOSTIC_ATTRIBUTE_KEYS: + value = event.get(key) + if value is None: + continue + attrs[f"hermes.{key}"] = _redact_string(value) if isinstance(value, str) else value + return attrs + @dataclass(slots=True) class GatewayHealthExportRuntime: @@ -32,27 +97,52 @@ class GatewayHealthExportRuntime: if self.stop_event is not None: self.stop_event.set() if self.thread is not None: - self.thread.join(timeout=2.0) + self.thread.join(timeout=0.25) if self.log_handler is not None: try: logging.getLogger().removeHandler(self.log_handler) except Exception: pass - if self.streamer is not None: - try: - self.streamer.shutdown() - except Exception: - pass - if self.log_streamer is not None: - try: - self.log_streamer.shutdown() - except Exception: - pass - if self.metric_provider is not None: - try: - self.metric_provider.shutdown() - except Exception: - pass + + # Detach synchronously so no new records are collected during a slow + # exporter shutdown. Network flush/close then runs under one bounded + # daemon-thread deadline and can never delay gateway teardown. + try: + from agent.monitoring.emitter import get_emitter + emitter = get_emitter() + if self.streamer is not None: + emitter.unsubscribe(self.streamer) + if self.log_streamer is not None: + emitter.unsubscribe(self.log_streamer) + except Exception: + pass + + closeables = [ + item for item in (self.streamer, self.log_streamer, self.metric_provider) + if item is not None + ] + + def _close() -> None: + for item in closeables: + try: + item.shutdown() + except Exception: + pass + + if closeables: + worker = threading.Thread( + target=_close, + name="hermes-gateway-health-export-shutdown", + daemon=True, + ) + worker.start() + worker.join(timeout=2.0) + + self.streamer = None + self.log_streamer = None + self.metric_provider = None + self.thread = None + self.stop_event = None def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]: @@ -209,9 +299,9 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None) interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) - resource_attrs = dict((gh.get("resource_attributes") or {})) - resource_attrs.setdefault("service.name", "hermes-gateway") - resource_attrs.setdefault("telemetry.scope", "gateway_health") + resource_attrs = _safe_resource_attributes(gh.get("resource_attributes")) + resource_attrs["service.name"] = "hermes-gateway" + resource_attrs["telemetry.scope"] = "gateway_health" provider = sdk["MeterProvider"]( metric_readers=[reader], resource=sdk["Resource"].create(resource_attrs), @@ -267,9 +357,9 @@ class GatewayDiagnosticLogStreamer: gh = _gateway_health_config(config) headers = _resolve_headers(otlp.get("headers_env")) endpoint = _logs_endpoint(str(otlp.get("endpoint"))) - resource_attrs = dict((gh.get("resource_attributes") or {})) - resource_attrs.setdefault("service.name", "hermes-gateway") - resource_attrs.setdefault("telemetry.scope", "gateway_diagnostics") + resource_attrs = _safe_resource_attributes(gh.get("resource_attributes")) + resource_attrs["service.name"] = "hermes-gateway" + resource_attrs["telemetry.scope"] = "gateway_diagnostics" self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs)) self._processor = sdk["BatchLogRecordProcessor"]( sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) @@ -284,11 +374,7 @@ class GatewayDiagnosticLogStreamer: for ev in batch: if ev.get("event") != "gateway_diagnostic": continue - attrs = { - f"hermes.{key}": val - for key, val in ev.items() - if key not in {"event", "redacted_message", "ts_ns"} and val is not None - } + attrs = _diagnostic_log_attributes(ev) body = ev.get("redacted_message") or ev.get("name") or "gateway diagnostic" record = self._LogRecord( timestamp=ev.get("ts_ns"), @@ -297,7 +383,7 @@ class GatewayDiagnosticLogStreamer: trace_flags=self._sdk["TraceFlags"].DEFAULT, severity_text=str(ev.get("severity") or "warning").upper(), severity_number=_severity_number(self._sdk, ev.get("severity")), - body=str(body), + body=_redact_string(body), attributes=attrs, ) self._logger.emit(record) @@ -337,7 +423,7 @@ def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) def _attach_log_handler(config: Dict[str, Any]) -> Any: gh = _gateway_health_config(config) - if not gh.get("warning_error_events_enabled", True): + if not gh.get("diagnostic_events_enabled", True) or not gh.get("warning_error_events_enabled", True): return None from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version()) @@ -382,20 +468,25 @@ def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRu try: from agent.monitoring import otlp_exporter runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event) + if runtime.streamer is None: + raise RuntimeError("gateway health span streamer did not start") runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk) except Exception: logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed") try: runtime.log_handler = _attach_log_handler(config) except Exception: logger.debug("gateway diagnostic log handler failed to attach", exc_info=True) - try: - _emit_snapshot_events(config) - runtime.stop_event = threading.Event() - runtime.thread = _start_snapshot_thread(config, runtime.stop_event) - except Exception: - logger.debug("gateway health snapshot thread failed to start", exc_info=True) + if gh.get("diagnostic_events_enabled", True): + try: + _emit_snapshot_events(config) + runtime.stop_event = threading.Event() + runtime.thread = _start_snapshot_thread(config, runtime.stop_event) + except Exception: + logger.debug("gateway health snapshot thread failed to start", exc_info=True) return runtime diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index 01eefa53506..27c600dcf8b 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -136,15 +136,21 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: "gateway_health": ("name", "gateway_state", "old_state", "new_state", "exit_reason", "restart_requested", "active_agents", "gateway_busy", "gateway_drainable", "platform_count", - "fatal_platform_count", "profile", "install_id", "version", + "fatal_platform_count", "version", "supervision_mode", "pid"), "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", "redacted_message", "platform", "old_state", "new_state", - "profile", "version", "severity"), + "version", "severity"), } for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] v = ev.get(col) if v is not None: + if isinstance(v, str): + try: + from agent.monitoring.redaction import redact_for_export + v = (redact_for_export(v) or "[redacted]")[:500] + except Exception: + v = "[redaction-unavailable]" attrs[f"hermes.{col}"] = v return attrs diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index 1ee9d4b5a8d..9ca8f2ce12a 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -19,8 +19,9 @@ trajectory capture is a separate plane served by the NeMo Relay integration | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | | Diagnostics | `/v1/logs` | Warning/error gateway log events with secrets AND PII scrubbed in-process before egress (`[redacted]` / `[email]`), bounded error classes | -Every signal carries resource attributes (`service.name`, profile, version, -install id, supervision mode) so an operator can tell instances apart. +Signals carry `service.name`, version, supervision mode, and a stable one-way +hash of the install id so an operator can distinguish instances without +exporting account/profile identity or the raw install identifier. ## Enabling diff --git a/hermes_cli/config.py b/hermes_cli/config.py index bd07d7d6a4e..1e2d1ba44a4 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3051,7 +3051,7 @@ DEFAULT_CONFIG = { "logs_export_interval_seconds": 5, "resource_attributes": { "service.name": "hermes-gateway", - "deployment.environment": "production", + "deployment.environment.name": "production", }, }, # OTLP destination. headers_env maps header names to ENVIRONMENT diff --git a/tests/monitoring/test_emitter.py b/tests/monitoring/test_emitter.py index a0f5c2d889d..98ad7b0389b 100644 --- a/tests/monitoring/test_emitter.py +++ b/tests/monitoring/test_emitter.py @@ -15,6 +15,25 @@ def test_emit_never_raises_when_disabled(): em.close() +def test_process_singleton_stays_dormant_until_subscribed(): + from agent.monitoring import emitter + + emitter.reset_emitter_for_tests() + try: + emitter.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + singleton = emitter.get_emitter() + assert singleton.stats()["queued"] == 0 + assert singleton._started is False + + subscriber = lambda _batch: None # noqa: E731 + singleton.subscribe(subscriber) + emitter.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + assert singleton._started is True + singleton.unsubscribe(subscriber) + finally: + emitter.reset_emitter_for_tests() + + def test_emit_accepts_dataclass_and_dict(tmp_path): em = MonitoringEmitter() seen: list = [] diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index 3fe8ff190b1..715de681bf8 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -14,6 +14,8 @@ def test_default_config_keeps_gateway_health_export_disabled(): assert cfg["warning_error_events_enabled"] is True assert cfg["export_interval_seconds"] == 60 assert cfg["logs_export_interval_seconds"] == 5 + assert cfg["resource_attributes"]["deployment.environment.name"] == "production" + assert "deployment.environment" not in cfg["resource_attributes"] # Redaction is always-on and deliberately NOT configurable. assert "redaction" not in cfg @@ -59,11 +61,12 @@ def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics( active = next(m for m in snapshot.metrics if m.name == "hermes.gateway.active_agents") assert active.value == 2 assert active.attributes == { - "hermes.profile": "default", - "service.instance.id": "install-1", + "service.instance.id": active.attributes["service.instance.id"], "service.version": "2026.7.test", "hermes.supervision_mode": "manual", } + assert active.attributes["service.instance.id"].startswith("sha256:") + assert "install-1" not in active.attributes["service.instance.id"] busy = next(m for m in snapshot.metrics if m.name == "hermes.gateway.busy") drainable = next(m for m in snapshot.metrics if m.name == "hermes.gateway.drainable") @@ -183,6 +186,7 @@ def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypat lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") assert lifecycle["old_state"] == "starting" assert lifecycle["new_state"] == "running" + assert lifecycle["exit_reason"] is None platform = next(e for e in captured if e["name"] == "platform.state_change") assert platform["old_state"] == "running" assert platform["new_state"] == "fatal" @@ -198,8 +202,21 @@ def test_runtime_status_transition_emits_startup_failed_and_exit(): old = emitter.emit emitter.emit = lambda event: captured.append(event.to_dict()) # type: ignore[assignment] try: - emit_runtime_status_transition({"gateway_state": "starting"}, {"gateway_state": "startup_failed", "exit_reason": "startup token ***"}) - emit_runtime_status_transition({"gateway_state": "running"}, {"gateway_state": "stopped", "exit_reason": "shutdown", "restart_requested": True}) + emit_runtime_status_transition( + {"gateway_state": "starting"}, + { + "gateway_state": "startup_failed", + "exit_reason": "Bearer top-secret-token rejected for user@example.com", + }, + ) + emit_runtime_status_transition( + {"gateway_state": "running"}, + { + "gateway_state": "stopped", + "exit_reason": "shutdown requested by user@example.com", + "restart_requested": True, + }, + ) finally: emitter.emit = old # type: ignore[assignment] @@ -208,8 +225,11 @@ def test_runtime_status_transition_emits_startup_failed_and_exit(): assert "gateway.exit" in names failed = next(e for e in captured if e["name"] == "gateway.startup_failed") assert "***" not in failed["redacted_message"] + lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") + assert lifecycle["exit_reason"] == "auth_failed" exit_event = next(e for e in captured if e["name"] == "gateway.exit") assert exit_event["restart_requested"] is True + assert exit_event["exit_reason"] == "restart_requested" def test_otlp_attrs_include_gateway_transition_fields(): @@ -230,6 +250,72 @@ def test_otlp_attrs_include_gateway_transition_fields(): assert attrs["hermes.restart_requested"] is True +def test_otlp_attrs_redact_strings_and_never_export_profile(): + from agent.monitoring.otlp_exporter import _span_attrs + + attrs = _span_attrs({ + "event": "gateway_health", + "name": "gateway.lifecycle", + "profile": "user@example.com", + "exit_reason": "Bearer top-secret-token for user@example.com", + }) + + assert "hermes.profile" not in attrs + assert "top-secret-token" not in str(attrs) + assert "user@example.com" not in str(attrs) + + +def test_resource_attributes_are_allowlisted_and_sanitized(): + from agent.monitoring.gateway_health_export import _safe_resource_attributes + + attrs = _safe_resource_attributes({ + "service.name": "hermes-gateway", + "service.instance.id": "install-1", + "deployment.environment.name": "staging", + "user.email": "user@example.com", + "authorization": "Bearer top-secret-token", + "custom.request.id": "unbounded", + }) + + assert attrs == { + "service.name": "hermes-gateway", + "service.instance.id": attrs["service.instance.id"], + "deployment.environment.name": "staging", + } + assert attrs["service.instance.id"].startswith("sha256:") + assert "install-1" not in attrs["service.instance.id"] + + +def test_instance_id_hash_is_stable_and_distinguishes_instances(): + from agent.monitoring.gateway_health import _safe_instance_id + + first = _safe_instance_id("install-1") + repeat = _safe_instance_id("install-1") + second = _safe_instance_id("install-2") + + assert first == repeat + assert first != second + assert first.startswith("sha256:") + assert "install-1" not in first + + +def test_diagnostic_log_attributes_are_allowlisted_redacted_and_profile_free(): + from agent.monitoring.gateway_health_export import _diagnostic_log_attributes + + attrs = _diagnostic_log_attributes({ + "event": "gateway_diagnostic", + "name": "platform.fatal", + "subsystem": "platform.slack", + "profile": "user@example.com", + "error_code": "Bearer top-secret-token", + "custom": "must-not-egress", + }) + + assert "hermes.profile" not in attrs + assert "hermes.custom" not in attrs + assert "top-secret-token" not in str(attrs) + + def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch): from agent.monitoring import gateway_health_export from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime @@ -259,6 +345,7 @@ def test_gateway_health_export_streams_only_gateway_events(monkeypatch): monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None) monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_diagnostic_log_streamer", lambda *a, **k: object()) monkeypatch.setattr(gateway_health_export, "_attach_log_handler", lambda *a, **k: None) monkeypatch.setattr(gateway_health_export, "_emit_snapshot_events", lambda *a, **k: None) monkeypatch.setattr(gateway_health_export, "_start_snapshot_thread", lambda *a, **k: None) @@ -301,6 +388,65 @@ def test_gateway_health_export_metric_failure_does_not_start_streamer(monkeypatc assert started == [] +def test_gateway_health_export_diagnostic_partial_start_cleans_up(monkeypatch): + from agent.monitoring import emitter, gateway_health_export, otlp_exporter + + class Streamer: + def __call__(self, _batch): + pass + + def shutdown(self): + pass + + streamer = Streamer() + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None) + monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: streamer) + monkeypatch.setattr( + gateway_health_export, + "_start_diagnostic_log_streamer", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")), + ) + emitter.get_emitter().subscribe(streamer) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True, "metrics_enabled": False}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is False + assert runtime.reason == "diagnostics_start_failed" + assert streamer not in emitter.get_emitter()._subscribers + + +def test_gateway_health_export_shutdown_is_bounded(): + import threading + import time + + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + release = threading.Event() + + class Blocking: + def shutdown(self): + release.wait(10) + + runtime = GatewayHealthExportRuntime( + enabled=True, + streamer=Blocking(), + log_streamer=Blocking(), + metric_provider=Blocking(), + ) + started = time.monotonic() + runtime.shutdown() + elapsed = time.monotonic() - started + release.set() + + assert elapsed < 2.5 + + def test_otlp_streamer_shutdown_unsubscribes(monkeypatch): from agent.monitoring import emitter from agent.monitoring.otlp_exporter import OTLPStreamer From 98a0260b0a10e79bfade47a4dd3434b0e5f666c9 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Wed, 22 Jul 2026 16:03:24 +0000 Subject: [PATCH 13/29] fix(otel): identify Hermes resources safely --- agent/monitoring/gateway_health_export.py | 27 ++++++++++++++----- agent/monitoring/otlp_exporter.py | 16 ++++++++--- .../monitoring/test_gateway_health_export.py | 22 +++++++++++++++ tests/monitoring/test_otlp_exporter.py | 12 +++++++++ 4 files changed, 66 insertions(+), 11 deletions(-) diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 9295072da98..04fff71a381 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -72,6 +72,20 @@ def _safe_resource_attributes(raw: Any) -> Dict[str, str]: return attrs +def _runtime_resource_attributes( + config: Dict[str, Any], *, telemetry_scope: str +) -> Dict[str, str]: + """Build the safe OTLP resource shared by metrics and diagnostic logs.""" + gh = _gateway_health_config(config) + attrs = _safe_resource_attributes(gh.get("resource_attributes")) + from agent.monitoring.gateway_health import _safe_instance_id + + attrs["service.name"] = "hermes-gateway" + attrs["service.instance.id"] = _safe_instance_id(_install_id(config)) + attrs["telemetry.scope"] = telemetry_scope + return attrs + + def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]: attrs: Dict[str, Any] = {} for key in _DIAGNOSTIC_ATTRIBUTE_KEYS: @@ -299,9 +313,9 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None) interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) - resource_attrs = _safe_resource_attributes(gh.get("resource_attributes")) - resource_attrs["service.name"] = "hermes-gateway" - resource_attrs["telemetry.scope"] = "gateway_health" + resource_attrs = _runtime_resource_attributes( + config, telemetry_scope="gateway_health" + ) provider = sdk["MeterProvider"]( metric_readers=[reader], resource=sdk["Resource"].create(resource_attrs), @@ -354,12 +368,11 @@ class GatewayDiagnosticLogStreamer: def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]): otlp = _otlp_config(config) - gh = _gateway_health_config(config) headers = _resolve_headers(otlp.get("headers_env")) endpoint = _logs_endpoint(str(otlp.get("endpoint"))) - resource_attrs = _safe_resource_attributes(gh.get("resource_attributes")) - resource_attrs["service.name"] = "hermes-gateway" - resource_attrs["telemetry.scope"] = "gateway_diagnostics" + resource_attrs = _runtime_resource_attributes( + config, telemetry_scope="gateway_diagnostics" + ) self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs)) self._processor = sdk["BatchLogRecordProcessor"]( sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index 27c600dcf8b..c9268b4ee5a 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -115,12 +115,20 @@ def build_exporter(config: Dict[str, Any]): return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) +def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]: + from agent.monitoring.gateway_health import _safe_instance_id + from agent.monitoring.policy import ensure_install_id + + return { + "service.name": "hermes-gateway", + "service.instance.id": _safe_instance_id(ensure_install_id(config)), + "telemetry.scope": "gateway_monitoring", + } + + def _make_provider(config: Dict[str, Any]): sdk = _require_sdk() - resource = sdk["Resource"].create({ - "service.name": "hermes-gateway", - "telemetry.scope": "gateway_monitoring", - }) + resource = sdk["Resource"].create(_resource_attributes(config)) provider = sdk["TracerProvider"](resource=resource) processor = sdk["BatchSpanProcessor"](build_exporter(config)) provider.add_span_processor(processor) diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index 715de681bf8..363b86cdfc4 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -299,6 +299,28 @@ def test_instance_id_hash_is_stable_and_distinguishes_instances(): assert "install-1" not in first +def test_runtime_resource_attributes_include_stable_hashed_instance(): + from agent.monitoring.gateway_health_export import _runtime_resource_attributes + + config = { + "monitoring": { + "install_id": "private-install-id", + "gateway_health_export": { + "resource_attributes": {"deployment.environment.name": "staging"} + }, + } + } + + attrs = _runtime_resource_attributes(config, telemetry_scope="gateway_health") + + assert attrs["service.name"] == "hermes-gateway" + assert attrs["service.instance.id"].startswith("sha256:") + assert len(attrs["service.instance.id"]) == len("sha256:") + 24 + assert "private-install-id" not in str(attrs) + assert attrs["deployment.environment.name"] == "staging" + assert attrs["telemetry.scope"] == "gateway_health" + + def test_diagnostic_log_attributes_are_allowlisted_redacted_and_profile_free(): from agent.monitoring.gateway_health_export import _diagnostic_log_attributes diff --git a/tests/monitoring/test_otlp_exporter.py b/tests/monitoring/test_otlp_exporter.py index 56edd566524..91ba7ede553 100644 --- a/tests/monitoring/test_otlp_exporter.py +++ b/tests/monitoring/test_otlp_exporter.py @@ -76,6 +76,18 @@ def test_is_enabled_requires_endpoint_and_flag(): assert not OE.is_enabled({}) +def test_trace_resource_includes_stable_hashed_instance(): + attrs = OE._resource_attributes( + {"monitoring": {"install_id": "private-install-id"}} + ) + + assert attrs["service.name"] == "hermes-gateway" + assert attrs["service.instance.id"].startswith("sha256:") + assert len(attrs["service.instance.id"]) == len("sha256:") + 24 + assert "private-install-id" not in str(attrs) + assert attrs["telemetry.scope"] == "gateway_monitoring" + + def test_export_otlp_feature_specs_match_pyproject(): from tools.lazy_deps import LAZY_DEPS import re From 73190cdbfe9948466823823ef5020d5056ab3634 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Wed, 22 Jul 2026 21:34:24 +0000 Subject: [PATCH 14/29] fix(monitoring): complete production OTLP runtime --- Dockerfile | 8 +++-- agent/monitoring/gateway_health.py | 33 ++++++++++++++----- .../monitoring/test_gateway_health_export.py | 32 ++++++++++++++++++ tests/tools/test_dockerfile_pid1_reaping.py | 14 ++++++++ 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 388056faacd..4d2c0f1972f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -152,7 +152,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # frontend stats the readme path during dep resolution, so we `touch` an # empty placeholder — the real README is restored by `COPY . .` below. # -# `uv sync --frozen --no-install-project --extra all --extra messaging` +# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp` # installs the deps reachable through the composite `[all]` extra # (handpicked set intended for the production image — excludes `[dev]`), # plus gateway messaging adapters that should work in the published image @@ -165,6 +165,10 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # so Docker users can use these providers without requiring runtime # lazy-install access to PyPI (often blocked in containerized envs). # +# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway +# Health export is enabled. Collector and observability-backend dependencies +# remain external and are not part of the Hermes production image. +# # The hindsight memory provider's client (hindsight-client) is baked in # for the same reason: it lazy-installs into /opt/hermes/.venv at first # use, which lives inside the (immutable) image layer rather than the @@ -182,7 +186,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # The editable link is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md -RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix +RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix # ---------- Frontend build (cached independently from Python source) ---------- # Copy only the frontend source trees first so that Python-only changes don't diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index 937b57ba442..ece881bca4e 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -31,8 +31,11 @@ class GatewayHealthSnapshot: _RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"} _FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"} -_KNOWN_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { +_KNOWN_GATEWAY_STATES = { "starting", "draining", "stopping", "stopped", "startup_failed", "unknown" +} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES +_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { + "connecting", "disconnected", "disabled", "paused", "retrying", "unknown" } _SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"} @@ -95,9 +98,9 @@ def classify_exit_reason( return classified -def _bounded_state(raw: Any) -> str: +def _bounded_state(raw: Any, *, allowed: set[str]) -> str: state = str(raw or "unknown").lower() - return state if state in _KNOWN_STATES else "unknown" + return state if state in allowed else "unknown" def _safe_metric_value(raw: Any, *, limit: int = 128) -> str: @@ -192,7 +195,9 @@ def build_gateway_health_snapshot( ) -> GatewayHealthSnapshot: """Convert gateway_state.json-compatible runtime state into P0 signals.""" runtime = runtime or {} - gateway_state = _bounded_state(runtime.get("gateway_state")) + gateway_state = _bounded_state( + runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) active_agents = _parse_active_agents(runtime.get("active_agents", 0)) busy = _derive_busy(gateway_running, gateway_state, active_agents) drainable = _derive_drainable(gateway_running, gateway_state) @@ -213,7 +218,9 @@ def build_gateway_health_snapshot( events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] for platform, pdata in platforms.items(): pdata = pdata if isinstance(pdata, dict) else {} - state = _bounded_state(pdata.get("state")) + state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) raw_error = pdata.get("error_code") or pdata.get("error_message") error_code = classify_gateway_error(raw_error) is_up = state in _RUNNING_PLATFORM_STATES @@ -289,8 +296,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] profile = _safe_profile() version = _safe_version() - old_gateway_state = _bounded_state((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None - new_gateway_state = _bounded_state(current.get("gateway_state")) if current.get("gateway_state") is not None else None + old_gateway_state = _bounded_state( + (previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = _bounded_state( + current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if current.get("gateway_state") is not None else None if old_gateway_state != new_gateway_state and new_gateway_state: out.append(GatewayHealthEvent( name="gateway.lifecycle", @@ -345,8 +356,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: pdata = pdata if isinstance(pdata, dict) else {} prev_raw = old_platforms.get(platform, {}) prev = prev_raw if isinstance(prev_raw, dict) else {} - old_state = _bounded_state(prev.get("state")) if prev.get("state") is not None else None - new_state = _bounded_state(pdata.get("state")) if pdata.get("state") is not None else None + old_state = _bounded_state( + prev.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if prev.get("state") is not None else None + new_state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if pdata.get("state") is not None else None if old_state == new_state or not new_state: continue error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index 363b86cdfc4..df50269b3f5 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -115,6 +115,38 @@ def test_gateway_health_snapshot_emits_content_free_diagnostic_event(): assert "Bearer" not in platform["redacted_message"] +def test_gateway_health_snapshot_preserves_real_bounded_platform_states(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + expected = { + "connecting", + "connected", + "disconnected", + "disabled", + "fatal", + "paused", + "retrying", + } + snapshot = build_gateway_health_snapshot( + { + "gateway_state": "running", + "platforms": {state: {"state": state} for state in expected}, + }, + gateway_running=True, + profile="default", + install_id="install-1", + version="v-test", + supervision_mode="container", + ) + + observed = { + metric.attributes["hermes.platform.state"] + for metric in snapshot.metrics + if metric.name == "hermes.platform.up" + } + assert observed == expected + + def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): from agent.monitoring import emitter from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler diff --git a/tests/tools/test_dockerfile_pid1_reaping.py b/tests/tools/test_dockerfile_pid1_reaping.py index 699fd5709a1..3b3e069c45f 100644 --- a/tests/tools/test_dockerfile_pid1_reaping.py +++ b/tests/tools/test_dockerfile_pid1_reaping.py @@ -172,6 +172,20 @@ def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text): ) +def test_dockerfile_preinstalls_gateway_monitoring_otlp_runtime(dockerfile_text): + sync_steps = [ + step for step in _run_steps(dockerfile_text) + if "uv sync" in step and "--no-install-project" in step + ] + + assert sync_steps, "Dockerfile must install Python dependencies with uv sync" + assert any("--extra otlp" in step for step in sync_steps), ( + "Published Docker images must preload the Hermes [otlp] runtime extra " + "so enabled Gateway Health export does not depend on first-boot package " + "installation into the immutable container environment." + ) + + def test_dockerfile_preinstalls_matrix_dependencies(dockerfile_text): sync_steps = [ step for step in _run_steps(dockerfile_text) From 4dc4f4302a318b3f6f361abd64b4f274a9fab915 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Wed, 22 Jul 2026 21:40:55 +0000 Subject: [PATCH 15/29] chore(monitoring): align smoke and attribution metadata --- contributors/emails/victor@nousresearch.com | 2 ++ scripts/observability/gateway_health_export_probe.py | 2 +- scripts/observability/otel_capture_collector.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 contributors/emails/victor@nousresearch.com diff --git a/contributors/emails/victor@nousresearch.com b/contributors/emails/victor@nousresearch.com new file mode 100644 index 00000000000..a899ee021d1 --- /dev/null +++ b/contributors/emails/victor@nousresearch.com @@ -0,0 +1,2 @@ +victor-kyriazakos +# PR #64536 Gateway Health OTLP production hardening diff --git a/scripts/observability/gateway_health_export_probe.py b/scripts/observability/gateway_health_export_probe.py index 1635f464888..e1b3bc0548d 100644 --- a/scripts/observability/gateway_health_export_probe.py +++ b/scripts/observability/gateway_health_export_probe.py @@ -43,7 +43,7 @@ def main() -> None: "logs_export_interval_seconds": 5, "resource_attributes": { "service.name": "hermes-gateway-smoke", - "deployment.environment": "local-smoke", + "deployment.environment.name": "local-smoke", }, }, "export": { diff --git a/scripts/observability/otel_capture_collector.py b/scripts/observability/otel_capture_collector.py index 0071328a4e2..cbfb659cb30 100644 --- a/scripts/observability/otel_capture_collector.py +++ b/scripts/observability/otel_capture_collector.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests. -This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces -and /v1/metrics, records request metadata as JSONL, and returns 200 so local -exporters can be exercised without Docker or a vendor backend. +This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces, +/v1/metrics, and /v1/logs, records request metadata as JSONL, and returns 200 so +local exporters can be exercised without Docker or a vendor backend. """ from __future__ import annotations From 18b46645437fb44c828458fe02eb4a6aa0bd3076 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Wed, 22 Jul 2026 22:08:51 +0000 Subject: [PATCH 16/29] fix(monitoring): drain terminal lifecycle events --- agent/monitoring/emitter.py | 33 ++++++++---- agent/monitoring/gateway_health_export.py | 9 ++-- gateway/run.py | 54 ++++++++++++------- .../gateway_health_export_probe.py | 1 + tests/gateway/test_startup_restart_race.py | 10 +++- tests/monitoring/test_emitter.py | 30 +++++++++++ .../monitoring/test_gateway_health_export.py | 35 ++++++++++++ 7 files changed, 139 insertions(+), 33 deletions(-) diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py index 52bee5e05cf..18a0d4851ae 100644 --- a/agent/monitoring/emitter.py +++ b/agent/monitoring/emitter.py @@ -67,6 +67,7 @@ class MonitoringEmitter: # Drop oldest to make room — bounded memory, newest-wins. try: self._q.get_nowait() + self._q.task_done() self._dropped += 1 self._q.put_nowait(payload) except Exception: @@ -99,7 +100,11 @@ class MonitoringEmitter: batch.append(self._q.get_nowait()) except queue.Empty: break - self._dispatch(batch) + try: + self._dispatch(batch) + finally: + for _ in batch: + self._q.task_done() def _dispatch(self, batch) -> None: # Fan-out to subscribers (OTLP streamers) — fully fail-isolated. @@ -126,15 +131,23 @@ class MonitoringEmitter: # ── introspection / shutdown (tests, CLI) ─────────────────────────────── def flush(self, timeout: float = 2.0) -> None: - """Block until the queue drains (test/CLI helper, NOT the hot path).""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if self._q.empty(): - # give the dispatcher a tick to finish the in-flight batch - time.sleep(0.05) - if self._q.empty(): - return - time.sleep(0.02) + """Wait boundedly for queued and in-flight batches to finish dispatch.""" + if timeout <= 0: + return + + finished = threading.Event() + + def _wait_for_completion() -> None: + self._q.join() + finished.set() + + waiter = threading.Thread( + target=_wait_for_completion, + name="hermes-monitoring-flush", + daemon=True, + ) + waiter.start() + finished.wait(timeout=timeout) def stats(self) -> Dict[str, int]: return { diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 04fff71a381..efb7575c692 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -118,12 +118,13 @@ class GatewayHealthExportRuntime: except Exception: pass - # Detach synchronously so no new records are collected during a slow - # exporter shutdown. Network flush/close then runs under one bounded - # daemon-thread deadline and can never delay gateway teardown. + # All producers above are now stopped. Drain queued and in-flight + # events before detaching subscribers so the terminal lifecycle event + # cannot race exporter shutdown. The barrier is bounded and fail-open. try: from agent.monitoring.emitter import get_emitter emitter = get_emitter() + emitter.flush(timeout=1.0) if self.streamer is not None: emitter.unsubscribe(self.streamer) if self.log_streamer is not None: @@ -131,6 +132,8 @@ class GatewayHealthExportRuntime: except Exception: pass + # Network flush/close runs under one bounded daemon-thread deadline and + # can never delay gateway teardown indefinitely. closeables = [ item for item in (self.streamer, self.log_streamer, self.metric_provider) if item is not None diff --git a/gateway/run.py b/gateway/run.py index 07d13b21c2e..600c234537b 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -9770,12 +9770,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._update_runtime_status("running", self._exit_reason) else: self._update_runtime_status("stopped", self._exit_reason) - try: - _gh_runtime = getattr(self, "_gateway_health_export_runtime", None) - if _gh_runtime is not None: - _gh_runtime.shutdown() - except Exception: - logger.debug("gateway health OTLP export shutdown failed", exc_info=True) + _shutdown_gateway_health_export(self) logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed()) self._stop_task = asyncio.create_task(_stop_impl()) @@ -23552,6 +23547,18 @@ async def _await_thread_exit( return not thread.is_alive() +def _shutdown_gateway_health_export(runner: Any) -> None: + """Idempotently drain and detach Gateway Health OTLP export.""" + runtime = getattr(runner, "_gateway_health_export_runtime", None) + if runtime is None: + return + runner._gateway_health_export_runtime = None + try: + runtime.shutdown() + except Exception: + logger.debug("gateway health OTLP export shutdown failed", exc_info=True) + + async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool: """ Start the gateway and run until interrupted. @@ -23999,10 +24006,16 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = logger.debug("MCP tool discovery failed: %s", e) # Start the gateway - success = await runner.start() + try: + success = await runner.start() + except BaseException: + _shutdown_gateway_health_export(runner) + raise if not success: + _shutdown_gateway_health_export(runner) return False if runner.should_exit_cleanly: + _shutdown_gateway_health_export(runner) if runner.exit_reason: logger.error("Gateway exiting cleanly: %s", runner.exit_reason) # A clean exit that carries an explicit exit code (e.g. a fatal @@ -24018,19 +24031,22 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = if not runner._running: # Startup was intentionally aborted by restart/shutdown before entering # running mode; preserve that lifecycle path without starting cron. - await runner.wait_for_shutdown() - if runner.should_exit_with_failure: - if runner.exit_reason: - logger.error("Gateway exiting with failure: %s", runner.exit_reason) - return False try: - from tools.mcp_tool import shutdown_mcp_servers - shutdown_mcp_servers() - except Exception: - pass - if runner.exit_code is not None: - raise SystemExit(runner.exit_code) - return True + await runner.wait_for_shutdown() + if runner.should_exit_with_failure: + if runner.exit_reason: + logger.error("Gateway exiting with failure: %s", runner.exit_reason) + return False + try: + from tools.mcp_tool import shutdown_mcp_servers + shutdown_mcp_servers() + except Exception: + pass + if runner.exit_code is not None: + raise SystemExit(runner.exit_code) + return True + finally: + _shutdown_gateway_health_export(runner) # Start the background cron scheduler via the resolved provider so # scheduled jobs fire automatically. The built-in provider is the diff --git a/scripts/observability/gateway_health_export_probe.py b/scripts/observability/gateway_health_export_probe.py index e1b3bc0548d..e1064088e2a 100644 --- a/scripts/observability/gateway_health_export_probe.py +++ b/scripts/observability/gateway_health_export_probe.py @@ -72,6 +72,7 @@ def main() -> None: logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com") emitter.get_emitter().flush(timeout=2.0) time.sleep(args.wait) + write_runtime_status(gateway_state="stopped", active_agents=0) runtime.shutdown() emitter.get_emitter().flush(timeout=2.0) diff --git a/tests/gateway/test_startup_restart_race.py b/tests/gateway/test_startup_restart_race.py index e303cba1c99..bdf5bbc42a4 100644 --- a/tests/gateway/test_startup_restart_race.py +++ b/tests/gateway/test_startup_restart_race.py @@ -250,16 +250,23 @@ async def test_startup_aborts_after_registered_adapter_restart(tmp_path, monkeyp async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) cron_started = False + export_shutdown_calls = 0 + + class ExportRuntime: + def shutdown(self): + nonlocal export_shutdown_calls + export_shutdown_calls += 1 class AbortedStartupRunner: def __init__(self, config): self.config = config self.adapters = {} self._running = False - self.should_exit_cleanly = False + self.should_exit_cleanly = True self.should_exit_with_failure = False self.exit_reason = None self.exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE + self._gateway_health_export_runtime = ExportRuntime() async def start(self): return True @@ -287,3 +294,4 @@ async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path, assert exc.value.code == GATEWAY_SERVICE_RESTART_EXIT_CODE assert cron_started is False + assert export_shutdown_calls == 1 diff --git a/tests/monitoring/test_emitter.py b/tests/monitoring/test_emitter.py index 98ad7b0389b..07fce136f31 100644 --- a/tests/monitoring/test_emitter.py +++ b/tests/monitoring/test_emitter.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +import threading from agent.monitoring.emitter import MonitoringEmitter from agent.monitoring.events import GatewayHealthEvent @@ -65,6 +66,35 @@ def test_subscriber_failure_is_isolated(): assert len(good) == 1 # the raising subscriber did not break fan-out +def test_flush_waits_for_in_flight_subscriber_delivery(): + em = MonitoringEmitter() + subscriber_started = threading.Event() + release_subscriber = threading.Event() + flush_finished = threading.Event() + + def blocking_subscriber(_batch): + subscriber_started.set() + release_subscriber.wait(timeout=2.0) + + em.subscribe(blocking_subscriber) + em.emit({"event": "gateway_health", "name": "gateway.exit"}) + assert subscriber_started.wait(timeout=1.0) + + flush_thread = threading.Thread( + target=lambda: (em.flush(timeout=1.0), flush_finished.set()), + daemon=True, + ) + flush_thread.start() + + try: + assert not flush_finished.wait(timeout=0.1) + finally: + release_subscriber.set() + + assert flush_finished.wait(timeout=1.0) + em.close() + + def test_unsubscribe_stops_delivery(): em = MonitoringEmitter() seen: list = [] diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index df50269b3f5..8e144c9abf9 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -388,6 +388,41 @@ def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch) assert runtime.reason == "otlp_unavailable" +def test_gateway_health_export_shutdown_flushes_before_unsubscribe(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + calls = [] + + class FakeEmitter: + def flush(self, timeout): + calls.append(("flush", timeout)) + + def unsubscribe(self, subscriber): + calls.append(("unsubscribe", subscriber)) + + class Streamer: + def shutdown(self): + calls.append(("shutdown", self)) + + streamer = Streamer() + log_streamer = Streamer() + monkeypatch.setattr(emitter, "get_emitter", lambda: FakeEmitter()) + + runtime = GatewayHealthExportRuntime( + enabled=True, + streamer=streamer, + log_streamer=log_streamer, + ) + runtime.shutdown() + + assert calls[0][0] == "flush" + assert calls[1:3] == [ + ("unsubscribe", streamer), + ("unsubscribe", log_streamer), + ] + + def test_gateway_health_export_streams_only_gateway_events(monkeypatch): from agent.monitoring import gateway_health_export From 42bc6c862642b68afd26518bbe388e8da1248a24 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Wed, 22 Jul 2026 23:34:21 +0000 Subject: [PATCH 17/29] fix(monitoring): keep diagnostics content-free --- agent/monitoring/events.py | 1 - agent/monitoring/gateway_health.py | 5 ----- agent/monitoring/gateway_health_export.py | 5 ++++- agent/monitoring/otlp_exporter.py | 2 +- docs/observability/monitoring.md | 8 ++++---- hermes_cli/main.py | 4 ++-- .../monitoring/test_gateway_health_export.py | 20 +++++++++++-------- tests/monitoring/test_otlp_exporter.py | 9 ++++++--- 8 files changed, 29 insertions(+), 25 deletions(-) diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py index 2151932b730..74b0aea2a7c 100644 --- a/agent/monitoring/events.py +++ b/agent/monitoring/events.py @@ -50,7 +50,6 @@ class GatewayDiagnosticEvent: subsystem: str error_class: str = "unknown" error_code: Optional[str] = None - redacted_message: Optional[str] = None platform: Optional[str] = None old_state: Optional[str] = None new_state: Optional[str] = None diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index ece881bca4e..cb02aac95c1 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -246,7 +246,6 @@ def build_gateway_health_snapshot( platform=str(platform), error_code=error_code, error_class=classify_gateway_error(error_code or pdata.get("error_message")), - redacted_message=redact_gateway_message(pdata.get("error_message")), profile=profile, version=version, severity="error" if state == "fatal" else "warning", @@ -325,7 +324,6 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: subsystem="gateway", error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"), error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"), - redacted_message=redact_gateway_message(current.get("exit_reason") or "startup failed"), profile=profile, version=version, severity="error", @@ -374,7 +372,6 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: new_state=new_state, error_code=error_code, error_class=error_code, - redacted_message=redact_gateway_message(pdata.get("error_message")), profile=profile, version=version, severity=severity, @@ -386,7 +383,6 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: platform=str(platform), error_code=error_code, error_class=error_code, - redacted_message=redact_gateway_message(pdata.get("error_message")), profile=profile, version=version, severity=severity, @@ -426,7 +422,6 @@ class GatewayDiagnosticLogHandler(logging.Handler): subsystem=subsystem, platform=platform_for_subsystem(subsystem), error_class=classify_gateway_error(message), - redacted_message=redact_gateway_message(message), profile=self.profile, version=self.version, severity=record.levelname.lower(), diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index efb7575c692..f8fefd24e2b 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -391,7 +391,10 @@ class GatewayDiagnosticLogStreamer: if ev.get("event") != "gateway_diagnostic": continue attrs = _diagnostic_log_attributes(ev) - body = ev.get("redacted_message") or ev.get("name") or "gateway diagnostic" + # Rendered Python log messages may contain arbitrary user IDs, names, + # paths, or configured strings. Keep the OTLP body content-free and + # carry only the structured, allowlisted attributes above. + body = "gateway diagnostic" record = self._LogRecord( timestamp=ev.get("ts_ns"), trace_id=self._sdk["INVALID_TRACE_ID"], diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index c9268b4ee5a..db584a3b098 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -147,7 +147,7 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: "fatal_platform_count", "version", "supervision_mode", "pid"), "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", - "redacted_message", "platform", "old_state", "new_state", + "platform", "old_state", "new_state", "version", "severity"), } for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index 9ca8f2ce12a..bb5088929d4 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -1,11 +1,11 @@ # Gateway Monitoring -Service health monitoring plus redacted operational diagnostics for the +Service health monitoring plus structured operational diagnostics for the Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver). This plane is content-free by construction. It exports gateway lifecycle -state, platform connector health, and redacted warning/error diagnostics. +state, platform connector health, and content-free warning/error diagnostics. It never exports prompts, messages, tool arguments or results, session history, usage analytics, audit logs, or execution traces. Run/model/tool trajectory capture is a separate plane served by the NeMo Relay integration @@ -17,7 +17,7 @@ trajectory capture is a separate plane served by the NeMo Relay integration | --- | --- | --- | | Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | -| Diagnostics | `/v1/logs` | Warning/error gateway log events with secrets AND PII scrubbed in-process before egress (`[redacted]` / `[email]`), bounded error classes | +| Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | Signals carry `service.name`, version, supervision mode, and a stable one-way hash of the install id so an operator can distinguish instances without @@ -85,7 +85,7 @@ python scripts/observability/otel_capture_collector.py \ --host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl # terminal 2: drive the real exporter through lifecycle transitions, -# a fatal platform, and a redacted warning log, then flush +# a fatal platform, and a structured warning event, then flush python scripts/observability/gateway_health_export_probe.py \ --endpoint http://127.0.0.1:4318/v1/traces \ --log /tmp/hermes_otel_capture.jsonl --wait 8 diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 3157d78870a..f3fb0fe581a 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -14340,8 +14340,8 @@ def cmd_monitoring(args): print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}") print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} " f"(interval {gh.get('logs_export_interval_seconds', 5)}s)") - print(" Redaction: always on " - "(secrets/PII scrubbed in-process before egress; not configurable)") + print(" Content safety: always on " + "(rendered messages are never exported; not configurable)") endpoint = otlp.get("endpoint") or "" if otlp.get("enabled") and endpoint: print(f" OTLP endpoint: {endpoint}") diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index 8e144c9abf9..e926c5ba2b7 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -111,8 +111,8 @@ def test_gateway_health_snapshot_emits_content_free_diagnostic_event(): assert health["fatal_platform_count"] == 1 assert platform["platform"] == "slack" assert platform["error_code"] == "auth_failed" - assert "secret" not in platform["redacted_message"].lower() - assert "Bearer" not in platform["redacted_message"] + assert "redacted_message" not in platform + assert "Bearer" not in str(platform) def test_gateway_health_snapshot_preserves_real_bounded_platform_states(): @@ -147,7 +147,7 @@ def test_gateway_health_snapshot_preserves_real_bounded_platform_states(): assert observed == expected -def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): +def test_gateway_diagnostic_log_handler_never_carries_rendered_message(caplog): from agent.monitoring import emitter from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler @@ -166,7 +166,10 @@ def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): logger.addHandler(handler) try: logger.info("ignore info token sk-live-secret") - logger.warning("Slack token sk-live-secret failed for user@example.com") + logger.warning( + "Unauthorized user: acct_7f3a (Alice Smith) on slack; " + "token «redacted:sk-…»" + ) finally: logger.removeHandler(handler) finally: @@ -178,8 +181,9 @@ def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): assert event["name"] == "gateway.log.warning" assert event["subsystem"] == "platform.slack" assert event["error_class"] == "auth_failed" - assert "***" not in event["redacted_message"] - assert "user@example.com" not in event["redacted_message"] + assert "redacted_message" not in event + assert "acct_7f3a" not in str(event) + assert "Alice Smith" not in str(event) def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch): @@ -223,7 +227,7 @@ def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypat assert platform["old_state"] == "running" assert platform["new_state"] == "fatal" assert platform["error_code"] == "auth_failed" - assert "Bearer" not in platform["redacted_message"] + assert "redacted_message" not in platform def test_runtime_status_transition_emits_startup_failed_and_exit(): @@ -256,7 +260,7 @@ def test_runtime_status_transition_emits_startup_failed_and_exit(): assert "gateway.startup_failed" in names assert "gateway.exit" in names failed = next(e for e in captured if e["name"] == "gateway.startup_failed") - assert "***" not in failed["redacted_message"] + assert "redacted_message" not in failed lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") assert lifecycle["exit_reason"] == "auth_failed" exit_event = next(e for e in captured if e["name"] == "gateway.exit") diff --git a/tests/monitoring/test_otlp_exporter.py b/tests/monitoring/test_otlp_exporter.py index 91ba7ede553..93749fcbfbb 100644 --- a/tests/monitoring/test_otlp_exporter.py +++ b/tests/monitoring/test_otlp_exporter.py @@ -42,16 +42,19 @@ def test_gateway_health_event_maps_to_span_with_attrs(): assert attrs["hermes.active_agents"] == 2 -def test_gateway_diagnostic_event_maps_redacted_message(): +def test_gateway_diagnostic_event_drops_arbitrary_message_content(): provider, mem = _mem_provider() OE.export_batch(provider, [{ "event": "gateway_diagnostic", "name": "platform.fatal", "subsystem": "platform.slack", "error_class": "auth_failed", - "redacted_message": "token [redacted] rejected", "severity": "error", + "redacted_message": "Unauthorized user: acct_7f3a (Alice Smith)", + "severity": "error", }]) attrs = dict(mem.get_finished_spans()[0].attributes or {}) assert attrs["hermes.error_class"] == "auth_failed" - assert attrs["hermes.redacted_message"] == "token [redacted] rejected" + assert "hermes.redacted_message" not in attrs + assert "acct_7f3a" not in str(attrs) + assert "Alice Smith" not in str(attrs) def test_unknown_event_kind_exports_no_attrs_beyond_kind(): From 9b098e7f78f16a9ded4fea5e7e32ea9a2109bd0c Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Thu, 23 Jul 2026 01:13:27 +0000 Subject: [PATCH 18/29] fix(monitoring): enrich gateway diagnostic scope --- agent/monitoring/events.py | 1 + agent/monitoring/gateway_health.py | 31 ++++- agent/monitoring/gateway_health_export.py | 24 +++- agent/monitoring/redaction.py | 5 +- .../monitoring/test_gateway_health_export.py | 126 ++++++++++++++++++ 5 files changed, 178 insertions(+), 9 deletions(-) diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py index 74b0aea2a7c..0d5b964556e 100644 --- a/agent/monitoring/events.py +++ b/agent/monitoring/events.py @@ -57,6 +57,7 @@ class GatewayDiagnosticEvent: version: Optional[str] = None severity: str = "warning" ts_ns: int = field(default_factory=_now_ns) + source_logger: Optional[str] = None def to_dict(self) -> Dict[str, Any]: return {"event": "gateway_diagnostic", **asdict(self)} diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index cb02aac95c1..752473fe31b 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -10,6 +10,7 @@ from __future__ import annotations import hashlib import logging +import re from dataclasses import dataclass from typing import Any, Dict, List, Optional @@ -38,12 +39,19 @@ _KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { "connecting", "disconnected", "disabled", "paused", "retrying", "unknown" } _SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"} +_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$") def _allowed_logger(name: str) -> bool: return name == "gateway" or name.startswith("gateway.") +def source_logger_for_export(name: Any) -> Optional[str]: + """Return a bounded source-controlled gateway logger name for OTLP scope.""" + value = str(name or "") + return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None + + def redact_gateway_message(message: Any) -> str: """Redact gateway diagnostic free text for operator-owned export. @@ -67,7 +75,20 @@ def classify_gateway_error(raw: Any) -> str: return "rate_limited" if "timeout" in s or "timed out" in s: return "timeout" - if any(k in s for k in ("network", "connection", "dns", "socket")): + if any( + k in s + for k in ( + "network", + "connection", + "dns", + "socket", + "connect call failed", + "failed to connect", + "cannot connect", + "unreachable", + "name resolution", + ) + ): return "network_error" if any(k in s for k in ("config", "missing", "invalid")): return "invalid_config" @@ -119,6 +140,8 @@ def _safe_instance_id(raw: Any) -> str: def subsystem_for_logger(logger_name: str) -> str: + if logger_name == "gateway.relay" or logger_name.startswith("gateway.relay."): + return "platform.relay" if logger_name.startswith("gateway.platforms."): parts = logger_name.split(".") if len(parts) >= 3 and parts[2]: @@ -417,11 +440,14 @@ class GatewayDiagnosticLogHandler(logging.Handler): return subsystem = subsystem_for_logger(record.name) message = record.getMessage() + error_class = classify_gateway_error(message) event = GatewayDiagnosticEvent( name=f"gateway.log.{record.levelname.lower()}", subsystem=subsystem, + source_logger=source_logger_for_export(record.name), platform=platform_for_subsystem(subsystem), - error_class=classify_gateway_error(message), + error_class=error_class, + error_code=error_class, profile=self.profile, version=self.version, severity=record.levelname.lower(), @@ -438,5 +464,6 @@ __all__ = [ "GatewayDiagnosticLogHandler", "build_gateway_health_snapshot", "classify_gateway_error", + "source_logger_for_export", "redact_gateway_message", ] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index f8fefd24e2b..e10043daba6 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -17,6 +17,8 @@ from typing import Any, Dict, Optional logger = logging.getLogger(__name__) +_DEFAULT_DIAGNOSTIC_SCOPE = "hermes.gateway.diagnostics" + _RESOURCE_ATTRIBUTE_KEYS = frozenset({ "service.name", "service.namespace", @@ -381,19 +383,31 @@ class GatewayDiagnosticLogStreamer: sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) ) self._provider.add_log_record_processor(self._processor) - self._logger = self._provider.get_logger("hermes.gateway.diagnostics") + self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE) self._LogRecord = sdk["LogRecord"] self._sdk = sdk self.exported = 0 def __call__(self, batch: list[Dict[str, Any]]) -> None: + from agent.monitoring.gateway_health import source_logger_for_export + for ev in batch: if ev.get("event") != "gateway_diagnostic": continue attrs = _diagnostic_log_attributes(ev) - # Rendered Python log messages may contain arbitrary user IDs, names, - # paths, or configured strings. Keep the OTLP body content-free and - # carry only the structured, allowlisted attributes above. + # Preserve the source-controlled Python logger as the OTel + # instrumentation scope. This adds precise code attribution without + # turning a fluid module layout into a maintained subsystem enum. + # Rendered messages stay out because they may contain arbitrary IDs, + # names, paths, or configured strings. A future, separately gated + # ``diagnostic_detail: redacted_message`` mode may add best-effort + # free text when an observability plane defines that privacy policy. + source_logger = source_logger_for_export(ev.get("source_logger")) + otel_logger = ( + self._provider.get_logger(source_logger) + if source_logger is not None + else self._logger + ) body = "gateway diagnostic" record = self._LogRecord( timestamp=ev.get("ts_ns"), @@ -405,7 +419,7 @@ class GatewayDiagnosticLogStreamer: body=_redact_string(body), attributes=attrs, ) - self._logger.emit(record) + otel_logger.emit(record) self.exported += 1 def shutdown(self) -> None: diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py index cf1564f0171..312875901e6 100644 --- a/agent/monitoring/redaction.py +++ b/agent/monitoring/redaction.py @@ -10,8 +10,9 @@ process passes through ``redact_for_export``: are rewritten to ``[email]`` / ``[phone]`` / ``[id]``. There is deliberately no setting to weaken this. The monitoring plane is -content-free by design, so the only free text it carries (log-derived -diagnostic messages) is always fully scrubbed. +content-free by design: rendered log messages are not exported, and bounded +structured strings are still scrubbed as defense-in-depth. This redactor also +remains available for a future, explicitly gated redacted-message detail mode. """ from __future__ import annotations diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index e926c5ba2b7..a48f585c3ba 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -2,6 +2,17 @@ from __future__ import annotations import logging +import pytest + + +def test_gateway_diagnostic_event_preserves_positional_error_class(): + from agent.monitoring.events import GatewayDiagnosticEvent + + event = GatewayDiagnosticEvent("gateway.log.warning", "gateway", "auth_failed") + + assert event.error_class == "auth_failed" + assert event.source_logger is None + def test_default_config_keeps_gateway_health_export_disabled(): from hermes_cli.config import DEFAULT_CONFIG @@ -180,12 +191,64 @@ def test_gateway_diagnostic_log_handler_never_carries_rendered_message(caplog): assert event["event"] == "gateway_diagnostic" assert event["name"] == "gateway.log.warning" assert event["subsystem"] == "platform.slack" + assert event["source_logger"] == "gateway.platforms.slack" assert event["error_class"] == "auth_failed" assert "redacted_message" not in event assert "acct_7f3a" not in str(event) assert "Alice Smith" not in str(event) +@pytest.mark.parametrize( + "message", + [ + "Connect call failed ('127.0.0.1', 9)", + "failed to connect to relay", + "connection refused", + "network is unreachable", + "temporary failure in name resolution", + ], +) +def test_gateway_error_classifier_recognizes_bounded_network_failures(message): + from agent.monitoring.gateway_health import classify_gateway_error + + assert classify_gateway_error(message) == "network_error" + + +def test_gateway_diagnostic_log_handler_enriches_relay_scope_without_message_content( + monkeypatch, +): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + monkeypatch.setattr(emitter, "get_emitter", lambda: DummyEmitter()) + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + logger = logging.getLogger("gateway.relay.adapter") + logger.addHandler(handler) + try: + logger.warning( + "Connect call failed for ws://alice@example.com/private; " + "credential=«redacted:sk-…»" + ) + finally: + logger.removeHandler(handler) + + assert len(captured) == 1 + event = captured[0] + assert event["subsystem"] == "platform.relay" + assert event["platform"] == "relay" + assert event["source_logger"] == "gateway.relay.adapter" + assert event["error_class"] == "network_error" + assert event["error_code"] == "network_error" + assert "alice@example.com" not in str(event) + assert "private" not in str(event) + + def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch): from agent.monitoring import emitter from agent.monitoring.gateway_health import emit_runtime_status_transition @@ -374,6 +437,69 @@ def test_diagnostic_log_attributes_are_allowlisted_redacted_and_profile_free(): assert "top-secret-token" not in str(attrs) +def test_diagnostic_log_streamer_uses_validated_source_as_otel_scope(): + from types import SimpleNamespace + + from agent.monitoring.gateway_health_export import GatewayDiagnosticLogStreamer + + class FakeLogger: + def __init__(self): + self.records = [] + + def emit(self, record): + self.records.append(record) + + class FakeProvider: + def __init__(self): + self.loggers = {} + + def get_logger(self, name): + return self.loggers.setdefault(name, FakeLogger()) + + provider = FakeProvider() + streamer = object.__new__(GatewayDiagnosticLogStreamer) + streamer._provider = provider + streamer._logger = provider.get_logger("hermes.gateway.diagnostics") + streamer._LogRecord = lambda **kwargs: SimpleNamespace(**kwargs) + streamer._sdk = { + "INVALID_TRACE_ID": 0, + "INVALID_SPAN_ID": 0, + "TraceFlags": SimpleNamespace(DEFAULT=0), + "SeverityNumber": SimpleNamespace( + FATAL="fatal", ERROR="error", WARN="warn", INFO="info", DEBUG="debug" + ), + } + streamer.exported = 0 + + streamer([ + { + "event": "gateway_diagnostic", + "name": "gateway.log.warning", + "subsystem": "platform.relay", + "platform": "relay", + "source_logger": "gateway.relay.adapter", + "error_class": "network_error", + "severity": "warning", + }, + { + "event": "gateway_diagnostic", + "name": "gateway.log.warning", + "subsystem": "gateway", + "source_logger": "gateway.relay.adapter\nalice@example.com", + "error_class": "unknown", + "severity": "warning", + }, + ]) + + precise = provider.loggers["gateway.relay.adapter"].records + fallback = provider.loggers["hermes.gateway.diagnostics"].records + assert len(precise) == 1 + assert len(fallback) == 1 + assert precise[0].body == "gateway diagnostic" + assert "source_logger" not in precise[0].attributes + assert "alice@example.com" not in str(provider.loggers) + + def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch): from agent.monitoring import gateway_health_export from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime From efbf2fd79c0cbed2fe332430b86f6f2072ef84d4 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Fri, 24 Jul 2026 18:23:40 +0000 Subject: [PATCH 19/29] feat(monitoring): add cron operational telemetry --- agent/monitoring/cron_health.py | 181 ++++++++++++++++++++ agent/monitoring/events.py | 17 ++ agent/monitoring/gateway_health_export.py | 26 ++- agent/monitoring/otlp_exporter.py | 2 + cron/executions.py | 34 +++- cron/scheduler.py | 13 +- docs/observability/monitoring.md | 20 ++- tests/monitoring/test_cron_health_export.py | 153 +++++++++++++++++ 8 files changed, 433 insertions(+), 13 deletions(-) create mode 100644 agent/monitoring/cron_health.py create mode 100644 tests/monitoring/test_cron_health_export.py diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py new file mode 100644 index 00000000000..03e8b8dd934 --- /dev/null +++ b/agent/monitoring/cron_health.py @@ -0,0 +1,181 @@ +"""Content-free cron service-health and execution telemetry projection.""" + +from __future__ import annotations + +import hashlib +import logging +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from agent.monitoring.events import CronExecutionEvent +from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric +from cron.jobs import ( + _compute_grace_seconds, + get_ticker_heartbeat_age, + get_ticker_success_age, + load_jobs, +) +from cron.scheduler import get_running_job_ids +from hermes_time import now as _hermes_now + +logger = logging.getLogger(__name__) +_KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} +_KNOWN_SOURCES = {"builtin", "direct", "external"} +_KNOWN_DELIVERY_OUTCOMES = {"delivered", "failed", "suppressed", "not_configured"} + + +@dataclass(frozen=True, slots=True) +class CronHealthSnapshot: + metrics: list[GatewayMetric] + events: list[CronExecutionEvent] + + +def _now() -> datetime: + return _hermes_now() + + +def _job_key(raw: Any) -> str: + value = str(raw or "unknown").encode("utf-8", errors="replace") + return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" + + +def classify_cron_error(raw: Any) -> str: + text = str(raw or "").lower() + if any(value in text for value in ("auth", "token", "unauthorized", "forbidden", "401", "403")): + return "auth_failed" + if "rate limit" in text or "429" in text or "quota" in text: + return "rate_limited" + if "timeout" in text or "timed out" in text: + return "timeout" + if any(value in text for value in ("network", "connection", "dns", "socket", "unreachable")): + return "network_error" + if "dispatch" in text or "executor" in text: + return "dispatch_failed" + if "interrupt" in text or "owner exited" in text or "restarted" in text: + return "interrupted" + if "empty response" in text: + return "empty_response" + if any(value in text for value in ("config", "missing", "invalid")): + return "invalid_config" + return "unknown" + + +def _parse_time(raw: Any) -> Optional[datetime]: + try: + return datetime.fromisoformat(str(raw)) if raw else None + except (TypeError, ValueError): + return None + + +def _duration_ms(record: dict[str, Any]) -> Optional[int]: + start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at")) + finish = _parse_time(record.get("finished_at")) + if start is None or finish is None: + return None + try: + duration = int((finish - start).total_seconds() * 1000) + except (TypeError, ValueError): + return None + return max(0, duration) + + +def project_execution_event( + record: dict[str, Any], *, delivery_outcome: Optional[str] = None +) -> CronExecutionEvent: + status = str(record.get("status") or "unknown").lower() + source = str(record.get("source") or "unknown").lower() + outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None + return CronExecutionEvent( + status=status if status in _KNOWN_STATUSES else "unknown", + job_key=_job_key(record.get("job_id")), + source=source if source in _KNOWN_SOURCES else "unknown", + duration_ms=_duration_ms(record), + delivery_outcome=( + outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None + ), + error_class=( + classify_cron_error(record.get("error")) + if status in {"failed", "unknown"} + else None + ), + ) + + +def emit_execution_state( + record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None +) -> None: + """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier.""" + if not record: + return + try: + from agent.monitoring import emitter + + event = project_execution_event(record, delivery_outcome=delivery_outcome) + target = emitter.get_emitter() + target.emit(event) + if event.status in {"completed", "failed", "unknown"}: + target.flush(timeout=1.0) + except Exception: + logger.debug("cron execution telemetry emit failed", exc_info=True) + + +def _is_overdue(job: dict[str, Any], now: datetime) -> bool: + if not job.get("enabled", True): + return False + next_run = _parse_time(job.get("next_run_at")) + schedule = job.get("schedule") + if next_run is None or not isinstance(schedule, dict): + return False + try: + if next_run.tzinfo is None and now.tzinfo is not None: + next_run = next_run.replace(tzinfo=now.tzinfo) + lateness = (now - next_run).total_seconds() + return lateness > _compute_grace_seconds(schedule) + except (TypeError, ValueError): + return False + + +def build_cron_health_snapshot() -> CronHealthSnapshot: + metrics: list[GatewayMetric] = [] + for name, reader in ( + ("hermes.cron.scheduler.heartbeat_age_seconds", get_ticker_heartbeat_age), + ("hermes.cron.scheduler.last_success_age_seconds", get_ticker_success_age), + ): + try: + value = reader() + if value is not None: + metrics.append(GatewayMetric(name, max(0.0, float(value)), {})) + except Exception: + logger.debug("cron freshness metric unavailable", exc_info=True) + + try: + jobs = load_jobs() + enabled = [job for job in jobs if job.get("enabled", True)] + metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {})) + metrics.append( + GatewayMetric( + "hermes.cron.jobs.overdue", + sum(1 for job in enabled if _is_overdue(job, _now())), + {}, + ) + ) + except Exception: + logger.debug("cron job metrics unavailable", exc_info=True) + + try: + metrics.append( + GatewayMetric("hermes.cron.jobs.running", len(get_running_job_ids()), {}) + ) + except Exception: + logger.debug("cron running-job metric unavailable", exc_info=True) + return CronHealthSnapshot(metrics=metrics, events=[]) + + +__all__ = [ + "CronHealthSnapshot", + "build_cron_health_snapshot", + "classify_cron_error", + "emit_execution_state", + "project_execution_event", +] diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py index 0d5b964556e..a17f4a49425 100644 --- a/agent/monitoring/events.py +++ b/agent/monitoring/events.py @@ -63,7 +63,24 @@ class GatewayDiagnosticEvent: return {"event": "gateway_diagnostic", **asdict(self)} +@dataclass(slots=True) +class CronExecutionEvent: + """Content-free durable cron execution lifecycle projection.""" + + status: str + job_key: str + source: str = "unknown" + duration_ms: Optional[int] = None + delivery_outcome: Optional[str] = None + error_class: Optional[str] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "cron_execution", **asdict(self)} + + __all__ = [ "GatewayHealthEvent", "GatewayDiagnosticEvent", + "CronExecutionEvent", ] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index e10043daba6..a0c851682e8 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -278,7 +278,7 @@ def _supervision_mode() -> str: return "manual" -def _read_runtime_snapshot(config: Dict[str, Any]): +def _read_gateway_snapshot(config: Dict[str, Any]): from agent.monitoring.gateway_health import build_gateway_health_snapshot try: from gateway.status import read_runtime_status @@ -295,6 +295,23 @@ def _read_runtime_snapshot(config: Dict[str, Any]): ) +def _read_cron_snapshot(): + from agent.monitoring.cron_health import build_cron_health_snapshot + + return build_cron_health_snapshot() + + +def _read_runtime_snapshot(config: Dict[str, Any]): + gateway_snapshot = _read_gateway_snapshot(config) + try: + cron_snapshot = _read_cron_snapshot() + except Exception: + logger.debug("cron health snapshot unavailable", exc_info=True) + return gateway_snapshot + gateway_snapshot.metrics.extend(cron_snapshot.metrics) + return gateway_snapshot + + def _emit_snapshot_events(config: Dict[str, Any]) -> None: gh = _gateway_health_config(config) if not gh.get("diagnostic_events_enabled", True): @@ -337,6 +354,11 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: "hermes.gateway.restart_requested", "hermes.platform.up", "hermes.platform.degraded", + "hermes.cron.scheduler.heartbeat_age_seconds", + "hermes.cron.scheduler.last_success_age_seconds", + "hermes.cron.jobs.enabled", + "hermes.cron.jobs.running", + "hermes.cron.jobs.overdue", ] def callback(name: str): @@ -467,7 +489,7 @@ def _attach_log_handler(config: Dict[str, Any]) -> Any: def _gateway_health_event(ev: Dict[str, Any]) -> bool: - return ev.get("event") == "gateway_health" + return ev.get("event") in {"gateway_health", "cron_execution"} def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime: diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index db584a3b098..cde95673ec5 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -149,6 +149,8 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", "platform", "old_state", "new_state", "version", "severity"), + "cron_execution": ("status", "job_key", "source", "duration_ms", + "delivery_outcome", "error_class"), } for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] v = ev.get(col) diff --git a/cron/executions.py b/cron/executions.py index 6ffe1eef1fc..d46eaf0d829 100644 --- a/cron/executions.py +++ b/cron/executions.py @@ -65,6 +65,18 @@ def _record(row: Optional[sqlite3.Row]) -> Optional[Dict[str, Any]]: return dict(row) if row is not None else None +def _emit_execution_state( + record: Optional[Dict[str, Any]], *, delivery_outcome: Optional[str] = None +) -> None: + """Project durable state to monitoring without affecting ledger behavior.""" + try: + from agent.monitoring.cron_health import emit_execution_state + + emit_execution_state(record, delivery_outcome=delivery_outcome) + except Exception: + pass + + def _process_start_time(pid: int) -> Optional[int]: try: from gateway.status import get_process_start_time @@ -115,7 +127,9 @@ def create_execution(job_id: str, *, source: str) -> Dict[str, Any]: row = conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) ).fetchone() - return _record(row) # type: ignore[return-value] + record = _record(row) + _emit_execution_state(record) + return record # type: ignore[return-value] def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]: @@ -129,13 +143,16 @@ def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]: ) if cur.rowcount != 1: return None - return _record(conn.execute( + record = _record(conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) ).fetchone()) + _emit_execution_state(record) + return record def finish_execution( execution_id: str, *, success: bool, error: Optional[str] = None, + delivery_outcome: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """Write a terminal result once; terminal attempts cannot be rewritten.""" now = _hermes_now().isoformat() @@ -150,15 +167,18 @@ def finish_execution( if cur.rowcount != 1: return None _prune_unlocked(conn) - return _record(conn.execute( + record = _record(conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) ).fetchone()) + _emit_execution_state(record, delivery_outcome=delivery_outcome) + return record def recover_interrupted_executions() -> int: """Mark provably abandoned attempts unknown without scheduling retries.""" now = _hermes_now().isoformat() changed = 0 + recovered: List[Dict[str, Any]] = [] with _lock, _connect() as conn: rows = conn.execute( """SELECT id, process_id, pid, process_started_at FROM executions @@ -178,8 +198,16 @@ def recover_interrupted_executions() -> int: row["id"]), ) changed += cur.rowcount + if cur.rowcount: + record = _record(conn.execute( + "SELECT * FROM executions WHERE id=?", (row["id"],) + ).fetchone()) + if record is not None: + recovered.append(record) if changed: _prune_unlocked(conn) + for record in recovered: + _emit_execution_state(record) return changed diff --git a/cron/scheduler.py b/cron/scheduler.py index c309c59c4e5..830c18aabab 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -3977,7 +3977,18 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - if not _consume_interrupted_flag(job["id"]): mark_job_run(job["id"], success, error, delivery_error=delivery_error) - finish_execution(execution_id, success=success, error=error) + if delivery_error: + delivery_outcome = "failed" + elif should_deliver and job.get("deliver", "local") != "local": + delivery_outcome = "delivered" + else: + delivery_outcome = "suppressed" + finish_execution( + execution_id, + success=success, + error=error, + delivery_outcome=delivery_outcome, + ) return True except Exception as e: diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index bb5088929d4..027ac9db8c7 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -4,11 +4,12 @@ Service health monitoring plus structured operational diagnostics for the Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver). -This plane is content-free by construction. It exports gateway lifecycle -state, platform connector health, and content-free warning/error diagnostics. -It never exports prompts, messages, tool arguments or results, session -history, usage analytics, audit logs, or execution traces. Run/model/tool -trajectory capture is a separate plane served by the NeMo Relay integration +This plane is content-free by construction. It exports gateway and cron +lifecycle state, platform connector health, and content-free warning/error +diagnostics. It never exports prompts, messages, tool arguments or results, +job names, destinations, schedules, raw errors, session history, usage +analytics, audit logs, or detailed execution traces. Run/model/tool trajectory +capture is a separate plane served by the NeMo Relay integration (`plugins/observability/nemo_relay/`) and its Hermes-owned subscribers. ## What gets exported @@ -18,6 +19,8 @@ trajectory capture is a separate plane served by the NeMo Relay integration | Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | | Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | +| Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | +| Cron execution lifecycle | `/v1/traces` | Durable `claimed/running/completed/failed/unknown` states, bounded source and error class, opaque hashed job key, elapsed duration when timestamps exist, and delivery outcome when the scheduler knows it; terminal states flush through a bounded fail-open barrier | Signals carry `service.name`, version, supervision mode, and a stable one-way hash of the install id so an operator can distinguish instances without @@ -94,8 +97,11 @@ python scripts/observability/gateway_health_export_probe.py \ ## Boundaries and roadmap -The `hermes monitoring` CLI intentionally exposes `status` only. Shared -client usage metrics and enterprise trace telemetry are being designed on +The `hermes monitoring` CLI intentionally exposes `status` only. This first +release covers only Hermes Agent-owned service-health and operational-diagnostic +signals. Team Gateway/shared-connector signals are explicitly out of scope, as +are product analytics, audit/quality reporting, and detailed execution traces. +Shared client usage metrics and enterprise trace telemetry are being designed on the NeMo Relay integration with their own consent, policy, and export boundaries; this monitoring plane stays narrow so an operator can enable it without touching any content-bearing signal. The telemetry surface may be diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py new file mode 100644 index 00000000000..8dfa4d41647 --- /dev/null +++ b/tests/monitoring/test_cron_health_export.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + + +def _metric(snapshot, name): + return next(metric for metric in snapshot.metrics if metric.name == name) + + +def test_cron_snapshot_projects_freshness_counts_and_overdue_without_content(monkeypatch): + from agent.monitoring import cron_health + + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) + secret = "Quarterly payroll for alice@example.com" + monkeypatch.setattr(cron_health, "_now", lambda: now) + monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: 4.5) + monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: 9.0) + monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset({"job-private-1"})) + monkeypatch.setattr( + cron_health, + "load_jobs", + lambda: [ + { + "id": "job-private-1", + "name": secret, + "prompt": secret, + "enabled": True, + "schedule": {"kind": "interval", "minutes": 10}, + "next_run_at": (now - timedelta(minutes=6)).isoformat(), + }, + { + "id": "job-private-2", + "name": "disabled private job", + "enabled": False, + "schedule": {"kind": "interval", "minutes": 10}, + "next_run_at": (now - timedelta(days=1)).isoformat(), + }, + ], + ) + + snapshot = cron_health.build_cron_health_snapshot() + + assert _metric(snapshot, "hermes.cron.scheduler.heartbeat_age_seconds").value == 4.5 + assert _metric(snapshot, "hermes.cron.scheduler.last_success_age_seconds").value == 9.0 + assert _metric(snapshot, "hermes.cron.jobs.enabled").value == 1 + assert _metric(snapshot, "hermes.cron.jobs.running").value == 1 + assert _metric(snapshot, "hermes.cron.jobs.overdue").value == 1 + assert secret not in str(snapshot) + assert "job-private-1" not in str(snapshot) + + +def test_cron_snapshot_omits_unknown_freshness_instead_of_inventing_values(monkeypatch): + from agent.monitoring import cron_health + + monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: None) + monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: None) + monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset()) + monkeypatch.setattr(cron_health, "load_jobs", lambda: []) + + names = {metric.name for metric in cron_health.build_cron_health_snapshot().metrics} + + assert "hermes.cron.scheduler.heartbeat_age_seconds" not in names + assert "hermes.cron.scheduler.last_success_age_seconds" not in names + + +def test_execution_projection_is_opaque_bounded_and_content_free(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + { + "id": "execution-private-id", + "job_id": "Payroll for alice@example.com and token top-secret-token", + "source": "builtin", + "status": "failed", + "claimed_at": "2026-07-24T12:00:00+00:00", + "started_at": "2026-07-24T12:00:01+00:00", + "finished_at": "2026-07-24T12:00:03.250000+00:00", + "error": "Bearer top-secret-token rejected for alice@example.com", + }, + delivery_outcome="failed", + ).to_dict() + + assert event["event"] == "cron_execution" + assert event["status"] == "failed" + assert event["job_key"].startswith("sha256:") + assert len(event["job_key"]) == len("sha256:") + 24 + assert event["duration_ms"] == 2250 + assert event["delivery_outcome"] == "failed" + assert event["error_class"] == "auth_failed" + assert "job_id" not in event + assert "error" not in event + assert "alice@example.com" not in str(event) + assert "top-secret-token" not in str(event) + + +def test_execution_projection_omits_duration_and_delivery_when_not_known(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + { + "job_id": "private", + "source": "external-value-must-not-leak", + "status": "claimed", + "claimed_at": "2026-07-24T12:00:00+00:00", + } + ).to_dict() + + assert event["status"] == "claimed" + assert event["source"] == "unknown" + assert event["duration_ms"] is None + assert event["delivery_outcome"] is None + + +def test_terminal_execution_emission_flushes_and_failures_are_fail_open(monkeypatch): + from agent.monitoring import cron_health, emitter + + calls = [] + + class FakeEmitter: + def emit(self, event): + calls.append(("emit", event.to_dict()["status"])) + + def flush(self, timeout): + calls.append(("flush", timeout)) + raise RuntimeError("collector unavailable") + + monkeypatch.setattr(emitter, "get_emitter", lambda: FakeEmitter()) + + cron_health.emit_execution_state( + {"job_id": "private", "source": "builtin", "status": "completed"} + ) + + assert calls == [("emit", "completed"), ("flush", 1.0)] + + +def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(monkeypatch): + from agent.monitoring import gateway_health_export + + gateway_snapshot = type("Snapshot", (), {"metrics": []})() + cron_snapshot = type( + "Snapshot", + (), + {"metrics": [type("Metric", (), {"name": "hermes.cron.jobs.enabled", "value": 2, "attributes": {}})()]}, + )() + monkeypatch.setattr(gateway_health_export, "_read_gateway_snapshot", lambda config: gateway_snapshot) + monkeypatch.setattr(gateway_health_export, "_read_cron_snapshot", lambda: cron_snapshot) + + snapshot = gateway_health_export._read_runtime_snapshot({}) + + assert [metric.name for metric in snapshot.metrics] == ["hermes.cron.jobs.enabled"] + assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True + assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True + assert gateway_health_export._gateway_health_event({"event": "run"}) is False From a65a647b04db1eaf6ccbc7ce4bebc09e81389acb Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Fri, 24 Jul 2026 18:42:39 +0000 Subject: [PATCH 20/29] fix(monitoring): correct cron operational signals --- agent/monitoring/cron_health.py | 22 +++++++- agent/monitoring/gateway_health_export.py | 1 + cron/jobs.py | 41 +++++++++++++++ cron/scheduler.py | 7 +++ docs/observability/monitoring.md | 7 +-- tests/cron/test_execution_ledger.py | 31 ++++++++++++ tests/cron/test_jobs.py | 18 +++++++ tests/monitoring/test_cron_health_export.py | 56 ++++++++++++++++++++- 8 files changed, 178 insertions(+), 5 deletions(-) diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py index 03e8b8dd934..2ed0f1cb9b2 100644 --- a/agent/monitoring/cron_health.py +++ b/agent/monitoring/cron_health.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import logging +import re from dataclasses import dataclass from datetime import datetime from typing import Any, Optional @@ -12,6 +13,7 @@ from agent.monitoring.events import CronExecutionEvent from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric from cron.jobs import ( _compute_grace_seconds, + get_catch_up_occurrence_count, get_ticker_heartbeat_age, get_ticker_success_age, load_jobs, @@ -42,7 +44,12 @@ def _job_key(raw: Any) -> str: def classify_cron_error(raw: Any) -> str: text = str(raw or "").lower() - if any(value in text for value in ("auth", "token", "unauthorized", "forbidden", "401", "403")): + if ( + re.search(r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden)\b", text) + or re.search(r"\bbearer\b", text) + or re.search(r"\b(?:access|api|refresh) token\b", text) + or re.search(r"\b(?:401|403)\b", text) + ): return "auth_failed" if "rate limit" in text or "429" in text or "quota" in text: return "rate_limited" @@ -85,6 +92,8 @@ def project_execution_event( ) -> CronExecutionEvent: status = str(record.get("status") or "unknown").lower() source = str(record.get("source") or "unknown").lower() + if source not in _KNOWN_SOURCES and source != "unknown": + source = "external" outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None return CronExecutionEvent( status=status if status in _KNOWN_STATUSES else "unknown", @@ -149,6 +158,17 @@ def build_cron_health_snapshot() -> CronHealthSnapshot: except Exception: logger.debug("cron freshness metric unavailable", exc_info=True) + try: + metrics.append( + GatewayMetric( + "hermes.cron.scheduler.catch_up_occurrences", + get_catch_up_occurrence_count(), + {}, + ) + ) + except Exception: + logger.debug("cron catch-up metric unavailable", exc_info=True) + try: jobs = load_jobs() enabled = [job for job in jobs if job.get("enabled", True)] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index a0c851682e8..222ac599523 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -356,6 +356,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: "hermes.platform.degraded", "hermes.cron.scheduler.heartbeat_age_seconds", "hermes.cron.scheduler.last_success_age_seconds", + "hermes.cron.scheduler.catch_up_occurrences", "hermes.cron.jobs.enabled", "hermes.cron.jobs.running", "hermes.cron.jobs.overdue", diff --git a/cron/jobs.py b/cron/jobs.py index 5a21f6d5246..00e0d5c40c3 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -806,6 +806,24 @@ def _atomic_write_epoch(path: Path) -> None: raise +def _atomic_write_counter(path: Path, value: int) -> None: + """Atomically persist a non-negative integer counter.""" + ensure_dirs() + fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".count_") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(str(max(0, value))) + f.flush() + os.fsync(f.fileno()) + atomic_replace(tmp_path, path) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + def record_ticker_heartbeat(success: bool = False) -> None: """Record a ticker liveness signal, and optionally a successful-tick signal. @@ -867,6 +885,28 @@ def get_ticker_success_age() -> Optional[float]: return _epoch_file_age(store.cron_dir / "ticker_last_success") +def record_catch_up_occurrence() -> None: + """Increment the profile-local stale-schedule catch-up counter, best effort.""" + path = _current_cron_store().cron_dir / "catch_up_occurrences" + try: + try: + value = int(path.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + value = 0 + _atomic_write_counter(path, max(0, value) + 1) + except Exception: + pass + + +def get_catch_up_occurrence_count() -> int: + """Return the profile-local stale-schedule catch-up count.""" + path = _current_cron_store().cron_dir / "catch_up_occurrences" + try: + return max(0, int(path.read_text(encoding="utf-8").strip())) + except (OSError, ValueError): + return 0 + + # ============================================================================= # Job CRUD Operations # ============================================================================= @@ -2088,6 +2128,7 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: rj["next_run_at"] = new_next needs_save = True break + record_catch_up_occurrence() # Fall through to due.append(job) — execute once now # One-shot dispatch-limit guard (issue #38758): a finite one-shot diff --git a/cron/scheduler.py b/cron/scheduler.py index 830c18aabab..e1cb061af38 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -3945,6 +3945,7 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - # responses: do not deliver a blank message, and let the # empty-response guard below mark the run as a soft failure. should_deliver = bool(deliver_content.strip()) + unresolved_origin = False # Cron silence suppression — see _is_cron_silence_response. Replaces the # old `SILENT_MARKER in ...upper()` substring check, which both leaked # bracketless near-markers ("SILENT" / "NO_REPLY") and wrongly swallowed @@ -3956,6 +3957,10 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - should_deliver = False if should_deliver: + unresolved_origin = ( + _normalize_deliver_value(job.get("deliver", "local")) == "origin" + and not _resolve_delivery_targets(job) + ) try: delivery_error = _deliver_result(job, deliver_content, adapters=adapters, loop=loop) except Exception as de: @@ -3979,6 +3984,8 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - mark_job_run(job["id"], success, error, delivery_error=delivery_error) if delivery_error: delivery_outcome = "failed" + elif should_deliver and unresolved_origin: + delivery_outcome = "not_configured" elif should_deliver and job.get("deliver", "local") != "local": delivery_outcome = "delivered" else: diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index 027ac9db8c7..cdec47c496e 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -19,8 +19,8 @@ capture is a separate plane served by the NeMo Relay integration | Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | | Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | -| Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | -| Cron execution lifecycle | `/v1/traces` | Durable `claimed/running/completed/failed/unknown` states, bounded source and error class, opaque hashed job key, elapsed duration when timestamps exist, and delivery outcome when the scheduler knows it; terminal states flush through a bounded fail-open barrier | +| Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | +| Cron execution lifecycle | `/v1/traces` | Durable `claimed/running/completed/failed/unknown` states, bounded source and error class, opaque hashed job key, elapsed duration when timestamps exist, and delivery outcome when the scheduler knows it; terminal states make a fail-open flush attempt that can delay completion by up to one second | Signals carry `service.name`, version, supervision mode, and a stable one-way hash of the install id so an operator can distinguish instances without @@ -99,7 +99,8 @@ python scripts/observability/gateway_health_export_probe.py \ The `hermes monitoring` CLI intentionally exposes `status` only. This first release covers only Hermes Agent-owned service-health and operational-diagnostic -signals. Team Gateway/shared-connector signals are explicitly out of scope, as +signals, including Hermes Agent-owned Relay transport health. Team Gateway's +authoritative shared connector/platform state is explicitly out of scope, as are product analytics, audit/quality reporting, and detailed execution traces. Shared client usage metrics and enterprise trace telemetry are being designed on the NeMo Relay integration with their own consent, policy, and export diff --git a/tests/cron/test_execution_ledger.py b/tests/cron/test_execution_ledger.py index 7d032b06a85..28c9b3a330d 100644 --- a/tests/cron/test_execution_ledger.py +++ b/tests/cron/test_execution_ledger.py @@ -269,6 +269,37 @@ def test_run_one_job_records_running_then_terminal(monkeypatch): assert events[-1][2]["success"] is True +def test_run_one_job_records_unresolved_origin_as_not_configured(monkeypatch): + import cron.scheduler as scheduler + + finished = [] + monkeypatch.setattr(scheduler, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr( + scheduler, + "finish_execution", + lambda execution_id, **kwargs: finished.append((execution_id, kwargs)), + ) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr( + scheduler, + "run_job", + lambda job, *, defer_agent_teardown=None: (True, "output", "response", None), + ) + monkeypatch.setattr(scheduler, "save_job_output", lambda *_args: None) + monkeypatch.setattr(scheduler, "_resolve_delivery_targets", lambda _job: []) + monkeypatch.setattr(scheduler, "_deliver_result", lambda *_args, **_kwargs: None) + monkeypatch.setattr(scheduler, "mark_job_run", lambda *_args, **_kwargs: None) + + job = { + "id": "job-unresolved-origin", + "execution_id": "exec-unresolved-origin", + "deliver": "origin", + } + assert scheduler.run_one_job(job) is True + + assert finished[-1][1]["delivery_outcome"] == "not_configured" + + def test_provider_start_recovers_interrupted_records_before_tick(monkeypatch): import cron.scheduler_provider as provider diff --git a/tests/cron/test_jobs.py b/tests/cron/test_jobs.py index 09fc7d2faab..704363a57ea 100644 --- a/tests/cron/test_jobs.py +++ b/tests/cron/test_jobs.py @@ -836,6 +836,24 @@ class TestGetDueJobs: next_dt = _ensure_aware(datetime.fromisoformat(updated["next_run_at"])) assert next_dt > _hermes_now() + def test_stale_past_due_records_one_catch_up_occurrence(self, tmp_cron_dir, monkeypatch): + import cron.jobs as jobs_module + + recorded = [] + monkeypatch.setattr( + jobs_module, + "record_catch_up_occurrence", + lambda: recorded.append("catch-up"), + raising=False, + ) + create_job(prompt="Stale", schedule="every 1h") + jobs = load_jobs() + jobs[0]["next_run_at"] = (datetime.now() - timedelta(minutes=35)).isoformat() + save_jobs(jobs) + + assert len(get_due_jobs()) == 1 + assert recorded == ["catch-up"] + def test_idless_job_does_not_crash_or_block_sibling_jobs(self, tmp_cron_dir): """A job missing its 'id' key must not crash the tick or freeze siblings. diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py index 8dfa4d41647..a44fa5ef2fc 100644 --- a/tests/monitoring/test_cron_health_export.py +++ b/tests/monitoring/test_cron_health_export.py @@ -2,6 +2,8 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone +import pytest + def _metric(snapshot, name): return next(metric for metric in snapshot.metrics if metric.name == name) @@ -106,11 +108,52 @@ def test_execution_projection_omits_duration_and_delivery_when_not_known(): ).to_dict() assert event["status"] == "claimed" - assert event["source"] == "unknown" + assert event["source"] == "external" assert event["duration_ms"] is None assert event["delivery_outcome"] is None +def test_external_provider_source_is_normalized_to_external(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + {"job_id": "private", "source": "Chronos", "status": "claimed"} + ) + + assert event.source == "external" + + +@pytest.mark.parametrize("message", ["oauth refresh failed", "tokenizer crashed", "HTTP 4015"]) +def test_error_classification_avoids_auth_substring_false_positives(message): + from agent.monitoring.cron_health import classify_cron_error + + assert classify_cron_error(message) == "unknown" + + +@pytest.mark.parametrize( + "message", + ["authentication failed", "not authorized", "access token expired", "HTTP 401"], +) +def test_error_classification_recognizes_auth_terms_and_status_tokens(message): + from agent.monitoring.cron_health import classify_cron_error + + assert classify_cron_error(message) == "auth_failed" + + +def test_cron_snapshot_exports_catch_up_occurrence_counter(monkeypatch): + from agent.monitoring import cron_health + + monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: None) + monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: None) + monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset()) + monkeypatch.setattr(cron_health, "load_jobs", lambda: []) + monkeypatch.setattr(cron_health, "get_catch_up_occurrence_count", lambda: 3) + + snapshot = cron_health.build_cron_health_snapshot() + + assert _metric(snapshot, "hermes.cron.scheduler.catch_up_occurrences").value == 3 + + def test_terminal_execution_emission_flushes_and_failures_are_fail_open(monkeypatch): from agent.monitoring import cron_health, emitter @@ -151,3 +194,14 @@ def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(mon assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True assert gateway_health_export._gateway_health_event({"event": "run"}) is False + + +def test_monitoring_docs_distinguish_relay_health_scope_and_terminal_flush(): + from pathlib import Path + + text = Path("docs/observability/monitoring.md").read_text(encoding="utf-8") + + assert "Hermes Agent-owned Relay transport health" in text + assert "authoritative shared connector/platform state" in text + assert "up to one second" in text + assert "terminal" in text From 2a46acc9be6089dcc631a07b9ff452230ba417cd Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Fri, 24 Jul 2026 18:44:43 +0000 Subject: [PATCH 21/29] docs(monitoring): clarify bounded cron flush --- docs/observability/monitoring.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index cdec47c496e..f13ebc161b9 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -48,9 +48,9 @@ hermes monitoring status The OpenTelemetry SDK is an optional extra (`pip install 'hermes-agent[otlp]'`), lazy-installed on first use. When the SDK is missing or the endpoint is down, -the gateway runs unaffected: every export path is fail-open and off the hot -path (events flow through a fire-and-forget in-process emitter; a slow or -failing exporter can never block gateway code). +the gateway runs unaffected: metric collection and ordinary event export stay +off the hot path, while terminal cron events make one bounded fail-open flush +attempt of up to one second so the final state is less likely to be lost. Works identically under systemd/launchd/s6 supervision, containers, tmux, or a plain `hermes gateway run` — the exporter lives in the gateway process, so From 2e2c77c30f7674d2c96456c745a0c949f611c377 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Fri, 24 Jul 2026 18:52:00 +0000 Subject: [PATCH 22/29] fix(monitoring): normalize cron delivery outcomes --- cron/scheduler.py | 3 ++- tests/cron/test_execution_ledger.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cron/scheduler.py b/cron/scheduler.py index e1cb061af38..3e8c8cb42e7 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -3982,11 +3982,12 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - if not _consume_interrupted_flag(job["id"]): mark_job_run(job["id"], success, error, delivery_error=delivery_error) + normalized_deliver = _normalize_deliver_value(job.get("deliver", "local")) if delivery_error: delivery_outcome = "failed" elif should_deliver and unresolved_origin: delivery_outcome = "not_configured" - elif should_deliver and job.get("deliver", "local") != "local": + elif should_deliver and normalized_deliver != "local": delivery_outcome = "delivered" else: delivery_outcome = "suppressed" diff --git a/tests/cron/test_execution_ledger.py b/tests/cron/test_execution_ledger.py index 28c9b3a330d..4fa15541722 100644 --- a/tests/cron/test_execution_ledger.py +++ b/tests/cron/test_execution_ledger.py @@ -300,6 +300,38 @@ def test_run_one_job_records_unresolved_origin_as_not_configured(monkeypatch): assert finished[-1][1]["delivery_outcome"] == "not_configured" +def test_run_one_job_normalizes_legacy_local_delivery_as_suppressed(monkeypatch): + import cron.scheduler as scheduler + + finished = [] + monkeypatch.setattr( + scheduler, + "run_job", + lambda job, *, defer_agent_teardown=None: (True, "output", "response", None), + ) + monkeypatch.setattr(scheduler, "save_job_output", lambda *_args: None) + monkeypatch.setattr(scheduler, "mark_job_run", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + scheduler, + "finish_execution", + lambda execution_id, **kwargs: finished.append((execution_id, kwargs)), + ) + monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_args: None) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scheduler, "_consume_interrupted_flag", lambda *_args: False) + + job = { + "id": "legacy-local", + "name": "Legacy local", + "schedule": {"kind": "interval", "minutes": 10}, + "deliver": ["local"], + "execution_id": "exec-legacy-local", + } + + assert scheduler.run_one_job(job) is True + assert finished[-1][1]["delivery_outcome"] == "suppressed" + + def test_provider_start_recovers_interrupted_records_before_tick(monkeypatch): import cron.scheduler_provider as provider From 58a7491ecd821990192ca612301fbfc37d175952 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Fri, 24 Jul 2026 19:54:08 +0000 Subject: [PATCH 23/29] docs(monitoring): add fleet operations guide --- docs/observability/monitoring.md | 87 +++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index f13ebc161b9..2f7a72ccbff 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -53,7 +53,7 @@ off the hot path, while terminal cron events make one bounded fail-open flush attempt of up to one second so the final state is less likely to be lost. Works identically under systemd/launchd/s6 supervision, containers, tmux, or -a plain `hermes gateway run` — the exporter lives in the gateway process, so +a plain `hermes gateway run`: the exporter lives in the gateway process, so no sidecar, agent, or collector is required on the host. ## Collecting into DataDog @@ -80,6 +80,91 @@ service: Point `monitoring.export.otlp.endpoint` at the collector. Alerts belong on `hermes.gateway.up`, `hermes.platform.up`, and `hermes.platform.degraded`. +## Generic fleet queries and alerts + +The exact syntax depends on the customer's observability backend. The examples +below use PromQL-style expressions and intentionally avoid vendor-specific +routing, destinations, or customer inventory. + +Group fleet views by the opaque `service.instance.id` resource attribute. A +process that has died cannot emit its own zero, so every deployment needs both +explicit-state and missing-series detection. + +```promql +# Explicit gateway failure. +hermes_gateway_up == 0 + +# Box disappeared or stopped exporting. Choose a window longer than the +# configured export interval and collector retry allowance. +absent_over_time(hermes_gateway_up[5m]) + +# Locally owned bridge is explicitly down. +hermes_platform_up == 0 + +# Scheduler thread is stale even though the gateway may still be alive. +hermes_cron_scheduler_heartbeat_age_seconds > 180 + +# Ticker loops but has not completed a successful tick recently. +hermes_cron_scheduler_last_success_age_seconds > 300 + +# One or more jobs are beyond their existing scheduler grace window. +hermes_cron_jobs_overdue > 0 + +# Catch-up counter increased, proving at least one stale occurrence was +# collapsed and run once after a delay. +increase(hermes_cron_scheduler_catch_up_occurrences[15m]) > 0 +``` + +Cron execution lifecycle records arrive as `hermes.cron_execution` spans. +Alert or derive events from bounded attributes such as: + +```text +hermes.status = failed|unknown +hermes.delivery_outcome = failed|not_configured +hermes.error_class = auth_failed|rate_limited|timeout|network_error| + dispatch_failed|interrupted|empty_response| + invalid_config|unknown +``` + +Recommended operator views: + +1. one row per `service.instance.id` with gateway and configured local-platform + state; +2. scheduler heartbeat, last-success age, running count, overdue count, and + catch-up increase; +3. a cron lifecycle feed keyed only by opaque `hermes.job_key`; +4. separate alerts for box absence, local bridge down, scheduler stale, cron + failed/unknown, delivery failure, and overdue/catch-up activity. + +Keep alert thresholds and routing in deployment-owned configuration. Do not add +job names, prompts, outputs, schedules, destinations, raw errors, profile names, +or account identity merely to make a dashboard easier to read. + +## Release-validation scenarios + +Before accepting a deployment, force and verify all five cases through the real +collector and backend: + +1. **Cron success:** observe `claimed -> running -> completed`, duration, and a + truthful delivery outcome. +2. **Cron failure:** observe `failed` plus a bounded error class, with no raw + exception or content in the decoded OTLP payload. +3. **Cron interruption:** stop the owning gateway during execution, restart it, + and observe recovery to `unknown`. +4. **Locally owned bridge outage:** break one native connector, observe its + bounded down/retrying/fatal state and recovery, and verify unaffected boxes + remain healthy. +5. **Killed gateway:** terminate one canary, verify missing-series detection, + restart it, and confirm the same opaque instance identity returns. + +Hermes Agent-owned Relay transport health remains in scope. A separate gateway +or connector service remains authoritative for any shared connected-platform +state that it owns and should export that state through its own telemetry path. + +For every scenario, verify the signal and alert clear on recovery, other boxes +remain unaffected, collector failure stays fail-open, and decoded metrics, +spans, logs, and resource attributes remain content-free. + ## Local smoke test (no Docker) ```bash From 6de6e36dbe599ee942aeecedc439de228c050aef Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Fri, 24 Jul 2026 21:58:46 +0000 Subject: [PATCH 24/29] fix(monitoring): surface cron snapshot failures at WARNING (content-free) The cron health snapshot failure path logged only at DEBUG, so a cron telemetry regression would silently drop all hermes.cron.* metrics while gateway health metrics kept flowing. Promote to WARNING with the exception *type* name only (no message text, preserving the no-raw-error contract); keep the traceback on DEBUG. (cherry picked from commit 1c9a3e737b7c57fa9886bc927f1623753a9a811b) --- agent/monitoring/gateway_health_export.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 222ac599523..ce9c2e4e7d6 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -305,8 +305,16 @@ def _read_runtime_snapshot(config: Dict[str, Any]): gateway_snapshot = _read_gateway_snapshot(config) try: cron_snapshot = _read_cron_snapshot() - except Exception: - logger.debug("cron health snapshot unavailable", exc_info=True) + except Exception as exc: + # Content-free visibility: cron telemetry silently dropping out is a + # release-relevant regression, so surface it at WARNING with only the + # exception *type* name (never the message, which could carry paths or + # other environment detail). exc_info stays on the DEBUG record. + logger.warning( + "cron health snapshot unavailable; cron telemetry not exported (error_type=%s)", + type(exc).__name__, + ) + logger.debug("cron health snapshot traceback", exc_info=True) return gateway_snapshot gateway_snapshot.metrics.extend(cron_snapshot.metrics) return gateway_snapshot From 9edcb7b09113b291aa6cee7131411119704aa8af Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Sat, 25 Jul 2026 01:33:36 +0000 Subject: [PATCH 25/29] feat(monitoring): emit hermes.gateway.background_work (subagent/bg jobs) active_agents counts foreground turns + cron + API runs but never the backgrounded delegate_task subagents / terminal(background) processes / kanban workers tracked only for scale-to-zero. Emit them as a distinct content-free gauge so the fleet dashboard can show subagent/background load per peer. Best-effort, sums async_delegation.active_count + process_registry.count_running; 0 if a source is unavailable. (cherry picked from commit 1b7ec684e253651a5aa5760b3bd527a3073ce207) --- agent/monitoring/gateway_health_export.py | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index ce9c2e4e7d6..846132137c5 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -301,8 +301,56 @@ def _read_cron_snapshot(): return build_cron_health_snapshot() +def _read_background_work_count() -> int: + """Count live background/subagent work that ``active_agents`` does NOT include. + + ``hermes.gateway.active_agents`` counts foreground turns + in-flight cron + jobs + API runs, but deliberately excludes backgrounded ``delegate_task`` + subagents, ``terminal(background=true)`` processes, kanban workers, and the + runner's own background tasks (they are tracked only for the scale-to-zero + suspend guard, ``_scale_to_zero_has_live_background_work``). Without this + metric a peer churning through delegated subagents shows ``active_agents=0`` + on the fleet dashboard. Best-effort and content-free: a single integer, + no job/task identity. Returns 0 if a source can't be imported. + """ + total = 0 + try: + from tools.async_delegation import active_count + + total += max(0, int(active_count())) + except Exception: + logger.debug("background-work async-delegation count failed", exc_info=True) + try: + from tools.process_registry import process_registry + + total += max(0, int(process_registry.count_running())) + except Exception: + logger.debug("background-work process-registry count failed", exc_info=True) + return total + + def _read_runtime_snapshot(config: Dict[str, Any]): gateway_snapshot = _read_gateway_snapshot(config) + # Background/subagent work — a distinct metric from active_agents (which + # never counts it). Appended to the gateway snapshot so it rides the same + # base resource attributes (service.instance.id etc.). + try: + from agent.monitoring.gateway_health import GatewayMetric + + base = dict(gateway_snapshot.metrics[0].attributes) if gateway_snapshot.metrics else {} + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_work", + value=_read_background_work_count(), + attributes=base, + ) + ) + except Exception as exc: + logger.warning( + "background-work snapshot unavailable; metric not exported (error_type=%s)", + type(exc).__name__, + ) + logger.debug("background-work snapshot traceback", exc_info=True) try: cron_snapshot = _read_cron_snapshot() except Exception as exc: @@ -360,6 +408,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: "hermes.gateway.busy", "hermes.gateway.drainable", "hermes.gateway.restart_requested", + "hermes.gateway.background_work", "hermes.platform.up", "hermes.platform.degraded", "hermes.cron.scheduler.heartbeat_age_seconds", From b49b78b7388c4d933bbfe38c563e262785f0f69d Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Sat, 25 Jul 2026 01:55:36 +0000 Subject: [PATCH 26/29] docs(monitoring): background_work signal + plane maintenance/extension guide Document hermes.gateway.background_work in the export table, and add a 'Maintaining and extending this plane' section: the content-free invariant, and per-change checklists for adding a metric, a new subsystem, extending the error-class/status/source/state enums, and adding a content-free span attribute. Each calls out the layers that silently drop an undeclared signal (observable metric_names registration, the emitter keep_by_kind allowlist, the closed enums, and any collector name/keep_keys allowlist) plus whole-chain verification. --- docs/observability/monitoring.md | 98 +++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index 2f7a72ccbff..e12ec0ebdb8 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -16,7 +16,7 @@ capture is a separate plane served by the NeMo Relay integration | Signal | OTLP route | Content | | --- | --- | --- | -| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | +| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | | Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | | Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | @@ -180,6 +180,102 @@ python scripts/observability/gateway_health_export_probe.py \ # exit 0 prints: {"requests": 6, "paths": ["/v1/logs", "/v1/metrics", "/v1/traces"]} ``` +## Maintaining and extending this plane + +This plane is a **fixed, enumerated, content-free vocabulary** by design. Adding +a signal is not just "emit a new metric" — every new name and attribute must be +declared in each layer that enforces the bounded vocabulary, or it is silently +dropped downstream. Follow the checklist for the change you are making. The +golden rule: **a new signal that is emitted but not declared in every layer +looks like a code bug but is a vocabulary-registration bug — nothing errors, the +signal just never arrives.** + +### Content-free invariant (applies to every change) + +Before adding anything, confirm it cannot carry content. Numbers, booleans, +ages, durations, monotonic counts, and one-way hashes are safe. **Never** add an +attribute that can hold a job name, prompt, output, schedule, destination, raw +exception text, file path, profile name, account id, or free-form string. When +you must key a record to a job/entity, hash it (`sha256(...)[:24]`, see +`_job_key` in `agent/monitoring/cron_health.py`) — never emit the raw id. All +string attributes that could touch user input must pass through +`redaction.redact_for_export` and be truncated (see `_span_attrs` in +`agent/monitoring/otlp_exporter.py`). + +### Adding a new gauge/metric + +1. Emit it in the snapshot builder (`agent/monitoring/gateway_health.py` + `build_gateway_health_snapshot`, `cron_health.py` `build_cron_health_snapshot`, + or a sibling reader wired into `_read_runtime_snapshot` in + `gateway_health_export.py`). Best-effort: never let a reader raise into the + collection loop — wrap it and log a **content-free WARNING with the exception + TYPE name only** (the pattern the cron and background-work readers use), so a + future regression is visible instead of silently dropping the signal. +2. Register the dotted metric name in the observable-gauge `metric_names` list in + `gateway_health_export.py::_start_metric_provider`. **A gauge that is emitted + in the snapshot but not registered here is never observed.** +3. Add the export-table row and an alert example in this file. +4. If the deployment fronts the exporter with an OpenTelemetry Collector that + uses a metric-name allowlist (a `filter/...` processor with `name != "..."` + guards), add the new name there too — otherwise the collector drops it before + the backend. This is not repo code, but it is the single most common reason a + correctly-emitted new metric never appears; call it out in the PR so the + deploying operator updates their collector config. + +### Adding a new subsystem (a new family of signals) + +Mirror the cron pattern (`cron_health.py` + its wiring): put the read/projection +logic in its own module, expose one `build__health_snapshot()` that +returns bounded `GatewayMetric`s (and events if any), and extend it into +`_read_runtime_snapshot` with the same best-effort try/except-WARNING guard. +Then do the "adding a metric" checklist for each new name, and the "adding an +attribute" checklist for each new event attribute. Add a release-validation +scenario below for the subsystem's failure mode. + +### Extending the error-class / status / source / state vocabularies + +These are the closed enums that keep the plane bounded. Extend the SET, then the +classifier, never one without the other: + +- **Cron** (`agent/monitoring/cron_health.py`): `_KNOWN_STATUSES`, + `_KNOWN_SOURCES`, `_KNOWN_DELIVERY_OUTCOMES`, and the `classify_cron_error` + keyword buckets. Anything not in the set is coerced to `unknown` on the way + out, so a new value that is not added to the set is invisible. +- **Gateway/platform** (`agent/monitoring/gateway_health.py`): + `_KNOWN_GATEWAY_STATES`, `_KNOWN_PLATFORM_STATES`, and `classify_gateway_error`. + +Rules: keep the vocabulary SMALL and operationally meaningful (an error class +should map to an operator action, not to an exception subclass); a new bucket +must match on a stable keyword, not on message text that could vary; update the +`hermes.error_class = ...` list in this file's alert section and the enum's unit +test so the contract is asserted, not frozen as a count. + +### Adding a content-free attribute to an existing event/span + +Add the key to the emitter's per-kind `keep_by_kind` allowlist in +`agent/monitoring/otlp_exporter.py::_span_attrs` (unlisted keys are dropped), run +it through redaction if it is ever string-shaped, and — as with metrics — if the +deployment's collector has a span-attribute `keep_keys(...)` allowlist, add the +attribute there too or it is stripped in transit. + +### Verify the whole chain, not just emission + +Emitting is necessary but not sufficient. Confirm the signal survives all the +way to the backend, because the enums, the `metric_names` registration, the +emitter attribute allowlist, and any collector allowlist each drop unlisted +values with no error: + +```bash +hermes monitoring status # posture +python scripts/observability/gateway_health_export_probe.py \ + --endpoint http://127.0.0.1:4318/v1/traces \ + --log /tmp/cap.jsonl --wait 8 # drive the real exporter +``` + +Decode the captured OTLP payload and assert the new name/attribute is present +AND that no content leaked. When a real collector sits in front, add its +allowlist entries and re-verify against the backend, not just the local capture. + ## Boundaries and roadmap The `hermes monitoring` CLI intentionally exposes `status` only. This first From 3c567f7c84a7e12c1429f5eb16ca58d7bb65d04c Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Sat, 25 Jul 2026 01:58:28 +0000 Subject: [PATCH 27/29] test(monitoring): assert background_work in snapshot + metric_names registration invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the cron-export test to assert background_work membership (behavior contract, not a frozen list), and add a regression guard that every gauge emitted in the runtime snapshot is also registered in the observable metric_names list — the silent-drop trap the extension guide documents. --- tests/monitoring/test_cron_health_export.py | 52 ++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py index a44fa5ef2fc..2b78b020e11 100644 --- a/tests/monitoring/test_cron_health_export.py +++ b/tests/monitoring/test_cron_health_export.py @@ -190,12 +190,62 @@ def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(mon snapshot = gateway_health_export._read_runtime_snapshot({}) - assert [metric.name for metric in snapshot.metrics] == ["hermes.cron.jobs.enabled"] + names = [metric.name for metric in snapshot.metrics] + # Cron metrics are folded into the gateway snapshot... + assert "hermes.cron.jobs.enabled" in names + # ...and the background/subagent-work gauge is appended (distinct from + # active_agents). Assert the relationship, not a frozen exact list. + assert "hermes.gateway.background_work" in names assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True assert gateway_health_export._gateway_health_event({"event": "run"}) is False +def test_registered_observable_metric_names_cover_snapshot_metrics(monkeypatch): + """Every gauge emitted in the runtime snapshot must also be registered in the + observable-gauge metric_names list, or the OTLP exporter never observes it. + + This asserts the vocabulary-registration invariant documented in + docs/observability/monitoring.md: an emitted-but-unregistered gauge is + silently dropped. Regression guard for background_work / cron additions. + """ + import inspect + from agent.monitoring import gateway_health_export + + # Build a representative snapshot (gateway + cron + background_work) without + # a live gateway by stubbing the gateway snapshot to the real metric names. + class _M: + def __init__(self, name): + self.name = name + self.value = 0 + self.attributes = {} + + gateway_snapshot = type("S", (), {"metrics": [ + _M("hermes.gateway.up"), _M("hermes.gateway.active_agents"), + _M("hermes.gateway.busy"), _M("hermes.gateway.drainable"), + _M("hermes.gateway.restart_requested"), + _M("hermes.platform.up"), _M("hermes.platform.degraded"), + ]})() + cron_snapshot = type("S", (), {"metrics": [ + _M("hermes.cron.scheduler.heartbeat_age_seconds"), + _M("hermes.cron.scheduler.last_success_age_seconds"), + _M("hermes.cron.scheduler.catch_up_occurrences"), + _M("hermes.cron.jobs.enabled"), _M("hermes.cron.jobs.running"), + _M("hermes.cron.jobs.overdue"), + ]})() + monkeypatch.setattr(gateway_health_export, "_read_gateway_snapshot", lambda config: gateway_snapshot) + monkeypatch.setattr(gateway_health_export, "_read_cron_snapshot", lambda: cron_snapshot) + + snapshot_names = {m.name for m in gateway_health_export._read_runtime_snapshot({}).metrics} + + # Extract the registered metric_names list literal from _start_metric_provider. + src = inspect.getsource(gateway_health_export._start_metric_provider) + registered = {n for n in snapshot_names if f'"{n}"' in src} + + missing = snapshot_names - registered + assert not missing, f"gauges emitted but NOT registered in metric_names (will be silently dropped): {sorted(missing)}" + + def test_monitoring_docs_distinguish_relay_health_scope_and_terminal_flush(): from pathlib import Path From 0ef0816284ca7e287b63b992cd55730cace26a40 Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Sat, 25 Jul 2026 02:12:12 +0000 Subject: [PATCH 28/29] feat(monitoring): count background_work task-granular (expand delegate batches) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delegate_task fan-out batch occupies ONE async-pool slot by design, so active_count() (unit/slot count) reports a 3-task batch as 1 — which undercounts real concurrent subagent load on the background_work metric. Add active_task_count() that expands a running batch to its child count (N-task batch -> N, single -> 1) and switch the background_work reader to it. active_count() is unchanged (capacity semantics preserved). Adds a contract test for the unit-vs-task distinction and documents both counts. --- agent/monitoring/gateway_health_export.py | 9 +++++-- docs/observability/monitoring.md | 11 ++++++++ tests/tools/test_async_delegation.py | 28 ++++++++++++++++++++ tools/async_delegation.py | 32 ++++++++++++++++++++++- 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 846132137c5..422650a5dcd 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -312,12 +312,17 @@ def _read_background_work_count() -> int: metric a peer churning through delegated subagents shows ``active_agents=0`` on the fleet dashboard. Best-effort and content-free: a single integer, no job/task identity. Returns 0 if a source can't be imported. + + Delegation is counted TASK-granular (``active_task_count``): a fan-out batch + of N subagents contributes N, not 1, so the metric reflects real concurrent + subagent load rather than dispatch-unit/pool-slot count. This intentionally + differs from the async pool's capacity accounting (one batch = one slot). """ total = 0 try: - from tools.async_delegation import active_count + from tools.async_delegation import active_task_count - total += max(0, int(active_count())) + total += max(0, int(active_task_count())) except Exception: logger.debug("background-work async-delegation count failed", exc_info=True) try: diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index e12ec0ebdb8..742497a6f33 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -26,6 +26,17 @@ Signals carry `service.name`, version, supervision mode, and a stable one-way hash of the install id so an operator can distinguish instances without exporting account/profile identity or the raw install identifier. +`hermes.gateway.active_agents` and `hermes.gateway.background_work` are +complementary and deliberately disjoint. `active_agents` counts foreground +message turns plus in-flight cron jobs plus API runs — the work the gateway +drains on shutdown. `background_work` counts detached work that `active_agents` +never includes: backgrounded `delegate_task` subagents, `terminal(background=true)` +processes, and kanban workers. `background_work` is **task-granular** — a +fan-out batch of N subagents counts as N, not as one dispatch unit — so it +reflects real concurrent subagent load rather than async-pool slot usage (a +batch occupies one slot but runs N children). Sum both for total live work per +instance. + ## Enabling ```yaml diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index cc6082ec66b..6940e61bf5c 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -244,6 +244,34 @@ def test_completed_records_pruned_to_cap(): assert len(ad.list_async_delegations()) <= ad._MAX_RETAINED_COMPLETED +def test_active_task_count_expands_batches_while_active_count_stays_unit(monkeypatch): + """active_count() counts dispatch UNITS (batch=1); active_task_count() + expands a batch to its child count. This is the batch-vs-single distinction + the background_work metric relies on so a 3-task fan-out isn't undercounted + as 1 running subagent. + """ + # Deterministic: install synthetic running records directly, no real spawn. + with ad._records_lock: + saved = dict(ad._records) + ad._records.clear() + ad._records["single_a"] = {"status": "running"} # single subagent + ad._records["batch_3"] = {"status": "running", "is_batch": True, + "goals": ["g1", "g2", "g3"]} # 3-task batch + ad._records["batch_missing"] = {"status": "running", "is_batch": True} # goals absent -> 1 + ad._records["done"] = {"status": "completed", "is_batch": True, + "goals": ["x", "y"]} # not running -> ignored + try: + # 3 running UNITS (single + 2 batches); the completed one is excluded. + assert ad.active_count() == 3 + # TASKS: single(1) + batch_3(3) + batch_missing(1, fallback) = 5. + assert ad.active_task_count() == 5 + finally: + with ad._records_lock: + ad._records.clear() + ad._records.update(saved) + + + def test_completion_is_persisted_and_delivery_can_be_acknowledged(tmp_path, monkeypatch): """A finished child remains pending on disk until its queue consumer acks it.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) diff --git a/tools/async_delegation.py b/tools/async_delegation.py index e456aed50f8..73b223bfcaf 100644 --- a/tools/async_delegation.py +++ b/tools/async_delegation.py @@ -473,11 +473,41 @@ def _get_executor(max_workers: int) -> ThreadPoolExecutor: def active_count() -> int: - """Number of async delegations currently running.""" + """Number of async delegation UNITS currently running. + + A unit is one dispatch: a single subagent OR a whole fan-out batch. A batch + counts as ONE here because it occupies one async-pool slot (the capacity + semantics ``dispatch_async_delegation_batch`` relies on). For the count of + actual concurrent child subagents (batch expanded), use + ``active_task_count()``. + """ with _records_lock: return sum(1 for r in _records.values() if r.get("status") in {"running", "finalizing"}) +def active_task_count() -> int: + """Number of async delegation TASKS (child subagents) currently running. + + Unlike ``active_count()`` (units/slots), this expands a batch to its child + count: a running batch of N tasks contributes N, a single subagent + contributes 1. This is the truthful "how many subagents are actually + working right now" figure for observability, where a 3-task batch shown as + "1" undercounts real concurrent work. Falls back to counting a batch as 1 + if its goal list is missing. + """ + with _records_lock: + total = 0 + for r in _records.values(): + if r.get("status") not in {"running", "finalizing"}: + continue + if r.get("is_batch"): + goals = r.get("goals") + total += len(goals) if isinstance(goals, (list, tuple)) and goals else 1 + else: + total += 1 + return total + + def _new_delegation_id() -> str: return f"deleg_{uuid.uuid4().hex[:8]}" From cc80dba96fa9d05cfc62a9115a1993ba153b5b6b Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Sat, 25 Jul 2026 02:30:20 +0000 Subject: [PATCH 29/29] feat(monitoring): add hermes.gateway.background_delegations (unit/slot count) Complements the task-granular background_work with the async-delegation UNIT count (each dispatch/batch = 1), recovering the pool-slot semantics active_count() gives. Together: background_work = real concurrent subagent load (batch expanded), background_delegations = slot pressure to alert against delegation.max_concurrent_children. Registered in metric_names, documented, and covered by a task-vs-unit contract test. --- agent/monitoring/gateway_health_export.py | 29 +++++++++++++++++++++ docs/observability/monitoring.md | 25 ++++++++++-------- tests/monitoring/test_cron_health_export.py | 29 ++++++++++++++++++++- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py index 422650a5dcd..0a377c99ab3 100644 --- a/agent/monitoring/gateway_health_export.py +++ b/agent/monitoring/gateway_health_export.py @@ -334,6 +334,27 @@ def _read_background_work_count() -> int: return total +def _read_background_delegations_count() -> int: + """Count live async delegation UNITS (dispatch/pool slots). + + Complements ``_read_background_work_count`` (which is task-granular): this + counts each ``delegate_task`` dispatch as ONE regardless of fan-out width, + matching the async pool's capacity accounting (a batch = one slot). Together + the two metrics let an operator see both slot pressure + (``background_delegations``, alert vs ``max_concurrent_children``) and real + concurrent subagent load (``background_work``). Delegations only — it does + not include ``terminal(background)`` / kanban work, which are already folded + into ``background_work``. Best-effort; 0 if the source can't be imported. + """ + try: + from tools.async_delegation import active_count + + return max(0, int(active_count())) + except Exception: + logger.debug("background-delegations count failed", exc_info=True) + return 0 + + def _read_runtime_snapshot(config: Dict[str, Any]): gateway_snapshot = _read_gateway_snapshot(config) # Background/subagent work — a distinct metric from active_agents (which @@ -350,6 +371,13 @@ def _read_runtime_snapshot(config: Dict[str, Any]): attributes=base, ) ) + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_delegations", + value=_read_background_delegations_count(), + attributes=base, + ) + ) except Exception as exc: logger.warning( "background-work snapshot unavailable; metric not exported (error_type=%s)", @@ -414,6 +442,7 @@ def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: "hermes.gateway.drainable", "hermes.gateway.restart_requested", "hermes.gateway.background_work", + "hermes.gateway.background_delegations", "hermes.platform.up", "hermes.platform.degraded", "hermes.cron.scheduler.heartbeat_age_seconds", diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md index 742497a6f33..f798bf2dbe6 100644 --- a/docs/observability/monitoring.md +++ b/docs/observability/monitoring.md @@ -16,7 +16,7 @@ capture is a separate plane served by the NeMo Relay integration | Signal | OTLP route | Content | | --- | --- | --- | -| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | +| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/background_delegations/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | | Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | | Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | | Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | @@ -26,16 +26,19 @@ Signals carry `service.name`, version, supervision mode, and a stable one-way hash of the install id so an operator can distinguish instances without exporting account/profile identity or the raw install identifier. -`hermes.gateway.active_agents` and `hermes.gateway.background_work` are -complementary and deliberately disjoint. `active_agents` counts foreground -message turns plus in-flight cron jobs plus API runs — the work the gateway -drains on shutdown. `background_work` counts detached work that `active_agents` -never includes: backgrounded `delegate_task` subagents, `terminal(background=true)` -processes, and kanban workers. `background_work` is **task-granular** — a -fan-out batch of N subagents counts as N, not as one dispatch unit — so it -reflects real concurrent subagent load rather than async-pool slot usage (a -batch occupies one slot but runs N children). Sum both for total live work per -instance. +`hermes.gateway.active_agents`, `hermes.gateway.background_work`, and +`hermes.gateway.background_delegations` are complementary. `active_agents` +counts foreground message turns plus in-flight cron jobs plus API runs — the +work the gateway drains on shutdown. `background_work` counts detached work that +`active_agents` never includes: backgrounded `delegate_task` subagents, +`terminal(background=true)` processes, and kanban workers; it is +**task-granular** — a fan-out batch of N subagents counts as N — so it reflects +real concurrent subagent load. `background_delegations` counts only async +delegation **units** (each `delegate_task` dispatch is one, a fan-out batch is +one), matching the async pool's capacity accounting; alert it against +`delegation.max_concurrent_children` to see slot pressure. Sum `active_agents` +and `background_work` for total live work per instance; use +`background_delegations` for pool-saturation. ## Enabling diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py index 2b78b020e11..adc0e91c303 100644 --- a/tests/monitoring/test_cron_health_export.py +++ b/tests/monitoring/test_cron_health_export.py @@ -193,14 +193,41 @@ def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(mon names = [metric.name for metric in snapshot.metrics] # Cron metrics are folded into the gateway snapshot... assert "hermes.cron.jobs.enabled" in names - # ...and the background/subagent-work gauge is appended (distinct from + # ...and the background/subagent-work gauges are appended (distinct from # active_agents). Assert the relationship, not a frozen exact list. assert "hermes.gateway.background_work" in names + assert "hermes.gateway.background_delegations" in names assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True assert gateway_health_export._gateway_health_event({"event": "run"}) is False +def test_background_work_is_task_granular_and_delegations_is_unit_granular(monkeypatch): + """background_work expands batches to child tasks; background_delegations + counts dispatch units. A 3-task batch => work +3, delegations +1. + """ + from agent.monitoring import gateway_health_export + from tools import async_delegation as ad + + with ad._records_lock: + saved = dict(ad._records) + ad._records.clear() + ad._records["single"] = {"status": "running"} + ad._records["batch3"] = {"status": "running", "is_batch": True, "goals": ["a", "b", "c"]} + # Isolate from process_registry so we measure only the delegation contribution. + monkeypatch.setattr( + "tools.process_registry.process_registry.count_running", lambda: 0, raising=False + ) + try: + # work = single(1) + batch(3) = 4 tasks; delegations = 2 units. + assert gateway_health_export._read_background_work_count() == 4 + assert gateway_health_export._read_background_delegations_count() == 2 + finally: + with ad._records_lock: + ad._records.clear() + ad._records.update(saved) + + def test_registered_observable_metric_names_cover_snapshot_metrics(monkeypatch): """Every gauge emitted in the runtime snapshot must also be registered in the observable-gauge metric_names list, or the OTLP exporter never observes it.