hermes-agent/plugins/telemetry/__init__.py

"""Telemetry plugin — wires Hermes lifecycle hooks to the local telemetry emitter.

This is the *only* instrumentation seam. It registers observational hooks (which core
already invokes fail-open) and translates each into a typed local telemetry event
handed to ``agent.telemetry.emitter``. There are zero edits to core call sites:
the hooks already carry model/provider/usage/duration/tool data.

Everything here is best-effort and fail-open — a raised exception in a hook callback is
swallowed by core, and we additionally guard each callback so a telemetry bug can never
disturb a session. No content, no network: local telemetry only.

Hooks consumed:
  on_session_start    -> begin a run context (trace_id/run_id + root span id)
  post_api_request    -> one model_call event + its timing span (tokens, latency)
  api_request_error   -> one error event
  post_tool_call      -> one tool_call event + its timing span (duration, result class)
  on_session_finalize -> finalize the run row + emit the run's root span
  subagent_start/stop -> (reserved) lineage markers

Each model/tool call emits a SpanEvent (timing + parent = the run's root span) keyed by
the same span_id as its detail row, so tel_spans reconstructs a run -> calls trace tree.
"""

from __future__ import annotations

import logging
import threading
import time
from typing import Any, Dict, Optional

logger = logging.getLogger(__name__)

# Per-run accumulators keyed by run_id, so on_session_finalize can roll up counts.
_runs: Dict[str, Dict[str, Any]] = {}
_runs_lock = threading.Lock()


def _safe(fn):
    """Decorator: never let a telemetry hook raise into core."""
    def wrapper(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except Exception:
            logger.debug("telemetry hook %s failed", getattr(fn, "__name__", "?"), exc_info=True)
            return None
    wrapper.__name__ = getattr(fn, "__name__", "wrapper")
    return wrapper


def _run_key(session_id: Optional[str], task_id: Optional[str]) -> str:
    return session_id or task_id or "default"


# ── on_session_start ────────────────────────────────────────────────────────
@_safe
def _on_session_start(**kw: Any) -> None:
    from agent.telemetry import spans

    session_id = kw.get("session_id") or ""
    platform = kw.get("platform") or kw.get("source") or ""
    ctx = spans.start_run()
    now = time.time_ns()
    root_span_id = spans.new_span_id()
    key = _run_key(session_id, kw.get("task_id"))
    with _runs_lock:
        _runs[key] = {
            "run_id": ctx.run_id,
            "trace_id": ctx.trace_id,
            "root_span_id": root_span_id,
            "session_id": session_id or None,
            "entrypoint": _entrypoint_for(platform, kw.get("source")),
            "platform": platform or None,
            "start_ns": now,
            "model_call_count": 0,
            "tool_call_count": 0,
            "error_count": 0,
        }


def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str = "") -> Dict[str, Any]:
    """Return the run accumulator, lazily creating one if session_start was missed."""
    from agent.telemetry import spans

    key = _run_key(session_id, task_id)
    with _runs_lock:
        run = _runs.get(key)
        if run is None:
            rid = spans.current_run_id() or spans.new_id()
            tid = spans.current_trace_id() or spans.new_id()
            run = {
                "run_id": rid,
                "trace_id": tid,
                "root_span_id": spans.new_span_id(),
                "session_id": session_id or None,
                "entrypoint": _entrypoint_for(platform),
                "platform": platform or None,
                "start_ns": time.time_ns(),
                "model_call_count": 0,
                "tool_call_count": 0,
                "error_count": 0,
            }
            _runs[key] = run
        return run


def _emit_call_span(run: Dict[str, Any], span_id: str, name: str, kind: str,
                    duration_ms: Optional[int], status: Optional[str]) -> None:
    """Emit the timing/lineage span for a model or tool call.

    The call hooks fire on completion, so end_ns is ~now and start_ns is reconstructed
    from the measured duration (end - duration). The span is parented to the run's root
    so a 2-level run -> calls waterfall can be reconstructed from tel_spans.
    """
    from agent.telemetry import emitter
    from agent.telemetry.events import SpanEvent

    end_ns = time.time_ns()
    dur_ns = int(duration_ms) * 1_000_000 if isinstance(duration_ms, (int, float)) else 0
    start_ns = end_ns - dur_ns
    emitter.emit(SpanEvent(
        span_id=span_id,
        trace_id=run["trace_id"],
        run_id=run["run_id"],
        parent_span_id=run.get("root_span_id"),
        name=name,
        kind=kind,
        start_ns=start_ns,
        end_ns=end_ns,
        status=status,
    ))


def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str:
    """Coarse entrypoint label (cli / gateway / tui / api / cron …).

    This is a workflow *surface* label, not model/tool anonymization — it answers
    "where did this run come from", which is genuinely categorical.
    """
    s = (source or platform or "").lower()
    if s in ("", "chat", "interactive", "cli", "desktop"):
        return "cli"
    if s in ("telegram", "discord", "slack", "whatsapp", "signal", "matrix",
             "email", "sms", "teams", "feishu", "wecom", "line", "google_chat"):
        return "gateway"
    if s == "tui":
        return "tui"
    if s in ("api", "api_server", "openai_api"):
        return "api"
    if s == "cron":
        return "cron"
    if s == "batch":
        return "batch"
    if s == "acp":
        return "acp"
    return "cli"


# ── post_api_request -> model_call event ────────────────────────────────────
@_safe
def _on_post_api_request(**kw: Any) -> None:
    from agent.telemetry import emitter, spans
    from agent.telemetry.events import ModelCallEvent

    session_id = kw.get("session_id") or ""
    platform = kw.get("platform") or ""
    run = _ensure_run(session_id, kw.get("task_id"), platform)

    usage = kw.get("usage") or {}
    duration = kw.get("api_duration")
    latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None

    span_id = spans.new_span_id()
    evt = ModelCallEvent(
        span_id=span_id,
        run_id=run["run_id"],
        provider=kw.get("provider"),       # raw
        model=kw.get("model"),             # raw
        base_url=kw.get("base_url"),
        input_tokens=int(usage.get("input_tokens") or 0),
        output_tokens=int(usage.get("output_tokens") or 0),
        cache_read_tokens=int(usage.get("cache_read_tokens") or 0),
        cache_write_tokens=int(usage.get("cache_write_tokens") or 0),
        reasoning_tokens=int(usage.get("reasoning_tokens") or 0),
        latency_ms=latency_ms,
        end_reason="completed",
    )
    with _runs_lock:
        run["model_call_count"] += 1
    _emit_call_span(run, span_id, name=kw.get("model") or "model_call",
                    kind="model", duration_ms=latency_ms, status="ok")
    emitter.emit(evt)


# ── api_request_error -> error event ────────────────────────────────────────
@_safe
def _on_api_request_error(**kw: Any) -> None:
    from agent.telemetry import emitter
    from agent.telemetry.events import ErrorEvent

    session_id = kw.get("session_id") or ""
    run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "")
    error_class = _coarse_error_class(kw.get("error_type") or kw.get("error") or "")
    with _runs_lock:
        run["error_count"] += 1
    emitter.emit(ErrorEvent(
        run_id=run["run_id"],
        error_class=error_class,
        subsystem="model_api",
        recovery=None,
    ))


def _coarse_error_class(raw: Any) -> str:
    s = str(raw).lower()
    if "timeout" in s:
        return "provider_timeout"
    if "rate" in s and "limit" in s:
        return "rate_limit"
    if any(k in s for k in ("auth", "401", "403", "unauthorized", "forbidden")):
        return "auth"
    if any(k in s for k in ("connection", "network", "dns", "socket")):
        return "network"
    if "context" in s and ("length" in s or "overflow" in s or "token" in s):
        return "context_overflow"
    if any(k in s for k in ("500", "502", "503", "server error", "provider")):
        return "provider_error"
    return "unknown"


# ── post_tool_call -> tool_call event ───────────────────────────────────────
@_safe
def _on_post_tool_call(**kw: Any) -> None:
    from agent.telemetry import emitter, spans
    from agent.telemetry.events import ToolCallEvent

    session_id = kw.get("session_id") or ""
    run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "")

    function_name = kw.get("function_name") or kw.get("tool_name")
    duration_ms = kw.get("duration_ms")
    result = kw.get("result")
    result_class = _tool_result_class(result)

    with _runs_lock:
        run["tool_call_count"] += 1
        if result_class == "error":
            run["error_count"] += 1

    dur_int = int(duration_ms) if isinstance(duration_ms, (int, float)) else None
    span_id = spans.new_span_id()
    _emit_call_span(run, span_id, name=function_name or "tool_call",
                    kind="tool", duration_ms=dur_int, status=result_class)
    emitter.emit(ToolCallEvent(
        span_id=span_id,
        run_id=run["run_id"],
        tool_name=function_name,             # raw tool name
        duration_ms=dur_int,
        result_class=result_class,
    ))


def _tool_result_class(result: Any) -> str:
    """Classify a tool result without retaining content — error vs ok vs blocked."""
    try:
        import json
        if isinstance(result, str):
            r = result.strip()
            if r.startswith("{"):
                obj = json.loads(r)
                if isinstance(obj, dict):
                    if obj.get("error") or obj.get("blocked"):
                        return "blocked" if obj.get("blocked") else "error"
                    if obj.get("timeout"):
                        return "timeout"
            return "ok"
        if isinstance(result, dict):
            if result.get("error"):
                return "error"
            return "ok"
    except Exception:
        return "ok"
    return "ok"


# ── on_session_finalize -> finalize the run row ─────────────────────────────
@_safe
def _on_session_finalize(**kw: Any) -> None:
    from agent.telemetry import emitter, spans
    from agent.telemetry.events import RunEvent, SpanEvent

    session_id = kw.get("session_id") or ""
    key = _run_key(session_id, kw.get("task_id"))
    with _runs_lock:
        run = _runs.pop(key, None)
    if run is None:
        run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "")
        with _runs_lock:
            _runs.pop(key, None)

    end_ns = time.time_ns()
    start_ns = run.get("start_ns", end_ns)
    end_reason = _coarse_end_reason(kw)
    # Root span for the run — the trace root the call spans hang off of.
    emitter.emit(SpanEvent(
        span_id=run.get("root_span_id") or spans.new_span_id(),
        trace_id=run["trace_id"],
        run_id=run["run_id"],
        parent_span_id=None,
        name=f"run:{run.get('entrypoint', 'cli')}",
        kind="run",
        start_ns=start_ns,
        end_ns=end_ns,
        status=end_reason,
    ))
    emitter.emit(RunEvent(
        run_id=run["run_id"],
        trace_id=run["trace_id"],
        entrypoint=run.get("entrypoint", "cli"),
        session_id=run.get("session_id"),
        platform=run.get("platform"),
        start_ns=start_ns,
        end_ns=end_ns,
        end_reason=end_reason,
        model_call_count=run.get("model_call_count", 0),
        tool_call_count=run.get("tool_call_count", 0),
        error_count=run.get("error_count", 0),
        estimated_cost_usd=_as_float(kw.get("estimated_cost_usd")),
        cost_status=kw.get("cost_status"),
    ))
    spans.clear_run()


def _coarse_end_reason(kw: Dict[str, Any]) -> str:
    if kw.get("interrupted"):
        return "interrupted"
    if kw.get("failed"):
        return "failed"
    reason = str(kw.get("turn_exit_reason") or "").lower()
    if "max_iteration" in reason:
        return "max_iterations"
    if "timeout" in reason:
        return "timeout"
    return "completed"


def _as_float(v: Any) -> Optional[float]:
    try:
        return float(v) if v is not None else None
    except (TypeError, ValueError):
        return None


# ── subagent lineage (reserved) ─────────────────────────────────────────────
# A delegated subagent runs its own ``run_conversation`` with its own session id, so
# its model/tool calls are already captured as a separate tel_runs row via the normal
# hooks — no subagent activity is lost. These hooks fire with the parent<->child bridge
# (parent_session_id, child_session_id, child_role, child_goal); they are reserved for
# recording parent->child *lineage* (linking a child run back to its parent), which
# needs a tel_runs.parent_run_id column. Deferred until a consumer needs the delegation
# tree; left registered as the attachment point.
@_safe
def _on_subagent_start(**kw: Any) -> None:
    return None


@_safe
def _on_subagent_stop(**kw: Any) -> None:
    return None


# ── registration ────────────────────────────────────────────────────────────
def register(ctx) -> None:
    ctx.register_hook("on_session_start", _on_session_start)
    ctx.register_hook("post_api_request", _on_post_api_request)
    ctx.register_hook("api_request_error", _on_api_request_error)
    ctx.register_hook("post_tool_call", _on_post_tool_call)
    ctx.register_hook("on_session_finalize", _on_session_finalize)
    ctx.register_hook("subagent_start", _on_subagent_start)
    ctx.register_hook("subagent_stop", _on_subagent_stop)
    logger.debug("telemetry plugin registered 7 hooks")