hermes-agent/agent/monitoring/otlp_exporter.py

"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.

Maps gateway monitoring events to OTel spans and sends them to the endpoint
configured under ``monitoring.export.otlp``. Lets an operator stream Hermes
gateway health into their own observability stack (OTEL Collector, DataDog,
and similar).

Notes:
  * The destination is operator-configured; this module only sends to that
    endpoint. No default destination ships.
  * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an
    optional extra (``pip install hermes-agent[otlp]``), imported lazily so the
    dependency is only required when OTLP export is actually used.
  * ``headers_env`` maps a header name to an environment variable name; values
    are read from the environment at export time and never logged or stored.
  * The continuous subscriber runs in the emitter's dispatcher thread and is
    fail-isolated, so an export error cannot affect the gateway.

Only monitoring events (gateway_health / gateway_diagnostic) exist on this
plane; the ``event_filter`` seam is kept so future planes sharing the emitter
cannot silently ride along on this exporter.
"""

from __future__ import annotations

import logging
import os
from typing import Any, Callable, Dict, List, Optional

logger = logging.getLogger(__name__)


class OTLPUnavailable(RuntimeError):
    """Raised when the optional OpenTelemetry SDK isn't installed."""


def _require_sdk(*, auto_install: bool = True, prompt: bool = True):
    """Import the OTel SDK, lazily installing it on first use if needed.

    Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK
    triggers the standard venv install flow — same as every other optional
    backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back
    to OTLPUnavailable (with a manual install hint) when the SDK can't be made
    importable (lazy installs disabled, install failed, or auto_install=False).

    ``auto_install``: attempt the lazy install when missing (default True).
    ``prompt``: ask before installing when interactive (default True); pass
    False from non-interactive contexts like the continuous streamer.
    """
    if auto_install:
        try:
            from tools.lazy_deps import ensure as _lazy_ensure
            _lazy_ensure("export.otlp", prompt=prompt)
        except ImportError:
            pass  # lazy_deps unavailable — fall through to the import attempt
        except Exception:
            # FeatureUnavailable (lazy installs disabled / declined / failed) —
            # fall through; the import below raises OTLPUnavailable with the hint.
            pass
    try:
        from opentelemetry.sdk.trace import TracerProvider
        from opentelemetry.sdk.trace.export import BatchSpanProcessor
        from opentelemetry.sdk.resources import Resource
        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
            OTLPSpanExporter,
        )
        from opentelemetry.trace import SpanKind
        return {
            "TracerProvider": TracerProvider,
            "BatchSpanProcessor": BatchSpanProcessor,
            "Resource": Resource,
            "OTLPSpanExporter": OTLPSpanExporter,
            "SpanKind": SpanKind,
        }
    except Exception as e:  # ImportError or partial install
        raise OTLPUnavailable(
            "OTLP export requires the optional dependency. Install with:\n"
            "    pip install 'hermes-agent[otlp]'\n"
            f"(import error: {e})"
        )


def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
    """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env.

    The config stores environment variable names, not secret values; values are
    read from the environment here. Missing variables are skipped (and noted at
    debug level without the value).
    """
    resolved: Dict[str, str] = {}
    for header_name, env_name in (headers_env or {}).items():
        val = os.environ.get(str(env_name))
        if val:
            resolved[str(header_name)] = val
        else:
            logger.debug("OTLP header %s: env var %s not set; skipping",
                         header_name, env_name)
    return resolved


def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
    mon = (config or {}).get("monitoring") or {}
    export = mon.get("export") or {}
    return export.get("otlp") or {}


def build_exporter(config: Dict[str, Any]):
    """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
    sdk = _require_sdk()
    otlp = _otlp_config(config)
    endpoint = otlp.get("endpoint")
    if not endpoint:
        raise ValueError("monitoring.export.otlp.endpoint is not set")
    headers = _resolve_headers(otlp.get("headers_env"))
    return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None)


def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]:
    from agent.monitoring.gateway_health import _safe_instance_id
    from agent.monitoring.policy import ensure_install_id

    return {
        "service.name": "hermes-gateway",
        "service.instance.id": _safe_instance_id(ensure_install_id(config)),
        "telemetry.scope": "gateway_monitoring",
    }


def _make_provider(config: Dict[str, Any]):
    sdk = _require_sdk()
    resource = sdk["Resource"].create(_resource_attributes(config))
    provider = sdk["TracerProvider"](resource=resource)
    processor = sdk["BatchSpanProcessor"](build_exporter(config))
    provider.add_span_processor(processor)
    return provider, processor


# ── event -> span attribute mapping ──────────────────────────────────────────
def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
    """Span attributes for a monitoring event (content-free by construction)."""
    kind = ev.get("event")
    attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"}
    keep_by_kind = {
        "gateway_health": ("name", "gateway_state", "old_state", "new_state",
                           "exit_reason", "restart_requested", "active_agents",
                           "gateway_busy", "gateway_drainable", "platform_count",
                           "fatal_platform_count", "version",
                           "supervision_mode", "pid"),
        "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
                               "platform", "old_state", "new_state",
                               "version", "severity"),
        "cron_execution": ("status", "job_key", "source", "duration_ms",
                           "delivery_outcome", "error_class"),
    }
    for col in keep_by_kind.get(kind, ()):  # type: ignore[arg-type]
        v = ev.get(col)
        if v is not None:
            if isinstance(v, str):
                try:
                    from agent.monitoring.redaction import redact_for_export
                    v = (redact_for_export(v) or "[redacted]")[:500]
                except Exception:
                    v = "[redaction-unavailable]"
            attrs[f"hermes.{col}"] = v
    return attrs


def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
    """Map a batch of events to OTel spans. Returns spans created."""
    tracer = provider.get_tracer("hermes.monitoring")
    n = 0
    for ev in batch:
        try:
            name = f"hermes.{ev.get('event', 'event')}"
            span = tracer.start_span(name, attributes=_span_attrs(ev))
            span.end()
            n += 1
        except Exception:
            logger.debug("OTLP span map failed", exc_info=True)
    return n


# ── continuous streaming subscriber ─────────────────────────────────────────
class OTLPStreamer:
    """A live subscriber that pushes each emitter batch to OTLP as it lands.

    Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.
    """

    def __init__(
        self,
        config: Dict[str, Any],
        *,
        event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
    ):
        self._provider, self._processor = _make_provider(config)
        self._event_filter = event_filter
        self.exported = 0

    def __call__(self, batch: List[Dict[str, Any]]) -> None:
        if self._event_filter is not None:
            batch = [ev for ev in batch if self._event_filter(ev)]
        if not batch:
            return
        self.exported += export_batch(self._provider, batch)

    def shutdown(self) -> None:
        try:
            from agent.monitoring.emitter import get_emitter
            get_emitter().unsubscribe(self)
        except Exception:
            pass
        try:
            self._processor.force_flush()
            self._provider.shutdown()
        except Exception:
            pass


def is_available() -> bool:
    """True when the OTel SDK is already importable. Does NOT auto-install —
    this is a pure check (e.g. for status display)."""
    try:
        _require_sdk(auto_install=False)
        return True
    except OTLPUnavailable:
        return False


def is_enabled(config: Dict[str, Any]) -> bool:
    otlp = _otlp_config(config)
    return bool(otlp.get("enabled") and otlp.get("endpoint"))


def start_streaming(
    config: Dict[str, Any],
    *,
    event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
) -> Optional[OTLPStreamer]:
    """If OTLP is enabled, attach a streamer to the singleton emitter.

    ``event_filter`` scopes the exporter to its plane, e.g. gateway-health
    export, so enabling one plane cannot silently export unrelated events.

    Non-interactive context (startup): attempts a lazy install with prompt=False
    so a configured-but-missing SDK is installed once (gated by
    security.allow_lazy_installs), then streams. If it still can't load, logs and
    no-ops — never blocks or raises into startup.
    """
    if not is_enabled(config):
        return None
    try:
        _require_sdk(prompt=False)
    except OTLPUnavailable:
        logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
                       "be installed/imported; install 'hermes-agent[otlp]'")
        return None
    from agent.monitoring.emitter import get_emitter
    streamer = OTLPStreamer(config, event_filter=event_filter)
    get_emitter().subscribe(streamer)
    return streamer


__all__ = [
    "OTLPUnavailable",
    "OTLPStreamer",
    "build_exporter",
    "export_batch",
    "is_available",
    "is_enabled",
    "start_streaming",
]