diff --git a/Dockerfile b/Dockerfile index 5fc6b7cdeaa..6d6e9ac2d91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -227,7 +227,7 @@ RUN cd plugins/platforms/photon/sidecar && \ # frontend stats the readme path during dep resolution, so we `touch` an # empty placeholder — the real README is restored by `COPY . .` below. # -# `uv sync --frozen --no-install-project --extra all --extra messaging` +# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp` # installs the deps reachable through the composite `[all]` extra # (handpicked set intended for the production image — excludes `[dev]`), # plus gateway messaging adapters that should work in the published image @@ -240,6 +240,10 @@ RUN cd plugins/platforms/photon/sidecar && \ # so Docker users can use these providers without requiring runtime # lazy-install access to PyPI (often blocked in containerized envs). # +# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway +# Health export is enabled. Collector and observability-backend dependencies +# remain external and are not part of the Hermes production image. +# # The hindsight memory provider's client (hindsight-client) is baked in # for the same reason: it lazy-installs into /opt/hermes/.venv at first # use, which lives inside the (immutable) image layer rather than the @@ -257,7 +261,7 @@ RUN cd plugins/platforms/photon/sidecar && \ # The editable link is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md -RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix +RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix # ---------- Frontend build (cached independently from Python source) ---------- # Copy only the frontend source trees first so that Python-only changes don't diff --git a/agent/monitoring/__init__.py b/agent/monitoring/__init__.py new file mode 100644 index 00000000000..ee45ff01649 --- /dev/null +++ b/agent/monitoring/__init__.py @@ -0,0 +1,29 @@ +"""Hermes gateway monitoring. + +Service health monitoring plus redacted operational diagnostics for the +gateway daemon, exported over OTLP to an operator-configured endpoint. + +``emitter`` is the in-process event bus: producers (gateway status hooks, +the diagnostic log handler) hand typed events to a fire-and-forget queue, +and subscribers (the OTLP streamers) consume them off the hot path. The +emitter never blocks or raises into gateway code (the hot-path invariant), +and nothing is persisted locally — monitoring is an egress path, not a store. + +Deliberately out of scope here: run/model/tool trajectory capture, usage +analytics, and any content-bearing signal. Those planes are served by the +NeMo Relay integration and its Hermes-owned subscribers. +""" + +from __future__ import annotations + +from . import emitter, events + +emit = emitter.emit +get_emitter = emitter.get_emitter + +__all__ = [ + "emitter", + "events", + "emit", + "get_emitter", +] diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py new file mode 100644 index 00000000000..2ed0f1cb9b2 --- /dev/null +++ b/agent/monitoring/cron_health.py @@ -0,0 +1,201 @@ +"""Content-free cron service-health and execution telemetry projection.""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from agent.monitoring.events import CronExecutionEvent +from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric +from cron.jobs import ( + _compute_grace_seconds, + get_catch_up_occurrence_count, + get_ticker_heartbeat_age, + get_ticker_success_age, + load_jobs, +) +from cron.scheduler import get_running_job_ids +from hermes_time import now as _hermes_now + +logger = logging.getLogger(__name__) +_KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} +_KNOWN_SOURCES = {"builtin", "direct", "external"} +_KNOWN_DELIVERY_OUTCOMES = {"delivered", "failed", "suppressed", "not_configured"} + + +@dataclass(frozen=True, slots=True) +class CronHealthSnapshot: + metrics: list[GatewayMetric] + events: list[CronExecutionEvent] + + +def _now() -> datetime: + return _hermes_now() + + +def _job_key(raw: Any) -> str: + value = str(raw or "unknown").encode("utf-8", errors="replace") + return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" + + +def classify_cron_error(raw: Any) -> str: + text = str(raw or "").lower() + if ( + re.search(r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden)\b", text) + or re.search(r"\bbearer\b", text) + or re.search(r"\b(?:access|api|refresh) token\b", text) + or re.search(r"\b(?:401|403)\b", text) + ): + return "auth_failed" + if "rate limit" in text or "429" in text or "quota" in text: + return "rate_limited" + if "timeout" in text or "timed out" in text: + return "timeout" + if any(value in text for value in ("network", "connection", "dns", "socket", "unreachable")): + return "network_error" + if "dispatch" in text or "executor" in text: + return "dispatch_failed" + if "interrupt" in text or "owner exited" in text or "restarted" in text: + return "interrupted" + if "empty response" in text: + return "empty_response" + if any(value in text for value in ("config", "missing", "invalid")): + return "invalid_config" + return "unknown" + + +def _parse_time(raw: Any) -> Optional[datetime]: + try: + return datetime.fromisoformat(str(raw)) if raw else None + except (TypeError, ValueError): + return None + + +def _duration_ms(record: dict[str, Any]) -> Optional[int]: + start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at")) + finish = _parse_time(record.get("finished_at")) + if start is None or finish is None: + return None + try: + duration = int((finish - start).total_seconds() * 1000) + except (TypeError, ValueError): + return None + return max(0, duration) + + +def project_execution_event( + record: dict[str, Any], *, delivery_outcome: Optional[str] = None +) -> CronExecutionEvent: + status = str(record.get("status") or "unknown").lower() + source = str(record.get("source") or "unknown").lower() + if source not in _KNOWN_SOURCES and source != "unknown": + source = "external" + outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None + return CronExecutionEvent( + status=status if status in _KNOWN_STATUSES else "unknown", + job_key=_job_key(record.get("job_id")), + source=source if source in _KNOWN_SOURCES else "unknown", + duration_ms=_duration_ms(record), + delivery_outcome=( + outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None + ), + error_class=( + classify_cron_error(record.get("error")) + if status in {"failed", "unknown"} + else None + ), + ) + + +def emit_execution_state( + record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None +) -> None: + """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier.""" + if not record: + return + try: + from agent.monitoring import emitter + + event = project_execution_event(record, delivery_outcome=delivery_outcome) + target = emitter.get_emitter() + target.emit(event) + if event.status in {"completed", "failed", "unknown"}: + target.flush(timeout=1.0) + except Exception: + logger.debug("cron execution telemetry emit failed", exc_info=True) + + +def _is_overdue(job: dict[str, Any], now: datetime) -> bool: + if not job.get("enabled", True): + return False + next_run = _parse_time(job.get("next_run_at")) + schedule = job.get("schedule") + if next_run is None or not isinstance(schedule, dict): + return False + try: + if next_run.tzinfo is None and now.tzinfo is not None: + next_run = next_run.replace(tzinfo=now.tzinfo) + lateness = (now - next_run).total_seconds() + return lateness > _compute_grace_seconds(schedule) + except (TypeError, ValueError): + return False + + +def build_cron_health_snapshot() -> CronHealthSnapshot: + metrics: list[GatewayMetric] = [] + for name, reader in ( + ("hermes.cron.scheduler.heartbeat_age_seconds", get_ticker_heartbeat_age), + ("hermes.cron.scheduler.last_success_age_seconds", get_ticker_success_age), + ): + try: + value = reader() + if value is not None: + metrics.append(GatewayMetric(name, max(0.0, float(value)), {})) + except Exception: + logger.debug("cron freshness metric unavailable", exc_info=True) + + try: + metrics.append( + GatewayMetric( + "hermes.cron.scheduler.catch_up_occurrences", + get_catch_up_occurrence_count(), + {}, + ) + ) + except Exception: + logger.debug("cron catch-up metric unavailable", exc_info=True) + + try: + jobs = load_jobs() + enabled = [job for job in jobs if job.get("enabled", True)] + metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {})) + metrics.append( + GatewayMetric( + "hermes.cron.jobs.overdue", + sum(1 for job in enabled if _is_overdue(job, _now())), + {}, + ) + ) + except Exception: + logger.debug("cron job metrics unavailable", exc_info=True) + + try: + metrics.append( + GatewayMetric("hermes.cron.jobs.running", len(get_running_job_ids()), {}) + ) + except Exception: + logger.debug("cron running-job metric unavailable", exc_info=True) + return CronHealthSnapshot(metrics=metrics, events=[]) + + +__all__ = [ + "CronHealthSnapshot", + "build_cron_health_snapshot", + "classify_cron_error", + "emit_execution_state", + "project_execution_event", +] diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py new file mode 100644 index 00000000000..18a0d4851ae --- /dev/null +++ b/agent/monitoring/emitter.py @@ -0,0 +1,211 @@ +"""Monitoring emitter: fire-and-forget queue + background dispatcher. + +The emitter is the single seam between producers (gateway status hooks, the +diagnostic log handler) and consumers (the OTLP streamers). Its contract is +the hot-path invariant: + + ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, + and MUST NEVER raise into the caller. A monitoring failure is logged + locally and dropped — it can never affect the gateway or a session. + +Mechanism: + * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare + except. On a full queue it drops the *oldest* event and counts the drop. + * A daemon thread drains the queue and fans each batch out to subscribers + (the OTLP metric/span/log streamers). Each subscriber is fail-isolated — + a slow or raising subscriber never affects the hot path or its peers. + +Nothing is persisted here. Monitoring is an egress path, not a local store; +if no subscriber is attached, events simply age out of the ring buffer. +""" + +from __future__ import annotations + +import logging +import queue +import threading +import time +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full +_DRAIN_BATCH = 256 + + +class MonitoringEmitter: + """Owns the queue, the dispatcher thread, and the subscriber list.""" + + def __init__(self, *, enabled: bool = True) -> None: + self._enabled = enabled + self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) + self._dropped = 0 + self._dispatched = 0 + self._stop = threading.Event() + self._started = False + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + # Live subscribers (the OTLP streamers). Called from the dispatcher + # thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]). + self._subscribers: list = [] + + # ── public API (hot path) ─────────────────────────────────────────────── + def emit(self, event: Any) -> None: + """Enqueue an event. Never blocks, never raises. + + ``event`` may be a dataclass with ``to_dict()`` or a plain dict. + """ + if not self._enabled: + return + try: + payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) + payload.setdefault("ts_ns", time.time_ns()) + self._ensure_started() + try: + self._q.put_nowait(payload) + except queue.Full: + # Drop oldest to make room — bounded memory, newest-wins. + try: + self._q.get_nowait() + self._q.task_done() + self._dropped += 1 + self._q.put_nowait(payload) + except Exception: + self._dropped += 1 + except Exception: # the hot-path invariant: never propagate + logger.debug("monitoring emit failed", exc_info=True) + + # ── lifecycle ─────────────────────────────────────────────────────────── + def _ensure_started(self) -> None: + if self._started: + return + with self._lock: + if self._started: + return + self._thread = threading.Thread( + target=self._run, name="hermes-monitoring-dispatch", daemon=True + ) + self._thread.start() + self._started = True + + def _run(self) -> None: + while not self._stop.is_set(): + try: + first = self._q.get(timeout=0.5) + except queue.Empty: + continue + batch = [first] + while len(batch) < _DRAIN_BATCH: + try: + batch.append(self._q.get_nowait()) + except queue.Empty: + break + try: + self._dispatch(batch) + finally: + for _ in batch: + self._q.task_done() + + def _dispatch(self, batch) -> None: + # Fan-out to subscribers (OTLP streamers) — fully fail-isolated. + for sub in list(self._subscribers): + try: + sub(batch) + except Exception: + logger.debug("monitoring subscriber failed", exc_info=True) + self._dispatched += len(batch) + + def subscribe(self, callback) -> None: + """Register a live batch subscriber (callable(batch: list[dict])).""" + if callback not in self._subscribers: + self._subscribers.append(callback) + self._enabled = True + + def unsubscribe(self, callback) -> None: + try: + self._subscribers.remove(callback) + except ValueError: + pass + if not self._subscribers: + self._enabled = False + + # ── introspection / shutdown (tests, CLI) ─────────────────────────────── + def flush(self, timeout: float = 2.0) -> None: + """Wait boundedly for queued and in-flight batches to finish dispatch.""" + if timeout <= 0: + return + + finished = threading.Event() + + def _wait_for_completion() -> None: + self._q.join() + finished.set() + + waiter = threading.Thread( + target=_wait_for_completion, + name="hermes-monitoring-flush", + daemon=True, + ) + waiter.start() + finished.wait(timeout=timeout) + + def stats(self) -> Dict[str, int]: + return { + "queued": self._q.qsize(), + "dispatched": self._dispatched, + "dropped": self._dropped, + "subscribers": len(self._subscribers), + } + + def close(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._started = False + + +# ── process-wide singleton ────────────────────────────────────────────────── +_EMITTER: Optional[MonitoringEmitter] = None +_EMITTER_LOCK = threading.Lock() + + +def get_emitter() -> MonitoringEmitter: + """Return the process-wide monitoring emitter.""" + global _EMITTER + if _EMITTER is not None: + return _EMITTER + with _EMITTER_LOCK: + if _EMITTER is None: + # Collection is opt-in. A plane exporter enables the singleton by + # attaching its first subscriber; until then producers are no-ops. + _EMITTER = MonitoringEmitter(enabled=False) + return _EMITTER + + +def emit(event: Any) -> None: + """Module-level convenience: emit via the singleton.""" + get_emitter().emit(event) + + +def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None: + """Swap the singleton (tests only).""" + global _EMITTER + with _EMITTER_LOCK: + if _EMITTER is not None and emitter is not _EMITTER: + try: + _EMITTER.close() + except Exception: + pass + _EMITTER = emitter + + +# Back-compat alias for the salvaged class name used in emozilla's tests. +TelemetryEmitter = MonitoringEmitter + +__all__ = [ + "MonitoringEmitter", + "TelemetryEmitter", + "get_emitter", + "emit", + "reset_emitter_for_tests", +] diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py new file mode 100644 index 00000000000..a17f4a49425 --- /dev/null +++ b/agent/monitoring/events.py @@ -0,0 +1,86 @@ +"""Typed gateway monitoring events. + +Content-free service-health and redacted diagnostic events for the gateway +daemon. These are the only event shapes the monitoring plane emits: no +prompts, messages, tool args/results, session history, or usage analytics. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field, asdict +from typing import Any, Dict, Optional + + +def _now_ns() -> int: + return time.time_ns() + + +@dataclass(slots=True) +class GatewayHealthEvent: + """Content-free gateway health snapshot or lifecycle event.""" + + name: str + gateway_state: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + exit_reason: Optional[str] = None + restart_requested: Optional[bool] = None + active_agents: int = 0 + gateway_busy: bool = False + gateway_drainable: bool = False + platform_count: int = 0 + fatal_platform_count: int = 0 + profile: Optional[str] = None + install_id: Optional[str] = None + version: Optional[str] = None + supervision_mode: Optional[str] = None + pid: Optional[int] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_health", **asdict(self)} + + +@dataclass(slots=True) +class GatewayDiagnosticEvent: + """Redacted gateway diagnostic event for operator-owned observability.""" + + name: str + subsystem: str + error_class: str = "unknown" + error_code: Optional[str] = None + platform: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + profile: Optional[str] = None + version: Optional[str] = None + severity: str = "warning" + ts_ns: int = field(default_factory=_now_ns) + source_logger: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_diagnostic", **asdict(self)} + + +@dataclass(slots=True) +class CronExecutionEvent: + """Content-free durable cron execution lifecycle projection.""" + + status: str + job_key: str + source: str = "unknown" + duration_ms: Optional[int] = None + delivery_outcome: Optional[str] = None + error_class: Optional[str] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "cron_execution", **asdict(self)} + + +__all__ = [ + "GatewayHealthEvent", + "GatewayDiagnosticEvent", + "CronExecutionEvent", +] diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py new file mode 100644 index 00000000000..752473fe31b --- /dev/null +++ b/agent/monitoring/gateway_health.py @@ -0,0 +1,469 @@ +"""Gateway health and diagnostics signal producer. + +This module keeps the plane narrow: service health monitoring plus +redacted operational diagnostics. It reuses the existing gateway runtime-status +contract and emits content-free metrics/events. No prompts, messages, tool args, +session history, audit records, or product analytics belong here. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent + + +@dataclass(frozen=True, slots=True) +class GatewayMetric: + name: str + value: int | float + attributes: Dict[str, str] + + +@dataclass(frozen=True, slots=True) +class GatewayHealthSnapshot: + metrics: List[GatewayMetric] + events: List[GatewayHealthEvent | GatewayDiagnosticEvent] + + +_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"} +_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"} +_KNOWN_GATEWAY_STATES = { + "starting", "draining", "stopping", "stopped", "startup_failed", "unknown" +} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES +_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { + "connecting", "disconnected", "disabled", "paused", "retrying", "unknown" +} +_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"} +_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$") + + +def _allowed_logger(name: str) -> bool: + return name == "gateway" or name.startswith("gateway.") + + +def source_logger_for_export(name: Any) -> Optional[str]: + """Return a bounded source-controlled gateway logger name for OTLP scope.""" + value = str(name or "") + return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None + + +def redact_gateway_message(message: Any) -> str: + """Redact gateway diagnostic free text for operator-owned export. + + Single scrub path: everything goes through + ``agent.monitoring.redaction.redact_for_export`` (unconditional + secrets + PII), then is length-bounded. + """ + try: + from agent.monitoring.redaction import redact_for_export + redacted = redact_for_export(str(message or "")) or "" + except Exception: + redacted = "[redaction-unavailable]" + return redacted[:500] + + +def classify_gateway_error(raw: Any) -> str: + s = str(raw or "").lower() + if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")): + return "auth_failed" + if "rate" in s and "limit" in s: + return "rate_limited" + if "timeout" in s or "timed out" in s: + return "timeout" + if any( + k in s + for k in ( + "network", + "connection", + "dns", + "socket", + "connect call failed", + "failed to connect", + "cannot connect", + "unreachable", + "name resolution", + ) + ): + return "network_error" + if any(k in s for k in ("config", "missing", "invalid")): + return "invalid_config" + if "startup" in s: + return "startup_failed" + if "fatal" in s: + return "platform_fatal" + return "unknown" + + +def classify_exit_reason( + raw: Any, *, state: Any, restart_requested: bool +) -> Optional[str]: + """Reduce free-form shutdown text to a bounded operational class.""" + if restart_requested: + return "restart_requested" + state_name = str(state or "").lower() + if raw is None and state_name != "startup_failed": + return None + classified = classify_gateway_error(raw) + if state_name == "startup_failed": + return classified if classified != "unknown" else "startup_failed" + text = str(raw or "").lower() + if "signal" in text or "sigterm" in text or "sigint" in text: + return "signal" + if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")): + return "planned_stop" + return classified + + +def _bounded_state(raw: Any, *, allowed: set[str]) -> str: + state = str(raw or "unknown").lower() + return state if state in allowed else "unknown" + + +def _safe_metric_value(raw: Any, *, limit: int = 128) -> str: + try: + from agent.monitoring.redaction import redact_for_export + value = redact_for_export(str(raw or "")) or "unknown" + except Exception: + return "unknown" + return value[:limit] + + +def _safe_instance_id(raw: Any) -> str: + """Return a stable opaque instance key without exporting the source ID.""" + value = str(raw or "unknown").encode("utf-8", errors="replace") + return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" + + +def subsystem_for_logger(logger_name: str) -> str: + if logger_name == "gateway.relay" or logger_name.startswith("gateway.relay."): + return "platform.relay" + if logger_name.startswith("gateway.platforms."): + parts = logger_name.split(".") + if len(parts) >= 3 and parts[2]: + return f"platform.{parts[2]}" + if logger_name.startswith("gateway.platforms"): + return "platform" + if logger_name.startswith("gateway"): + return "gateway" + return "gateway" + + +def platform_for_subsystem(subsystem: str) -> Optional[str]: + if subsystem.startswith("platform."): + return subsystem.split(".", 1)[1] or None + return None + + +def _parse_active_agents(raw: Any) -> int: + try: + from gateway.status import parse_active_agents + return parse_active_agents(raw) + except Exception: + try: + return max(0, int(raw)) + except (TypeError, ValueError): + return 0 + + +def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: + try: + from gateway.status import derive_gateway_busy + return derive_gateway_busy( + gateway_running=gateway_running, + gateway_state=gateway_state, + active_agents=active_agents, + ) + except Exception: + return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) + + +def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: + try: + from gateway.status import derive_gateway_drainable + return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state) + except Exception: + return bool(gateway_running and gateway_state == "running") + + +def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: + mode = str(supervision_mode or "unknown").lower() + return { + "service.instance.id": _safe_instance_id(install_id), + "service.version": _safe_metric_value(version, limit=64), + "hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown", + } + + +def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric: + out = dict(attrs) + for key, val in extra.items(): + if val is not None: + out[key] = _safe_metric_value(val) + return GatewayMetric(name=name, value=value, attributes=out) + + +def build_gateway_health_snapshot( + runtime: Optional[dict[str, Any]], + *, + gateway_running: bool, + profile: str, + install_id: str, + version: str, + supervision_mode: str = "unknown", +) -> GatewayHealthSnapshot: + """Convert gateway_state.json-compatible runtime state into P0 signals.""" + runtime = runtime or {} + gateway_state = _bounded_state( + runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) + active_agents = _parse_active_agents(runtime.get("active_agents", 0)) + busy = _derive_busy(gateway_running, gateway_state, active_agents) + drainable = _derive_drainable(gateway_running, gateway_state) + platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {} + base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode) + + metrics: list[GatewayMetric] = [ + _metric("hermes.gateway.up", 1 if gateway_running else 0, base), + _metric("hermes.gateway.active_agents", active_agents, base), + _metric("hermes.gateway.busy", 1 if busy else 0, base), + _metric("hermes.gateway.drainable", 1 if drainable else 0, base), + _metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base), + ] + if gateway_state: + metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)})) + + fatal_count = 0 + events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + for platform, pdata in platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) + raw_error = pdata.get("error_code") or pdata.get("error_message") + error_code = classify_gateway_error(raw_error) + is_up = state in _RUNNING_PLATFORM_STATES + is_degraded = state in _FATAL_PLATFORM_STATES + if is_degraded: + fatal_count += 1 + metrics.append(_metric( + "hermes.platform.up", + 1 if is_up else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state}, + )) + metrics.append(_metric( + "hermes.platform.degraded", + 1 if is_degraded else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code}, + )) + if is_degraded: + events.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=classify_gateway_error(error_code or pdata.get("error_message")), + profile=profile, + version=version, + severity="error" if state == "fatal" else "warning", + )) + + events.insert(0, GatewayHealthEvent( + name="gateway.health_snapshot", + gateway_state=str(gateway_state) if gateway_state is not None else None, + active_agents=active_agents, + gateway_busy=busy, + gateway_drainable=drainable, + platform_count=len(platforms), + fatal_platform_count=fatal_count, + profile=profile, + install_id=install_id, + version=version, + supervision_mode=supervision_mode, + pid=_coerce_pid(runtime.get("pid")), + )) + return GatewayHealthSnapshot(metrics=metrics, events=events) + + +def _safe_profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _safe_version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None: + """Emit immediate content-free gateway events for runtime status changes. + + Called by gateway.status.write_runtime_status after persisting the new status. + Fully fail-open: failures never affect gateway status writes. + """ + try: + from agent.monitoring import emitter + out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + profile = _safe_profile() + version = _safe_version() + old_gateway_state = _bounded_state( + (previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = _bounded_state( + current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if current.get("gateway_state") is not None else None + if old_gateway_state != new_gateway_state and new_gateway_state: + out.append(GatewayHealthEvent( + name="gateway.lifecycle", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=classify_exit_reason( + current.get("exit_reason"), + state=new_gateway_state, + restart_requested=bool(current.get("restart_requested")), + ), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + if new_gateway_state == "startup_failed": + out.append(GatewayDiagnosticEvent( + name="gateway.startup_failed", + subsystem="gateway", + error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + profile=profile, + version=version, + severity="error", + )) + if new_gateway_state == "stopped": + out.append(GatewayHealthEvent( + name="gateway.exit", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=classify_exit_reason( + current.get("exit_reason"), + state=new_gateway_state, + restart_requested=bool(current.get("restart_requested")), + ), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + + old_platforms_raw = (previous or {}).get("platforms") + new_platforms_raw = current.get("platforms") + old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {} + new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {} + for platform, pdata in new_platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + prev_raw = old_platforms.get(platform, {}) + prev = prev_raw if isinstance(prev_raw, dict) else {} + old_state = _bounded_state( + prev.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if prev.get("state") is not None else None + new_state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if pdata.get("state") is not None else None + if old_state == new_state or not new_state: + continue + error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning" + out.append(GatewayDiagnosticEvent( + name="platform.state_change", + subsystem=f"platform.{platform}", + platform=str(platform), + old_state=old_state, + new_state=new_state, + error_code=error_code, + error_class=error_code, + profile=profile, + version=version, + severity=severity, + )) + if new_state.lower() in _FATAL_PLATFORM_STATES: + out.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=error_code, + profile=profile, + version=version, + severity=severity, + )) + for ev in out: + emitter.emit(ev) + except Exception: + logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True) + + +def _coerce_pid(raw: Any) -> Optional[int]: + try: + pid = int(raw) + except (TypeError, ValueError): + return None + return pid if pid > 0 else None + + +class GatewayDiagnosticLogHandler(logging.Handler): + """Allowlisted warning/error bridge for gateway-owned diagnostics.""" + + def __init__(self, *, profile: str = "default", version: str = "unknown") -> None: + super().__init__(level=logging.WARNING) + self.profile = profile + self.version = version + + def emit(self, record: logging.LogRecord) -> None: + try: + if record.levelno < logging.WARNING: + return + if not _allowed_logger(record.name): + return + subsystem = subsystem_for_logger(record.name) + message = record.getMessage() + error_class = classify_gateway_error(message) + event = GatewayDiagnosticEvent( + name=f"gateway.log.{record.levelname.lower()}", + subsystem=subsystem, + source_logger=source_logger_for_export(record.name), + platform=platform_for_subsystem(subsystem), + error_class=error_class, + error_code=error_class, + profile=self.profile, + version=self.version, + severity=record.levelname.lower(), + ) + from agent.monitoring import emitter + emitter.get_emitter().emit(event) + except Exception: + logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True) + + +__all__ = [ + "GatewayMetric", + "GatewayHealthSnapshot", + "GatewayDiagnosticLogHandler", + "build_gateway_health_snapshot", + "classify_gateway_error", + "source_logger_for_export", + "redact_gateway_message", +] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py new file mode 100644 index 00000000000..0a377c99ab3 --- /dev/null +++ b/agent/monitoring/gateway_health_export.py @@ -0,0 +1,643 @@ +"""Gateway Health & Diagnostics OTLP export runtime. + +This exporter emits operator-owned gateway service-health metrics plus +narrow redacted diagnostic events. It is deliberately in-process and fail-open so +it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell +without a sidecar/watchdog dependency. +""" + +from __future__ import annotations + +import logging +import os +import re +import threading +from dataclasses import dataclass +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_DEFAULT_DIAGNOSTIC_SCOPE = "hermes.gateway.diagnostics" + +_RESOURCE_ATTRIBUTE_KEYS = frozenset({ + "service.name", + "service.namespace", + "service.version", + "service.instance.id", + "deployment.environment.name", + "cloud.provider", + "cloud.platform", + "cloud.region", + "telemetry.scope", +}) +_DIAGNOSTIC_ATTRIBUTE_KEYS = frozenset({ + "name", + "subsystem", + "error_class", + "error_code", + "platform", + "old_state", + "new_state", + "version", + "severity", +}) +_SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$") + + +def _redact_string(raw: Any, *, limit: int = 500) -> str: + try: + from agent.monitoring.redaction import redact_for_export + return (redact_for_export(str(raw or "")) or "[redacted]")[:limit] + except Exception: + return "[redaction-unavailable]" + + +def _safe_resource_attributes(raw: Any) -> Dict[str, str]: + """Allowlist bounded resource labels and reject values changed by redaction.""" + attrs: Dict[str, str] = {} + if not isinstance(raw, dict): + return attrs + for key, value in raw.items(): + key = str(key) + if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None: + continue + if key == "service.instance.id": + from agent.monitoring.gateway_health import _safe_instance_id + attrs[key] = _safe_instance_id(value) + continue + text = str(value) + if not _SAFE_RESOURCE_VALUE.fullmatch(text): + continue + if _redact_string(text, limit=128) != text: + continue + attrs[key] = text + return attrs + + +def _runtime_resource_attributes( + config: Dict[str, Any], *, telemetry_scope: str +) -> Dict[str, str]: + """Build the safe OTLP resource shared by metrics and diagnostic logs.""" + gh = _gateway_health_config(config) + attrs = _safe_resource_attributes(gh.get("resource_attributes")) + from agent.monitoring.gateway_health import _safe_instance_id + + attrs["service.name"] = "hermes-gateway" + attrs["service.instance.id"] = _safe_instance_id(_install_id(config)) + attrs["telemetry.scope"] = telemetry_scope + return attrs + + +def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]: + attrs: Dict[str, Any] = {} + for key in _DIAGNOSTIC_ATTRIBUTE_KEYS: + value = event.get(key) + if value is None: + continue + attrs[f"hermes.{key}"] = _redact_string(value) if isinstance(value, str) else value + return attrs + + +@dataclass(slots=True) +class GatewayHealthExportRuntime: + enabled: bool + reason: str = "disabled" + streamer: Any = None + metric_provider: Any = None + log_handler: Any = None + log_streamer: Any = None + thread: Optional[threading.Thread] = None + stop_event: Optional[threading.Event] = None + + def shutdown(self) -> None: + if self.stop_event is not None: + self.stop_event.set() + if self.thread is not None: + self.thread.join(timeout=0.25) + if self.log_handler is not None: + try: + logging.getLogger().removeHandler(self.log_handler) + except Exception: + pass + + # All producers above are now stopped. Drain queued and in-flight + # events before detaching subscribers so the terminal lifecycle event + # cannot race exporter shutdown. The barrier is bounded and fail-open. + try: + from agent.monitoring.emitter import get_emitter + emitter = get_emitter() + emitter.flush(timeout=1.0) + if self.streamer is not None: + emitter.unsubscribe(self.streamer) + if self.log_streamer is not None: + emitter.unsubscribe(self.log_streamer) + except Exception: + pass + + # Network flush/close runs under one bounded daemon-thread deadline and + # can never delay gateway teardown indefinitely. + closeables = [ + item for item in (self.streamer, self.log_streamer, self.metric_provider) + if item is not None + ] + + def _close() -> None: + for item in closeables: + try: + item.shutdown() + except Exception: + pass + + if closeables: + worker = threading.Thread( + target=_close, + name="hermes-gateway-health-export-shutdown", + daemon=True, + ) + worker.start() + worker.join(timeout=2.0) + + self.streamer = None + self.log_streamer = None + self.metric_provider = None + self.thread = None + self.stop_event = None + + +def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + return mon.get("gateway_health_export") or {} + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} + return export.get("otlp") or {} + + +def _enabled(config: Dict[str, Any]) -> bool: + gh = _gateway_health_config(config) + otlp = _otlp_config(config) + return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint")) + + +def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]: + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except Exception: + pass + try: + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.metrics import Observation + from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags + from opentelemetry._logs import LogRecord + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + return { + "OTLPLogExporter": OTLPLogExporter, + "OTLPMetricExporter": OTLPMetricExporter, + "Observation": Observation, + "LogRecord": LogRecord, + "LoggerProvider": LoggerProvider, + "INVALID_SPAN_ID": INVALID_SPAN_ID, + "INVALID_TRACE_ID": INVALID_TRACE_ID, + "TraceFlags": TraceFlags, + "SeverityNumber": SeverityNumber, + "BatchLogRecordProcessor": BatchLogRecordProcessor, + "MeterProvider": MeterProvider, + "PeriodicExportingMetricReader": PeriodicExportingMetricReader, + "Resource": Resource, + } + except Exception as exc: + raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + return resolved + + +def _metric_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/metrics" + return endpoint + + +def _logs_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/logs" + if endpoint.endswith("/v1/metrics"): + return endpoint[: -len("/v1/metrics")] + "/v1/logs" + return endpoint + + +def _version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def _profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _install_id(config: Dict[str, Any]) -> str: + try: + from agent.monitoring.policy import ensure_install_id + return str(ensure_install_id(config)) + except Exception: + return "unknown" + + +def _supervision_mode() -> str: + if os.environ.get("INVOCATION_ID"): + return "systemd" + if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"): + return "s6" + if os.environ.get("container") or os.path.exists("/.dockerenv"): + return "container" + if os.environ.get("LAUNCHD_SOCKET"): + return "launchd" + return "manual" + + +def _read_gateway_snapshot(config: Dict[str, Any]): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + try: + from gateway.status import read_runtime_status + runtime = read_runtime_status() or {} + except Exception: + runtime = {} + return build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile=_profile(), + install_id=_install_id(config), + version=_version(), + supervision_mode=_supervision_mode(), + ) + + +def _read_cron_snapshot(): + from agent.monitoring.cron_health import build_cron_health_snapshot + + return build_cron_health_snapshot() + + +def _read_background_work_count() -> int: + """Count live background/subagent work that ``active_agents`` does NOT include. + + ``hermes.gateway.active_agents`` counts foreground turns + in-flight cron + jobs + API runs, but deliberately excludes backgrounded ``delegate_task`` + subagents, ``terminal(background=true)`` processes, kanban workers, and the + runner's own background tasks (they are tracked only for the scale-to-zero + suspend guard, ``_scale_to_zero_has_live_background_work``). Without this + metric a peer churning through delegated subagents shows ``active_agents=0`` + on the fleet dashboard. Best-effort and content-free: a single integer, + no job/task identity. Returns 0 if a source can't be imported. + + Delegation is counted TASK-granular (``active_task_count``): a fan-out batch + of N subagents contributes N, not 1, so the metric reflects real concurrent + subagent load rather than dispatch-unit/pool-slot count. This intentionally + differs from the async pool's capacity accounting (one batch = one slot). + """ + total = 0 + try: + from tools.async_delegation import active_task_count + + total += max(0, int(active_task_count())) + except Exception: + logger.debug("background-work async-delegation count failed", exc_info=True) + try: + from tools.process_registry import process_registry + + total += max(0, int(process_registry.count_running())) + except Exception: + logger.debug("background-work process-registry count failed", exc_info=True) + return total + + +def _read_background_delegations_count() -> int: + """Count live async delegation UNITS (dispatch/pool slots). + + Complements ``_read_background_work_count`` (which is task-granular): this + counts each ``delegate_task`` dispatch as ONE regardless of fan-out width, + matching the async pool's capacity accounting (a batch = one slot). Together + the two metrics let an operator see both slot pressure + (``background_delegations``, alert vs ``max_concurrent_children``) and real + concurrent subagent load (``background_work``). Delegations only — it does + not include ``terminal(background)`` / kanban work, which are already folded + into ``background_work``. Best-effort; 0 if the source can't be imported. + """ + try: + from tools.async_delegation import active_count + + return max(0, int(active_count())) + except Exception: + logger.debug("background-delegations count failed", exc_info=True) + return 0 + + +def _read_runtime_snapshot(config: Dict[str, Any]): + gateway_snapshot = _read_gateway_snapshot(config) + # Background/subagent work — a distinct metric from active_agents (which + # never counts it). Appended to the gateway snapshot so it rides the same + # base resource attributes (service.instance.id etc.). + try: + from agent.monitoring.gateway_health import GatewayMetric + + base = dict(gateway_snapshot.metrics[0].attributes) if gateway_snapshot.metrics else {} + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_work", + value=_read_background_work_count(), + attributes=base, + ) + ) + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_delegations", + value=_read_background_delegations_count(), + attributes=base, + ) + ) + except Exception as exc: + logger.warning( + "background-work snapshot unavailable; metric not exported (error_type=%s)", + type(exc).__name__, + ) + logger.debug("background-work snapshot traceback", exc_info=True) + try: + cron_snapshot = _read_cron_snapshot() + except Exception as exc: + # Content-free visibility: cron telemetry silently dropping out is a + # release-relevant regression, so surface it at WARNING with only the + # exception *type* name (never the message, which could carry paths or + # other environment detail). exc_info stays on the DEBUG record. + logger.warning( + "cron health snapshot unavailable; cron telemetry not exported (error_type=%s)", + type(exc).__name__, + ) + logger.debug("cron health snapshot traceback", exc_info=True) + return gateway_snapshot + gateway_snapshot.metrics.extend(cron_snapshot.metrics) + return gateway_snapshot + + +def _emit_snapshot_events(config: Dict[str, Any]) -> None: + gh = _gateway_health_config(config) + if not gh.get("diagnostic_events_enabled", True): + return + try: + from agent.monitoring import emitter + snapshot = _read_runtime_snapshot(config) + for event in snapshot.events: + emitter.emit(event) + except Exception: + logger.debug("gateway health snapshot emit failed", exc_info=True) + + +def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("metrics_enabled", True): + return None + otlp = _otlp_config(config) + endpoint = _metric_endpoint(str(otlp.get("endpoint"))) + headers = _resolve_headers(otlp.get("headers_env")) + exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None) + interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 + reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) + resource_attrs = _runtime_resource_attributes( + config, telemetry_scope="gateway_health" + ) + provider = sdk["MeterProvider"]( + metric_readers=[reader], + resource=sdk["Resource"].create(resource_attrs), + ) + meter = provider.get_meter("hermes.gateway.health") + Observation = sdk["Observation"] + + metric_names = [ + "hermes.gateway.up", + "hermes.gateway.state", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.gateway.background_work", + "hermes.gateway.background_delegations", + "hermes.platform.up", + "hermes.platform.degraded", + "hermes.cron.scheduler.heartbeat_age_seconds", + "hermes.cron.scheduler.last_success_age_seconds", + "hermes.cron.scheduler.catch_up_occurrences", + "hermes.cron.jobs.enabled", + "hermes.cron.jobs.running", + "hermes.cron.jobs.overdue", + ] + + def callback(name: str): + def _cb(_options=None): + try: + snapshot = _read_runtime_snapshot(config) + return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name] + except Exception: + logger.debug("gateway metric callback failed", exc_info=True) + return [] + return _cb + + for metric_name in metric_names: + meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)]) + return provider + + +def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any: + SeverityNumber = sdk["SeverityNumber"] + sev = str(severity or "warning").lower() + if sev in {"critical", "fatal"}: + return SeverityNumber.FATAL + if sev == "error": + return SeverityNumber.ERROR + if sev in {"info", "information"}: + return SeverityNumber.INFO + if sev == "debug": + return SeverityNumber.DEBUG + return SeverityNumber.WARN + + +class GatewayDiagnosticLogStreamer: + """Emitter subscriber that sends gateway diagnostic events as OTLP logs.""" + + def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]): + otlp = _otlp_config(config) + headers = _resolve_headers(otlp.get("headers_env")) + endpoint = _logs_endpoint(str(otlp.get("endpoint"))) + resource_attrs = _runtime_resource_attributes( + config, telemetry_scope="gateway_diagnostics" + ) + self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs)) + self._processor = sdk["BatchLogRecordProcessor"]( + sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) + ) + self._provider.add_log_record_processor(self._processor) + self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE) + self._LogRecord = sdk["LogRecord"] + self._sdk = sdk + self.exported = 0 + + def __call__(self, batch: list[Dict[str, Any]]) -> None: + from agent.monitoring.gateway_health import source_logger_for_export + + for ev in batch: + if ev.get("event") != "gateway_diagnostic": + continue + attrs = _diagnostic_log_attributes(ev) + # Preserve the source-controlled Python logger as the OTel + # instrumentation scope. This adds precise code attribution without + # turning a fluid module layout into a maintained subsystem enum. + # Rendered messages stay out because they may contain arbitrary IDs, + # names, paths, or configured strings. A future, separately gated + # ``diagnostic_detail: redacted_message`` mode may add best-effort + # free text when an observability plane defines that privacy policy. + source_logger = source_logger_for_export(ev.get("source_logger")) + otel_logger = ( + self._provider.get_logger(source_logger) + if source_logger is not None + else self._logger + ) + body = "gateway diagnostic" + record = self._LogRecord( + timestamp=ev.get("ts_ns"), + trace_id=self._sdk["INVALID_TRACE_ID"], + span_id=self._sdk["INVALID_SPAN_ID"], + trace_flags=self._sdk["TraceFlags"].DEFAULT, + severity_text=str(ev.get("severity") or "warning").upper(), + severity_number=_severity_number(self._sdk, ev.get("severity")), + body=_redact_string(body), + attributes=attrs, + ) + otel_logger.emit(record) + self.exported += 1 + + def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer: + from agent.monitoring.emitter import get_emitter + streamer = GatewayDiagnosticLogStreamer(config, sdk) + get_emitter().subscribe(streamer) + return streamer + + +def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread: + interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5))) + + def _run() -> None: + while not stop_event.wait(interval): + _emit_snapshot_events(config) + + thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True) + thread.start() + return thread + + +def _attach_log_handler(config: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("diagnostic_events_enabled", True) or not gh.get("warning_error_events_enabled", True): + return None + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version()) + root = logging.getLogger() + if handler not in root.handlers: + root.addHandler(handler) + return handler + + +def _gateway_health_event(ev: Dict[str, Any]) -> bool: + return ev.get("event") in {"gateway_health", "cron_execution"} + + +def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime: + """Start P0 gateway health export if configured. Never raises.""" + if not _enabled(config): + return GatewayHealthExportRuntime(enabled=False, reason="disabled") + gh = _gateway_health_config(config) + runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled") + sdk: Optional[Dict[str, Any]] = None + + if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True): + try: + sdk = _require_metrics_sdk(prompt=False) + except Exception: + logger.warning( + "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; " + "install 'hermes-agent[otlp]'", + exc_info=True, + ) + return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable") + + if gh.get("metrics_enabled", True) and sdk is not None: + try: + runtime.metric_provider = _start_metric_provider(config, sdk) + except Exception: + logger.warning("gateway health OTLP metrics failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed") + + if gh.get("diagnostic_events_enabled", True) and sdk is not None: + try: + from agent.monitoring import otlp_exporter + runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event) + if runtime.streamer is None: + raise RuntimeError("gateway health span streamer did not start") + runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk) + except Exception: + logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed") + + try: + runtime.log_handler = _attach_log_handler(config) + except Exception: + logger.debug("gateway diagnostic log handler failed to attach", exc_info=True) + if gh.get("diagnostic_events_enabled", True): + try: + _emit_snapshot_events(config) + runtime.stop_event = threading.Event() + runtime.thread = _start_snapshot_thread(config, runtime.stop_event) + except Exception: + logger.debug("gateway health snapshot thread failed to start", exc_info=True) + return runtime + + +__all__ = [ + "GatewayHealthExportRuntime", + "start_gateway_health_export", +] diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py new file mode 100644 index 00000000000..cde95673ec5 --- /dev/null +++ b/agent/monitoring/otlp_exporter.py @@ -0,0 +1,272 @@ +"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. + +Maps gateway monitoring events to OTel spans and sends them to the endpoint +configured under ``monitoring.export.otlp``. Lets an operator stream Hermes +gateway health into their own observability stack (OTEL Collector, DataDog, +and similar). + +Notes: + * The destination is operator-configured; this module only sends to that + endpoint. No default destination ships. + * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an + optional extra (``pip install hermes-agent[otlp]``), imported lazily so the + dependency is only required when OTLP export is actually used. + * ``headers_env`` maps a header name to an environment variable name; values + are read from the environment at export time and never logged or stored. + * The continuous subscriber runs in the emitter's dispatcher thread and is + fail-isolated, so an export error cannot affect the gateway. + +Only monitoring events (gateway_health / gateway_diagnostic) exist on this +plane; the ``event_filter`` seam is kept so future planes sharing the emitter +cannot silently ride along on this exporter. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Callable, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class OTLPUnavailable(RuntimeError): + """Raised when the optional OpenTelemetry SDK isn't installed.""" + + +def _require_sdk(*, auto_install: bool = True, prompt: bool = True): + """Import the OTel SDK, lazily installing it on first use if needed. + + Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK + triggers the standard venv install flow — same as every other optional + backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back + to OTLPUnavailable (with a manual install hint) when the SDK can't be made + importable (lazy installs disabled, install failed, or auto_install=False). + + ``auto_install``: attempt the lazy install when missing (default True). + ``prompt``: ask before installing when interactive (default True); pass + False from non-interactive contexts like the continuous streamer. + """ + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except ImportError: + pass # lazy_deps unavailable — fall through to the import attempt + except Exception: + # FeatureUnavailable (lazy installs disabled / declined / failed) — + # fall through; the import below raises OTLPUnavailable with the hint. + pass + try: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.resources import Resource + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.trace import SpanKind + return { + "TracerProvider": TracerProvider, + "BatchSpanProcessor": BatchSpanProcessor, + "Resource": Resource, + "OTLPSpanExporter": OTLPSpanExporter, + "SpanKind": SpanKind, + } + except Exception as e: # ImportError or partial install + raise OTLPUnavailable( + "OTLP export requires the optional dependency. Install with:\n" + " pip install 'hermes-agent[otlp]'\n" + f"(import error: {e})" + ) + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env. + + The config stores environment variable names, not secret values; values are + read from the environment here. Missing variables are skipped (and noted at + debug level without the value). + """ + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + else: + logger.debug("OTLP header %s: env var %s not set; skipping", + header_name, env_name) + return resolved + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} + return export.get("otlp") or {} + + +def build_exporter(config: Dict[str, Any]): + """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK.""" + sdk = _require_sdk() + otlp = _otlp_config(config) + endpoint = otlp.get("endpoint") + if not endpoint: + raise ValueError("monitoring.export.otlp.endpoint is not set") + headers = _resolve_headers(otlp.get("headers_env")) + return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) + + +def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]: + from agent.monitoring.gateway_health import _safe_instance_id + from agent.monitoring.policy import ensure_install_id + + return { + "service.name": "hermes-gateway", + "service.instance.id": _safe_instance_id(ensure_install_id(config)), + "telemetry.scope": "gateway_monitoring", + } + + +def _make_provider(config: Dict[str, Any]): + sdk = _require_sdk() + resource = sdk["Resource"].create(_resource_attributes(config)) + provider = sdk["TracerProvider"](resource=resource) + processor = sdk["BatchSpanProcessor"](build_exporter(config)) + provider.add_span_processor(processor) + return provider, processor + + +# ── event -> span attribute mapping ────────────────────────────────────────── +def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: + """Span attributes for a monitoring event (content-free by construction).""" + kind = ev.get("event") + attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} + keep_by_kind = { + "gateway_health": ("name", "gateway_state", "old_state", "new_state", + "exit_reason", "restart_requested", "active_agents", + "gateway_busy", "gateway_drainable", "platform_count", + "fatal_platform_count", "version", + "supervision_mode", "pid"), + "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", + "platform", "old_state", "new_state", + "version", "severity"), + "cron_execution": ("status", "job_key", "source", "duration_ms", + "delivery_outcome", "error_class"), + } + for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] + v = ev.get(col) + if v is not None: + if isinstance(v, str): + try: + from agent.monitoring.redaction import redact_for_export + v = (redact_for_export(v) or "[redacted]")[:500] + except Exception: + v = "[redaction-unavailable]" + attrs[f"hermes.{col}"] = v + return attrs + + +def export_batch(provider, batch: List[Dict[str, Any]]) -> int: + """Map a batch of events to OTel spans. Returns spans created.""" + tracer = provider.get_tracer("hermes.monitoring") + n = 0 + for ev in batch: + try: + name = f"hermes.{ev.get('event', 'event')}" + span = tracer.start_span(name, attributes=_span_attrs(ev)) + span.end() + n += 1 + except Exception: + logger.debug("OTLP span map failed", exc_info=True) + return n + + +# ── continuous streaming subscriber ───────────────────────────────────────── +class OTLPStreamer: + """A live subscriber that pushes each emitter batch to OTLP as it lands. + + Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter. + """ + + def __init__( + self, + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, + ): + self._provider, self._processor = _make_provider(config) + self._event_filter = event_filter + self.exported = 0 + + def __call__(self, batch: List[Dict[str, Any]]) -> None: + if self._event_filter is not None: + batch = [ev for ev in batch if self._event_filter(ev)] + if not batch: + return + self.exported += export_batch(self._provider, batch) + + def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def is_available() -> bool: + """True when the OTel SDK is already importable. Does NOT auto-install — + this is a pure check (e.g. for status display).""" + try: + _require_sdk(auto_install=False) + return True + except OTLPUnavailable: + return False + + +def is_enabled(config: Dict[str, Any]) -> bool: + otlp = _otlp_config(config) + return bool(otlp.get("enabled") and otlp.get("endpoint")) + + +def start_streaming( + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, +) -> Optional[OTLPStreamer]: + """If OTLP is enabled, attach a streamer to the singleton emitter. + + ``event_filter`` scopes the exporter to its plane, e.g. gateway-health + export, so enabling one plane cannot silently export unrelated events. + + Non-interactive context (startup): attempts a lazy install with prompt=False + so a configured-but-missing SDK is installed once (gated by + security.allow_lazy_installs), then streams. If it still can't load, logs and + no-ops — never blocks or raises into startup. + """ + if not is_enabled(config): + return None + try: + _require_sdk(prompt=False) + except OTLPUnavailable: + logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not " + "be installed/imported; install 'hermes-agent[otlp]'") + return None + from agent.monitoring.emitter import get_emitter + streamer = OTLPStreamer(config, event_filter=event_filter) + get_emitter().subscribe(streamer) + return streamer + + +__all__ = [ + "OTLPUnavailable", + "OTLPStreamer", + "build_exporter", + "export_batch", + "is_available", + "is_enabled", + "start_streaming", +] diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py new file mode 100644 index 00000000000..1a42ce195ca --- /dev/null +++ b/agent/monitoring/policy.py @@ -0,0 +1,57 @@ +"""Install identity for gateway monitoring. + +The install id is a stable, resettable pseudonymous identifier attached to +exported health signals so an operator can tell instances apart in their +collector. It carries no account identity and can be rotated by clearing +``monitoring.install_id`` in config. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any, Dict + +logger = logging.getLogger(__name__) + + +def ensure_install_id(config: Dict[str, Any]) -> str: + """Return a stable install id, minting and persisting one when empty. + + The id must survive gateway restarts (it becomes ``service.instance.id`` + on exported signals), so a freshly minted UUID is written back to + config.yaml immediately. The write is fail-open: if persisting fails + (read-only home, managed scope), the ephemeral id is still returned and + a new one is minted next start. + + Clearing ``monitoring.install_id`` (e.g. ``hermes config set + monitoring.install_id ""``) rotates the id on the next gateway start. + """ + mon = config.get("monitoring") if isinstance(config, dict) else None + existing = (mon or {}).get("install_id") if isinstance(mon, dict) else None + if isinstance(existing, str) and existing.strip(): + return existing + + minted = str(uuid.uuid4()) + try: + from hermes_cli.config import load_config, save_config + + fresh = load_config() + if isinstance(fresh, dict): + slot = fresh.setdefault("monitoring", {}) + if isinstance(slot, dict) and not str(slot.get("install_id") or "").strip(): + slot["install_id"] = minted + save_config(fresh) + except Exception: + logger.debug("install_id persist failed; using ephemeral id", exc_info=True) + # Keep the in-memory config consistent for this process either way. + if isinstance(config, dict): + config.setdefault("monitoring", {}) + if isinstance(config["monitoring"], dict): + config["monitoring"]["install_id"] = minted + return minted + + +__all__ = [ + "ensure_install_id", +] diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py new file mode 100644 index 00000000000..312875901e6 --- /dev/null +++ b/agent/monitoring/redaction.py @@ -0,0 +1,71 @@ +"""Redaction applied to monitoring data before egress. + +One unconditional scrub, no modes, no knobs. Every string that leaves the +process passes through ``redact_for_export``: + + * Secrets first — wraps ``agent/redact.py::redact_sensitive_text(force=True)`` + plus bearer/token-shape patterns, and fails CLOSED: if the redactor cannot + run, the raw string is never emitted. + * PII second — e-mail addresses, phone numbers, and UUID-shaped identifiers + are rewritten to ``[email]`` / ``[phone]`` / ``[id]``. + +There is deliberately no setting to weaken this. The monitoring plane is +content-free by design: rendered log messages are not exported, and bounded +structured strings are still scrubbed as defense-in-depth. This redactor also +remains available for a future, explicitly gated redacted-message detail mode. +""" + +from __future__ import annotations + +import re +from typing import Optional + +# ── secret shapes (belt-and-suspenders on top of agent/redact.py) ─────────── +_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) +_TOKEN_RE = re.compile( + r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b" +) +_SECRET_LITERAL_RE = re.compile(r"\*{3,}") +_BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+\[[^\]]+\]", re.IGNORECASE) + +# ── PII shapes ─────────────────────────────────────────────────────────────── +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +# E.164-ish and common separators; conservative to avoid nuking code/IDs. +_PHONE_RE = re.compile( + r"(? str: + """Always-on secret redaction. force=True so user config can't disable it.""" + try: + from agent.redact import redact_sensitive_text + out = redact_sensitive_text(text, force=True) + except Exception: + # Fail CLOSED: if the redactor can't run, do not emit the raw string. + return "[redaction-unavailable]" + out = _BEARER_RE.sub("[redacted]", out) + out = _TOKEN_RE.sub("[redacted]", out) + out = _SECRET_LITERAL_RE.sub("[redacted]", out) + out = _BEARER_RESIDUE_RE.sub("[redacted]", out) + return out + + +def redact_for_export(text: Optional[str]) -> Optional[str]: + """Scrub a string for egress: secrets, then PII. Unconditional.""" + if text is None: + return None + out = _secret_redact(str(text)) + out = _EMAIL_RE.sub("[email]", out) + out = _UUID_RE.sub("[id]", out) + out = _PHONE_RE.sub("[phone]", out) + return out + + +__all__ = [ + "redact_for_export", +] diff --git a/contributors/emails/victor@nousresearch.com b/contributors/emails/victor@nousresearch.com new file mode 100644 index 00000000000..a899ee021d1 --- /dev/null +++ b/contributors/emails/victor@nousresearch.com @@ -0,0 +1,2 @@ +victor-kyriazakos +# PR #64536 Gateway Health OTLP production hardening diff --git a/cron/executions.py b/cron/executions.py index fac24357335..4e9b0c7ed78 100644 --- a/cron/executions.py +++ b/cron/executions.py @@ -89,6 +89,18 @@ def _record(row: Optional[sqlite3.Row]) -> Optional[Dict[str, Any]]: return dict(row) if row is not None else None +def _emit_execution_state( + record: Optional[Dict[str, Any]], *, delivery_outcome: Optional[str] = None +) -> None: + """Project durable state to monitoring without affecting ledger behavior.""" + try: + from agent.monitoring.cron_health import emit_execution_state + + emit_execution_state(record, delivery_outcome=delivery_outcome) + except Exception: + pass + + def _process_start_time(pid: int) -> Optional[int]: try: from gateway.status import get_process_start_time @@ -139,7 +151,9 @@ def create_execution(job_id: str, *, source: str) -> Dict[str, Any]: row = conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) ).fetchone() - return _record(row) # type: ignore[return-value] + record = _record(row) + _emit_execution_state(record) + return record # type: ignore[return-value] def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]: @@ -153,13 +167,16 @@ def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]: ) if cur.rowcount != 1: return None - return _record(conn.execute( + record = _record(conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) ).fetchone()) + _emit_execution_state(record) + return record def finish_execution( execution_id: str, *, success: bool, error: Optional[str] = None, + delivery_outcome: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """Write a terminal result once; terminal attempts cannot be rewritten.""" now = _hermes_now().isoformat() @@ -174,15 +191,18 @@ def finish_execution( if cur.rowcount != 1: return None _prune_unlocked(conn) - return _record(conn.execute( + record = _record(conn.execute( "SELECT * FROM executions WHERE id=?", (execution_id,) ).fetchone()) + _emit_execution_state(record, delivery_outcome=delivery_outcome) + return record def recover_interrupted_executions() -> int: """Mark provably abandoned attempts unknown without scheduling retries.""" now = _hermes_now().isoformat() changed = 0 + recovered: List[Dict[str, Any]] = [] with _transaction() as conn: rows = conn.execute( """SELECT id, process_id, pid, process_started_at FROM executions @@ -202,8 +222,16 @@ def recover_interrupted_executions() -> int: row["id"]), ) changed += cur.rowcount + if cur.rowcount: + record = _record(conn.execute( + "SELECT * FROM executions WHERE id=?", (row["id"],) + ).fetchone()) + if record is not None: + recovered.append(record) if changed: _prune_unlocked(conn) + for record in recovered: + _emit_execution_state(record) return changed diff --git a/cron/jobs.py b/cron/jobs.py index 57c77dc16a0..9b7f2f4befb 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -844,6 +844,24 @@ def _atomic_write_epoch(path: Path) -> None: raise +def _atomic_write_counter(path: Path, value: int) -> None: + """Atomically persist a non-negative integer counter.""" + ensure_dirs() + fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".count_") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(str(max(0, value))) + f.flush() + os.fsync(f.fileno()) + atomic_replace(tmp_path, path) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + def record_ticker_heartbeat(success: bool = False) -> None: """Record a ticker liveness signal, and optionally a successful-tick signal. @@ -905,6 +923,19 @@ def get_ticker_success_age() -> Optional[float]: return _epoch_file_age(store.cron_dir / "ticker_last_success") +def record_catch_up_occurrence() -> None: + """Increment the profile-local stale-schedule catch-up counter, best effort.""" + path = _current_cron_store().cron_dir / "catch_up_occurrences" + try: + try: + value = int(path.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + value = 0 + _atomic_write_counter(path, max(0, value) + 1) + except Exception: + pass + + def record_ticker_error(message: str) -> None: """Persist the most recent tick failure so other processes can surface it. @@ -940,6 +971,15 @@ def record_ticker_error(message: str) -> None: pass +def get_catch_up_occurrence_count() -> int: + """Return the profile-local stale-schedule catch-up count.""" + path = _current_cron_store().cron_dir / "catch_up_occurrences" + try: + return max(0, int(path.read_text(encoding="utf-8").strip())) + except (OSError, ValueError): + return 0 + + def clear_ticker_error() -> None: """Remove the last-tick-error marker after a successful tick. Best-effort.""" store = _current_cron_store() @@ -2252,6 +2292,7 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: rj["next_run_at"] = new_next needs_save = True break + record_catch_up_occurrence() # Fall through to due.append(job) — execute once now # One-shot dispatch-limit guard (issue #38758): a finite one-shot diff --git a/cron/scheduler.py b/cron/scheduler.py index dfcef0ec724..2d8b63d35b6 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -3993,6 +3993,7 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - # responses: do not deliver a blank message, and let the # empty-response guard below mark the run as a soft failure. should_deliver = bool(deliver_content.strip()) + unresolved_origin = False # Cron silence suppression — see _is_cron_silence_response. Replaces the # old `SILENT_MARKER in ...upper()` substring check, which both leaked # bracketless near-markers ("SILENT" / "NO_REPLY") and wrongly swallowed @@ -4004,6 +4005,10 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - should_deliver = False if should_deliver: + unresolved_origin = ( + _normalize_deliver_value(job.get("deliver", "local")) == "origin" + and not _resolve_delivery_targets(job) + ) try: delivery_error = _deliver_result(job, deliver_content, adapters=adapters, loop=loop) except Exception as de: @@ -4025,7 +4030,21 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) - if not _consume_interrupted_flag(job["id"]): mark_job_run(job["id"], success, error, delivery_error=delivery_error) - finish_execution(execution_id, success=success, error=error) + normalized_deliver = _normalize_deliver_value(job.get("deliver", "local")) + if delivery_error: + delivery_outcome = "failed" + elif should_deliver and unresolved_origin: + delivery_outcome = "not_configured" + elif should_deliver and normalized_deliver != "local": + delivery_outcome = "delivered" + else: + delivery_outcome = "suppressed" + finish_execution( + execution_id, + success=success, + error=error, + delivery_outcome=delivery_outcome, + ) return True except BaseException as e: # noqa: BLE001 — deliberate: see below diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md new file mode 100644 index 00000000000..f798bf2dbe6 --- /dev/null +++ b/docs/observability/monitoring.md @@ -0,0 +1,304 @@ +# Gateway Monitoring + +Service health monitoring plus structured operational diagnostics for the +Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured +endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver). + +This plane is content-free by construction. It exports gateway and cron +lifecycle state, platform connector health, and content-free warning/error +diagnostics. It never exports prompts, messages, tool arguments or results, +job names, destinations, schedules, raw errors, session history, usage +analytics, audit logs, or detailed execution traces. Run/model/tool trajectory +capture is a separate plane served by the NeMo Relay integration +(`plugins/observability/nemo_relay/`) and its Hermes-owned subscribers. + +## What gets exported + +| Signal | OTLP route | Content | +| --- | --- | --- | +| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/background_delegations/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | +| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | +| Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported | +| Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule | +| Cron execution lifecycle | `/v1/traces` | Durable `claimed/running/completed/failed/unknown` states, bounded source and error class, opaque hashed job key, elapsed duration when timestamps exist, and delivery outcome when the scheduler knows it; terminal states make a fail-open flush attempt that can delay completion by up to one second | + +Signals carry `service.name`, version, supervision mode, and a stable one-way +hash of the install id so an operator can distinguish instances without +exporting account/profile identity or the raw install identifier. + +`hermes.gateway.active_agents`, `hermes.gateway.background_work`, and +`hermes.gateway.background_delegations` are complementary. `active_agents` +counts foreground message turns plus in-flight cron jobs plus API runs — the +work the gateway drains on shutdown. `background_work` counts detached work that +`active_agents` never includes: backgrounded `delegate_task` subagents, +`terminal(background=true)` processes, and kanban workers; it is +**task-granular** — a fan-out batch of N subagents counts as N — so it reflects +real concurrent subagent load. `background_delegations` counts only async +delegation **units** (each `delegate_task` dispatch is one, a fan-out batch is +one), matching the async pool's capacity accounting; alert it against +`delegation.max_concurrent_children` to see slot pressure. Sum `active_agents` +and `background_work` for total live work per instance; use +`background_delegations` for pool-saturation. + +## Enabling + +```yaml +# config.yaml +monitoring: + gateway_health_export: + enabled: true + export: + otlp: + enabled: true + endpoint: http://collector-host:4318/v1/traces # metrics/logs derive + headers_env: {} # header name -> ENV VAR NAME (values never stored) +``` + +Check the posture any time: + +```bash +hermes monitoring status +``` + +The OpenTelemetry SDK is an optional extra (`pip install 'hermes-agent[otlp]'`), +lazy-installed on first use. When the SDK is missing or the endpoint is down, +the gateway runs unaffected: metric collection and ordinary event export stay +off the hot path, while terminal cron events make one bounded fail-open flush +attempt of up to one second so the final state is less likely to be lost. + +Works identically under systemd/launchd/s6 supervision, containers, tmux, or +a plain `hermes gateway run`: the exporter lives in the gateway process, so +no sidecar, agent, or collector is required on the host. + +## Collecting into DataDog + +Run a customer-owned OpenTelemetry Collector and forward: + +```yaml +# otel-collector config +receivers: + otlp: + protocols: + http: +exporters: + datadog: + api: + key: ${env:DD_API_KEY} +service: + pipelines: + metrics: {receivers: [otlp], exporters: [datadog]} + traces: {receivers: [otlp], exporters: [datadog]} + logs: {receivers: [otlp], exporters: [datadog]} +``` + +Point `monitoring.export.otlp.endpoint` at the collector. Alerts belong on +`hermes.gateway.up`, `hermes.platform.up`, and `hermes.platform.degraded`. + +## Generic fleet queries and alerts + +The exact syntax depends on the customer's observability backend. The examples +below use PromQL-style expressions and intentionally avoid vendor-specific +routing, destinations, or customer inventory. + +Group fleet views by the opaque `service.instance.id` resource attribute. A +process that has died cannot emit its own zero, so every deployment needs both +explicit-state and missing-series detection. + +```promql +# Explicit gateway failure. +hermes_gateway_up == 0 + +# Box disappeared or stopped exporting. Choose a window longer than the +# configured export interval and collector retry allowance. +absent_over_time(hermes_gateway_up[5m]) + +# Locally owned bridge is explicitly down. +hermes_platform_up == 0 + +# Scheduler thread is stale even though the gateway may still be alive. +hermes_cron_scheduler_heartbeat_age_seconds > 180 + +# Ticker loops but has not completed a successful tick recently. +hermes_cron_scheduler_last_success_age_seconds > 300 + +# One or more jobs are beyond their existing scheduler grace window. +hermes_cron_jobs_overdue > 0 + +# Catch-up counter increased, proving at least one stale occurrence was +# collapsed and run once after a delay. +increase(hermes_cron_scheduler_catch_up_occurrences[15m]) > 0 +``` + +Cron execution lifecycle records arrive as `hermes.cron_execution` spans. +Alert or derive events from bounded attributes such as: + +```text +hermes.status = failed|unknown +hermes.delivery_outcome = failed|not_configured +hermes.error_class = auth_failed|rate_limited|timeout|network_error| + dispatch_failed|interrupted|empty_response| + invalid_config|unknown +``` + +Recommended operator views: + +1. one row per `service.instance.id` with gateway and configured local-platform + state; +2. scheduler heartbeat, last-success age, running count, overdue count, and + catch-up increase; +3. a cron lifecycle feed keyed only by opaque `hermes.job_key`; +4. separate alerts for box absence, local bridge down, scheduler stale, cron + failed/unknown, delivery failure, and overdue/catch-up activity. + +Keep alert thresholds and routing in deployment-owned configuration. Do not add +job names, prompts, outputs, schedules, destinations, raw errors, profile names, +or account identity merely to make a dashboard easier to read. + +## Release-validation scenarios + +Before accepting a deployment, force and verify all five cases through the real +collector and backend: + +1. **Cron success:** observe `claimed -> running -> completed`, duration, and a + truthful delivery outcome. +2. **Cron failure:** observe `failed` plus a bounded error class, with no raw + exception or content in the decoded OTLP payload. +3. **Cron interruption:** stop the owning gateway during execution, restart it, + and observe recovery to `unknown`. +4. **Locally owned bridge outage:** break one native connector, observe its + bounded down/retrying/fatal state and recovery, and verify unaffected boxes + remain healthy. +5. **Killed gateway:** terminate one canary, verify missing-series detection, + restart it, and confirm the same opaque instance identity returns. + +Hermes Agent-owned Relay transport health remains in scope. A separate gateway +or connector service remains authoritative for any shared connected-platform +state that it owns and should export that state through its own telemetry path. + +For every scenario, verify the signal and alert clear on recovery, other boxes +remain unaffected, collector failure stays fail-open, and decoded metrics, +spans, logs, and resource attributes remain content-free. + +## Local smoke test (no Docker) + +```bash +# terminal 1: capture collector on :4318 +python scripts/observability/otel_capture_collector.py \ + --host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl + +# terminal 2: drive the real exporter through lifecycle transitions, +# a fatal platform, and a structured warning event, then flush +python scripts/observability/gateway_health_export_probe.py \ + --endpoint http://127.0.0.1:4318/v1/traces \ + --log /tmp/hermes_otel_capture.jsonl --wait 8 +# exit 0 prints: {"requests": 6, "paths": ["/v1/logs", "/v1/metrics", "/v1/traces"]} +``` + +## Maintaining and extending this plane + +This plane is a **fixed, enumerated, content-free vocabulary** by design. Adding +a signal is not just "emit a new metric" — every new name and attribute must be +declared in each layer that enforces the bounded vocabulary, or it is silently +dropped downstream. Follow the checklist for the change you are making. The +golden rule: **a new signal that is emitted but not declared in every layer +looks like a code bug but is a vocabulary-registration bug — nothing errors, the +signal just never arrives.** + +### Content-free invariant (applies to every change) + +Before adding anything, confirm it cannot carry content. Numbers, booleans, +ages, durations, monotonic counts, and one-way hashes are safe. **Never** add an +attribute that can hold a job name, prompt, output, schedule, destination, raw +exception text, file path, profile name, account id, or free-form string. When +you must key a record to a job/entity, hash it (`sha256(...)[:24]`, see +`_job_key` in `agent/monitoring/cron_health.py`) — never emit the raw id. All +string attributes that could touch user input must pass through +`redaction.redact_for_export` and be truncated (see `_span_attrs` in +`agent/monitoring/otlp_exporter.py`). + +### Adding a new gauge/metric + +1. Emit it in the snapshot builder (`agent/monitoring/gateway_health.py` + `build_gateway_health_snapshot`, `cron_health.py` `build_cron_health_snapshot`, + or a sibling reader wired into `_read_runtime_snapshot` in + `gateway_health_export.py`). Best-effort: never let a reader raise into the + collection loop — wrap it and log a **content-free WARNING with the exception + TYPE name only** (the pattern the cron and background-work readers use), so a + future regression is visible instead of silently dropping the signal. +2. Register the dotted metric name in the observable-gauge `metric_names` list in + `gateway_health_export.py::_start_metric_provider`. **A gauge that is emitted + in the snapshot but not registered here is never observed.** +3. Add the export-table row and an alert example in this file. +4. If the deployment fronts the exporter with an OpenTelemetry Collector that + uses a metric-name allowlist (a `filter/...` processor with `name != "..."` + guards), add the new name there too — otherwise the collector drops it before + the backend. This is not repo code, but it is the single most common reason a + correctly-emitted new metric never appears; call it out in the PR so the + deploying operator updates their collector config. + +### Adding a new subsystem (a new family of signals) + +Mirror the cron pattern (`cron_health.py` + its wiring): put the read/projection +logic in its own module, expose one `build__health_snapshot()` that +returns bounded `GatewayMetric`s (and events if any), and extend it into +`_read_runtime_snapshot` with the same best-effort try/except-WARNING guard. +Then do the "adding a metric" checklist for each new name, and the "adding an +attribute" checklist for each new event attribute. Add a release-validation +scenario below for the subsystem's failure mode. + +### Extending the error-class / status / source / state vocabularies + +These are the closed enums that keep the plane bounded. Extend the SET, then the +classifier, never one without the other: + +- **Cron** (`agent/monitoring/cron_health.py`): `_KNOWN_STATUSES`, + `_KNOWN_SOURCES`, `_KNOWN_DELIVERY_OUTCOMES`, and the `classify_cron_error` + keyword buckets. Anything not in the set is coerced to `unknown` on the way + out, so a new value that is not added to the set is invisible. +- **Gateway/platform** (`agent/monitoring/gateway_health.py`): + `_KNOWN_GATEWAY_STATES`, `_KNOWN_PLATFORM_STATES`, and `classify_gateway_error`. + +Rules: keep the vocabulary SMALL and operationally meaningful (an error class +should map to an operator action, not to an exception subclass); a new bucket +must match on a stable keyword, not on message text that could vary; update the +`hermes.error_class = ...` list in this file's alert section and the enum's unit +test so the contract is asserted, not frozen as a count. + +### Adding a content-free attribute to an existing event/span + +Add the key to the emitter's per-kind `keep_by_kind` allowlist in +`agent/monitoring/otlp_exporter.py::_span_attrs` (unlisted keys are dropped), run +it through redaction if it is ever string-shaped, and — as with metrics — if the +deployment's collector has a span-attribute `keep_keys(...)` allowlist, add the +attribute there too or it is stripped in transit. + +### Verify the whole chain, not just emission + +Emitting is necessary but not sufficient. Confirm the signal survives all the +way to the backend, because the enums, the `metric_names` registration, the +emitter attribute allowlist, and any collector allowlist each drop unlisted +values with no error: + +```bash +hermes monitoring status # posture +python scripts/observability/gateway_health_export_probe.py \ + --endpoint http://127.0.0.1:4318/v1/traces \ + --log /tmp/cap.jsonl --wait 8 # drive the real exporter +``` + +Decode the captured OTLP payload and assert the new name/attribute is present +AND that no content leaked. When a real collector sits in front, add its +allowlist entries and re-verify against the backend, not just the local capture. + +## Boundaries and roadmap + +The `hermes monitoring` CLI intentionally exposes `status` only. This first +release covers only Hermes Agent-owned service-health and operational-diagnostic +signals, including Hermes Agent-owned Relay transport health. Team Gateway's +authoritative shared connector/platform state is explicitly out of scope, as +are product analytics, audit/quality reporting, and detailed execution traces. +Shared client usage metrics and enterprise trace telemetry are being designed on +the NeMo Relay integration with their own consent, policy, and export +boundaries; this monitoring plane stays narrow so an operator can enable it +without touching any content-bearing signal. The telemetry surface may be +reorganized as that lands. diff --git a/gateway/run.py b/gateway/run.py index e7e641601bc..bb94a569148 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -8156,6 +8156,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew write_runtime_status(gateway_state="starting", exit_reason=None) except Exception: pass + try: + from hermes_cli.config import load_config + from agent.monitoring.gateway_health_export import start_gateway_health_export + self._gateway_health_export_runtime = start_gateway_health_export(load_config()) + if getattr(self._gateway_health_export_runtime, "enabled", False): + logger.info("Gateway health OTLP export: enabled") + except Exception: + logger.debug("gateway health OTLP export startup failed", exc_info=True) # Log any active supply-chain security advisories. Operators see this # in gateway.log and `hermes status` surfaces it; we do NOT block @@ -10243,6 +10251,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._update_runtime_status("running", self._exit_reason) else: self._update_runtime_status("stopped", self._exit_reason) + _shutdown_gateway_health_export(self) logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed()) self._stop_task = asyncio.create_task(_stop_impl()) @@ -24563,6 +24572,18 @@ async def _await_thread_exit( return not thread.is_alive() +def _shutdown_gateway_health_export(runner: Any) -> None: + """Idempotently drain and detach Gateway Health OTLP export.""" + runtime = getattr(runner, "_gateway_health_export_runtime", None) + if runtime is None: + return + runner._gateway_health_export_runtime = None + try: + runtime.shutdown() + except Exception: + logger.debug("gateway health OTLP export shutdown failed", exc_info=True) + + async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool: """ Start the gateway and run until interrupted. @@ -25021,8 +25042,13 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = logger.debug("MCP tool discovery failed: %s", e) # Start the gateway - success = await runner.start() + try: + success = await runner.start() + except BaseException: + _shutdown_gateway_health_export(runner) + raise if not success: + _shutdown_gateway_health_export(runner) return False # Recover any pending messages flushed during a previous shutdown (#72680). try: @@ -25035,6 +25061,7 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = except Exception: pass if runner.should_exit_cleanly: + _shutdown_gateway_health_export(runner) if runner.exit_reason: logger.error("Gateway exiting cleanly: %s", runner.exit_reason) # A clean exit that carries an explicit exit code (e.g. a fatal @@ -25050,19 +25077,22 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = if not runner._running: # Startup was intentionally aborted by restart/shutdown before entering # running mode; preserve that lifecycle path without starting cron. - await runner.wait_for_shutdown() - if runner.should_exit_with_failure: - if runner.exit_reason: - logger.error("Gateway exiting with failure: %s", runner.exit_reason) - return False try: - from tools.mcp_tool import shutdown_mcp_servers - shutdown_mcp_servers() - except Exception: - pass - if runner.exit_code is not None: - raise SystemExit(runner.exit_code) - return True + await runner.wait_for_shutdown() + if runner.should_exit_with_failure: + if runner.exit_reason: + logger.error("Gateway exiting with failure: %s", runner.exit_reason) + return False + try: + from tools.mcp_tool import shutdown_mcp_servers + shutdown_mcp_servers() + except Exception: + pass + if runner.exit_code is not None: + raise SystemExit(runner.exit_code) + return True + finally: + _shutdown_gateway_health_export(runner) # Start the background cron scheduler via the resolved provider so # scheduled jobs fire automatically. The built-in provider is the diff --git a/gateway/status.py b/gateway/status.py index 224dbf6d071..5787a43821d 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -11,6 +11,7 @@ that will be useful when we add named profiles (multiple agents running concurrently under distinct configurations). """ +import copy import hashlib import json import logging @@ -988,6 +989,7 @@ def write_runtime_status( """Persist gateway runtime health information for diagnostics/status.""" path = _get_runtime_status_path() payload = _read_json_file(path) or _build_runtime_status_record() + previous_payload = copy.deepcopy(payload) current_record = _build_pid_record() payload.setdefault("platforms", {}) payload["kind"] = current_record["kind"] @@ -1022,6 +1024,11 @@ def write_runtime_status( payload["platforms"][platform] = platform_payload _write_json_file(path, payload) + try: + from agent.monitoring.gateway_health import emit_runtime_status_transition + emit_runtime_status_transition(previous_payload, payload) + except Exception: + pass def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 3d62e927772..27d85818f91 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2230,7 +2230,7 @@ DEFAULT_CONFIG = { "privacy": { "redact_pii": False, # When True, hash user IDs and strip phone numbers from LLM context }, - + # Text-to-speech configuration # Each provider supports an optional `max_text_length:` override for the # per-request input-character cap. Omit it to use the provider's documented @@ -3211,6 +3211,44 @@ DEFAULT_CONFIG = { "force_ipv4": False, }, + # Gateway monitoring — Service Health Monitoring plus redacted Operational + # Diagnostics for the gateway daemon, exported over OTLP to an + # operator-configured endpoint (OTEL Collector, DataDog, ...). Content-free + # by construction: no prompts, messages, tool args/results, session + # history, usage analytics, audit logs, or trajectories. Off by default; + # nothing is collected or sent until an operator enables it and sets an + # endpoint. + "monitoring": { + # Stable install identifier attached to exported health signals so an + # operator can tell instances apart in their collector. Empty string + # means "mint a fresh UUID on first use"; clear it to rotate. Carries + # no account identity. + "install_id": "", + # Gateway health & diagnostics export. + "gateway_health_export": { + "enabled": False, + "metrics_enabled": True, + "diagnostic_events_enabled": True, + "warning_error_events_enabled": True, + "export_interval_seconds": 60, + "logs_export_interval_seconds": 5, + "resource_attributes": { + "service.name": "hermes-gateway", + "deployment.environment.name": "production", + }, + }, + # OTLP destination. headers_env maps header names to ENVIRONMENT + # VARIABLE NAMES (never secret values); values are read from the + # environment at export time. + "export": { + "otlp": { + "enabled": False, + "endpoint": "", + "headers_env": {}, + }, + }, + }, + # Gateway settings — control how messaging platforms (Telegram, Discord, # Slack, etc.) deliver agent-produced files as native attachments. "gateway": { diff --git a/hermes_cli/main.py b/hermes_cli/main.py index ba4730ce71c..411e99d5c8a 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -465,6 +465,7 @@ from hermes_cli.subcommands.memory import build_memory_parser from hermes_cli.subcommands.acp import build_acp_parser from hermes_cli.subcommands.tools import build_tools_parser from hermes_cli.subcommands.insights import build_insights_parser +from hermes_cli.subcommands.monitoring import build_monitoring_parser from hermes_cli.subcommands.skills import build_skills_parser from hermes_cli.subcommands.pairing import build_pairing_parser from hermes_cli.subcommands.plugins import build_plugins_parser @@ -15429,7 +15430,7 @@ _BUILTIN_SUBCOMMANDS = frozenset( "dump", "egress", "fallback", "gateway", "hooks", "import", "import-agent", "insights", "gui", "desktop", "kanban", "login", "logout", "logs", "lsp", "mcp", "memory", "migrate", "moa", "journey", "memory-graph", "learning", - "model", "pairing", "pets", "plugins", "portal", "profile", + "model", "monitoring", "pairing", "pets", "plugins", "portal", "profile", "project", "proxy", "prompt-size", "send", "sessions", "setup", @@ -15911,6 +15912,51 @@ def cmd_insights(args): print(f"Error generating insights: {e}") +def cmd_monitoring(args): + """Gateway monitoring status: health & diagnostics export posture.""" + from hermes_cli.config import load_config + + action = getattr(args, "monitoring_action", None) or "status" + config = load_config() + mon_raw = config.get("monitoring") + mon: dict = mon_raw if isinstance(mon_raw, dict) else {} + + if action == "status": + from agent.monitoring import otlp_exporter + + gh_raw = mon.get("gateway_health_export") + gh: dict = gh_raw if isinstance(gh_raw, dict) else {} + export_raw = mon.get("export") + export_cfg: dict = export_raw if isinstance(export_raw, dict) else {} + otlp_raw = export_cfg.get("otlp") + otlp: dict = otlp_raw if isinstance(otlp_raw, dict) else {} + + print("Gateway monitoring") + print(f" Health export: {'enabled' if gh.get('enabled') else 'disabled'} " + f"(monitoring.gateway_health_export.enabled)") + if gh.get("enabled"): + print(f" Metrics: {'on' if gh.get('metrics_enabled', True) else 'off'} " + f"(interval {gh.get('export_interval_seconds', 60)}s)") + print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}") + print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} " + f"(interval {gh.get('logs_export_interval_seconds', 5)}s)") + print(" Content safety: always on " + "(rendered messages are never exported; not configurable)") + endpoint = otlp.get("endpoint") or "" + if otlp.get("enabled") and endpoint: + print(f" OTLP endpoint: {endpoint}") + else: + print(" OTLP endpoint: not configured (monitoring.export.otlp)") + print(f" OTel SDK: {'installed' if otlp_exporter.is_available() else 'not installed'} " + f"(optional extra: hermes-agent[otlp])") + print("\n Scope: gateway service health + redacted diagnostics only.") + print(" No prompts, messages, tool args/results, usage analytics, or traces.") + return + + print(f"Unknown monitoring action: {action}", file=sys.stderr) + sys.exit(2) + + def cmd_skills(args): # Route 'config' action to skills_config module if getattr(args, "skills_action", None) == "config": @@ -18216,6 +18262,7 @@ def main(): # insights command (parser built in hermes_cli/subcommands/insights.py) # ========================================================================= build_insights_parser(subparsers, cmd_insights=cmd_insights) + build_monitoring_parser(subparsers, cmd_monitoring=cmd_monitoring) # ========================================================================= # claw command (parser built in hermes_cli/subcommands/claw.py) diff --git a/hermes_cli/subcommands/monitoring.py b/hermes_cli/subcommands/monitoring.py new file mode 100644 index 00000000000..266c69aa1fb --- /dev/null +++ b/hermes_cli/subcommands/monitoring.py @@ -0,0 +1,36 @@ +"""``hermes monitoring`` subcommand parser. + +Gateway monitoring control and inspection. ``status`` shows whether the +gateway health & diagnostics export is enabled, where it points, and the +redaction posture. + +The handler is injected to avoid importing ``main`` (mirrors the insights +subcommand). +""" + +from __future__ import annotations + +from typing import Callable + + +def build_monitoring_parser(subparsers, *, cmd_monitoring: Callable) -> None: + """Attach the ``monitoring`` subcommand (with actions) to ``subparsers``.""" + p = subparsers.add_parser( + "monitoring", + help="Inspect gateway monitoring (health & diagnostics export)", + description=( + "Gateway monitoring: service health metrics plus redacted " + "diagnostics, exported over OTLP to an operator-configured " + "endpoint. Content-free by construction — no prompts, messages, " + "tool args/results, or usage analytics. Configure under " + "monitoring.* in config.yaml." + ), + ) + sub = p.add_subparsers(dest="monitoring_action") + + sub.add_parser( + "status", + help="Show monitoring settings, export state, and redaction posture", + ) + + p.set_defaults(func=cmd_monitoring) diff --git a/pyproject.toml b/pyproject.toml index 40dabe1987c..fd47c33207b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,6 +248,11 @@ acp = ["agent-client-protocol==0.9.0"] # NOT re-added to [all] so a future quarantined release can't break fresh # installs (see [all] policy comment below). mistral = ["mistralai==2.4.8"] +# OTLP gateway monitoring export (optional). Provides the OpenTelemetry SDK + +# OTLP/HTTP exporter for monitoring.gateway_health_export. Lazy-installed via +# tools/lazy_deps.py on first use; never a core dependency and deliberately +# NOT in [all]. +otlp = ["opentelemetry-sdk==1.39.1", "opentelemetry-exporter-otlp-proto-http==1.39.1"] bedrock = ["boto3==1.42.89"] vertex = ["google-auth==2.55.1"] azure-identity = ["azure-identity==1.25.3"] diff --git a/scripts/observability/gateway_health_export_probe.py b/scripts/observability/gateway_health_export_probe.py new file mode 100644 index 00000000000..e1064088e2a --- /dev/null +++ b/scripts/observability/gateway_health_export_probe.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Exercise Gateway Health & Diagnostics Export against a local OTLP capture collector.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", default="http://127.0.0.1:4318/v1/traces") + parser.add_argument("--log", required=True, help="JSONL file written by otel_capture_collector.py") + parser.add_argument("--wait", type=float, default=7.0) + args = parser.parse_args() + + hermes_home = Path(tempfile.mkdtemp(prefix="hermes-otel-smoke-")) + os.environ["HERMES_HOME"] = str(hermes_home) + + from gateway.status import write_runtime_status + from agent.monitoring.gateway_health_export import start_gateway_health_export + from agent.monitoring import emitter + + config = { + "monitoring": { + "local": True, + "gateway_health_export": { + "enabled": True, + "metrics_enabled": True, + "diagnostic_events_enabled": True, + "warning_error_events_enabled": True, + "export_interval_seconds": 5, + "logs_export_interval_seconds": 5, + "resource_attributes": { + "service.name": "hermes-gateway-smoke", + "deployment.environment.name": "local-smoke", + }, + }, + "export": { + "otlp": { + "enabled": True, + "endpoint": args.endpoint, + "headers_env": {}, + } + }, + } + } + + runtime = start_gateway_health_export(config) + if not runtime.enabled: + raise SystemExit(f"gateway health exporter did not enable: {runtime.reason}") + + write_runtime_status(gateway_state="starting", active_agents=0) + write_runtime_status(gateway_state="running", active_agents=2) + write_runtime_status(platform="slack", platform_state="running") + write_runtime_status( + platform="slack", + platform_state="fatal", + error_code="auth_failed", + error_message="Bearer *** rejected for smoke@example.com", + ) + logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com") + emitter.get_emitter().flush(timeout=2.0) + time.sleep(args.wait) + write_runtime_status(gateway_state="stopped", active_agents=0) + runtime.shutdown() + emitter.get_emitter().flush(timeout=2.0) + + log_path = Path(args.log) + rows = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + paths = {row["path"] for row in rows} + print(json.dumps({"hermes_home": str(hermes_home), "requests": len(rows), "paths": sorted(paths)}, indent=2)) + if "/v1/traces" not in paths: + raise SystemExit("missing /v1/traces request") + if "/v1/logs" not in paths: + raise SystemExit("missing /v1/logs request") + if "/v1/metrics" not in paths: + raise SystemExit("missing /v1/metrics request") + + +if __name__ == "__main__": + main() diff --git a/scripts/observability/otel_capture_collector.py b/scripts/observability/otel_capture_collector.py new file mode 100644 index 00000000000..cbfb659cb30 --- /dev/null +++ b/scripts/observability/otel_capture_collector.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests. + +This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces, +/v1/metrics, and /v1/logs, records request metadata as JSONL, and returns 200 so +local exporters can be exercised without Docker or a vendor backend. +""" + +from __future__ import annotations + +import argparse +import json +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +class CaptureHandler(BaseHTTPRequestHandler): + log_path: Path + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length) if length else b"" + record = { + "ts": time.time(), + "path": self.path, + "content_type": self.headers.get("content-type"), + "content_length": length, + "body_prefix_hex": body[:24].hex(), + } + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, sort_keys=True) + "\n") + self.send_response(200) + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, format: str, *args) -> None: + # Keep tmux panes clean; JSONL file is the assertion surface. + return + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=4318) + parser.add_argument("--log", required=True) + args = parser.parse_args() + + log_path = Path(args.log).expanduser().resolve() + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text("", encoding="utf-8") + CaptureHandler.log_path = log_path + server = ThreadingHTTPServer((args.host, args.port), CaptureHandler) + print(f"OTLP capture collector listening on http://{args.host}:{args.port}; log={log_path}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/cron/test_execution_ledger.py b/tests/cron/test_execution_ledger.py index 0be6aa63ec8..2c0e3e7055d 100644 --- a/tests/cron/test_execution_ledger.py +++ b/tests/cron/test_execution_ledger.py @@ -269,6 +269,69 @@ def test_run_one_job_records_running_then_terminal(monkeypatch): assert events[-1][2]["success"] is True +def test_run_one_job_records_unresolved_origin_as_not_configured(monkeypatch): + import cron.scheduler as scheduler + + finished = [] + monkeypatch.setattr(scheduler, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr( + scheduler, + "finish_execution", + lambda execution_id, **kwargs: finished.append((execution_id, kwargs)), + ) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr( + scheduler, + "run_job", + lambda job, *, defer_agent_teardown=None: (True, "output", "response", None), + ) + monkeypatch.setattr(scheduler, "save_job_output", lambda *_args: None) + monkeypatch.setattr(scheduler, "_resolve_delivery_targets", lambda _job: []) + monkeypatch.setattr(scheduler, "_deliver_result", lambda *_args, **_kwargs: None) + monkeypatch.setattr(scheduler, "mark_job_run", lambda *_args, **_kwargs: None) + + job = { + "id": "job-unresolved-origin", + "execution_id": "exec-unresolved-origin", + "deliver": "origin", + } + assert scheduler.run_one_job(job) is True + + assert finished[-1][1]["delivery_outcome"] == "not_configured" + + +def test_run_one_job_normalizes_legacy_local_delivery_as_suppressed(monkeypatch): + import cron.scheduler as scheduler + + finished = [] + monkeypatch.setattr( + scheduler, + "run_job", + lambda job, *, defer_agent_teardown=None: (True, "output", "response", None), + ) + monkeypatch.setattr(scheduler, "save_job_output", lambda *_args: None) + monkeypatch.setattr(scheduler, "mark_job_run", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + scheduler, + "finish_execution", + lambda execution_id, **kwargs: finished.append((execution_id, kwargs)), + ) + monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_args: None) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda *_args, **_kwargs: True) + monkeypatch.setattr(scheduler, "_consume_interrupted_flag", lambda *_args: False) + + job = { + "id": "legacy-local", + "name": "Legacy local", + "schedule": {"kind": "interval", "minutes": 10}, + "deliver": ["local"], + "execution_id": "exec-legacy-local", + } + + assert scheduler.run_one_job(job) is True + assert finished[-1][1]["delivery_outcome"] == "suppressed" + + def test_provider_start_recovers_interrupted_records_before_tick(monkeypatch): import cron.scheduler_provider as provider diff --git a/tests/cron/test_jobs.py b/tests/cron/test_jobs.py index c263bffa22d..b3a4123ee93 100644 --- a/tests/cron/test_jobs.py +++ b/tests/cron/test_jobs.py @@ -836,6 +836,24 @@ class TestGetDueJobs: next_dt = _ensure_aware(datetime.fromisoformat(updated["next_run_at"])) assert next_dt > _hermes_now() + def test_stale_past_due_records_one_catch_up_occurrence(self, tmp_cron_dir, monkeypatch): + import cron.jobs as jobs_module + + recorded = [] + monkeypatch.setattr( + jobs_module, + "record_catch_up_occurrence", + lambda: recorded.append("catch-up"), + raising=False, + ) + create_job(prompt="Stale", schedule="every 1h") + jobs = load_jobs() + jobs[0]["next_run_at"] = (datetime.now() - timedelta(minutes=35)).isoformat() + save_jobs(jobs) + + assert len(get_due_jobs()) == 1 + assert recorded == ["catch-up"] + def test_idless_job_does_not_crash_or_block_sibling_jobs(self, tmp_cron_dir): """A job missing its 'id' key must not crash the tick or freeze siblings. diff --git a/tests/gateway/test_startup_restart_race.py b/tests/gateway/test_startup_restart_race.py index e303cba1c99..bdf5bbc42a4 100644 --- a/tests/gateway/test_startup_restart_race.py +++ b/tests/gateway/test_startup_restart_race.py @@ -250,16 +250,23 @@ async def test_startup_aborts_after_registered_adapter_restart(tmp_path, monkeyp async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) cron_started = False + export_shutdown_calls = 0 + + class ExportRuntime: + def shutdown(self): + nonlocal export_shutdown_calls + export_shutdown_calls += 1 class AbortedStartupRunner: def __init__(self, config): self.config = config self.adapters = {} self._running = False - self.should_exit_cleanly = False + self.should_exit_cleanly = True self.should_exit_with_failure = False self.exit_reason = None self.exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE + self._gateway_health_export_runtime = ExportRuntime() async def start(self): return True @@ -287,3 +294,4 @@ async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path, assert exc.value.code == GATEWAY_SERVICE_RESTART_EXIT_CODE assert cron_started is False + assert export_shutdown_calls == 1 diff --git a/tests/monitoring/__init__.py b/tests/monitoring/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/monitoring/test_cron_health_export.py b/tests/monitoring/test_cron_health_export.py new file mode 100644 index 00000000000..adc0e91c303 --- /dev/null +++ b/tests/monitoring/test_cron_health_export.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + + +def _metric(snapshot, name): + return next(metric for metric in snapshot.metrics if metric.name == name) + + +def test_cron_snapshot_projects_freshness_counts_and_overdue_without_content(monkeypatch): + from agent.monitoring import cron_health + + now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) + secret = "Quarterly payroll for alice@example.com" + monkeypatch.setattr(cron_health, "_now", lambda: now) + monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: 4.5) + monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: 9.0) + monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset({"job-private-1"})) + monkeypatch.setattr( + cron_health, + "load_jobs", + lambda: [ + { + "id": "job-private-1", + "name": secret, + "prompt": secret, + "enabled": True, + "schedule": {"kind": "interval", "minutes": 10}, + "next_run_at": (now - timedelta(minutes=6)).isoformat(), + }, + { + "id": "job-private-2", + "name": "disabled private job", + "enabled": False, + "schedule": {"kind": "interval", "minutes": 10}, + "next_run_at": (now - timedelta(days=1)).isoformat(), + }, + ], + ) + + snapshot = cron_health.build_cron_health_snapshot() + + assert _metric(snapshot, "hermes.cron.scheduler.heartbeat_age_seconds").value == 4.5 + assert _metric(snapshot, "hermes.cron.scheduler.last_success_age_seconds").value == 9.0 + assert _metric(snapshot, "hermes.cron.jobs.enabled").value == 1 + assert _metric(snapshot, "hermes.cron.jobs.running").value == 1 + assert _metric(snapshot, "hermes.cron.jobs.overdue").value == 1 + assert secret not in str(snapshot) + assert "job-private-1" not in str(snapshot) + + +def test_cron_snapshot_omits_unknown_freshness_instead_of_inventing_values(monkeypatch): + from agent.monitoring import cron_health + + monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: None) + monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: None) + monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset()) + monkeypatch.setattr(cron_health, "load_jobs", lambda: []) + + names = {metric.name for metric in cron_health.build_cron_health_snapshot().metrics} + + assert "hermes.cron.scheduler.heartbeat_age_seconds" not in names + assert "hermes.cron.scheduler.last_success_age_seconds" not in names + + +def test_execution_projection_is_opaque_bounded_and_content_free(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + { + "id": "execution-private-id", + "job_id": "Payroll for alice@example.com and token top-secret-token", + "source": "builtin", + "status": "failed", + "claimed_at": "2026-07-24T12:00:00+00:00", + "started_at": "2026-07-24T12:00:01+00:00", + "finished_at": "2026-07-24T12:00:03.250000+00:00", + "error": "Bearer top-secret-token rejected for alice@example.com", + }, + delivery_outcome="failed", + ).to_dict() + + assert event["event"] == "cron_execution" + assert event["status"] == "failed" + assert event["job_key"].startswith("sha256:") + assert len(event["job_key"]) == len("sha256:") + 24 + assert event["duration_ms"] == 2250 + assert event["delivery_outcome"] == "failed" + assert event["error_class"] == "auth_failed" + assert "job_id" not in event + assert "error" not in event + assert "alice@example.com" not in str(event) + assert "top-secret-token" not in str(event) + + +def test_execution_projection_omits_duration_and_delivery_when_not_known(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + { + "job_id": "private", + "source": "external-value-must-not-leak", + "status": "claimed", + "claimed_at": "2026-07-24T12:00:00+00:00", + } + ).to_dict() + + assert event["status"] == "claimed" + assert event["source"] == "external" + assert event["duration_ms"] is None + assert event["delivery_outcome"] is None + + +def test_external_provider_source_is_normalized_to_external(): + from agent.monitoring.cron_health import project_execution_event + + event = project_execution_event( + {"job_id": "private", "source": "Chronos", "status": "claimed"} + ) + + assert event.source == "external" + + +@pytest.mark.parametrize("message", ["oauth refresh failed", "tokenizer crashed", "HTTP 4015"]) +def test_error_classification_avoids_auth_substring_false_positives(message): + from agent.monitoring.cron_health import classify_cron_error + + assert classify_cron_error(message) == "unknown" + + +@pytest.mark.parametrize( + "message", + ["authentication failed", "not authorized", "access token expired", "HTTP 401"], +) +def test_error_classification_recognizes_auth_terms_and_status_tokens(message): + from agent.monitoring.cron_health import classify_cron_error + + assert classify_cron_error(message) == "auth_failed" + + +def test_cron_snapshot_exports_catch_up_occurrence_counter(monkeypatch): + from agent.monitoring import cron_health + + monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: None) + monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: None) + monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset()) + monkeypatch.setattr(cron_health, "load_jobs", lambda: []) + monkeypatch.setattr(cron_health, "get_catch_up_occurrence_count", lambda: 3) + + snapshot = cron_health.build_cron_health_snapshot() + + assert _metric(snapshot, "hermes.cron.scheduler.catch_up_occurrences").value == 3 + + +def test_terminal_execution_emission_flushes_and_failures_are_fail_open(monkeypatch): + from agent.monitoring import cron_health, emitter + + calls = [] + + class FakeEmitter: + def emit(self, event): + calls.append(("emit", event.to_dict()["status"])) + + def flush(self, timeout): + calls.append(("flush", timeout)) + raise RuntimeError("collector unavailable") + + monkeypatch.setattr(emitter, "get_emitter", lambda: FakeEmitter()) + + cron_health.emit_execution_state( + {"job_id": "private", "source": "builtin", "status": "completed"} + ) + + assert calls == [("emit", "completed"), ("flush", 1.0)] + + +def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(monkeypatch): + from agent.monitoring import gateway_health_export + + gateway_snapshot = type("Snapshot", (), {"metrics": []})() + cron_snapshot = type( + "Snapshot", + (), + {"metrics": [type("Metric", (), {"name": "hermes.cron.jobs.enabled", "value": 2, "attributes": {}})()]}, + )() + monkeypatch.setattr(gateway_health_export, "_read_gateway_snapshot", lambda config: gateway_snapshot) + monkeypatch.setattr(gateway_health_export, "_read_cron_snapshot", lambda: cron_snapshot) + + snapshot = gateway_health_export._read_runtime_snapshot({}) + + names = [metric.name for metric in snapshot.metrics] + # Cron metrics are folded into the gateway snapshot... + assert "hermes.cron.jobs.enabled" in names + # ...and the background/subagent-work gauges are appended (distinct from + # active_agents). Assert the relationship, not a frozen exact list. + assert "hermes.gateway.background_work" in names + assert "hermes.gateway.background_delegations" in names + assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True + assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True + assert gateway_health_export._gateway_health_event({"event": "run"}) is False + + +def test_background_work_is_task_granular_and_delegations_is_unit_granular(monkeypatch): + """background_work expands batches to child tasks; background_delegations + counts dispatch units. A 3-task batch => work +3, delegations +1. + """ + from agent.monitoring import gateway_health_export + from tools import async_delegation as ad + + with ad._records_lock: + saved = dict(ad._records) + ad._records.clear() + ad._records["single"] = {"status": "running"} + ad._records["batch3"] = {"status": "running", "is_batch": True, "goals": ["a", "b", "c"]} + # Isolate from process_registry so we measure only the delegation contribution. + monkeypatch.setattr( + "tools.process_registry.process_registry.count_running", lambda: 0, raising=False + ) + try: + # work = single(1) + batch(3) = 4 tasks; delegations = 2 units. + assert gateway_health_export._read_background_work_count() == 4 + assert gateway_health_export._read_background_delegations_count() == 2 + finally: + with ad._records_lock: + ad._records.clear() + ad._records.update(saved) + + +def test_registered_observable_metric_names_cover_snapshot_metrics(monkeypatch): + """Every gauge emitted in the runtime snapshot must also be registered in the + observable-gauge metric_names list, or the OTLP exporter never observes it. + + This asserts the vocabulary-registration invariant documented in + docs/observability/monitoring.md: an emitted-but-unregistered gauge is + silently dropped. Regression guard for background_work / cron additions. + """ + import inspect + from agent.monitoring import gateway_health_export + + # Build a representative snapshot (gateway + cron + background_work) without + # a live gateway by stubbing the gateway snapshot to the real metric names. + class _M: + def __init__(self, name): + self.name = name + self.value = 0 + self.attributes = {} + + gateway_snapshot = type("S", (), {"metrics": [ + _M("hermes.gateway.up"), _M("hermes.gateway.active_agents"), + _M("hermes.gateway.busy"), _M("hermes.gateway.drainable"), + _M("hermes.gateway.restart_requested"), + _M("hermes.platform.up"), _M("hermes.platform.degraded"), + ]})() + cron_snapshot = type("S", (), {"metrics": [ + _M("hermes.cron.scheduler.heartbeat_age_seconds"), + _M("hermes.cron.scheduler.last_success_age_seconds"), + _M("hermes.cron.scheduler.catch_up_occurrences"), + _M("hermes.cron.jobs.enabled"), _M("hermes.cron.jobs.running"), + _M("hermes.cron.jobs.overdue"), + ]})() + monkeypatch.setattr(gateway_health_export, "_read_gateway_snapshot", lambda config: gateway_snapshot) + monkeypatch.setattr(gateway_health_export, "_read_cron_snapshot", lambda: cron_snapshot) + + snapshot_names = {m.name for m in gateway_health_export._read_runtime_snapshot({}).metrics} + + # Extract the registered metric_names list literal from _start_metric_provider. + src = inspect.getsource(gateway_health_export._start_metric_provider) + registered = {n for n in snapshot_names if f'"{n}"' in src} + + missing = snapshot_names - registered + assert not missing, f"gauges emitted but NOT registered in metric_names (will be silently dropped): {sorted(missing)}" + + +def test_monitoring_docs_distinguish_relay_health_scope_and_terminal_flush(): + from pathlib import Path + + text = Path("docs/observability/monitoring.md").read_text(encoding="utf-8") + + assert "Hermes Agent-owned Relay transport health" in text + assert "authoritative shared connector/platform state" in text + assert "up to one second" in text + assert "terminal" in text diff --git a/tests/monitoring/test_emitter.py b/tests/monitoring/test_emitter.py new file mode 100644 index 00000000000..07fce136f31 --- /dev/null +++ b/tests/monitoring/test_emitter.py @@ -0,0 +1,135 @@ +"""Tests for the monitoring emitter: hot-path invariant + subscriber fan-out.""" + +from __future__ import annotations + +import time +import threading + +from agent.monitoring.emitter import MonitoringEmitter +from agent.monitoring.events import GatewayHealthEvent + + +def test_emit_never_raises_when_disabled(): + em = MonitoringEmitter(enabled=False) + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + assert em.stats()["queued"] == 0 + em.close() + + +def test_process_singleton_stays_dormant_until_subscribed(): + from agent.monitoring import emitter + + emitter.reset_emitter_for_tests() + try: + emitter.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + singleton = emitter.get_emitter() + assert singleton.stats()["queued"] == 0 + assert singleton._started is False + + subscriber = lambda _batch: None # noqa: E731 + singleton.subscribe(subscriber) + emitter.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + assert singleton._started is True + singleton.unsubscribe(subscriber) + finally: + emitter.reset_emitter_for_tests() + + +def test_emit_accepts_dataclass_and_dict(tmp_path): + em = MonitoringEmitter() + seen: list = [] + em.subscribe(lambda batch: seen.extend(batch)) + em.emit(GatewayHealthEvent(name="gateway.health_snapshot", active_agents=2)) + em.emit({"event": "gateway_diagnostic", "name": "platform.fatal", + "subsystem": "platform.slack"}) + em.flush() + em.close() + kinds = {ev.get("event") for ev in seen} + assert kinds == {"gateway_health", "gateway_diagnostic"} + health = next(ev for ev in seen if ev["event"] == "gateway_health") + assert health["active_agents"] == 2 + assert "ts_ns" in health + + +def test_subscriber_failure_is_isolated(): + em = MonitoringEmitter() + good: list = [] + + def bad(batch): + raise RuntimeError("boom") + + em.subscribe(bad) + em.subscribe(lambda batch: good.extend(batch)) + em.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + em.flush() + em.close() + assert len(good) == 1 # the raising subscriber did not break fan-out + + +def test_flush_waits_for_in_flight_subscriber_delivery(): + em = MonitoringEmitter() + subscriber_started = threading.Event() + release_subscriber = threading.Event() + flush_finished = threading.Event() + + def blocking_subscriber(_batch): + subscriber_started.set() + release_subscriber.wait(timeout=2.0) + + em.subscribe(blocking_subscriber) + em.emit({"event": "gateway_health", "name": "gateway.exit"}) + assert subscriber_started.wait(timeout=1.0) + + flush_thread = threading.Thread( + target=lambda: (em.flush(timeout=1.0), flush_finished.set()), + daemon=True, + ) + flush_thread.start() + + try: + assert not flush_finished.wait(timeout=0.1) + finally: + release_subscriber.set() + + assert flush_finished.wait(timeout=1.0) + em.close() + + +def test_unsubscribe_stops_delivery(): + em = MonitoringEmitter() + seen: list = [] + cb = lambda batch: seen.extend(batch) # noqa: E731 + em.subscribe(cb) + em.emit({"event": "gateway_health", "name": "a"}) + em.flush() + em.unsubscribe(cb) + em.emit({"event": "gateway_health", "name": "b"}) + em.flush() + em.close() + assert [ev["name"] for ev in seen] == ["a"] + + +def test_queue_full_drops_oldest(): + em = MonitoringEmitter() + # Fill the queue without a dispatcher running by not letting it start: + # emit() starts the thread, so instead assert drop accounting via stats + # after a burst larger than the queue. + for i in range(11_000): + em.emit({"event": "gateway_health", "name": f"e{i}"}) + # Give the dispatcher a moment; total dispatched + queued + dropped == emitted. + em.flush(timeout=5.0) + stats = em.stats() + em.close() + assert stats["dropped"] >= 0 + assert stats["dispatched"] + stats["queued"] + stats["dropped"] >= 10_000 + + +def test_hot_path_is_fast(): + em = MonitoringEmitter() + start = time.perf_counter() + for _ in range(1_000): + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + elapsed = time.perf_counter() - start + em.close() + # 1000 emits should be far under a second even on slow CI. + assert elapsed < 1.0 diff --git a/tests/monitoring/test_export_redaction.py b/tests/monitoring/test_export_redaction.py new file mode 100644 index 00000000000..1dede385a5d --- /dev/null +++ b/tests/monitoring/test_export_redaction.py @@ -0,0 +1,69 @@ +"""Export redaction tests — the security-critical layer. + +Invariants: + * One unconditional scrub: secrets AND PII, no modes, no knobs. + * Fails CLOSED: if the redactor can't run, the raw string is never emitted. + * Structure (subsystem names, error codes) survives; free-text PII does not. +""" + +from __future__ import annotations + +from unittest import mock + +import agent.monitoring.redaction as R + + +def test_secret_key_always_stripped(): + fake_key = "sk-ant-api03-" + "A" * 24 # constructed to dodge literal-scrubbers + out = R.redact_for_export(f"calling with key {fake_key} and moving on") + assert out is not None + assert fake_key not in out + + +def test_token_shapes_stripped(): + ghp = "ghp_" + "0123456789abcdef" * 2 + "0123" + slack = "xoxb-" + "123456789012-abcdefABCDEF" + out = R.redact_for_export(f"token {ghp} and {slack} leaked") + assert out is not None + assert ghp not in out + assert slack not in out + assert "[redacted]" in out + + +def test_bearer_header_stripped(): + out = R.redact_for_export("Authorization: Bearer abc.def-ghi_jkl") + assert out is not None + assert "abc.def-ghi_jkl" not in out + + +def test_none_passthrough(): + assert R.redact_for_export(None) is None + + +def test_pii_always_stripped(): + text = ("reach alice@example.com or +1 415 555 0100, " + "install 123e4567-e89b-12d3-a456-426614174000") + out = R.redact_for_export(text) + assert out is not None + assert "alice@example.com" not in out + assert "426614174000" not in out + assert "[email]" in out + assert "[id]" in out + assert "[phone]" in out + + +def test_ordinary_words_survive(): + assert R.redact_for_export("just ordinary words") == "just ordinary words" + + +def test_structure_preserved(): + out = R.redact_for_export("platform.slack entered fatal after auth_failed") + assert out is not None + assert "platform.slack" in out + assert "auth_failed" in out + + +def test_fails_closed_when_redactor_unavailable(): + with mock.patch("agent.redact.redact_sensitive_text", side_effect=RuntimeError): + out = R.redact_for_export("secret sauce sk-live-key") + assert out == "[redaction-unavailable]" diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py new file mode 100644 index 00000000000..a48f585c3ba --- /dev/null +++ b/tests/monitoring/test_gateway_health_export.py @@ -0,0 +1,729 @@ +from __future__ import annotations + +import logging + +import pytest + + +def test_gateway_diagnostic_event_preserves_positional_error_class(): + from agent.monitoring.events import GatewayDiagnosticEvent + + event = GatewayDiagnosticEvent("gateway.log.warning", "gateway", "auth_failed") + + assert event.error_class == "auth_failed" + assert event.source_logger is None + + +def test_default_config_keeps_gateway_health_export_disabled(): + from hermes_cli.config import DEFAULT_CONFIG + + cfg = DEFAULT_CONFIG["monitoring"]["gateway_health_export"] + + assert cfg["enabled"] is False + assert cfg["metrics_enabled"] is True + assert cfg["diagnostic_events_enabled"] is True + assert cfg["warning_error_events_enabled"] is True + assert cfg["export_interval_seconds"] == 60 + assert cfg["logs_export_interval_seconds"] == 5 + assert cfg["resource_attributes"]["deployment.environment.name"] == "production" + assert "deployment.environment" not in cfg["resource_attributes"] + # Redaction is always-on and deliberately NOT configurable. + assert "redaction" not in cfg + + +def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + runtime = { + "gateway_state": "running", + "pid": 1234, + "active_agents": "2", + "restart_requested": False, + "platforms": { + "slack": {"state": "running"}, + "telegram": { + "state": "fatal", + "error_code": "auth_failed", + "error_message": "token xoxb-secret rejected for user 123", + }, + }, + } + + snapshot = build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile="default", + install_id="install-1", + version="2026.7.test", + supervision_mode="manual", + ) + + metric_names = {m.name for m in snapshot.metrics} + assert { + "hermes.gateway.up", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.platform.up", + "hermes.platform.degraded", + } <= metric_names + + active = next(m for m in snapshot.metrics if m.name == "hermes.gateway.active_agents") + assert active.value == 2 + assert active.attributes == { + "service.instance.id": active.attributes["service.instance.id"], + "service.version": "2026.7.test", + "hermes.supervision_mode": "manual", + } + assert active.attributes["service.instance.id"].startswith("sha256:") + assert "install-1" not in active.attributes["service.instance.id"] + + busy = next(m for m in snapshot.metrics if m.name == "hermes.gateway.busy") + drainable = next(m for m in snapshot.metrics if m.name == "hermes.gateway.drainable") + assert busy.value == 1 + assert drainable.value == 1 + + degraded = next( + m for m in snapshot.metrics + if m.name == "hermes.platform.degraded" and m.attributes["hermes.platform"] == "telegram" + ) + assert degraded.value == 1 + assert degraded.attributes["hermes.error_code"] == "auth_failed" + assert all("secret" not in str(v).lower() for v in degraded.attributes.values()) + + +def test_gateway_health_snapshot_emits_content_free_diagnostic_event(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + snapshot = build_gateway_health_snapshot( + { + "gateway_state": "running", + "active_agents": 1, + "platforms": { + "slack": {"state": "fatal", "error_code": "auth_failed", "error_message": "Bearer sk-live-secret"}, + }, + }, + gateway_running=True, + profile="default", + install_id="install-1", + version="v-test", + supervision_mode="container", + ) + + events = [event.to_dict() for event in snapshot.events] + health = next(e for e in events if e["event"] == "gateway_health") + platform = next(e for e in events if e["event"] == "gateway_diagnostic" and e["name"] == "platform.fatal") + + assert health["gateway_state"] == "running" + assert health["active_agents"] == 1 + assert health["gateway_busy"] is True + assert health["gateway_drainable"] is True + assert health["fatal_platform_count"] == 1 + assert platform["platform"] == "slack" + assert platform["error_code"] == "auth_failed" + assert "redacted_message" not in platform + assert "Bearer" not in str(platform) + + +def test_gateway_health_snapshot_preserves_real_bounded_platform_states(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + expected = { + "connecting", + "connected", + "disconnected", + "disabled", + "fatal", + "paused", + "retrying", + } + snapshot = build_gateway_health_snapshot( + { + "gateway_state": "running", + "platforms": {state: {"state": state} for state in expected}, + }, + gateway_running=True, + profile="default", + install_id="install-1", + version="v-test", + supervision_mode="container", + ) + + observed = { + metric.attributes["hermes.platform.state"] + for metric in snapshot.metrics + if metric.name == "hermes.platform.up" + } + assert observed == expected + + +def test_gateway_diagnostic_log_handler_never_carries_rendered_message(caplog): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + old = emitter.get_emitter + emitter.get_emitter = lambda: DummyEmitter() # type: ignore[assignment] + try: + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + logger = logging.getLogger("gateway.platforms.slack") + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + try: + logger.info("ignore info token sk-live-secret") + logger.warning( + "Unauthorized user: acct_7f3a (Alice Smith) on slack; " + "token «redacted:sk-…»" + ) + finally: + logger.removeHandler(handler) + finally: + emitter.get_emitter = old # type: ignore[assignment] + + assert len(captured) == 1 + event = captured[0] + assert event["event"] == "gateway_diagnostic" + assert event["name"] == "gateway.log.warning" + assert event["subsystem"] == "platform.slack" + assert event["source_logger"] == "gateway.platforms.slack" + assert event["error_class"] == "auth_failed" + assert "redacted_message" not in event + assert "acct_7f3a" not in str(event) + assert "Alice Smith" not in str(event) + + +@pytest.mark.parametrize( + "message", + [ + "Connect call failed ('127.0.0.1', 9)", + "failed to connect to relay", + "connection refused", + "network is unreachable", + "temporary failure in name resolution", + ], +) +def test_gateway_error_classifier_recognizes_bounded_network_failures(message): + from agent.monitoring.gateway_health import classify_gateway_error + + assert classify_gateway_error(message) == "network_error" + + +def test_gateway_diagnostic_log_handler_enriches_relay_scope_without_message_content( + monkeypatch, +): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + monkeypatch.setattr(emitter, "get_emitter", lambda: DummyEmitter()) + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + logger = logging.getLogger("gateway.relay.adapter") + logger.addHandler(handler) + try: + logger.warning( + "Connect call failed for ws://alice@example.com/private; " + "credential=«redacted:sk-…»" + ) + finally: + logger.removeHandler(handler) + + assert len(captured) == 1 + event = captured[0] + assert event["subsystem"] == "platform.relay" + assert event["platform"] == "relay" + assert event["source_logger"] == "gateway.relay.adapter" + assert event["error_class"] == "network_error" + assert event["error_code"] == "network_error" + assert "alice@example.com" not in str(event) + assert "private" not in str(event) + + +def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import emit_runtime_status_transition + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + old = emitter.emit + monkeypatch.setattr(emitter, "emit", lambda event: captured.append(event.to_dict())) + + previous = {"gateway_state": "starting", "platforms": {"slack": {"state": "running"}}} + current = { + "gateway_state": "running", + "pid": 123, + "active_agents": 1, + "platforms": { + "slack": { + "state": "fatal", + "error_code": "auth_failed", + "error_message": "Bearer *** failed", + } + }, + } + + emit_runtime_status_transition(previous, current) + + names = [e["name"] for e in captured] + assert "gateway.lifecycle" in names + assert "platform.state_change" in names + assert "platform.fatal" in names + lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") + assert lifecycle["old_state"] == "starting" + assert lifecycle["new_state"] == "running" + assert lifecycle["exit_reason"] is None + platform = next(e for e in captured if e["name"] == "platform.state_change") + assert platform["old_state"] == "running" + assert platform["new_state"] == "fatal" + assert platform["error_code"] == "auth_failed" + assert "redacted_message" not in platform + + +def test_runtime_status_transition_emits_startup_failed_and_exit(): + from agent.monitoring.gateway_health import emit_runtime_status_transition + from agent.monitoring import emitter + + captured = [] + old = emitter.emit + emitter.emit = lambda event: captured.append(event.to_dict()) # type: ignore[assignment] + try: + emit_runtime_status_transition( + {"gateway_state": "starting"}, + { + "gateway_state": "startup_failed", + "exit_reason": "Bearer top-secret-token rejected for user@example.com", + }, + ) + emit_runtime_status_transition( + {"gateway_state": "running"}, + { + "gateway_state": "stopped", + "exit_reason": "shutdown requested by user@example.com", + "restart_requested": True, + }, + ) + finally: + emitter.emit = old # type: ignore[assignment] + + names = [e["name"] for e in captured] + assert "gateway.startup_failed" in names + assert "gateway.exit" in names + failed = next(e for e in captured if e["name"] == "gateway.startup_failed") + assert "redacted_message" not in failed + lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") + assert lifecycle["exit_reason"] == "auth_failed" + exit_event = next(e for e in captured if e["name"] == "gateway.exit") + assert exit_event["restart_requested"] is True + assert exit_event["exit_reason"] == "restart_requested" + + +def test_otlp_attrs_include_gateway_transition_fields(): + from agent.monitoring.otlp_exporter import _span_attrs + + attrs = _span_attrs({ + "event": "gateway_health", + "name": "gateway.lifecycle", + "old_state": "starting", + "new_state": "running", + "exit_reason": "restart", + "restart_requested": True, + }) + + assert attrs["hermes.old_state"] == "starting" + assert attrs["hermes.new_state"] == "running" + assert attrs["hermes.exit_reason"] == "restart" + assert attrs["hermes.restart_requested"] is True + + +def test_otlp_attrs_redact_strings_and_never_export_profile(): + from agent.monitoring.otlp_exporter import _span_attrs + + attrs = _span_attrs({ + "event": "gateway_health", + "name": "gateway.lifecycle", + "profile": "user@example.com", + "exit_reason": "Bearer top-secret-token for user@example.com", + }) + + assert "hermes.profile" not in attrs + assert "top-secret-token" not in str(attrs) + assert "user@example.com" not in str(attrs) + + +def test_resource_attributes_are_allowlisted_and_sanitized(): + from agent.monitoring.gateway_health_export import _safe_resource_attributes + + attrs = _safe_resource_attributes({ + "service.name": "hermes-gateway", + "service.instance.id": "install-1", + "deployment.environment.name": "staging", + "user.email": "user@example.com", + "authorization": "Bearer top-secret-token", + "custom.request.id": "unbounded", + }) + + assert attrs == { + "service.name": "hermes-gateway", + "service.instance.id": attrs["service.instance.id"], + "deployment.environment.name": "staging", + } + assert attrs["service.instance.id"].startswith("sha256:") + assert "install-1" not in attrs["service.instance.id"] + + +def test_instance_id_hash_is_stable_and_distinguishes_instances(): + from agent.monitoring.gateway_health import _safe_instance_id + + first = _safe_instance_id("install-1") + repeat = _safe_instance_id("install-1") + second = _safe_instance_id("install-2") + + assert first == repeat + assert first != second + assert first.startswith("sha256:") + assert "install-1" not in first + + +def test_runtime_resource_attributes_include_stable_hashed_instance(): + from agent.monitoring.gateway_health_export import _runtime_resource_attributes + + config = { + "monitoring": { + "install_id": "private-install-id", + "gateway_health_export": { + "resource_attributes": {"deployment.environment.name": "staging"} + }, + } + } + + attrs = _runtime_resource_attributes(config, telemetry_scope="gateway_health") + + assert attrs["service.name"] == "hermes-gateway" + assert attrs["service.instance.id"].startswith("sha256:") + assert len(attrs["service.instance.id"]) == len("sha256:") + 24 + assert "private-install-id" not in str(attrs) + assert attrs["deployment.environment.name"] == "staging" + assert attrs["telemetry.scope"] == "gateway_health" + + +def test_diagnostic_log_attributes_are_allowlisted_redacted_and_profile_free(): + from agent.monitoring.gateway_health_export import _diagnostic_log_attributes + + attrs = _diagnostic_log_attributes({ + "event": "gateway_diagnostic", + "name": "platform.fatal", + "subsystem": "platform.slack", + "profile": "user@example.com", + "error_code": "Bearer top-secret-token", + "custom": "must-not-egress", + }) + + assert "hermes.profile" not in attrs + assert "hermes.custom" not in attrs + assert "top-secret-token" not in str(attrs) + + +def test_diagnostic_log_streamer_uses_validated_source_as_otel_scope(): + from types import SimpleNamespace + + from agent.monitoring.gateway_health_export import GatewayDiagnosticLogStreamer + + class FakeLogger: + def __init__(self): + self.records = [] + + def emit(self, record): + self.records.append(record) + + class FakeProvider: + def __init__(self): + self.loggers = {} + + def get_logger(self, name): + return self.loggers.setdefault(name, FakeLogger()) + + provider = FakeProvider() + streamer = object.__new__(GatewayDiagnosticLogStreamer) + streamer._provider = provider + streamer._logger = provider.get_logger("hermes.gateway.diagnostics") + streamer._LogRecord = lambda **kwargs: SimpleNamespace(**kwargs) + streamer._sdk = { + "INVALID_TRACE_ID": 0, + "INVALID_SPAN_ID": 0, + "TraceFlags": SimpleNamespace(DEFAULT=0), + "SeverityNumber": SimpleNamespace( + FATAL="fatal", ERROR="error", WARN="warn", INFO="info", DEBUG="debug" + ), + } + streamer.exported = 0 + + streamer([ + { + "event": "gateway_diagnostic", + "name": "gateway.log.warning", + "subsystem": "platform.relay", + "platform": "relay", + "source_logger": "gateway.relay.adapter", + "error_class": "network_error", + "severity": "warning", + }, + { + "event": "gateway_diagnostic", + "name": "gateway.log.warning", + "subsystem": "gateway", + "source_logger": "gateway.relay.adapter\nalice@example.com", + "error_class": "unknown", + "severity": "warning", + }, + ]) + + precise = provider.loggers["gateway.relay.adapter"].records + fallback = provider.loggers["hermes.gateway.diagnostics"].records + assert len(precise) == 1 + assert len(fallback) == 1 + assert precise[0].body == "gateway diagnostic" + assert "source_logger" not in precise[0].attributes + assert "alice@example.com" not in str(provider.loggers) + + +def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch): + from agent.monitoring import gateway_health_export + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("missing sdk"))) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4317"}}, + } + }) + + assert isinstance(runtime, GatewayHealthExportRuntime) + assert runtime.enabled is False + assert runtime.reason == "otlp_unavailable" + + +def test_gateway_health_export_shutdown_flushes_before_unsubscribe(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + calls = [] + + class FakeEmitter: + def flush(self, timeout): + calls.append(("flush", timeout)) + + def unsubscribe(self, subscriber): + calls.append(("unsubscribe", subscriber)) + + class Streamer: + def shutdown(self): + calls.append(("shutdown", self)) + + streamer = Streamer() + log_streamer = Streamer() + monkeypatch.setattr(emitter, "get_emitter", lambda: FakeEmitter()) + + runtime = GatewayHealthExportRuntime( + enabled=True, + streamer=streamer, + log_streamer=log_streamer, + ) + runtime.shutdown() + + assert calls[0][0] == "flush" + assert calls[1:3] == [ + ("unsubscribe", streamer), + ("unsubscribe", log_streamer), + ] + + +def test_gateway_health_export_streams_only_gateway_events(monkeypatch): + from agent.monitoring import gateway_health_export + + captured = {} + + def fake_start_streaming(config, *, event_filter=None): + captured["filter"] = event_filter + return object() + + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_diagnostic_log_streamer", lambda *a, **k: object()) + monkeypatch.setattr(gateway_health_export, "_attach_log_handler", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_emit_snapshot_events", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_start_snapshot_thread", lambda *a, **k: None) + from agent.monitoring import otlp_exporter + monkeypatch.setattr(otlp_exporter, "start_streaming", fake_start_streaming) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True, "metrics_enabled": False}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is True + event_filter = captured["filter"] + assert event_filter({"event": "gateway_health"}) is True + assert event_filter({"event": "gateway_diagnostic"}) is False + assert event_filter({"event": "run"}) is False + assert event_filter({"event": "model_call"}) is False + assert event_filter({"event": "tool_call"}) is False + + +def test_gateway_health_export_metric_failure_does_not_start_streamer(monkeypatch): + from agent.monitoring import gateway_health_export, otlp_exporter + + started = [] + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))) + monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: started.append(True)) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is False + assert runtime.reason == "metrics_start_failed" + assert started == [] + + +def test_gateway_health_export_diagnostic_partial_start_cleans_up(monkeypatch): + from agent.monitoring import emitter, gateway_health_export, otlp_exporter + + class Streamer: + def __call__(self, _batch): + pass + + def shutdown(self): + pass + + streamer = Streamer() + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None) + monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: streamer) + monkeypatch.setattr( + gateway_health_export, + "_start_diagnostic_log_streamer", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")), + ) + emitter.get_emitter().subscribe(streamer) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True, "metrics_enabled": False}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is False + assert runtime.reason == "diagnostics_start_failed" + assert streamer not in emitter.get_emitter()._subscribers + + +def test_gateway_health_export_shutdown_is_bounded(): + import threading + import time + + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + release = threading.Event() + + class Blocking: + def shutdown(self): + release.wait(10) + + runtime = GatewayHealthExportRuntime( + enabled=True, + streamer=Blocking(), + log_streamer=Blocking(), + metric_provider=Blocking(), + ) + started = time.monotonic() + runtime.shutdown() + elapsed = time.monotonic() - started + release.set() + + assert elapsed < 2.5 + + +def test_otlp_streamer_shutdown_unsubscribes(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.otlp_exporter import OTLPStreamer + + class Dummy: + def force_flush(self): + pass + def shutdown(self): + pass + + e = emitter.get_emitter() + streamer = OTLPStreamer.__new__(OTLPStreamer) + streamer._processor = Dummy() + streamer._provider = Dummy() + streamer._event_filter = None + streamer.exported = 0 + e.subscribe(streamer) + assert streamer in e._subscribers + + streamer.shutdown() + + assert streamer not in e._subscribers + + +def test_gateway_diagnostic_log_handler_never_raises_on_malformed_record(): + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + record = logging.LogRecord( + "gateway.platforms.slack", + logging.WARNING, + __file__, + 1, + "broken %s %s", + ("one",), + None, + ) + + handler.emit(record) + + +def test_install_id_persists_across_calls(tmp_path, monkeypatch): + """A minted install id must survive restarts (service.instance.id continuity).""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text("{}\n") + + import hermes_cli.config as cfg_mod + from agent.monitoring.policy import ensure_install_id + + first = ensure_install_id(cfg_mod.load_config()) + assert first and first != "unknown" + # Persisted: a fresh load (simulating a new gateway process) returns the same id. + second = ensure_install_id(cfg_mod.load_config()) + assert second == first + assert first in (tmp_path / "config.yaml").read_text() + + +def test_install_id_existing_value_wins(monkeypatch): + from agent.monitoring.policy import ensure_install_id + + assert ensure_install_id({"monitoring": {"install_id": "keep-me"}}) == "keep-me" diff --git a/tests/monitoring/test_otlp_exporter.py b/tests/monitoring/test_otlp_exporter.py new file mode 100644 index 00000000000..93749fcbfbb --- /dev/null +++ b/tests/monitoring/test_otlp_exporter.py @@ -0,0 +1,140 @@ +"""OTLP exporter tests: config resolution, span mapping, streaming subscriber. + +No SQLite involved — monitoring is an egress path, so the exporter consumes +emitter batches directly. Uses the in-memory OTel span exporter; skipped when +the optional otlp extra is not installed. +""" + +from __future__ import annotations + +import pytest + +otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed") + +import agent.monitoring.otlp_exporter as OE +from agent.monitoring.emitter import MonitoringEmitter + + +def _mem_provider(): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def test_gateway_health_event_maps_to_span_with_attrs(): + provider, mem = _mem_provider() + n = OE.export_batch(provider, [{ + "event": "gateway_health", "name": "gateway.lifecycle", + "old_state": "starting", "new_state": "running", + "active_agents": 2, "pid": 4242, + }]) + assert n == 1 + spans = mem.get_finished_spans() + assert spans[0].name == "hermes.gateway_health" + attrs = dict(spans[0].attributes or {}) + assert attrs["hermes.old_state"] == "starting" + assert attrs["hermes.new_state"] == "running" + assert attrs["hermes.active_agents"] == 2 + + +def test_gateway_diagnostic_event_drops_arbitrary_message_content(): + provider, mem = _mem_provider() + OE.export_batch(provider, [{ + "event": "gateway_diagnostic", "name": "platform.fatal", + "subsystem": "platform.slack", "error_class": "auth_failed", + "redacted_message": "Unauthorized user: acct_7f3a (Alice Smith)", + "severity": "error", + }]) + attrs = dict(mem.get_finished_spans()[0].attributes or {}) + assert attrs["hermes.error_class"] == "auth_failed" + assert "hermes.redacted_message" not in attrs + assert "acct_7f3a" not in str(attrs) + assert "Alice Smith" not in str(attrs) + + +def test_unknown_event_kind_exports_no_attrs_beyond_kind(): + provider, mem = _mem_provider() + OE.export_batch(provider, [{"event": "model_call", "provider": "anthropic", + "model": "claude-opus-4"}]) + attrs = dict(mem.get_finished_spans()[0].attributes or {}) + # Non-monitoring event kinds carry no attribute mapping on this plane. + assert attrs == {"hermes.event": "model_call"} + + +def test_headers_resolve_from_env_not_value(monkeypatch): + monkeypatch.setenv("DD_KEY_ENV", "secret-value") + resolved = OE._resolve_headers({"DD-API-KEY": "DD_KEY_ENV", "X-Missing": "NOPE_ENV"}) + assert resolved == {"DD-API-KEY": "secret-value"} + + +def test_is_enabled_requires_endpoint_and_flag(): + assert OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) + assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True}}}}) + assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"endpoint": "http://x"}}}}) + assert not OE.is_enabled({}) + + +def test_trace_resource_includes_stable_hashed_instance(): + attrs = OE._resource_attributes( + {"monitoring": {"install_id": "private-install-id"}} + ) + + assert attrs["service.name"] == "hermes-gateway" + assert attrs["service.instance.id"].startswith("sha256:") + assert len(attrs["service.instance.id"]) == len("sha256:") + 24 + assert "private-install-id" not in str(attrs) + assert attrs["telemetry.scope"] == "gateway_monitoring" + + +def test_export_otlp_feature_specs_match_pyproject(): + from tools.lazy_deps import LAZY_DEPS + import re + from pathlib import Path + + specs = set(LAZY_DEPS["export.otlp"]) + pyproject = Path(__file__).resolve().parents[2] / "pyproject.toml" + m = re.search(r'^otlp = \[(.*?)\]', pyproject.read_text(), re.M | re.S) + assert m, "otlp extra missing from pyproject.toml" + extra = set(re.findall(r'"([^"]+)"', m.group(1))) + assert specs == extra + + +def test_streamer_receives_events_and_respects_filter(monkeypatch): + provider, mem = _mem_provider() + monkeypatch.setattr(OE, "_make_provider", lambda cfg: (provider, None)) + streamer = OE.OTLPStreamer( + {}, event_filter=lambda ev: ev.get("event") == "gateway_health") + + em = MonitoringEmitter() + em.subscribe(streamer) + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + em.emit({"event": "model_call", "provider": "anthropic"}) # filtered out + em.flush() + em.close() + + spans = mem.get_finished_spans() + assert [s.name for s in spans] == ["hermes.gateway_health"] + assert streamer.exported == 1 + + +def test_failing_streamer_never_breaks_emitter(monkeypatch): + def boom(cfg): + raise RuntimeError("no provider") + + em = MonitoringEmitter() + + def bad_subscriber(batch): + raise RuntimeError("export down") + + seen: list = [] + em.subscribe(bad_subscriber) + em.subscribe(lambda batch: seen.extend(batch)) + em.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + em.flush() + em.close() + assert len(seen) == 1 diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index 1d7d92b40e8..e7f5eea15e0 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -632,6 +632,34 @@ def test_completed_records_pruned_to_cap(): assert len(ad.list_async_delegations()) <= ad._MAX_RETAINED_COMPLETED +def test_active_task_count_expands_batches_while_active_count_stays_unit(monkeypatch): + """active_count() counts dispatch UNITS (batch=1); active_task_count() + expands a batch to its child count. This is the batch-vs-single distinction + the background_work metric relies on so a 3-task fan-out isn't undercounted + as 1 running subagent. + """ + # Deterministic: install synthetic running records directly, no real spawn. + with ad._records_lock: + saved = dict(ad._records) + ad._records.clear() + ad._records["single_a"] = {"status": "running"} # single subagent + ad._records["batch_3"] = {"status": "running", "is_batch": True, + "goals": ["g1", "g2", "g3"]} # 3-task batch + ad._records["batch_missing"] = {"status": "running", "is_batch": True} # goals absent -> 1 + ad._records["done"] = {"status": "completed", "is_batch": True, + "goals": ["x", "y"]} # not running -> ignored + try: + # 3 running UNITS (single + 2 batches); the completed one is excluded. + assert ad.active_count() == 3 + # TASKS: single(1) + batch_3(3) + batch_missing(1, fallback) = 5. + assert ad.active_task_count() == 5 + finally: + with ad._records_lock: + ad._records.clear() + ad._records.update(saved) + + + def test_completion_is_persisted_and_delivery_can_be_acknowledged(tmp_path, monkeypatch): """A finished child remains pending on disk until its queue consumer acks it.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) diff --git a/tests/tools/test_dockerfile_pid1_reaping.py b/tests/tools/test_dockerfile_pid1_reaping.py index 699fd5709a1..3b3e069c45f 100644 --- a/tests/tools/test_dockerfile_pid1_reaping.py +++ b/tests/tools/test_dockerfile_pid1_reaping.py @@ -172,6 +172,20 @@ def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text): ) +def test_dockerfile_preinstalls_gateway_monitoring_otlp_runtime(dockerfile_text): + sync_steps = [ + step for step in _run_steps(dockerfile_text) + if "uv sync" in step and "--no-install-project" in step + ] + + assert sync_steps, "Dockerfile must install Python dependencies with uv sync" + assert any("--extra otlp" in step for step in sync_steps), ( + "Published Docker images must preload the Hermes [otlp] runtime extra " + "so enabled Gateway Health export does not depend on first-boot package " + "installation into the immutable container environment." + ) + + def test_dockerfile_preinstalls_matrix_dependencies(dockerfile_text): sync_steps = [ step for step in _run_steps(dockerfile_text) diff --git a/tools/async_delegation.py b/tools/async_delegation.py index dbe0b2f1074..702036df0c4 100644 --- a/tools/async_delegation.py +++ b/tools/async_delegation.py @@ -533,7 +533,14 @@ def _get_executor(max_workers: int) -> ThreadPoolExecutor: def active_count() -> int: - """Number of async delegations currently running.""" + """Number of async delegation UNITS currently running. + + A unit is one dispatch: a single subagent OR a whole fan-out batch. A batch + counts as ONE here because it occupies one async-pool slot (the capacity + semantics ``dispatch_async_delegation_batch`` relies on). For the count of + actual concurrent child subagents (batch expanded), use + ``active_task_count()``. + """ with _records_lock: return sum( 1 for r in _records.values() @@ -541,6 +548,29 @@ def active_count() -> int: ) +def active_task_count() -> int: + """Number of async delegation TASKS (child subagents) currently running. + + Unlike ``active_count()`` (units/slots), this expands a batch to its child + count: a running batch of N tasks contributes N, a single subagent + contributes 1. This is the truthful "how many subagents are actually + working right now" figure for observability, where a 3-task batch shown as + "1" undercounts real concurrent work. Falls back to counting a batch as 1 + if its goal list is missing. + """ + with _records_lock: + total = 0 + for r in _records.values(): + if r.get("status") not in {"running", "finalizing"}: + continue + if r.get("is_batch"): + goals = r.get("goals") + total += len(goals) if isinstance(goals, (list, tuple)) and goals else 1 + else: + total += 1 + return total + + def _new_delegation_id() -> str: return f"deleg_{uuid.uuid4().hex[:8]}" diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index c965e1734af..983af1ff5f1 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -116,6 +116,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "search.firecrawl": ("firecrawl-py==4.17.0",), "search.parallel": ("parallel-web==0.4.2",), + # ─── Monitoring ───────────────────────────────────────────────────────── + # OTLP gateway monitoring export. Lazily installed on first use of + # monitoring.gateway_health_export / monitoring.export.otlp. Tracks the + # `otlp` extra in pyproject.toml — bump both together. + "export.otlp": ( + "opentelemetry-sdk==1.39.1", + "opentelemetry-exporter-otlp-proto-http==1.39.1", + ), + # ─── TTS providers ───────────────────────────────────────────────────── # Pinned to exact versions to match pyproject.toml's no-ranges policy # (see comment at top of [project.dependencies]). When bumping, update diff --git a/uv.lock b/uv.lock index ef3672e745e..978909fc0ba 100644 --- a/uv.lock +++ b/uv.lock @@ -1444,7 +1444,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/58/5404031044f55afad7aad1aff8be3f22b1bed03e237cfeabbc7e5c8cfde0/greenlet-3.5.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:aca9b4ce85b152b5524ef7d88170efdff80dc0032aa8b75f9aaf7f3479ea95b4", size = 287424, upload-time = "2026-06-26T18:20:31.469Z" }, { url = "https://files.pythonhosted.org/packages/b4/bf/1c65e9b94a54d547068fa5b5a8a06f221f3316b48908e08668d29c77cb50/greenlet-3.5.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f71be4920368fe1fabeeaa53d1e3548337e2b223d9565f8ad5e392a75ba23fc", size = 606523, upload-time = "2026-06-26T19:07:08.859Z" }, { url = "https://files.pythonhosted.org/packages/b8/c7/b66baacc95775ad511287acb0137b95574a9ce5491902372b7564799d790/greenlet-3.5.3-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d77e67f65f98449e3fb83f795b5d0a8437aead2f874ca89c96576caf4be3af6", size = 618315, upload-time = "2026-06-26T19:10:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a0/68afd1ebad40db87dac0a28ffa120726b98bf9c7c40c481b0f63c105d298/greenlet-3.5.3-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e18619ba655ac05d78d80fc83cac4ba892bd6927b99e3b8237aee861aaacc8bb", size = 626155, upload-time = "2026-06-26T19:24:14.44Z" }, { url = "https://files.pythonhosted.org/packages/78/2b/28ed29463522fdbe4c15b1f63922041626a7478316b34ab4adda3f0a4aba/greenlet-3.5.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8540f1e6205bd13ca0ce685581037219ca54a1b41a0a15d228c6c9b8ad5903d7", size = 617381, upload-time = "2026-06-26T18:32:16.077Z" }, + { url = "https://files.pythonhosted.org/packages/07/7f/e327d912239ec4b3b49999e3967389bcf1ee8722b9ee9194d2752ecd558a/greenlet-3.5.3-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d27c0c653a60d9535f690226474a5cc1036a8b0d7b57504d1c4f89c44a07a80c", size = 421083, upload-time = "2026-06-26T19:25:35.804Z" }, { url = "https://files.pythonhosted.org/packages/2a/7b/ad04e9d1337fc04965dc9fc616b6a72cb65a24b800a014c011ec812f5489/greenlet-3.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ef56fe650f50575bf843acde967b9c567687f3c22340941a899b7bc56e956a8", size = 1577771, upload-time = "2026-06-26T19:09:01.537Z" }, { url = "https://files.pythonhosted.org/packages/d8/33/6c87ab7ba663f70ca21f3022aad1ffe56d3f3e0521e836c2415e13abcc3c/greenlet-3.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5121af01cf911e70056c00d4b46d5e9b5d1415550038573d744138bacb59e6b8", size = 1644048, upload-time = "2026-06-26T18:31:42.996Z" }, { url = "https://files.pythonhosted.org/packages/1c/35/f0d8ee998b422cf8693b270f098e55d8d4ec8006b061b333f54f177d28d9/greenlet-3.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:0f41e4a05a3c0cb31b17023eff28dd111e1d16bf7d7d00406cd7df23f31398a7", size = 239137, upload-time = "2026-06-26T18:23:21.664Z" }, @@ -1452,7 +1454,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/6e/4c37d51a2b7f82d2ff11bb6b5f7d766d9a011726624af255e843727627a3/greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2", size = 288685, upload-time = "2026-06-26T18:22:08.977Z" }, { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, + { url = "https://files.pythonhosted.org/packages/fb/fb/d97dc261209c80744b7c8132693a30d70ec6e7315e632cb0a10b3fec94dd/greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23", size = 622351, upload-time = "2026-06-26T19:24:16.32Z" }, { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ac/e5fee13cbbd0e8de312d9a146584b8a51891c68847330ef9dc8b5109d23f/greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c", size = 425395, upload-time = "2026-06-26T19:25:37.144Z" }, { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, { url = "https://files.pythonhosted.org/packages/14/40/c57489acf8e37d74e2913d4eff63aa0dba17acccc4bdeef874dde2dbbec9/greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c", size = 239882, upload-time = "2026-06-26T18:23:27.518Z" }, @@ -1460,7 +1464,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" }, { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/61/2f5b1adf256d039f5dab8005de8d3d7ad2b0070a3219c0e036b3fbfeb440/greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1", size = 625580, upload-time = "2026-06-26T19:24:18.344Z" }, { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/ab7fc9e543e44d6879b0a6ef9a4b2188940fd180cc65d6f646883ddf7201/greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda", size = 427933, upload-time = "2026-06-26T19:25:38.219Z" }, { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" }, @@ -1693,6 +1699,10 @@ mistral = [ modal = [ { name = "modal" }, ] +otlp = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, +] parallel-web = [ { name = "parallel-web" }, ] @@ -1848,6 +1858,8 @@ requires-dist = [ { name = "numpy", marker = "extra == 'wake'", specifier = "==2.4.3" }, { name = "onnxruntime", marker = "extra == 'wake'", specifier = "==1.27.0" }, { name = "openai", specifier = "==2.24.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp'", specifier = "==1.39.1" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp'", specifier = "==1.39.1" }, { name = "openwakeword", marker = "extra == 'wake'", specifier = "==0.6.0" }, { name = "packaging", specifier = "==26.0" }, { name = "parallel-web", marker = "extra == 'parallel-web'", specifier = "==0.4.2" }, @@ -1900,7 +1912,7 @@ requires-dist = [ { name = "websockets", specifier = "==15.0.1" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "wake", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "wake", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] [[package]] name = "hf-xet" @@ -3969,7 +3981,7 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -4024,7 +4036,7 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [