Merge pull request #64536 from victor-kyriazakos/feat/gateway-health-diagnostics

feat(monitoring): gateway health & diagnostics OTLP export

Dockerfile

@@ -227,7 +227,7 @@ RUN cd plugins/platforms/photon/sidecar && \
 # frontend stats the readme path during dep resolution, so we `touch` an
 # empty placeholder — the real README is restored by `COPY . .` below.
 #
-# `uv sync --frozen --no-install-project --extra all --extra messaging`
+# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp`
 # installs the deps reachable through the composite `[all]` extra
 # (handpicked set intended for the production image — excludes `[dev]`),
 # plus gateway messaging adapters that should work in the published image
@@ -240,6 +240,10 @@ RUN cd plugins/platforms/photon/sidecar && \
 # so Docker users can use these providers without requiring runtime
 # lazy-install access to PyPI (often blocked in containerized envs).
 #
+# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway
+# Health export is enabled. Collector and observability-backend dependencies
+# remain external and are not part of the Hermes production image.
+#
 # The hindsight memory provider's client (hindsight-client) is baked in
 # for the same reason: it lazy-installs into /opt/hermes/.venv at first
 # use, which lives inside the (immutable) image layer rather than the
@@ -257,7 +261,7 @@ RUN cd plugins/platforms/photon/sidecar && \
 # The editable link is created after the source copy below.
 COPY pyproject.toml uv.lock ./
 RUN touch ./README.md
-RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix
+RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix
 
 # ---------- Frontend build (cached independently from Python source) ----------
 # Copy only the frontend source trees first so that Python-only changes don't

agent/monitoring/__init__.py

@@ -0,0 +1,29 @@
+"""Hermes gateway monitoring.
+
+Service health monitoring plus redacted operational diagnostics for the
+gateway daemon, exported over OTLP to an operator-configured endpoint.
+
+``emitter`` is the in-process event bus: producers (gateway status hooks,
+the diagnostic log handler) hand typed events to a fire-and-forget queue,
+and subscribers (the OTLP streamers) consume them off the hot path. The
+emitter never blocks or raises into gateway code (the hot-path invariant),
+and nothing is persisted locally — monitoring is an egress path, not a store.
+
+Deliberately out of scope here: run/model/tool trajectory capture, usage
+analytics, and any content-bearing signal. Those planes are served by the
+NeMo Relay integration and its Hermes-owned subscribers.
+"""
+
+from __future__ import annotations
+
+from . import emitter, events
+
+emit = emitter.emit
+get_emitter = emitter.get_emitter
+
+__all__ = [
+    "emitter",
+    "events",
+    "emit",
+    "get_emitter",
+]

agent/monitoring/cron_health.py

@@ -0,0 +1,201 @@
+"""Content-free cron service-health and execution telemetry projection."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Optional
+
+from agent.monitoring.events import CronExecutionEvent
+from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric
+from cron.jobs import (
+    _compute_grace_seconds,
+    get_catch_up_occurrence_count,
+    get_ticker_heartbeat_age,
+    get_ticker_success_age,
+    load_jobs,
+)
+from cron.scheduler import get_running_job_ids
+from hermes_time import now as _hermes_now
+
+logger = logging.getLogger(__name__)
+_KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"}
+_KNOWN_SOURCES = {"builtin", "direct", "external"}
+_KNOWN_DELIVERY_OUTCOMES = {"delivered", "failed", "suppressed", "not_configured"}
+
+
+@dataclass(frozen=True, slots=True)
+class CronHealthSnapshot:
+    metrics: list[GatewayMetric]
+    events: list[CronExecutionEvent]
+
+
+def _now() -> datetime:
+    return _hermes_now()
+
+
+def _job_key(raw: Any) -> str:
+    value = str(raw or "unknown").encode("utf-8", errors="replace")
+    return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
+
+
+def classify_cron_error(raw: Any) -> str:
+    text = str(raw or "").lower()
+    if (
+        re.search(r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden)\b", text)
+        or re.search(r"\bbearer\b", text)
+        or re.search(r"\b(?:access|api|refresh) token\b", text)
+        or re.search(r"\b(?:401|403)\b", text)
+    ):
+        return "auth_failed"
+    if "rate limit" in text or "429" in text or "quota" in text:
+        return "rate_limited"
+    if "timeout" in text or "timed out" in text:
+        return "timeout"
+    if any(value in text for value in ("network", "connection", "dns", "socket", "unreachable")):
+        return "network_error"
+    if "dispatch" in text or "executor" in text:
+        return "dispatch_failed"
+    if "interrupt" in text or "owner exited" in text or "restarted" in text:
+        return "interrupted"
+    if "empty response" in text:
+        return "empty_response"
+    if any(value in text for value in ("config", "missing", "invalid")):
+        return "invalid_config"
+    return "unknown"
+
+
+def _parse_time(raw: Any) -> Optional[datetime]:
+    try:
+        return datetime.fromisoformat(str(raw)) if raw else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _duration_ms(record: dict[str, Any]) -> Optional[int]:
+    start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at"))
+    finish = _parse_time(record.get("finished_at"))
+    if start is None or finish is None:
+        return None
+    try:
+        duration = int((finish - start).total_seconds() * 1000)
+    except (TypeError, ValueError):
+        return None
+    return max(0, duration)
+
+
+def project_execution_event(
+    record: dict[str, Any], *, delivery_outcome: Optional[str] = None
+) -> CronExecutionEvent:
+    status = str(record.get("status") or "unknown").lower()
+    source = str(record.get("source") or "unknown").lower()
+    if source not in _KNOWN_SOURCES and source != "unknown":
+        source = "external"
+    outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None
+    return CronExecutionEvent(
+        status=status if status in _KNOWN_STATUSES else "unknown",
+        job_key=_job_key(record.get("job_id")),
+        source=source if source in _KNOWN_SOURCES else "unknown",
+        duration_ms=_duration_ms(record),
+        delivery_outcome=(
+            outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None
+        ),
+        error_class=(
+            classify_cron_error(record.get("error"))
+            if status in {"failed", "unknown"}
+            else None
+        ),
+    )
+
+
+def emit_execution_state(
+    record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None
+) -> None:
+    """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier."""
+    if not record:
+        return
+    try:
+        from agent.monitoring import emitter
+
+        event = project_execution_event(record, delivery_outcome=delivery_outcome)
+        target = emitter.get_emitter()
+        target.emit(event)
+        if event.status in {"completed", "failed", "unknown"}:
+            target.flush(timeout=1.0)
+    except Exception:
+        logger.debug("cron execution telemetry emit failed", exc_info=True)
+
+
+def _is_overdue(job: dict[str, Any], now: datetime) -> bool:
+    if not job.get("enabled", True):
+        return False
+    next_run = _parse_time(job.get("next_run_at"))
+    schedule = job.get("schedule")
+    if next_run is None or not isinstance(schedule, dict):
+        return False
+    try:
+        if next_run.tzinfo is None and now.tzinfo is not None:
+            next_run = next_run.replace(tzinfo=now.tzinfo)
+        lateness = (now - next_run).total_seconds()
+        return lateness > _compute_grace_seconds(schedule)
+    except (TypeError, ValueError):
+        return False
+
+
+def build_cron_health_snapshot() -> CronHealthSnapshot:
+    metrics: list[GatewayMetric] = []
+    for name, reader in (
+        ("hermes.cron.scheduler.heartbeat_age_seconds", get_ticker_heartbeat_age),
+        ("hermes.cron.scheduler.last_success_age_seconds", get_ticker_success_age),
+    ):
+        try:
+            value = reader()
+            if value is not None:
+                metrics.append(GatewayMetric(name, max(0.0, float(value)), {}))
+        except Exception:
+            logger.debug("cron freshness metric unavailable", exc_info=True)
+
+    try:
+        metrics.append(
+            GatewayMetric(
+                "hermes.cron.scheduler.catch_up_occurrences",
+                get_catch_up_occurrence_count(),
+                {},
+            )
+        )
+    except Exception:
+        logger.debug("cron catch-up metric unavailable", exc_info=True)
+
+    try:
+        jobs = load_jobs()
+        enabled = [job for job in jobs if job.get("enabled", True)]
+        metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {}))
+        metrics.append(
+            GatewayMetric(
+                "hermes.cron.jobs.overdue",
+                sum(1 for job in enabled if _is_overdue(job, _now())),
+                {},
+            )
+        )
+    except Exception:
+        logger.debug("cron job metrics unavailable", exc_info=True)
+
+    try:
+        metrics.append(
+            GatewayMetric("hermes.cron.jobs.running", len(get_running_job_ids()), {})
+        )
+    except Exception:
+        logger.debug("cron running-job metric unavailable", exc_info=True)
+    return CronHealthSnapshot(metrics=metrics, events=[])
+
+
+__all__ = [
+    "CronHealthSnapshot",
+    "build_cron_health_snapshot",
+    "classify_cron_error",
+    "emit_execution_state",
+    "project_execution_event",
+]

agent/monitoring/emitter.py

@@ -0,0 +1,211 @@
+"""Monitoring emitter: fire-and-forget queue + background dispatcher.
+
+The emitter is the single seam between producers (gateway status hooks, the
+diagnostic log handler) and consumers (the OTLP streamers). Its contract is
+the hot-path invariant:
+
+    ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network,
+    and MUST NEVER raise into the caller. A monitoring failure is logged
+    locally and dropped — it can never affect the gateway or a session.
+
+Mechanism:
+  * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare
+    except. On a full queue it drops the *oldest* event and counts the drop.
+  * A daemon thread drains the queue and fans each batch out to subscribers
+    (the OTLP metric/span/log streamers). Each subscriber is fail-isolated —
+    a slow or raising subscriber never affects the hot path or its peers.
+
+Nothing is persisted here. Monitoring is an egress path, not a local store;
+if no subscriber is attached, events simply age out of the ring buffer.
+"""
+
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+import time
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+_MAX_QUEUE = 10_000  # ring-buffer depth; oldest dropped when full
+_DRAIN_BATCH = 256
+
+
+class MonitoringEmitter:
+    """Owns the queue, the dispatcher thread, and the subscriber list."""
+
+    def __init__(self, *, enabled: bool = True) -> None:
+        self._enabled = enabled
+        self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE)
+        self._dropped = 0
+        self._dispatched = 0
+        self._stop = threading.Event()
+        self._started = False
+        self._lock = threading.Lock()
+        self._thread: Optional[threading.Thread] = None
+        # Live subscribers (the OTLP streamers). Called from the dispatcher
+        # thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]).
+        self._subscribers: list = []
+
+    # ── public API (hot path) ───────────────────────────────────────────────
+    def emit(self, event: Any) -> None:
+        """Enqueue an event. Never blocks, never raises.
+
+        ``event`` may be a dataclass with ``to_dict()`` or a plain dict.
+        """
+        if not self._enabled:
+            return
+        try:
+            payload = event.to_dict() if hasattr(event, "to_dict") else dict(event)
+            payload.setdefault("ts_ns", time.time_ns())
+            self._ensure_started()
+            try:
+                self._q.put_nowait(payload)
+            except queue.Full:
+                # Drop oldest to make room — bounded memory, newest-wins.
+                try:
+                    self._q.get_nowait()
+                    self._q.task_done()
+                    self._dropped += 1
+                    self._q.put_nowait(payload)
+                except Exception:
+                    self._dropped += 1
+        except Exception:  # the hot-path invariant: never propagate
+            logger.debug("monitoring emit failed", exc_info=True)
+
+    # ── lifecycle ───────────────────────────────────────────────────────────
+    def _ensure_started(self) -> None:
+        if self._started:
+            return
+        with self._lock:
+            if self._started:
+                return
+            self._thread = threading.Thread(
+                target=self._run, name="hermes-monitoring-dispatch", daemon=True
+            )
+            self._thread.start()
+            self._started = True
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            try:
+                first = self._q.get(timeout=0.5)
+            except queue.Empty:
+                continue
+            batch = [first]
+            while len(batch) < _DRAIN_BATCH:
+                try:
+                    batch.append(self._q.get_nowait())
+                except queue.Empty:
+                    break
+            try:
+                self._dispatch(batch)
+            finally:
+                for _ in batch:
+                    self._q.task_done()
+
+    def _dispatch(self, batch) -> None:
+        # Fan-out to subscribers (OTLP streamers) — fully fail-isolated.
+        for sub in list(self._subscribers):
+            try:
+                sub(batch)
+            except Exception:
+                logger.debug("monitoring subscriber failed", exc_info=True)
+        self._dispatched += len(batch)
+
+    def subscribe(self, callback) -> None:
+        """Register a live batch subscriber (callable(batch: list[dict]))."""
+        if callback not in self._subscribers:
+            self._subscribers.append(callback)
+        self._enabled = True
+
+    def unsubscribe(self, callback) -> None:
+        try:
+            self._subscribers.remove(callback)
+        except ValueError:
+            pass
+        if not self._subscribers:
+            self._enabled = False
+
+    # ── introspection / shutdown (tests, CLI) ───────────────────────────────
+    def flush(self, timeout: float = 2.0) -> None:
+        """Wait boundedly for queued and in-flight batches to finish dispatch."""
+        if timeout <= 0:
+            return
+
+        finished = threading.Event()
+
+        def _wait_for_completion() -> None:
+            self._q.join()
+            finished.set()
+
+        waiter = threading.Thread(
+            target=_wait_for_completion,
+            name="hermes-monitoring-flush",
+            daemon=True,
+        )
+        waiter.start()
+        finished.wait(timeout=timeout)
+
+    def stats(self) -> Dict[str, int]:
+        return {
+            "queued": self._q.qsize(),
+            "dispatched": self._dispatched,
+            "dropped": self._dropped,
+            "subscribers": len(self._subscribers),
+        }
+
+    def close(self) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=2.0)
+        self._started = False
+
+
+# ── process-wide singleton ──────────────────────────────────────────────────
+_EMITTER: Optional[MonitoringEmitter] = None
+_EMITTER_LOCK = threading.Lock()
+
+
+def get_emitter() -> MonitoringEmitter:
+    """Return the process-wide monitoring emitter."""
+    global _EMITTER
+    if _EMITTER is not None:
+        return _EMITTER
+    with _EMITTER_LOCK:
+        if _EMITTER is None:
+            # Collection is opt-in. A plane exporter enables the singleton by
+            # attaching its first subscriber; until then producers are no-ops.
+            _EMITTER = MonitoringEmitter(enabled=False)
+    return _EMITTER
+
+
+def emit(event: Any) -> None:
+    """Module-level convenience: emit via the singleton."""
+    get_emitter().emit(event)
+
+
+def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None:
+    """Swap the singleton (tests only)."""
+    global _EMITTER
+    with _EMITTER_LOCK:
+        if _EMITTER is not None and emitter is not _EMITTER:
+            try:
+                _EMITTER.close()
+            except Exception:
+                pass
+        _EMITTER = emitter
+
+
+# Back-compat alias for the salvaged class name used in emozilla's tests.
+TelemetryEmitter = MonitoringEmitter
+
+__all__ = [
+    "MonitoringEmitter",
+    "TelemetryEmitter",
+    "get_emitter",
+    "emit",
+    "reset_emitter_for_tests",
+]

agent/monitoring/events.py

@@ -0,0 +1,86 @@
+"""Typed gateway monitoring events.
+
+Content-free service-health and redacted diagnostic events for the gateway
+daemon. These are the only event shapes the monitoring plane emits: no
+prompts, messages, tool args/results, session history, or usage analytics.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field, asdict
+from typing import Any, Dict, Optional
+
+
+def _now_ns() -> int:
+    return time.time_ns()
+
+
+@dataclass(slots=True)
+class GatewayHealthEvent:
+    """Content-free gateway health snapshot or lifecycle event."""
+
+    name: str
+    gateway_state: Optional[str] = None
+    old_state: Optional[str] = None
+    new_state: Optional[str] = None
+    exit_reason: Optional[str] = None
+    restart_requested: Optional[bool] = None
+    active_agents: int = 0
+    gateway_busy: bool = False
+    gateway_drainable: bool = False
+    platform_count: int = 0
+    fatal_platform_count: int = 0
+    profile: Optional[str] = None
+    install_id: Optional[str] = None
+    version: Optional[str] = None
+    supervision_mode: Optional[str] = None
+    pid: Optional[int] = None
+    ts_ns: int = field(default_factory=_now_ns)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"event": "gateway_health", **asdict(self)}
+
+
+@dataclass(slots=True)
+class GatewayDiagnosticEvent:
+    """Redacted gateway diagnostic event for operator-owned observability."""
+
+    name: str
+    subsystem: str
+    error_class: str = "unknown"
+    error_code: Optional[str] = None
+    platform: Optional[str] = None
+    old_state: Optional[str] = None
+    new_state: Optional[str] = None
+    profile: Optional[str] = None
+    version: Optional[str] = None
+    severity: str = "warning"
+    ts_ns: int = field(default_factory=_now_ns)
+    source_logger: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"event": "gateway_diagnostic", **asdict(self)}
+
+
+@dataclass(slots=True)
+class CronExecutionEvent:
+    """Content-free durable cron execution lifecycle projection."""
+
+    status: str
+    job_key: str
+    source: str = "unknown"
+    duration_ms: Optional[int] = None
+    delivery_outcome: Optional[str] = None
+    error_class: Optional[str] = None
+    ts_ns: int = field(default_factory=_now_ns)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"event": "cron_execution", **asdict(self)}
+
+
+__all__ = [
+    "GatewayHealthEvent",
+    "GatewayDiagnosticEvent",
+    "CronExecutionEvent",
+]

agent/monitoring/gateway_health.py

@@ -0,0 +1,469 @@
+"""Gateway health and diagnostics signal producer.
+
+This module keeps the plane narrow: service health monitoring plus
+redacted operational diagnostics. It reuses the existing gateway runtime-status
+contract and emits content-free metrics/events. No prompts, messages, tool args,
+session history, audit records, or product analytics belong here.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent
+
+
+@dataclass(frozen=True, slots=True)
+class GatewayMetric:
+    name: str
+    value: int | float
+    attributes: Dict[str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class GatewayHealthSnapshot:
+    metrics: List[GatewayMetric]
+    events: List[GatewayHealthEvent | GatewayDiagnosticEvent]
+
+
+_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
+_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
+_KNOWN_GATEWAY_STATES = {
+    "starting", "draining", "stopping", "stopped", "startup_failed", "unknown"
+} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES
+_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
+    "connecting", "disconnected", "disabled", "paused", "retrying", "unknown"
+}
+_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
+_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$")
+
+
+def _allowed_logger(name: str) -> bool:
+    return name == "gateway" or name.startswith("gateway.")
+
+
+def source_logger_for_export(name: Any) -> Optional[str]:
+    """Return a bounded source-controlled gateway logger name for OTLP scope."""
+    value = str(name or "")
+    return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None
+
+
+def redact_gateway_message(message: Any) -> str:
+    """Redact gateway diagnostic free text for operator-owned export.
+
+    Single scrub path: everything goes through
+    ``agent.monitoring.redaction.redact_for_export`` (unconditional
+    secrets + PII), then is length-bounded.
+    """
+    try:
+        from agent.monitoring.redaction import redact_for_export
+        redacted = redact_for_export(str(message or "")) or ""
+    except Exception:
+        redacted = "[redaction-unavailable]"
+    return redacted[:500]
+
+
+def classify_gateway_error(raw: Any) -> str:
+    s = str(raw or "").lower()
+    if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")):
+        return "auth_failed"
+    if "rate" in s and "limit" in s:
+        return "rate_limited"
+    if "timeout" in s or "timed out" in s:
+        return "timeout"
+    if any(
+        k in s
+        for k in (
+            "network",
+            "connection",
+            "dns",
+            "socket",
+            "connect call failed",
+            "failed to connect",
+            "cannot connect",
+            "unreachable",
+            "name resolution",
+        )
+    ):
+        return "network_error"
+    if any(k in s for k in ("config", "missing", "invalid")):
+        return "invalid_config"
+    if "startup" in s:
+        return "startup_failed"
+    if "fatal" in s:
+        return "platform_fatal"
+    return "unknown"
+
+
+def classify_exit_reason(
+    raw: Any, *, state: Any, restart_requested: bool
+) -> Optional[str]:
+    """Reduce free-form shutdown text to a bounded operational class."""
+    if restart_requested:
+        return "restart_requested"
+    state_name = str(state or "").lower()
+    if raw is None and state_name != "startup_failed":
+        return None
+    classified = classify_gateway_error(raw)
+    if state_name == "startup_failed":
+        return classified if classified != "unknown" else "startup_failed"
+    text = str(raw or "").lower()
+    if "signal" in text or "sigterm" in text or "sigint" in text:
+        return "signal"
+    if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")):
+        return "planned_stop"
+    return classified
+
+
+def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
+    state = str(raw or "unknown").lower()
+    return state if state in allowed else "unknown"
+
+
+def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
+    try:
+        from agent.monitoring.redaction import redact_for_export
+        value = redact_for_export(str(raw or "")) or "unknown"
+    except Exception:
+        return "unknown"
+    return value[:limit]
+
+
+def _safe_instance_id(raw: Any) -> str:
+    """Return a stable opaque instance key without exporting the source ID."""
+    value = str(raw or "unknown").encode("utf-8", errors="replace")
+    return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
+
+
+def subsystem_for_logger(logger_name: str) -> str:
+    if logger_name == "gateway.relay" or logger_name.startswith("gateway.relay."):
+        return "platform.relay"
+    if logger_name.startswith("gateway.platforms."):
+        parts = logger_name.split(".")
+        if len(parts) >= 3 and parts[2]:
+            return f"platform.{parts[2]}"
+    if logger_name.startswith("gateway.platforms"):
+        return "platform"
+    if logger_name.startswith("gateway"):
+        return "gateway"
+    return "gateway"
+
+
+def platform_for_subsystem(subsystem: str) -> Optional[str]:
+    if subsystem.startswith("platform."):
+        return subsystem.split(".", 1)[1] or None
+    return None
+
+
+def _parse_active_agents(raw: Any) -> int:
+    try:
+        from gateway.status import parse_active_agents
+        return parse_active_agents(raw)
+    except Exception:
+        try:
+            return max(0, int(raw))
+        except (TypeError, ValueError):
+            return 0
+
+
+def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
+    try:
+        from gateway.status import derive_gateway_busy
+        return derive_gateway_busy(
+            gateway_running=gateway_running,
+            gateway_state=gateway_state,
+            active_agents=active_agents,
+        )
+    except Exception:
+        return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0)
+
+
+def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool:
+    try:
+        from gateway.status import derive_gateway_drainable
+        return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state)
+    except Exception:
+        return bool(gateway_running and gateway_state == "running")
+
+
+def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]:
+    mode = str(supervision_mode or "unknown").lower()
+    return {
+        "service.instance.id": _safe_instance_id(install_id),
+        "service.version": _safe_metric_value(version, limit=64),
+        "hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown",
+    }
+
+
+def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric:
+    out = dict(attrs)
+    for key, val in extra.items():
+        if val is not None:
+            out[key] = _safe_metric_value(val)
+    return GatewayMetric(name=name, value=value, attributes=out)
+
+
+def build_gateway_health_snapshot(
+    runtime: Optional[dict[str, Any]],
+    *,
+    gateway_running: bool,
+    profile: str,
+    install_id: str,
+    version: str,
+    supervision_mode: str = "unknown",
+) -> GatewayHealthSnapshot:
+    """Convert gateway_state.json-compatible runtime state into P0 signals."""
+    runtime = runtime or {}
+    gateway_state = _bounded_state(
+        runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
+    )
+    active_agents = _parse_active_agents(runtime.get("active_agents", 0))
+    busy = _derive_busy(gateway_running, gateway_state, active_agents)
+    drainable = _derive_drainable(gateway_running, gateway_state)
+    platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {}
+    base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode)
+
+    metrics: list[GatewayMetric] = [
+        _metric("hermes.gateway.up", 1 if gateway_running else 0, base),
+        _metric("hermes.gateway.active_agents", active_agents, base),
+        _metric("hermes.gateway.busy", 1 if busy else 0, base),
+        _metric("hermes.gateway.drainable", 1 if drainable else 0, base),
+        _metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base),
+    ]
+    if gateway_state:
+        metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)}))
+
+    fatal_count = 0
+    events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
+    for platform, pdata in platforms.items():
+        pdata = pdata if isinstance(pdata, dict) else {}
+        state = _bounded_state(
+            pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
+        )
+        raw_error = pdata.get("error_code") or pdata.get("error_message")
+        error_code = classify_gateway_error(raw_error)
+        is_up = state in _RUNNING_PLATFORM_STATES
+        is_degraded = state in _FATAL_PLATFORM_STATES
+        if is_degraded:
+            fatal_count += 1
+        metrics.append(_metric(
+            "hermes.platform.up",
+            1 if is_up else 0,
+            base,
+            **{"hermes.platform": str(platform), "hermes.platform.state": state},
+        ))
+        metrics.append(_metric(
+            "hermes.platform.degraded",
+            1 if is_degraded else 0,
+            base,
+            **{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code},
+        ))
+        if is_degraded:
+            events.append(GatewayDiagnosticEvent(
+                name="platform.fatal",
+                subsystem=f"platform.{platform}",
+                platform=str(platform),
+                error_code=error_code,
+                error_class=classify_gateway_error(error_code or pdata.get("error_message")),
+                profile=profile,
+                version=version,
+                severity="error" if state == "fatal" else "warning",
+            ))
+
+    events.insert(0, GatewayHealthEvent(
+        name="gateway.health_snapshot",
+        gateway_state=str(gateway_state) if gateway_state is not None else None,
+        active_agents=active_agents,
+        gateway_busy=busy,
+        gateway_drainable=drainable,
+        platform_count=len(platforms),
+        fatal_platform_count=fatal_count,
+        profile=profile,
+        install_id=install_id,
+        version=version,
+        supervision_mode=supervision_mode,
+        pid=_coerce_pid(runtime.get("pid")),
+    ))
+    return GatewayHealthSnapshot(metrics=metrics, events=events)
+
+
+def _safe_profile() -> str:
+    try:
+        from hermes_cli.profiles import get_active_profile_name
+        return str(get_active_profile_name() or "default")
+    except Exception:
+        return "default"
+
+
+def _safe_version() -> str:
+    try:
+        from hermes_cli import __version__
+        return str(__version__)
+    except Exception:
+        return "unknown"
+
+
+def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None:
+    """Emit immediate content-free gateway events for runtime status changes.
+
+    Called by gateway.status.write_runtime_status after persisting the new status.
+    Fully fail-open: failures never affect gateway status writes.
+    """
+    try:
+        from agent.monitoring import emitter
+        out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
+        profile = _safe_profile()
+        version = _safe_version()
+        old_gateway_state = _bounded_state(
+            (previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
+        ) if (previous or {}).get("gateway_state") is not None else None
+        new_gateway_state = _bounded_state(
+            current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
+        ) if current.get("gateway_state") is not None else None
+        if old_gateway_state != new_gateway_state and new_gateway_state:
+            out.append(GatewayHealthEvent(
+                name="gateway.lifecycle",
+                gateway_state=new_gateway_state,
+                old_state=old_gateway_state,
+                new_state=new_gateway_state,
+                exit_reason=classify_exit_reason(
+                    current.get("exit_reason"),
+                    state=new_gateway_state,
+                    restart_requested=bool(current.get("restart_requested")),
+                ),
+                restart_requested=bool(current.get("restart_requested")),
+                active_agents=_parse_active_agents(current.get("active_agents", 0)),
+                profile=profile,
+                version=version,
+                pid=_coerce_pid(current.get("pid")),
+            ))
+            if new_gateway_state == "startup_failed":
+                out.append(GatewayDiagnosticEvent(
+                    name="gateway.startup_failed",
+                    subsystem="gateway",
+                    error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
+                    error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
+                    profile=profile,
+                    version=version,
+                    severity="error",
+                ))
+            if new_gateway_state == "stopped":
+                out.append(GatewayHealthEvent(
+                    name="gateway.exit",
+                    gateway_state=new_gateway_state,
+                    old_state=old_gateway_state,
+                    new_state=new_gateway_state,
+                    exit_reason=classify_exit_reason(
+                        current.get("exit_reason"),
+                        state=new_gateway_state,
+                        restart_requested=bool(current.get("restart_requested")),
+                    ),
+                    restart_requested=bool(current.get("restart_requested")),
+                    active_agents=_parse_active_agents(current.get("active_agents", 0)),
+                    profile=profile,
+                    version=version,
+                    pid=_coerce_pid(current.get("pid")),
+                ))
+
+        old_platforms_raw = (previous or {}).get("platforms")
+        new_platforms_raw = current.get("platforms")
+        old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {}
+        new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {}
+        for platform, pdata in new_platforms.items():
+            pdata = pdata if isinstance(pdata, dict) else {}
+            prev_raw = old_platforms.get(platform, {})
+            prev = prev_raw if isinstance(prev_raw, dict) else {}
+            old_state = _bounded_state(
+                prev.get("state"), allowed=_KNOWN_PLATFORM_STATES
+            ) if prev.get("state") is not None else None
+            new_state = _bounded_state(
+                pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
+            ) if pdata.get("state") is not None else None
+            if old_state == new_state or not new_state:
+                continue
+            error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
+            severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning"
+            out.append(GatewayDiagnosticEvent(
+                name="platform.state_change",
+                subsystem=f"platform.{platform}",
+                platform=str(platform),
+                old_state=old_state,
+                new_state=new_state,
+                error_code=error_code,
+                error_class=error_code,
+                profile=profile,
+                version=version,
+                severity=severity,
+            ))
+            if new_state.lower() in _FATAL_PLATFORM_STATES:
+                out.append(GatewayDiagnosticEvent(
+                    name="platform.fatal",
+                    subsystem=f"platform.{platform}",
+                    platform=str(platform),
+                    error_code=error_code,
+                    error_class=error_code,
+                    profile=profile,
+                    version=version,
+                    severity=severity,
+                ))
+        for ev in out:
+            emitter.emit(ev)
+    except Exception:
+        logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True)
+
+
+def _coerce_pid(raw: Any) -> Optional[int]:
+    try:
+        pid = int(raw)
+    except (TypeError, ValueError):
+        return None
+    return pid if pid > 0 else None
+
+
+class GatewayDiagnosticLogHandler(logging.Handler):
+    """Allowlisted warning/error bridge for gateway-owned diagnostics."""
+
+    def __init__(self, *, profile: str = "default", version: str = "unknown") -> None:
+        super().__init__(level=logging.WARNING)
+        self.profile = profile
+        self.version = version
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            if record.levelno < logging.WARNING:
+                return
+            if not _allowed_logger(record.name):
+                return
+            subsystem = subsystem_for_logger(record.name)
+            message = record.getMessage()
+            error_class = classify_gateway_error(message)
+            event = GatewayDiagnosticEvent(
+                name=f"gateway.log.{record.levelname.lower()}",
+                subsystem=subsystem,
+                source_logger=source_logger_for_export(record.name),
+                platform=platform_for_subsystem(subsystem),
+                error_class=error_class,
+                error_code=error_class,
+                profile=self.profile,
+                version=self.version,
+                severity=record.levelname.lower(),
+            )
+            from agent.monitoring import emitter
+            emitter.get_emitter().emit(event)
+        except Exception:
+            logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True)
+
+
+__all__ = [
+    "GatewayMetric",
+    "GatewayHealthSnapshot",
+    "GatewayDiagnosticLogHandler",
+    "build_gateway_health_snapshot",
+    "classify_gateway_error",
+    "source_logger_for_export",
+    "redact_gateway_message",
+]

agent/monitoring/gateway_health_export.py

@@ -0,0 +1,643 @@
+"""Gateway Health & Diagnostics OTLP export runtime.
+
+This exporter emits operator-owned gateway service-health metrics plus
+narrow redacted diagnostic events. It is deliberately in-process and fail-open so
+it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell
+without a sidecar/watchdog dependency.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import threading
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_DIAGNOSTIC_SCOPE = "hermes.gateway.diagnostics"
+
+_RESOURCE_ATTRIBUTE_KEYS = frozenset({
+    "service.name",
+    "service.namespace",
+    "service.version",
+    "service.instance.id",
+    "deployment.environment.name",
+    "cloud.provider",
+    "cloud.platform",
+    "cloud.region",
+    "telemetry.scope",
+})
+_DIAGNOSTIC_ATTRIBUTE_KEYS = frozenset({
+    "name",
+    "subsystem",
+    "error_class",
+    "error_code",
+    "platform",
+    "old_state",
+    "new_state",
+    "version",
+    "severity",
+})
+_SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$")
+
+
+def _redact_string(raw: Any, *, limit: int = 500) -> str:
+    try:
+        from agent.monitoring.redaction import redact_for_export
+        return (redact_for_export(str(raw or "")) or "[redacted]")[:limit]
+    except Exception:
+        return "[redaction-unavailable]"
+
+
+def _safe_resource_attributes(raw: Any) -> Dict[str, str]:
+    """Allowlist bounded resource labels and reject values changed by redaction."""
+    attrs: Dict[str, str] = {}
+    if not isinstance(raw, dict):
+        return attrs
+    for key, value in raw.items():
+        key = str(key)
+        if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None:
+            continue
+        if key == "service.instance.id":
+            from agent.monitoring.gateway_health import _safe_instance_id
+            attrs[key] = _safe_instance_id(value)
+            continue
+        text = str(value)
+        if not _SAFE_RESOURCE_VALUE.fullmatch(text):
+            continue
+        if _redact_string(text, limit=128) != text:
+            continue
+        attrs[key] = text
+    return attrs
+
+
+def _runtime_resource_attributes(
+    config: Dict[str, Any], *, telemetry_scope: str
+) -> Dict[str, str]:
+    """Build the safe OTLP resource shared by metrics and diagnostic logs."""
+    gh = _gateway_health_config(config)
+    attrs = _safe_resource_attributes(gh.get("resource_attributes"))
+    from agent.monitoring.gateway_health import _safe_instance_id
+
+    attrs["service.name"] = "hermes-gateway"
+    attrs["service.instance.id"] = _safe_instance_id(_install_id(config))
+    attrs["telemetry.scope"] = telemetry_scope
+    return attrs
+
+
+def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]:
+    attrs: Dict[str, Any] = {}
+    for key in _DIAGNOSTIC_ATTRIBUTE_KEYS:
+        value = event.get(key)
+        if value is None:
+            continue
+        attrs[f"hermes.{key}"] = _redact_string(value) if isinstance(value, str) else value
+    return attrs
+
+
+@dataclass(slots=True)
+class GatewayHealthExportRuntime:
+    enabled: bool
+    reason: str = "disabled"
+    streamer: Any = None
+    metric_provider: Any = None
+    log_handler: Any = None
+    log_streamer: Any = None
+    thread: Optional[threading.Thread] = None
+    stop_event: Optional[threading.Event] = None
+
+    def shutdown(self) -> None:
+        if self.stop_event is not None:
+            self.stop_event.set()
+        if self.thread is not None:
+            self.thread.join(timeout=0.25)
+        if self.log_handler is not None:
+            try:
+                logging.getLogger().removeHandler(self.log_handler)
+            except Exception:
+                pass
+
+        # All producers above are now stopped. Drain queued and in-flight
+        # events before detaching subscribers so the terminal lifecycle event
+        # cannot race exporter shutdown. The barrier is bounded and fail-open.
+        try:
+            from agent.monitoring.emitter import get_emitter
+            emitter = get_emitter()
+            emitter.flush(timeout=1.0)
+            if self.streamer is not None:
+                emitter.unsubscribe(self.streamer)
+            if self.log_streamer is not None:
+                emitter.unsubscribe(self.log_streamer)
+        except Exception:
+            pass
+
+        # Network flush/close runs under one bounded daemon-thread deadline and
+        # can never delay gateway teardown indefinitely.
+        closeables = [
+            item for item in (self.streamer, self.log_streamer, self.metric_provider)
+            if item is not None
+        ]
+
+        def _close() -> None:
+            for item in closeables:
+                try:
+                    item.shutdown()
+                except Exception:
+                    pass
+
+        if closeables:
+            worker = threading.Thread(
+                target=_close,
+                name="hermes-gateway-health-export-shutdown",
+                daemon=True,
+            )
+            worker.start()
+            worker.join(timeout=2.0)
+
+        self.streamer = None
+        self.log_streamer = None
+        self.metric_provider = None
+        self.thread = None
+        self.stop_event = None
+
+
+def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    mon = (config or {}).get("monitoring") or {}
+    return mon.get("gateway_health_export") or {}
+
+
+def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    mon = (config or {}).get("monitoring") or {}
+    export = mon.get("export") or {}
+    return export.get("otlp") or {}
+
+
+def _enabled(config: Dict[str, Any]) -> bool:
+    gh = _gateway_health_config(config)
+    otlp = _otlp_config(config)
+    return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint"))
+
+
+def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]:
+    if auto_install:
+        try:
+            from tools.lazy_deps import ensure as _lazy_ensure
+            _lazy_ensure("export.otlp", prompt=prompt)
+        except Exception:
+            pass
+    try:
+        from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+        from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+        from opentelemetry.metrics import Observation
+        from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags
+        from opentelemetry._logs import LogRecord
+        from opentelemetry._logs.severity import SeverityNumber
+        from opentelemetry.sdk._logs import LoggerProvider
+        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+        from opentelemetry.sdk.metrics import MeterProvider
+        from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+        from opentelemetry.sdk.resources import Resource
+        return {
+            "OTLPLogExporter": OTLPLogExporter,
+            "OTLPMetricExporter": OTLPMetricExporter,
+            "Observation": Observation,
+            "LogRecord": LogRecord,
+            "LoggerProvider": LoggerProvider,
+            "INVALID_SPAN_ID": INVALID_SPAN_ID,
+            "INVALID_TRACE_ID": INVALID_TRACE_ID,
+            "TraceFlags": TraceFlags,
+            "SeverityNumber": SeverityNumber,
+            "BatchLogRecordProcessor": BatchLogRecordProcessor,
+            "MeterProvider": MeterProvider,
+            "PeriodicExportingMetricReader": PeriodicExportingMetricReader,
+            "Resource": Resource,
+        }
+    except Exception as exc:
+        raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc
+
+
+def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
+    resolved: Dict[str, str] = {}
+    for header_name, env_name in (headers_env or {}).items():
+        val = os.environ.get(str(env_name))
+        if val:
+            resolved[str(header_name)] = val
+    return resolved
+
+
+def _metric_endpoint(endpoint: str) -> str:
+    if endpoint.endswith("/v1/traces"):
+        return endpoint[: -len("/v1/traces")] + "/v1/metrics"
+    return endpoint
+
+
+def _logs_endpoint(endpoint: str) -> str:
+    if endpoint.endswith("/v1/traces"):
+        return endpoint[: -len("/v1/traces")] + "/v1/logs"
+    if endpoint.endswith("/v1/metrics"):
+        return endpoint[: -len("/v1/metrics")] + "/v1/logs"
+    return endpoint
+
+
+def _version() -> str:
+    try:
+        from hermes_cli import __version__
+        return str(__version__)
+    except Exception:
+        return "unknown"
+
+
+def _profile() -> str:
+    try:
+        from hermes_cli.profiles import get_active_profile_name
+        return str(get_active_profile_name() or "default")
+    except Exception:
+        return "default"
+
+
+def _install_id(config: Dict[str, Any]) -> str:
+    try:
+        from agent.monitoring.policy import ensure_install_id
+        return str(ensure_install_id(config))
+    except Exception:
+        return "unknown"
+
+
+def _supervision_mode() -> str:
+    if os.environ.get("INVOCATION_ID"):
+        return "systemd"
+    if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"):
+        return "s6"
+    if os.environ.get("container") or os.path.exists("/.dockerenv"):
+        return "container"
+    if os.environ.get("LAUNCHD_SOCKET"):
+        return "launchd"
+    return "manual"
+
+
+def _read_gateway_snapshot(config: Dict[str, Any]):
+    from agent.monitoring.gateway_health import build_gateway_health_snapshot
+    try:
+        from gateway.status import read_runtime_status
+        runtime = read_runtime_status() or {}
+    except Exception:
+        runtime = {}
+    return build_gateway_health_snapshot(
+        runtime,
+        gateway_running=True,
+        profile=_profile(),
+        install_id=_install_id(config),
+        version=_version(),
+        supervision_mode=_supervision_mode(),
+    )
+
+
+def _read_cron_snapshot():
+    from agent.monitoring.cron_health import build_cron_health_snapshot
+
+    return build_cron_health_snapshot()
+
+
+def _read_background_work_count() -> int:
+    """Count live background/subagent work that ``active_agents`` does NOT include.
+
+    ``hermes.gateway.active_agents`` counts foreground turns + in-flight cron
+    jobs + API runs, but deliberately excludes backgrounded ``delegate_task``
+    subagents, ``terminal(background=true)`` processes, kanban workers, and the
+    runner's own background tasks (they are tracked only for the scale-to-zero
+    suspend guard, ``_scale_to_zero_has_live_background_work``). Without this
+    metric a peer churning through delegated subagents shows ``active_agents=0``
+    on the fleet dashboard. Best-effort and content-free: a single integer,
+    no job/task identity. Returns 0 if a source can't be imported.
+
+    Delegation is counted TASK-granular (``active_task_count``): a fan-out batch
+    of N subagents contributes N, not 1, so the metric reflects real concurrent
+    subagent load rather than dispatch-unit/pool-slot count. This intentionally
+    differs from the async pool's capacity accounting (one batch = one slot).
+    """
+    total = 0
+    try:
+        from tools.async_delegation import active_task_count
+
+        total += max(0, int(active_task_count()))
+    except Exception:
+        logger.debug("background-work async-delegation count failed", exc_info=True)
+    try:
+        from tools.process_registry import process_registry
+
+        total += max(0, int(process_registry.count_running()))
+    except Exception:
+        logger.debug("background-work process-registry count failed", exc_info=True)
+    return total
+
+
+def _read_background_delegations_count() -> int:
+    """Count live async delegation UNITS (dispatch/pool slots).
+
+    Complements ``_read_background_work_count`` (which is task-granular): this
+    counts each ``delegate_task`` dispatch as ONE regardless of fan-out width,
+    matching the async pool's capacity accounting (a batch = one slot). Together
+    the two metrics let an operator see both slot pressure
+    (``background_delegations``, alert vs ``max_concurrent_children``) and real
+    concurrent subagent load (``background_work``). Delegations only — it does
+    not include ``terminal(background)`` / kanban work, which are already folded
+    into ``background_work``. Best-effort; 0 if the source can't be imported.
+    """
+    try:
+        from tools.async_delegation import active_count
+
+        return max(0, int(active_count()))
+    except Exception:
+        logger.debug("background-delegations count failed", exc_info=True)
+        return 0
+
+
+def _read_runtime_snapshot(config: Dict[str, Any]):
+    gateway_snapshot = _read_gateway_snapshot(config)
+    # Background/subagent work — a distinct metric from active_agents (which
+    # never counts it). Appended to the gateway snapshot so it rides the same
+    # base resource attributes (service.instance.id etc.).
+    try:
+        from agent.monitoring.gateway_health import GatewayMetric
+
+        base = dict(gateway_snapshot.metrics[0].attributes) if gateway_snapshot.metrics else {}
+        gateway_snapshot.metrics.append(
+            GatewayMetric(
+                name="hermes.gateway.background_work",
+                value=_read_background_work_count(),
+                attributes=base,
+            )
+        )
+        gateway_snapshot.metrics.append(
+            GatewayMetric(
+                name="hermes.gateway.background_delegations",
+                value=_read_background_delegations_count(),
+                attributes=base,
+            )
+        )
+    except Exception as exc:
+        logger.warning(
+            "background-work snapshot unavailable; metric not exported (error_type=%s)",
+            type(exc).__name__,
+        )
+        logger.debug("background-work snapshot traceback", exc_info=True)
+    try:
+        cron_snapshot = _read_cron_snapshot()
+    except Exception as exc:
+        # Content-free visibility: cron telemetry silently dropping out is a
+        # release-relevant regression, so surface it at WARNING with only the
+        # exception *type* name (never the message, which could carry paths or
+        # other environment detail). exc_info stays on the DEBUG record.
+        logger.warning(
+            "cron health snapshot unavailable; cron telemetry not exported (error_type=%s)",
+            type(exc).__name__,
+        )
+        logger.debug("cron health snapshot traceback", exc_info=True)
+        return gateway_snapshot
+    gateway_snapshot.metrics.extend(cron_snapshot.metrics)
+    return gateway_snapshot
+
+
+def _emit_snapshot_events(config: Dict[str, Any]) -> None:
+    gh = _gateway_health_config(config)
+    if not gh.get("diagnostic_events_enabled", True):
+        return
+    try:
+        from agent.monitoring import emitter
+        snapshot = _read_runtime_snapshot(config)
+        for event in snapshot.events:
+            emitter.emit(event)
+    except Exception:
+        logger.debug("gateway health snapshot emit failed", exc_info=True)
+
+
+def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any:
+    gh = _gateway_health_config(config)
+    if not gh.get("metrics_enabled", True):
+        return None
+    otlp = _otlp_config(config)
+    endpoint = _metric_endpoint(str(otlp.get("endpoint")))
+    headers = _resolve_headers(otlp.get("headers_env"))
+    exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None)
+    interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000
+    reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms)
+    resource_attrs = _runtime_resource_attributes(
+        config, telemetry_scope="gateway_health"
+    )
+    provider = sdk["MeterProvider"](
+        metric_readers=[reader],
+        resource=sdk["Resource"].create(resource_attrs),
+    )
+    meter = provider.get_meter("hermes.gateway.health")
+    Observation = sdk["Observation"]
+
+    metric_names = [
+        "hermes.gateway.up",
+        "hermes.gateway.state",
+        "hermes.gateway.active_agents",
+        "hermes.gateway.busy",
+        "hermes.gateway.drainable",
+        "hermes.gateway.restart_requested",
+        "hermes.gateway.background_work",
+        "hermes.gateway.background_delegations",
+        "hermes.platform.up",
+        "hermes.platform.degraded",
+        "hermes.cron.scheduler.heartbeat_age_seconds",
+        "hermes.cron.scheduler.last_success_age_seconds",
+        "hermes.cron.scheduler.catch_up_occurrences",
+        "hermes.cron.jobs.enabled",
+        "hermes.cron.jobs.running",
+        "hermes.cron.jobs.overdue",
+    ]
+
+    def callback(name: str):
+        def _cb(_options=None):
+            try:
+                snapshot = _read_runtime_snapshot(config)
+                return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name]
+            except Exception:
+                logger.debug("gateway metric callback failed", exc_info=True)
+                return []
+        return _cb
+
+    for metric_name in metric_names:
+        meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)])
+    return provider
+
+
+def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any:
+    SeverityNumber = sdk["SeverityNumber"]
+    sev = str(severity or "warning").lower()
+    if sev in {"critical", "fatal"}:
+        return SeverityNumber.FATAL
+    if sev == "error":
+        return SeverityNumber.ERROR
+    if sev in {"info", "information"}:
+        return SeverityNumber.INFO
+    if sev == "debug":
+        return SeverityNumber.DEBUG
+    return SeverityNumber.WARN
+
+
+class GatewayDiagnosticLogStreamer:
+    """Emitter subscriber that sends gateway diagnostic events as OTLP logs."""
+
+    def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]):
+        otlp = _otlp_config(config)
+        headers = _resolve_headers(otlp.get("headers_env"))
+        endpoint = _logs_endpoint(str(otlp.get("endpoint")))
+        resource_attrs = _runtime_resource_attributes(
+            config, telemetry_scope="gateway_diagnostics"
+        )
+        self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs))
+        self._processor = sdk["BatchLogRecordProcessor"](
+            sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None)
+        )
+        self._provider.add_log_record_processor(self._processor)
+        self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE)
+        self._LogRecord = sdk["LogRecord"]
+        self._sdk = sdk
+        self.exported = 0
+
+    def __call__(self, batch: list[Dict[str, Any]]) -> None:
+        from agent.monitoring.gateway_health import source_logger_for_export
+
+        for ev in batch:
+            if ev.get("event") != "gateway_diagnostic":
+                continue
+            attrs = _diagnostic_log_attributes(ev)
+            # Preserve the source-controlled Python logger as the OTel
+            # instrumentation scope. This adds precise code attribution without
+            # turning a fluid module layout into a maintained subsystem enum.
+            # Rendered messages stay out because they may contain arbitrary IDs,
+            # names, paths, or configured strings. A future, separately gated
+            # ``diagnostic_detail: redacted_message`` mode may add best-effort
+            # free text when an observability plane defines that privacy policy.
+            source_logger = source_logger_for_export(ev.get("source_logger"))
+            otel_logger = (
+                self._provider.get_logger(source_logger)
+                if source_logger is not None
+                else self._logger
+            )
+            body = "gateway diagnostic"
+            record = self._LogRecord(
+                timestamp=ev.get("ts_ns"),
+                trace_id=self._sdk["INVALID_TRACE_ID"],
+                span_id=self._sdk["INVALID_SPAN_ID"],
+                trace_flags=self._sdk["TraceFlags"].DEFAULT,
+                severity_text=str(ev.get("severity") or "warning").upper(),
+                severity_number=_severity_number(self._sdk, ev.get("severity")),
+                body=_redact_string(body),
+                attributes=attrs,
+            )
+            otel_logger.emit(record)
+            self.exported += 1
+
+    def shutdown(self) -> None:
+        try:
+            from agent.monitoring.emitter import get_emitter
+            get_emitter().unsubscribe(self)
+        except Exception:
+            pass
+        try:
+            self._processor.force_flush()
+            self._provider.shutdown()
+        except Exception:
+            pass
+
+
+def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer:
+    from agent.monitoring.emitter import get_emitter
+    streamer = GatewayDiagnosticLogStreamer(config, sdk)
+    get_emitter().subscribe(streamer)
+    return streamer
+
+
+def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread:
+    interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5)))
+
+    def _run() -> None:
+        while not stop_event.wait(interval):
+            _emit_snapshot_events(config)
+
+    thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True)
+    thread.start()
+    return thread
+
+
+def _attach_log_handler(config: Dict[str, Any]) -> Any:
+    gh = _gateway_health_config(config)
+    if not gh.get("diagnostic_events_enabled", True) or not gh.get("warning_error_events_enabled", True):
+        return None
+    from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
+    handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version())
+    root = logging.getLogger()
+    if handler not in root.handlers:
+        root.addHandler(handler)
+    return handler
+
+
+def _gateway_health_event(ev: Dict[str, Any]) -> bool:
+    return ev.get("event") in {"gateway_health", "cron_execution"}
+
+
+def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime:
+    """Start P0 gateway health export if configured. Never raises."""
+    if not _enabled(config):
+        return GatewayHealthExportRuntime(enabled=False, reason="disabled")
+    gh = _gateway_health_config(config)
+    runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled")
+    sdk: Optional[Dict[str, Any]] = None
+
+    if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True):
+        try:
+            sdk = _require_metrics_sdk(prompt=False)
+        except Exception:
+            logger.warning(
+                "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; "
+                "install 'hermes-agent[otlp]'",
+                exc_info=True,
+            )
+            return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable")
+
+    if gh.get("metrics_enabled", True) and sdk is not None:
+        try:
+            runtime.metric_provider = _start_metric_provider(config, sdk)
+        except Exception:
+            logger.warning("gateway health OTLP metrics failed to start", exc_info=True)
+            runtime.shutdown()
+            return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed")
+
+    if gh.get("diagnostic_events_enabled", True) and sdk is not None:
+        try:
+            from agent.monitoring import otlp_exporter
+            runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event)
+            if runtime.streamer is None:
+                raise RuntimeError("gateway health span streamer did not start")
+            runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk)
+        except Exception:
+            logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True)
+            runtime.shutdown()
+            return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed")
+
+    try:
+        runtime.log_handler = _attach_log_handler(config)
+    except Exception:
+        logger.debug("gateway diagnostic log handler failed to attach", exc_info=True)
+    if gh.get("diagnostic_events_enabled", True):
+        try:
+            _emit_snapshot_events(config)
+            runtime.stop_event = threading.Event()
+            runtime.thread = _start_snapshot_thread(config, runtime.stop_event)
+        except Exception:
+            logger.debug("gateway health snapshot thread failed to start", exc_info=True)
+    return runtime
+
+
+__all__ = [
+    "GatewayHealthExportRuntime",
+    "start_gateway_health_export",
+]

agent/monitoring/otlp_exporter.py

@@ -0,0 +1,272 @@
+"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
+
+Maps gateway monitoring events to OTel spans and sends them to the endpoint
+configured under ``monitoring.export.otlp``. Lets an operator stream Hermes
+gateway health into their own observability stack (OTEL Collector, DataDog,
+and similar).
+
+Notes:
+  * The destination is operator-configured; this module only sends to that
+    endpoint. No default destination ships.
+  * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an
+    optional extra (``pip install hermes-agent[otlp]``), imported lazily so the
+    dependency is only required when OTLP export is actually used.
+  * ``headers_env`` maps a header name to an environment variable name; values
+    are read from the environment at export time and never logged or stored.
+  * The continuous subscriber runs in the emitter's dispatcher thread and is
+    fail-isolated, so an export error cannot affect the gateway.
+
+Only monitoring events (gateway_health / gateway_diagnostic) exist on this
+plane; the ``event_filter`` seam is kept so future planes sharing the emitter
+cannot silently ride along on this exporter.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class OTLPUnavailable(RuntimeError):
+    """Raised when the optional OpenTelemetry SDK isn't installed."""
+
+
+def _require_sdk(*, auto_install: bool = True, prompt: bool = True):
+    """Import the OTel SDK, lazily installing it on first use if needed.
+
+    Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK
+    triggers the standard venv install flow — same as every other optional
+    backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back
+    to OTLPUnavailable (with a manual install hint) when the SDK can't be made
+    importable (lazy installs disabled, install failed, or auto_install=False).
+
+    ``auto_install``: attempt the lazy install when missing (default True).
+    ``prompt``: ask before installing when interactive (default True); pass
+    False from non-interactive contexts like the continuous streamer.
+    """
+    if auto_install:
+        try:
+            from tools.lazy_deps import ensure as _lazy_ensure
+            _lazy_ensure("export.otlp", prompt=prompt)
+        except ImportError:
+            pass  # lazy_deps unavailable — fall through to the import attempt
+        except Exception:
+            # FeatureUnavailable (lazy installs disabled / declined / failed) —
+            # fall through; the import below raises OTLPUnavailable with the hint.
+            pass
+    try:
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.sdk.resources import Resource
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
+        from opentelemetry.trace import SpanKind
+        return {
+            "TracerProvider": TracerProvider,
+            "BatchSpanProcessor": BatchSpanProcessor,
+            "Resource": Resource,
+            "OTLPSpanExporter": OTLPSpanExporter,
+            "SpanKind": SpanKind,
+        }
+    except Exception as e:  # ImportError or partial install
+        raise OTLPUnavailable(
+            "OTLP export requires the optional dependency. Install with:\n"
+            "    pip install 'hermes-agent[otlp]'\n"
+            f"(import error: {e})"
+        )
+
+
+def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
+    """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env.
+
+    The config stores environment variable names, not secret values; values are
+    read from the environment here. Missing variables are skipped (and noted at
+    debug level without the value).
+    """
+    resolved: Dict[str, str] = {}
+    for header_name, env_name in (headers_env or {}).items():
+        val = os.environ.get(str(env_name))
+        if val:
+            resolved[str(header_name)] = val
+        else:
+            logger.debug("OTLP header %s: env var %s not set; skipping",
+                         header_name, env_name)
+    return resolved
+
+
+def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    mon = (config or {}).get("monitoring") or {}
+    export = mon.get("export") or {}
+    return export.get("otlp") or {}
+
+
+def build_exporter(config: Dict[str, Any]):
+    """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
+    sdk = _require_sdk()
+    otlp = _otlp_config(config)
+    endpoint = otlp.get("endpoint")
+    if not endpoint:
+        raise ValueError("monitoring.export.otlp.endpoint is not set")
+    headers = _resolve_headers(otlp.get("headers_env"))
+    return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None)
+
+
+def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]:
+    from agent.monitoring.gateway_health import _safe_instance_id
+    from agent.monitoring.policy import ensure_install_id
+
+    return {
+        "service.name": "hermes-gateway",
+        "service.instance.id": _safe_instance_id(ensure_install_id(config)),
+        "telemetry.scope": "gateway_monitoring",
+    }
+
+
+def _make_provider(config: Dict[str, Any]):
+    sdk = _require_sdk()
+    resource = sdk["Resource"].create(_resource_attributes(config))
+    provider = sdk["TracerProvider"](resource=resource)
+    processor = sdk["BatchSpanProcessor"](build_exporter(config))
+    provider.add_span_processor(processor)
+    return provider, processor
+
+
+# ── event -> span attribute mapping ──────────────────────────────────────────
+def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
+    """Span attributes for a monitoring event (content-free by construction)."""
+    kind = ev.get("event")
+    attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"}
+    keep_by_kind = {
+        "gateway_health": ("name", "gateway_state", "old_state", "new_state",
+                           "exit_reason", "restart_requested", "active_agents",
+                           "gateway_busy", "gateway_drainable", "platform_count",
+                           "fatal_platform_count", "version",
+                           "supervision_mode", "pid"),
+        "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
+                               "platform", "old_state", "new_state",
+                               "version", "severity"),
+        "cron_execution": ("status", "job_key", "source", "duration_ms",
+                           "delivery_outcome", "error_class"),
+    }
+    for col in keep_by_kind.get(kind, ()):  # type: ignore[arg-type]
+        v = ev.get(col)
+        if v is not None:
+            if isinstance(v, str):
+                try:
+                    from agent.monitoring.redaction import redact_for_export
+                    v = (redact_for_export(v) or "[redacted]")[:500]
+                except Exception:
+                    v = "[redaction-unavailable]"
+            attrs[f"hermes.{col}"] = v
+    return attrs
+
+
+def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
+    """Map a batch of events to OTel spans. Returns spans created."""
+    tracer = provider.get_tracer("hermes.monitoring")
+    n = 0
+    for ev in batch:
+        try:
+            name = f"hermes.{ev.get('event', 'event')}"
+            span = tracer.start_span(name, attributes=_span_attrs(ev))
+            span.end()
+            n += 1
+        except Exception:
+            logger.debug("OTLP span map failed", exc_info=True)
+    return n
+
+
+# ── continuous streaming subscriber ─────────────────────────────────────────
+class OTLPStreamer:
+    """A live subscriber that pushes each emitter batch to OTLP as it lands.
+
+    Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.
+    """
+
+    def __init__(
+        self,
+        config: Dict[str, Any],
+        *,
+        event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
+    ):
+        self._provider, self._processor = _make_provider(config)
+        self._event_filter = event_filter
+        self.exported = 0
+
+    def __call__(self, batch: List[Dict[str, Any]]) -> None:
+        if self._event_filter is not None:
+            batch = [ev for ev in batch if self._event_filter(ev)]
+        if not batch:
+            return
+        self.exported += export_batch(self._provider, batch)
+
+    def shutdown(self) -> None:
+        try:
+            from agent.monitoring.emitter import get_emitter
+            get_emitter().unsubscribe(self)
+        except Exception:
+            pass
+        try:
+            self._processor.force_flush()
+            self._provider.shutdown()
+        except Exception:
+            pass
+
+
+def is_available() -> bool:
+    """True when the OTel SDK is already importable. Does NOT auto-install —
+    this is a pure check (e.g. for status display)."""
+    try:
+        _require_sdk(auto_install=False)
+        return True
+    except OTLPUnavailable:
+        return False
+
+
+def is_enabled(config: Dict[str, Any]) -> bool:
+    otlp = _otlp_config(config)
+    return bool(otlp.get("enabled") and otlp.get("endpoint"))
+
+
+def start_streaming(
+    config: Dict[str, Any],
+    *,
+    event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
+) -> Optional[OTLPStreamer]:
+    """If OTLP is enabled, attach a streamer to the singleton emitter.
+
+    ``event_filter`` scopes the exporter to its plane, e.g. gateway-health
+    export, so enabling one plane cannot silently export unrelated events.
+
+    Non-interactive context (startup): attempts a lazy install with prompt=False
+    so a configured-but-missing SDK is installed once (gated by
+    security.allow_lazy_installs), then streams. If it still can't load, logs and
+    no-ops — never blocks or raises into startup.
+    """
+    if not is_enabled(config):
+        return None
+    try:
+        _require_sdk(prompt=False)
+    except OTLPUnavailable:
+        logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
+                       "be installed/imported; install 'hermes-agent[otlp]'")
+        return None
+    from agent.monitoring.emitter import get_emitter
+    streamer = OTLPStreamer(config, event_filter=event_filter)
+    get_emitter().subscribe(streamer)
+    return streamer
+
+
+__all__ = [
+    "OTLPUnavailable",
+    "OTLPStreamer",
+    "build_exporter",
+    "export_batch",
+    "is_available",
+    "is_enabled",
+    "start_streaming",
+]

agent/monitoring/policy.py

@@ -0,0 +1,57 @@
+"""Install identity for gateway monitoring.
+
+The install id is a stable, resettable pseudonymous identifier attached to
+exported health signals so an operator can tell instances apart in their
+collector. It carries no account identity and can be rotated by clearing
+``monitoring.install_id`` in config.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any, Dict
+
+logger = logging.getLogger(__name__)
+
+
+def ensure_install_id(config: Dict[str, Any]) -> str:
+    """Return a stable install id, minting and persisting one when empty.
+
+    The id must survive gateway restarts (it becomes ``service.instance.id``
+    on exported signals), so a freshly minted UUID is written back to
+    config.yaml immediately. The write is fail-open: if persisting fails
+    (read-only home, managed scope), the ephemeral id is still returned and
+    a new one is minted next start.
+
+    Clearing ``monitoring.install_id`` (e.g. ``hermes config set
+    monitoring.install_id ""``) rotates the id on the next gateway start.
+    """
+    mon = config.get("monitoring") if isinstance(config, dict) else None
+    existing = (mon or {}).get("install_id") if isinstance(mon, dict) else None
+    if isinstance(existing, str) and existing.strip():
+        return existing
+
+    minted = str(uuid.uuid4())
+    try:
+        from hermes_cli.config import load_config, save_config
+
+        fresh = load_config()
+        if isinstance(fresh, dict):
+            slot = fresh.setdefault("monitoring", {})
+            if isinstance(slot, dict) and not str(slot.get("install_id") or "").strip():
+                slot["install_id"] = minted
+                save_config(fresh)
+    except Exception:
+        logger.debug("install_id persist failed; using ephemeral id", exc_info=True)
+    # Keep the in-memory config consistent for this process either way.
+    if isinstance(config, dict):
+        config.setdefault("monitoring", {})
+        if isinstance(config["monitoring"], dict):
+            config["monitoring"]["install_id"] = minted
+    return minted
+
+
+__all__ = [
+    "ensure_install_id",
+]

agent/monitoring/redaction.py

@@ -0,0 +1,71 @@
+"""Redaction applied to monitoring data before egress.
+
+One unconditional scrub, no modes, no knobs. Every string that leaves the
+process passes through ``redact_for_export``:
+
+  * Secrets first — wraps ``agent/redact.py::redact_sensitive_text(force=True)``
+    plus bearer/token-shape patterns, and fails CLOSED: if the redactor cannot
+    run, the raw string is never emitted.
+  * PII second — e-mail addresses, phone numbers, and UUID-shaped identifiers
+    are rewritten to ``[email]`` / ``[phone]`` / ``[id]``.
+
+There is deliberately no setting to weaken this. The monitoring plane is
+content-free by design: rendered log messages are not exported, and bounded
+structured strings are still scrubbed as defense-in-depth. This redactor also
+remains available for a future, explicitly gated redacted-message detail mode.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+# ── secret shapes (belt-and-suspenders on top of agent/redact.py) ───────────
+_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE)
+_TOKEN_RE = re.compile(
+    r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b"
+)
+_SECRET_LITERAL_RE = re.compile(r"\*{3,}")
+_BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+\[[^\]]+\]", re.IGNORECASE)
+
+# ── PII shapes ───────────────────────────────────────────────────────────────
+_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}")
+# E.164-ish and common separators; conservative to avoid nuking code/IDs.
+_PHONE_RE = re.compile(
+    r"(?<!\w)(?:\+?\d{1,3}[\s.\-]?)?(?:\(\d{2,4}\)[\s.\-]?)?\d{3}[\s.\-]?\d{3,4}(?:[\s.\-]?\d{2,4})?(?!\w)"
+)
+# Long opaque hex/uuid-ish user identifiers.
+_UUID_RE = re.compile(
+    r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
+)
+
+
+def _secret_redact(text: str) -> str:
+    """Always-on secret redaction. force=True so user config can't disable it."""
+    try:
+        from agent.redact import redact_sensitive_text
+        out = redact_sensitive_text(text, force=True)
+    except Exception:
+        # Fail CLOSED: if the redactor can't run, do not emit the raw string.
+        return "[redaction-unavailable]"
+    out = _BEARER_RE.sub("[redacted]", out)
+    out = _TOKEN_RE.sub("[redacted]", out)
+    out = _SECRET_LITERAL_RE.sub("[redacted]", out)
+    out = _BEARER_RESIDUE_RE.sub("[redacted]", out)
+    return out
+
+
+def redact_for_export(text: Optional[str]) -> Optional[str]:
+    """Scrub a string for egress: secrets, then PII. Unconditional."""
+    if text is None:
+        return None
+    out = _secret_redact(str(text))
+    out = _EMAIL_RE.sub("[email]", out)
+    out = _UUID_RE.sub("[id]", out)
+    out = _PHONE_RE.sub("[phone]", out)
+    return out
+
+
+__all__ = [
+    "redact_for_export",
+]

contributors/emails/victor@nousresearch.com

@@ -0,0 +1,2 @@
+victor-kyriazakos
+# PR #64536 Gateway Health OTLP production hardening

cron/executions.py

@@ -89,6 +89,18 @@ def _record(row: Optional[sqlite3.Row]) -> Optional[Dict[str, Any]]:
     return dict(row) if row is not None else None
 
 
+def _emit_execution_state(
+    record: Optional[Dict[str, Any]], *, delivery_outcome: Optional[str] = None
+) -> None:
+    """Project durable state to monitoring without affecting ledger behavior."""
+    try:
+        from agent.monitoring.cron_health import emit_execution_state
+
+        emit_execution_state(record, delivery_outcome=delivery_outcome)
+    except Exception:
+        pass
+
+
 def _process_start_time(pid: int) -> Optional[int]:
     try:
         from gateway.status import get_process_start_time
@@ -139,7 +151,9 @@ def create_execution(job_id: str, *, source: str) -> Dict[str, Any]:
         row = conn.execute(
             "SELECT * FROM executions WHERE id=?", (execution_id,)
         ).fetchone()
-    return _record(row)  # type: ignore[return-value]
+    record = _record(row)
+    _emit_execution_state(record)
+    return record  # type: ignore[return-value]
 
 
 def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]:
@@ -153,13 +167,16 @@ def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]:
         )
         if cur.rowcount != 1:
             return None
-        return _record(conn.execute(
+        record = _record(conn.execute(
             "SELECT * FROM executions WHERE id=?", (execution_id,)
         ).fetchone())
+    _emit_execution_state(record)
+    return record
 
 
 def finish_execution(
     execution_id: str, *, success: bool, error: Optional[str] = None,
+    delivery_outcome: Optional[str] = None,
 ) -> Optional[Dict[str, Any]]:
     """Write a terminal result once; terminal attempts cannot be rewritten."""
     now = _hermes_now().isoformat()
@@ -174,15 +191,18 @@ def finish_execution(
         if cur.rowcount != 1:
             return None
         _prune_unlocked(conn)
-        return _record(conn.execute(
+        record = _record(conn.execute(
             "SELECT * FROM executions WHERE id=?", (execution_id,)
         ).fetchone())
+    _emit_execution_state(record, delivery_outcome=delivery_outcome)
+    return record
 
 
 def recover_interrupted_executions() -> int:
     """Mark provably abandoned attempts unknown without scheduling retries."""
     now = _hermes_now().isoformat()
     changed = 0
+    recovered: List[Dict[str, Any]] = []
     with _transaction() as conn:
         rows = conn.execute(
             """SELECT id, process_id, pid, process_started_at FROM executions
@@ -202,8 +222,16 @@ def recover_interrupted_executions() -> int:
                  row["id"]),
             )
             changed += cur.rowcount
+            if cur.rowcount:
+                record = _record(conn.execute(
+                    "SELECT * FROM executions WHERE id=?", (row["id"],)
+                ).fetchone())
+                if record is not None:
+                    recovered.append(record)
         if changed:
             _prune_unlocked(conn)
+    for record in recovered:
+        _emit_execution_state(record)
     return changed
 
 

cron/jobs.py

@@ -844,6 +844,24 @@ def _atomic_write_epoch(path: Path) -> None:
         raise
 
 
+def _atomic_write_counter(path: Path, value: int) -> None:
+    """Atomically persist a non-negative integer counter."""
+    ensure_dirs()
+    fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp", prefix=".count_")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(str(max(0, value)))
+            f.flush()
+            os.fsync(f.fileno())
+        atomic_replace(tmp_path, path)
+    except BaseException:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
+
+
 def record_ticker_heartbeat(success: bool = False) -> None:
     """Record a ticker liveness signal, and optionally a successful-tick signal.
 
@@ -905,6 +923,19 @@ def get_ticker_success_age() -> Optional[float]:
     return _epoch_file_age(store.cron_dir / "ticker_last_success")
 
 
+def record_catch_up_occurrence() -> None:
+    """Increment the profile-local stale-schedule catch-up counter, best effort."""
+    path = _current_cron_store().cron_dir / "catch_up_occurrences"
+    try:
+        try:
+            value = int(path.read_text(encoding="utf-8").strip())
+        except (OSError, ValueError):
+            value = 0
+        _atomic_write_counter(path, max(0, value) + 1)
+    except Exception:
+        pass
+
+
 def record_ticker_error(message: str) -> None:
     """Persist the most recent tick failure so other processes can surface it.
 
@@ -940,6 +971,15 @@ def record_ticker_error(message: str) -> None:
         pass
 
 
+def get_catch_up_occurrence_count() -> int:
+    """Return the profile-local stale-schedule catch-up count."""
+    path = _current_cron_store().cron_dir / "catch_up_occurrences"
+    try:
+        return max(0, int(path.read_text(encoding="utf-8").strip()))
+    except (OSError, ValueError):
+        return 0
+
+
 def clear_ticker_error() -> None:
     """Remove the last-tick-error marker after a successful tick. Best-effort."""
     store = _current_cron_store()
@@ -2252,6 +2292,7 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:
                                 rj["next_run_at"] = new_next
                                 needs_save = True
                                 break
+                        record_catch_up_occurrence()
                         # Fall through to due.append(job) — execute once now
 
                 # One-shot dispatch-limit guard (issue #38758): a finite one-shot

cron/scheduler.py

@@ -3993,6 +3993,7 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) -
             # responses: do not deliver a blank message, and let the
             # empty-response guard below mark the run as a soft failure.
             should_deliver = bool(deliver_content.strip())
+            unresolved_origin = False
             # Cron silence suppression — see _is_cron_silence_response.  Replaces the
             # old `SILENT_MARKER in ...upper()` substring check, which both leaked
             # bracketless near-markers ("SILENT" / "NO_REPLY") and wrongly swallowed
@@ -4004,6 +4005,10 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) -
                 should_deliver = False
 
             if should_deliver:
+                unresolved_origin = (
+                    _normalize_deliver_value(job.get("deliver", "local")) == "origin"
+                    and not _resolve_delivery_targets(job)
+                )
                 try:
                     delivery_error = _deliver_result(job, deliver_content, adapters=adapters, loop=loop)
                 except Exception as de:
@@ -4025,7 +4030,21 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) -
 
         if not _consume_interrupted_flag(job["id"]):
             mark_job_run(job["id"], success, error, delivery_error=delivery_error)
-        finish_execution(execution_id, success=success, error=error)
+        normalized_deliver = _normalize_deliver_value(job.get("deliver", "local"))
+        if delivery_error:
+            delivery_outcome = "failed"
+        elif should_deliver and unresolved_origin:
+            delivery_outcome = "not_configured"
+        elif should_deliver and normalized_deliver != "local":
+            delivery_outcome = "delivered"
+        else:
+            delivery_outcome = "suppressed"
+        finish_execution(
+            execution_id,
+            success=success,
+            error=error,
+            delivery_outcome=delivery_outcome,
+        )
         return True
 
     except BaseException as e:  # noqa: BLE001 — deliberate: see below

docs/observability/monitoring.md

@@ -0,0 +1,304 @@
+# Gateway Monitoring
+
+Service health monitoring plus structured operational diagnostics for the
+Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured
+endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver).
+
+This plane is content-free by construction. It exports gateway and cron
+lifecycle state, platform connector health, and content-free warning/error
+diagnostics. It never exports prompts, messages, tool arguments or results,
+job names, destinations, schedules, raw errors, session history, usage
+analytics, audit logs, or detailed execution traces. Run/model/tool trajectory
+capture is a separate plane served by the NeMo Relay integration
+(`plugins/observability/nemo_relay/`) and its Hermes-owned subscribers.
+
+## What gets exported
+
+| Signal | OTLP route | Content |
+| --- | --- | --- |
+| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/background_work/background_delegations/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes |
+| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes |
+| Diagnostics | `/v1/logs` | Warning/error gateway events with a constant body and bounded subsystem, severity, error class, and error code attributes; rendered log messages are never exported |
+| Cron scheduler gauges | `/v1/metrics` | Ticker heartbeat and last-success age (omitted when unavailable), a monotonic catch-up-occurrence count from the scheduler's stale-window branch, enabled/running job counts, and overdue count derived from persisted `next_run_at` plus the scheduler's existing grace rule |
+| Cron execution lifecycle | `/v1/traces` | Durable `claimed/running/completed/failed/unknown` states, bounded source and error class, opaque hashed job key, elapsed duration when timestamps exist, and delivery outcome when the scheduler knows it; terminal states make a fail-open flush attempt that can delay completion by up to one second |
+
+Signals carry `service.name`, version, supervision mode, and a stable one-way
+hash of the install id so an operator can distinguish instances without
+exporting account/profile identity or the raw install identifier.
+
+`hermes.gateway.active_agents`, `hermes.gateway.background_work`, and
+`hermes.gateway.background_delegations` are complementary. `active_agents`
+counts foreground message turns plus in-flight cron jobs plus API runs — the
+work the gateway drains on shutdown. `background_work` counts detached work that
+`active_agents` never includes: backgrounded `delegate_task` subagents,
+`terminal(background=true)` processes, and kanban workers; it is
+**task-granular** — a fan-out batch of N subagents counts as N — so it reflects
+real concurrent subagent load. `background_delegations` counts only async
+delegation **units** (each `delegate_task` dispatch is one, a fan-out batch is
+one), matching the async pool's capacity accounting; alert it against
+`delegation.max_concurrent_children` to see slot pressure. Sum `active_agents`
+and `background_work` for total live work per instance; use
+`background_delegations` for pool-saturation.
+
+## Enabling
+
+```yaml
+# config.yaml
+monitoring:
+  gateway_health_export:
+    enabled: true
+  export:
+    otlp:
+      enabled: true
+      endpoint: http://collector-host:4318/v1/traces   # metrics/logs derive
+      headers_env: {}   # header name -> ENV VAR NAME (values never stored)
+```
+
+Check the posture any time:
+
+```bash
+hermes monitoring status
+```
+
+The OpenTelemetry SDK is an optional extra (`pip install 'hermes-agent[otlp]'`),
+lazy-installed on first use. When the SDK is missing or the endpoint is down,
+the gateway runs unaffected: metric collection and ordinary event export stay
+off the hot path, while terminal cron events make one bounded fail-open flush
+attempt of up to one second so the final state is less likely to be lost.
+
+Works identically under systemd/launchd/s6 supervision, containers, tmux, or
+a plain `hermes gateway run`: the exporter lives in the gateway process, so
+no sidecar, agent, or collector is required on the host.
+
+## Collecting into DataDog
+
+Run a customer-owned OpenTelemetry Collector and forward:
+
+```yaml
+# otel-collector config
+receivers:
+  otlp:
+    protocols:
+      http:
+exporters:
+  datadog:
+    api:
+      key: ${env:DD_API_KEY}
+service:
+  pipelines:
+    metrics:   {receivers: [otlp], exporters: [datadog]}
+    traces:    {receivers: [otlp], exporters: [datadog]}
+    logs:      {receivers: [otlp], exporters: [datadog]}
+```
+
+Point `monitoring.export.otlp.endpoint` at the collector. Alerts belong on
+`hermes.gateway.up`, `hermes.platform.up`, and `hermes.platform.degraded`.
+
+## Generic fleet queries and alerts
+
+The exact syntax depends on the customer's observability backend. The examples
+below use PromQL-style expressions and intentionally avoid vendor-specific
+routing, destinations, or customer inventory.
+
+Group fleet views by the opaque `service.instance.id` resource attribute. A
+process that has died cannot emit its own zero, so every deployment needs both
+explicit-state and missing-series detection.
+
+```promql
+# Explicit gateway failure.
+hermes_gateway_up == 0
+
+# Box disappeared or stopped exporting. Choose a window longer than the
+# configured export interval and collector retry allowance.
+absent_over_time(hermes_gateway_up[5m])
+
+# Locally owned bridge is explicitly down.
+hermes_platform_up == 0
+
+# Scheduler thread is stale even though the gateway may still be alive.
+hermes_cron_scheduler_heartbeat_age_seconds > 180
+
+# Ticker loops but has not completed a successful tick recently.
+hermes_cron_scheduler_last_success_age_seconds > 300
+
+# One or more jobs are beyond their existing scheduler grace window.
+hermes_cron_jobs_overdue > 0
+
+# Catch-up counter increased, proving at least one stale occurrence was
+# collapsed and run once after a delay.
+increase(hermes_cron_scheduler_catch_up_occurrences[15m]) > 0
+```
+
+Cron execution lifecycle records arrive as `hermes.cron_execution` spans.
+Alert or derive events from bounded attributes such as:
+
+```text
+hermes.status = failed|unknown
+hermes.delivery_outcome = failed|not_configured
+hermes.error_class = auth_failed|rate_limited|timeout|network_error|
+                     dispatch_failed|interrupted|empty_response|
+                     invalid_config|unknown
+```
+
+Recommended operator views:
+
+1. one row per `service.instance.id` with gateway and configured local-platform
+   state;
+2. scheduler heartbeat, last-success age, running count, overdue count, and
+   catch-up increase;
+3. a cron lifecycle feed keyed only by opaque `hermes.job_key`;
+4. separate alerts for box absence, local bridge down, scheduler stale, cron
+   failed/unknown, delivery failure, and overdue/catch-up activity.
+
+Keep alert thresholds and routing in deployment-owned configuration. Do not add
+job names, prompts, outputs, schedules, destinations, raw errors, profile names,
+or account identity merely to make a dashboard easier to read.
+
+## Release-validation scenarios
+
+Before accepting a deployment, force and verify all five cases through the real
+collector and backend:
+
+1. **Cron success:** observe `claimed -> running -> completed`, duration, and a
+   truthful delivery outcome.
+2. **Cron failure:** observe `failed` plus a bounded error class, with no raw
+   exception or content in the decoded OTLP payload.
+3. **Cron interruption:** stop the owning gateway during execution, restart it,
+   and observe recovery to `unknown`.
+4. **Locally owned bridge outage:** break one native connector, observe its
+   bounded down/retrying/fatal state and recovery, and verify unaffected boxes
+   remain healthy.
+5. **Killed gateway:** terminate one canary, verify missing-series detection,
+   restart it, and confirm the same opaque instance identity returns.
+
+Hermes Agent-owned Relay transport health remains in scope. A separate gateway
+or connector service remains authoritative for any shared connected-platform
+state that it owns and should export that state through its own telemetry path.
+
+For every scenario, verify the signal and alert clear on recovery, other boxes
+remain unaffected, collector failure stays fail-open, and decoded metrics,
+spans, logs, and resource attributes remain content-free.
+
+## Local smoke test (no Docker)
+
+```bash
+# terminal 1: capture collector on :4318
+python scripts/observability/otel_capture_collector.py \
+  --host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl
+
+# terminal 2: drive the real exporter through lifecycle transitions,
+# a fatal platform, and a structured warning event, then flush
+python scripts/observability/gateway_health_export_probe.py \
+  --endpoint http://127.0.0.1:4318/v1/traces \
+  --log /tmp/hermes_otel_capture.jsonl --wait 8
+# exit 0 prints: {"requests": 6, "paths": ["/v1/logs", "/v1/metrics", "/v1/traces"]}
+```
+
+## Maintaining and extending this plane
+
+This plane is a **fixed, enumerated, content-free vocabulary** by design. Adding
+a signal is not just "emit a new metric" — every new name and attribute must be
+declared in each layer that enforces the bounded vocabulary, or it is silently
+dropped downstream. Follow the checklist for the change you are making. The
+golden rule: **a new signal that is emitted but not declared in every layer
+looks like a code bug but is a vocabulary-registration bug — nothing errors, the
+signal just never arrives.**
+
+### Content-free invariant (applies to every change)
+
+Before adding anything, confirm it cannot carry content. Numbers, booleans,
+ages, durations, monotonic counts, and one-way hashes are safe. **Never** add an
+attribute that can hold a job name, prompt, output, schedule, destination, raw
+exception text, file path, profile name, account id, or free-form string. When
+you must key a record to a job/entity, hash it (`sha256(...)[:24]`, see
+`_job_key` in `agent/monitoring/cron_health.py`) — never emit the raw id. All
+string attributes that could touch user input must pass through
+`redaction.redact_for_export` and be truncated (see `_span_attrs` in
+`agent/monitoring/otlp_exporter.py`).
+
+### Adding a new gauge/metric
+
+1. Emit it in the snapshot builder (`agent/monitoring/gateway_health.py`
+   `build_gateway_health_snapshot`, `cron_health.py` `build_cron_health_snapshot`,
+   or a sibling reader wired into `_read_runtime_snapshot` in
+   `gateway_health_export.py`). Best-effort: never let a reader raise into the
+   collection loop — wrap it and log a **content-free WARNING with the exception
+   TYPE name only** (the pattern the cron and background-work readers use), so a
+   future regression is visible instead of silently dropping the signal.
+2. Register the dotted metric name in the observable-gauge `metric_names` list in
+   `gateway_health_export.py::_start_metric_provider`. **A gauge that is emitted
+   in the snapshot but not registered here is never observed.**
+3. Add the export-table row and an alert example in this file.
+4. If the deployment fronts the exporter with an OpenTelemetry Collector that
+   uses a metric-name allowlist (a `filter/...` processor with `name != "..."`
+   guards), add the new name there too — otherwise the collector drops it before
+   the backend. This is not repo code, but it is the single most common reason a
+   correctly-emitted new metric never appears; call it out in the PR so the
+   deploying operator updates their collector config.
+
+### Adding a new subsystem (a new family of signals)
+
+Mirror the cron pattern (`cron_health.py` + its wiring): put the read/projection
+logic in its own module, expose one `build_<subsystem>_health_snapshot()` that
+returns bounded `GatewayMetric`s (and events if any), and extend it into
+`_read_runtime_snapshot` with the same best-effort try/except-WARNING guard.
+Then do the "adding a metric" checklist for each new name, and the "adding an
+attribute" checklist for each new event attribute. Add a release-validation
+scenario below for the subsystem's failure mode.
+
+### Extending the error-class / status / source / state vocabularies
+
+These are the closed enums that keep the plane bounded. Extend the SET, then the
+classifier, never one without the other:
+
+- **Cron** (`agent/monitoring/cron_health.py`): `_KNOWN_STATUSES`,
+  `_KNOWN_SOURCES`, `_KNOWN_DELIVERY_OUTCOMES`, and the `classify_cron_error`
+  keyword buckets. Anything not in the set is coerced to `unknown` on the way
+  out, so a new value that is not added to the set is invisible.
+- **Gateway/platform** (`agent/monitoring/gateway_health.py`):
+  `_KNOWN_GATEWAY_STATES`, `_KNOWN_PLATFORM_STATES`, and `classify_gateway_error`.
+
+Rules: keep the vocabulary SMALL and operationally meaningful (an error class
+should map to an operator action, not to an exception subclass); a new bucket
+must match on a stable keyword, not on message text that could vary; update the
+`hermes.error_class = ...` list in this file's alert section and the enum's unit
+test so the contract is asserted, not frozen as a count.
+
+### Adding a content-free attribute to an existing event/span
+
+Add the key to the emitter's per-kind `keep_by_kind` allowlist in
+`agent/monitoring/otlp_exporter.py::_span_attrs` (unlisted keys are dropped), run
+it through redaction if it is ever string-shaped, and — as with metrics — if the
+deployment's collector has a span-attribute `keep_keys(...)` allowlist, add the
+attribute there too or it is stripped in transit.
+
+### Verify the whole chain, not just emission
+
+Emitting is necessary but not sufficient. Confirm the signal survives all the
+way to the backend, because the enums, the `metric_names` registration, the
+emitter attribute allowlist, and any collector allowlist each drop unlisted
+values with no error:
+
+```bash
+hermes monitoring status                 # posture
+python scripts/observability/gateway_health_export_probe.py \
+  --endpoint http://127.0.0.1:4318/v1/traces \
+  --log /tmp/cap.jsonl --wait 8          # drive the real exporter
+```
+
+Decode the captured OTLP payload and assert the new name/attribute is present
+AND that no content leaked. When a real collector sits in front, add its
+allowlist entries and re-verify against the backend, not just the local capture.
+
+## Boundaries and roadmap
+
+The `hermes monitoring` CLI intentionally exposes `status` only. This first
+release covers only Hermes Agent-owned service-health and operational-diagnostic
+signals, including Hermes Agent-owned Relay transport health. Team Gateway's
+authoritative shared connector/platform state is explicitly out of scope, as
+are product analytics, audit/quality reporting, and detailed execution traces.
+Shared client usage metrics and enterprise trace telemetry are being designed on
+the NeMo Relay integration with their own consent, policy, and export
+boundaries; this monitoring plane stays narrow so an operator can enable it
+without touching any content-bearing signal. The telemetry surface may be
+reorganized as that lands.

gateway/run.py

@@ -8156,6 +8156,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
             write_runtime_status(gateway_state="starting", exit_reason=None)
         except Exception:
             pass
+        try:
+            from hermes_cli.config import load_config
+            from agent.monitoring.gateway_health_export import start_gateway_health_export
+            self._gateway_health_export_runtime = start_gateway_health_export(load_config())
+            if getattr(self._gateway_health_export_runtime, "enabled", False):
+                logger.info("Gateway health OTLP export: enabled")
+        except Exception:
+            logger.debug("gateway health OTLP export startup failed", exc_info=True)
 
         # Log any active supply-chain security advisories. Operators see this
         # in gateway.log and `hermes status` surfaces it; we do NOT block
@@ -10243,6 +10251,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
                 self._update_runtime_status("running", self._exit_reason)
             else:
                 self._update_runtime_status("stopped", self._exit_reason)
+            _shutdown_gateway_health_export(self)
             logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed())
 
         self._stop_task = asyncio.create_task(_stop_impl())
@@ -24563,6 +24572,18 @@ async def _await_thread_exit(
     return not thread.is_alive()
 
 
+def _shutdown_gateway_health_export(runner: Any) -> None:
+    """Idempotently drain and detach Gateway Health OTLP export."""
+    runtime = getattr(runner, "_gateway_health_export_runtime", None)
+    if runtime is None:
+        return
+    runner._gateway_health_export_runtime = None
+    try:
+        runtime.shutdown()
+    except Exception:
+        logger.debug("gateway health OTLP export shutdown failed", exc_info=True)
+
+
 async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool:
     """
     Start the gateway and run until interrupted.
@@ -25021,8 +25042,13 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
         logger.debug("MCP tool discovery failed: %s", e)
 
     # Start the gateway
-    success = await runner.start()
+    try:
+        success = await runner.start()
+    except BaseException:
+        _shutdown_gateway_health_export(runner)
+        raise
     if not success:
+        _shutdown_gateway_health_export(runner)
         return False
     # Recover any pending messages flushed during a previous shutdown (#72680).
     try:
@@ -25035,6 +25061,7 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
     except Exception:
         pass
     if runner.should_exit_cleanly:
+        _shutdown_gateway_health_export(runner)
         if runner.exit_reason:
             logger.error("Gateway exiting cleanly: %s", runner.exit_reason)
         # A clean exit that carries an explicit exit code (e.g. a fatal
@@ -25050,19 +25077,22 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
     if not runner._running:
         # Startup was intentionally aborted by restart/shutdown before entering
         # running mode; preserve that lifecycle path without starting cron.
-        await runner.wait_for_shutdown()
-        if runner.should_exit_with_failure:
-            if runner.exit_reason:
-                logger.error("Gateway exiting with failure: %s", runner.exit_reason)
-            return False
         try:
-            from tools.mcp_tool import shutdown_mcp_servers
-            shutdown_mcp_servers()
-        except Exception:
-            pass
-        if runner.exit_code is not None:
-            raise SystemExit(runner.exit_code)
-        return True
+            await runner.wait_for_shutdown()
+            if runner.should_exit_with_failure:
+                if runner.exit_reason:
+                    logger.error("Gateway exiting with failure: %s", runner.exit_reason)
+                return False
+            try:
+                from tools.mcp_tool import shutdown_mcp_servers
+                shutdown_mcp_servers()
+            except Exception:
+                pass
+            if runner.exit_code is not None:
+                raise SystemExit(runner.exit_code)
+            return True
+        finally:
+            _shutdown_gateway_health_export(runner)
 
     # Start the background cron scheduler via the resolved provider so
     # scheduled jobs fire automatically. The built-in provider is the

gateway/status.py

@@ -11,6 +11,7 @@ that will be useful when we add named profiles (multiple agents running
 concurrently under distinct configurations).
 """
 
+import copy
 import hashlib
 import json
 import logging
@@ -988,6 +989,7 @@ def write_runtime_status(
     """Persist gateway runtime health information for diagnostics/status."""
     path = _get_runtime_status_path()
     payload = _read_json_file(path) or _build_runtime_status_record()
+    previous_payload = copy.deepcopy(payload)
     current_record = _build_pid_record()
     payload.setdefault("platforms", {})
     payload["kind"] = current_record["kind"]
@@ -1022,6 +1024,11 @@ def write_runtime_status(
         payload["platforms"][platform] = platform_payload
 
     _write_json_file(path, payload)
+    try:
+        from agent.monitoring.gateway_health import emit_runtime_status_transition
+        emit_runtime_status_transition(previous_payload, payload)
+    except Exception:
+        pass
 
 
 def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]:

hermes_cli/config.py

@@ -2230,7 +2230,7 @@ DEFAULT_CONFIG = {
     "privacy": {
         "redact_pii": False,  # When True, hash user IDs and strip phone numbers from LLM context
     },
-    
+
     # Text-to-speech configuration
     # Each provider supports an optional `max_text_length:` override for the
     # per-request input-character cap. Omit it to use the provider's documented
@@ -3211,6 +3211,44 @@ DEFAULT_CONFIG = {
         "force_ipv4": False,
     },
 
+    # Gateway monitoring — Service Health Monitoring plus redacted Operational
+    # Diagnostics for the gateway daemon, exported over OTLP to an
+    # operator-configured endpoint (OTEL Collector, DataDog, ...). Content-free
+    # by construction: no prompts, messages, tool args/results, session
+    # history, usage analytics, audit logs, or trajectories. Off by default;
+    # nothing is collected or sent until an operator enables it and sets an
+    # endpoint.
+    "monitoring": {
+        # Stable install identifier attached to exported health signals so an
+        # operator can tell instances apart in their collector. Empty string
+        # means "mint a fresh UUID on first use"; clear it to rotate. Carries
+        # no account identity.
+        "install_id": "",
+        # Gateway health & diagnostics export.
+        "gateway_health_export": {
+            "enabled": False,
+            "metrics_enabled": True,
+            "diagnostic_events_enabled": True,
+            "warning_error_events_enabled": True,
+            "export_interval_seconds": 60,
+            "logs_export_interval_seconds": 5,
+            "resource_attributes": {
+                "service.name": "hermes-gateway",
+                "deployment.environment.name": "production",
+            },
+        },
+        # OTLP destination. headers_env maps header names to ENVIRONMENT
+        # VARIABLE NAMES (never secret values); values are read from the
+        # environment at export time.
+        "export": {
+            "otlp": {
+                "enabled": False,
+                "endpoint": "",
+                "headers_env": {},
+            },
+        },
+    },
+
     # Gateway settings — control how messaging platforms (Telegram, Discord,
     # Slack, etc.) deliver agent-produced files as native attachments.
     "gateway": {

hermes_cli/main.py

@@ -465,6 +465,7 @@ from hermes_cli.subcommands.memory import build_memory_parser
 from hermes_cli.subcommands.acp import build_acp_parser
 from hermes_cli.subcommands.tools import build_tools_parser
 from hermes_cli.subcommands.insights import build_insights_parser
+from hermes_cli.subcommands.monitoring import build_monitoring_parser
 from hermes_cli.subcommands.skills import build_skills_parser
 from hermes_cli.subcommands.pairing import build_pairing_parser
 from hermes_cli.subcommands.plugins import build_plugins_parser
@@ -15429,7 +15430,7 @@ _BUILTIN_SUBCOMMANDS = frozenset(
         "dump", "egress", "fallback", "gateway", "hooks", "import", "import-agent", "insights",
         "gui", "desktop", "kanban", "login", "logout", "logs", "lsp", "mcp", "memory", "migrate", "moa",
         "journey", "memory-graph", "learning",
-        "model", "pairing", "pets", "plugins", "portal", "profile",
+        "model", "monitoring", "pairing", "pets", "plugins", "portal", "profile",
         "project", "proxy",
         "prompt-size",
         "send", "sessions", "setup",
@@ -15911,6 +15912,51 @@ def cmd_insights(args):
         print(f"Error generating insights: {e}")
 
 
+def cmd_monitoring(args):
+    """Gateway monitoring status: health & diagnostics export posture."""
+    from hermes_cli.config import load_config
+
+    action = getattr(args, "monitoring_action", None) or "status"
+    config = load_config()
+    mon_raw = config.get("monitoring")
+    mon: dict = mon_raw if isinstance(mon_raw, dict) else {}
+
+    if action == "status":
+        from agent.monitoring import otlp_exporter
+
+        gh_raw = mon.get("gateway_health_export")
+        gh: dict = gh_raw if isinstance(gh_raw, dict) else {}
+        export_raw = mon.get("export")
+        export_cfg: dict = export_raw if isinstance(export_raw, dict) else {}
+        otlp_raw = export_cfg.get("otlp")
+        otlp: dict = otlp_raw if isinstance(otlp_raw, dict) else {}
+
+        print("Gateway monitoring")
+        print(f"  Health export:  {'enabled' if gh.get('enabled') else 'disabled'} "
+              f"(monitoring.gateway_health_export.enabled)")
+        if gh.get("enabled"):
+            print(f"    Metrics:            {'on' if gh.get('metrics_enabled', True) else 'off'} "
+                  f"(interval {gh.get('export_interval_seconds', 60)}s)")
+            print(f"    Diagnostic events:  {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}")
+            print(f"    Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} "
+                  f"(interval {gh.get('logs_export_interval_seconds', 5)}s)")
+            print("    Content safety:     always on "
+                  "(rendered messages are never exported; not configurable)")
+        endpoint = otlp.get("endpoint") or ""
+        if otlp.get("enabled") and endpoint:
+            print(f"  OTLP endpoint:  {endpoint}")
+        else:
+            print("  OTLP endpoint:  not configured (monitoring.export.otlp)")
+        print(f"  OTel SDK:       {'installed' if otlp_exporter.is_available() else 'not installed'} "
+              f"(optional extra: hermes-agent[otlp])")
+        print("\n  Scope: gateway service health + redacted diagnostics only.")
+        print("  No prompts, messages, tool args/results, usage analytics, or traces.")
+        return
+
+    print(f"Unknown monitoring action: {action}", file=sys.stderr)
+    sys.exit(2)
+
+
 def cmd_skills(args):
     # Route 'config' action to skills_config module
     if getattr(args, "skills_action", None) == "config":
@@ -18216,6 +18262,7 @@ def main():
     # insights command  (parser built in hermes_cli/subcommands/insights.py)
     # =========================================================================
     build_insights_parser(subparsers, cmd_insights=cmd_insights)
+    build_monitoring_parser(subparsers, cmd_monitoring=cmd_monitoring)
 
     # =========================================================================
     # claw command  (parser built in hermes_cli/subcommands/claw.py)

hermes_cli/subcommands/monitoring.py

@@ -0,0 +1,36 @@
+"""``hermes monitoring`` subcommand parser.
+
+Gateway monitoring control and inspection. ``status`` shows whether the
+gateway health & diagnostics export is enabled, where it points, and the
+redaction posture.
+
+The handler is injected to avoid importing ``main`` (mirrors the insights
+subcommand).
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+
+def build_monitoring_parser(subparsers, *, cmd_monitoring: Callable) -> None:
+    """Attach the ``monitoring`` subcommand (with actions) to ``subparsers``."""
+    p = subparsers.add_parser(
+        "monitoring",
+        help="Inspect gateway monitoring (health & diagnostics export)",
+        description=(
+            "Gateway monitoring: service health metrics plus redacted "
+            "diagnostics, exported over OTLP to an operator-configured "
+            "endpoint. Content-free by construction — no prompts, messages, "
+            "tool args/results, or usage analytics. Configure under "
+            "monitoring.* in config.yaml."
+        ),
+    )
+    sub = p.add_subparsers(dest="monitoring_action")
+
+    sub.add_parser(
+        "status",
+        help="Show monitoring settings, export state, and redaction posture",
+    )
+
+    p.set_defaults(func=cmd_monitoring)

pyproject.toml

@@ -248,6 +248,11 @@ acp = ["agent-client-protocol==0.9.0"]
 # NOT re-added to [all] so a future quarantined release can't break fresh
 # installs (see [all] policy comment below).
 mistral = ["mistralai==2.4.8"]
+# OTLP gateway monitoring export (optional). Provides the OpenTelemetry SDK +
+# OTLP/HTTP exporter for monitoring.gateway_health_export. Lazy-installed via
+# tools/lazy_deps.py on first use; never a core dependency and deliberately
+# NOT in [all].
+otlp = ["opentelemetry-sdk==1.39.1", "opentelemetry-exporter-otlp-proto-http==1.39.1"]
 bedrock = ["boto3==1.42.89"]
 vertex = ["google-auth==2.55.1"]
 azure-identity = ["azure-identity==1.25.3"]

scripts/observability/gateway_health_export_probe.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Exercise Gateway Health & Diagnostics Export against a local OTLP capture collector."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--endpoint", default="http://127.0.0.1:4318/v1/traces")
+    parser.add_argument("--log", required=True, help="JSONL file written by otel_capture_collector.py")
+    parser.add_argument("--wait", type=float, default=7.0)
+    args = parser.parse_args()
+
+    hermes_home = Path(tempfile.mkdtemp(prefix="hermes-otel-smoke-"))
+    os.environ["HERMES_HOME"] = str(hermes_home)
+
+    from gateway.status import write_runtime_status
+    from agent.monitoring.gateway_health_export import start_gateway_health_export
+    from agent.monitoring import emitter
+
+    config = {
+        "monitoring": {
+            "local": True,
+            "gateway_health_export": {
+                "enabled": True,
+                "metrics_enabled": True,
+                "diagnostic_events_enabled": True,
+                "warning_error_events_enabled": True,
+                "export_interval_seconds": 5,
+                "logs_export_interval_seconds": 5,
+                "resource_attributes": {
+                    "service.name": "hermes-gateway-smoke",
+                    "deployment.environment.name": "local-smoke",
+                },
+            },
+            "export": {
+                "otlp": {
+                    "enabled": True,
+                    "endpoint": args.endpoint,
+                    "headers_env": {},
+                }
+            },
+        }
+    }
+
+    runtime = start_gateway_health_export(config)
+    if not runtime.enabled:
+        raise SystemExit(f"gateway health exporter did not enable: {runtime.reason}")
+
+    write_runtime_status(gateway_state="starting", active_agents=0)
+    write_runtime_status(gateway_state="running", active_agents=2)
+    write_runtime_status(platform="slack", platform_state="running")
+    write_runtime_status(
+        platform="slack",
+        platform_state="fatal",
+        error_code="auth_failed",
+        error_message="Bearer *** rejected for smoke@example.com",
+    )
+    logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com")
+    emitter.get_emitter().flush(timeout=2.0)
+    time.sleep(args.wait)
+    write_runtime_status(gateway_state="stopped", active_agents=0)
+    runtime.shutdown()
+    emitter.get_emitter().flush(timeout=2.0)
+
+    log_path = Path(args.log)
+    rows = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    paths = {row["path"] for row in rows}
+    print(json.dumps({"hermes_home": str(hermes_home), "requests": len(rows), "paths": sorted(paths)}, indent=2))
+    if "/v1/traces" not in paths:
+        raise SystemExit("missing /v1/traces request")
+    if "/v1/logs" not in paths:
+        raise SystemExit("missing /v1/logs request")
+    if "/v1/metrics" not in paths:
+        raise SystemExit("missing /v1/metrics request")
+
+
+if __name__ == "__main__":
+    main()

scripts/observability/otel_capture_collector.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests.
+
+This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces,
+/v1/metrics, and /v1/logs, records request metadata as JSONL, and returns 200 so
+local exporters can be exercised without Docker or a vendor backend.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+
+class CaptureHandler(BaseHTTPRequestHandler):
+    log_path: Path
+
+    def do_POST(self) -> None:  # noqa: N802 - BaseHTTPRequestHandler API
+        length = int(self.headers.get("content-length") or 0)
+        body = self.rfile.read(length) if length else b""
+        record = {
+            "ts": time.time(),
+            "path": self.path,
+            "content_type": self.headers.get("content-type"),
+            "content_length": length,
+            "body_prefix_hex": body[:24].hex(),
+        }
+        with self.log_path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(record, sort_keys=True) + "\n")
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"{}")
+
+    def log_message(self, format: str, *args) -> None:
+        # Keep tmux panes clean; JSONL file is the assertion surface.
+        return
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=4318)
+    parser.add_argument("--log", required=True)
+    args = parser.parse_args()
+
+    log_path = Path(args.log).expanduser().resolve()
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    log_path.write_text("", encoding="utf-8")
+    CaptureHandler.log_path = log_path
+    server = ThreadingHTTPServer((args.host, args.port), CaptureHandler)
+    print(f"OTLP capture collector listening on http://{args.host}:{args.port}; log={log_path}", flush=True)
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

tests/cron/test_execution_ledger.py

@@ -269,6 +269,69 @@ def test_run_one_job_records_running_then_terminal(monkeypatch):
     assert events[-1][2]["success"] is True
 
 
+def test_run_one_job_records_unresolved_origin_as_not_configured(monkeypatch):
+    import cron.scheduler as scheduler
+
+    finished = []
+    monkeypatch.setattr(scheduler, "mark_execution_running", lambda _execution_id: None)
+    monkeypatch.setattr(
+        scheduler,
+        "finish_execution",
+        lambda execution_id, **kwargs: finished.append((execution_id, kwargs)),
+    )
+    monkeypatch.setattr(scheduler, "claim_dispatch", lambda _job_id: True)
+    monkeypatch.setattr(
+        scheduler,
+        "run_job",
+        lambda job, *, defer_agent_teardown=None: (True, "output", "response", None),
+    )
+    monkeypatch.setattr(scheduler, "save_job_output", lambda *_args: None)
+    monkeypatch.setattr(scheduler, "_resolve_delivery_targets", lambda _job: [])
+    monkeypatch.setattr(scheduler, "_deliver_result", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(scheduler, "mark_job_run", lambda *_args, **_kwargs: None)
+
+    job = {
+        "id": "job-unresolved-origin",
+        "execution_id": "exec-unresolved-origin",
+        "deliver": "origin",
+    }
+    assert scheduler.run_one_job(job) is True
+
+    assert finished[-1][1]["delivery_outcome"] == "not_configured"
+
+
+def test_run_one_job_normalizes_legacy_local_delivery_as_suppressed(monkeypatch):
+    import cron.scheduler as scheduler
+
+    finished = []
+    monkeypatch.setattr(
+        scheduler,
+        "run_job",
+        lambda job, *, defer_agent_teardown=None: (True, "output", "response", None),
+    )
+    monkeypatch.setattr(scheduler, "save_job_output", lambda *_args: None)
+    monkeypatch.setattr(scheduler, "mark_job_run", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(
+        scheduler,
+        "finish_execution",
+        lambda execution_id, **kwargs: finished.append((execution_id, kwargs)),
+    )
+    monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_args: None)
+    monkeypatch.setattr(scheduler, "claim_dispatch", lambda *_args, **_kwargs: True)
+    monkeypatch.setattr(scheduler, "_consume_interrupted_flag", lambda *_args: False)
+
+    job = {
+        "id": "legacy-local",
+        "name": "Legacy local",
+        "schedule": {"kind": "interval", "minutes": 10},
+        "deliver": ["local"],
+        "execution_id": "exec-legacy-local",
+    }
+
+    assert scheduler.run_one_job(job) is True
+    assert finished[-1][1]["delivery_outcome"] == "suppressed"
+
+
 def test_provider_start_recovers_interrupted_records_before_tick(monkeypatch):
     import cron.scheduler_provider as provider
 

tests/cron/test_jobs.py

@@ -836,6 +836,24 @@ class TestGetDueJobs:
         next_dt = _ensure_aware(datetime.fromisoformat(updated["next_run_at"]))
         assert next_dt > _hermes_now()
 
+    def test_stale_past_due_records_one_catch_up_occurrence(self, tmp_cron_dir, monkeypatch):
+        import cron.jobs as jobs_module
+
+        recorded = []
+        monkeypatch.setattr(
+            jobs_module,
+            "record_catch_up_occurrence",
+            lambda: recorded.append("catch-up"),
+            raising=False,
+        )
+        create_job(prompt="Stale", schedule="every 1h")
+        jobs = load_jobs()
+        jobs[0]["next_run_at"] = (datetime.now() - timedelta(minutes=35)).isoformat()
+        save_jobs(jobs)
+
+        assert len(get_due_jobs()) == 1
+        assert recorded == ["catch-up"]
+
     def test_idless_job_does_not_crash_or_block_sibling_jobs(self, tmp_cron_dir):
         """A job missing its 'id' key must not crash the tick or freeze siblings.
 

tests/gateway/test_startup_restart_race.py

@@ -250,16 +250,23 @@ async def test_startup_aborts_after_registered_adapter_restart(tmp_path, monkeyp
 async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path, monkeypatch):
     monkeypatch.setenv("HERMES_HOME", str(tmp_path))
     cron_started = False
+    export_shutdown_calls = 0
+
+    class ExportRuntime:
+        def shutdown(self):
+            nonlocal export_shutdown_calls
+            export_shutdown_calls += 1
 
     class AbortedStartupRunner:
         def __init__(self, config):
             self.config = config
             self.adapters = {}
             self._running = False
-            self.should_exit_cleanly = False
+            self.should_exit_cleanly = True
             self.should_exit_with_failure = False
             self.exit_reason = None
             self.exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE
+            self._gateway_health_export_runtime = ExportRuntime()
 
         async def start(self):
             return True
@@ -287,3 +294,4 @@ async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path,
 
     assert exc.value.code == GATEWAY_SERVICE_RESTART_EXIT_CODE
     assert cron_started is False
+    assert export_shutdown_calls == 1

tests/monitoring/test_cron_health_export.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+
+def _metric(snapshot, name):
+    return next(metric for metric in snapshot.metrics if metric.name == name)
+
+
+def test_cron_snapshot_projects_freshness_counts_and_overdue_without_content(monkeypatch):
+    from agent.monitoring import cron_health
+
+    now = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc)
+    secret = "Quarterly payroll for alice@example.com"
+    monkeypatch.setattr(cron_health, "_now", lambda: now)
+    monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: 4.5)
+    monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: 9.0)
+    monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset({"job-private-1"}))
+    monkeypatch.setattr(
+        cron_health,
+        "load_jobs",
+        lambda: [
+            {
+                "id": "job-private-1",
+                "name": secret,
+                "prompt": secret,
+                "enabled": True,
+                "schedule": {"kind": "interval", "minutes": 10},
+                "next_run_at": (now - timedelta(minutes=6)).isoformat(),
+            },
+            {
+                "id": "job-private-2",
+                "name": "disabled private job",
+                "enabled": False,
+                "schedule": {"kind": "interval", "minutes": 10},
+                "next_run_at": (now - timedelta(days=1)).isoformat(),
+            },
+        ],
+    )
+
+    snapshot = cron_health.build_cron_health_snapshot()
+
+    assert _metric(snapshot, "hermes.cron.scheduler.heartbeat_age_seconds").value == 4.5
+    assert _metric(snapshot, "hermes.cron.scheduler.last_success_age_seconds").value == 9.0
+    assert _metric(snapshot, "hermes.cron.jobs.enabled").value == 1
+    assert _metric(snapshot, "hermes.cron.jobs.running").value == 1
+    assert _metric(snapshot, "hermes.cron.jobs.overdue").value == 1
+    assert secret not in str(snapshot)
+    assert "job-private-1" not in str(snapshot)
+
+
+def test_cron_snapshot_omits_unknown_freshness_instead_of_inventing_values(monkeypatch):
+    from agent.monitoring import cron_health
+
+    monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: None)
+    monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: None)
+    monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset())
+    monkeypatch.setattr(cron_health, "load_jobs", lambda: [])
+
+    names = {metric.name for metric in cron_health.build_cron_health_snapshot().metrics}
+
+    assert "hermes.cron.scheduler.heartbeat_age_seconds" not in names
+    assert "hermes.cron.scheduler.last_success_age_seconds" not in names
+
+
+def test_execution_projection_is_opaque_bounded_and_content_free():
+    from agent.monitoring.cron_health import project_execution_event
+
+    event = project_execution_event(
+        {
+            "id": "execution-private-id",
+            "job_id": "Payroll for alice@example.com and token top-secret-token",
+            "source": "builtin",
+            "status": "failed",
+            "claimed_at": "2026-07-24T12:00:00+00:00",
+            "started_at": "2026-07-24T12:00:01+00:00",
+            "finished_at": "2026-07-24T12:00:03.250000+00:00",
+            "error": "Bearer top-secret-token rejected for alice@example.com",
+        },
+        delivery_outcome="failed",
+    ).to_dict()
+
+    assert event["event"] == "cron_execution"
+    assert event["status"] == "failed"
+    assert event["job_key"].startswith("sha256:")
+    assert len(event["job_key"]) == len("sha256:") + 24
+    assert event["duration_ms"] == 2250
+    assert event["delivery_outcome"] == "failed"
+    assert event["error_class"] == "auth_failed"
+    assert "job_id" not in event
+    assert "error" not in event
+    assert "alice@example.com" not in str(event)
+    assert "top-secret-token" not in str(event)
+
+
+def test_execution_projection_omits_duration_and_delivery_when_not_known():
+    from agent.monitoring.cron_health import project_execution_event
+
+    event = project_execution_event(
+        {
+            "job_id": "private",
+            "source": "external-value-must-not-leak",
+            "status": "claimed",
+            "claimed_at": "2026-07-24T12:00:00+00:00",
+        }
+    ).to_dict()
+
+    assert event["status"] == "claimed"
+    assert event["source"] == "external"
+    assert event["duration_ms"] is None
+    assert event["delivery_outcome"] is None
+
+
+def test_external_provider_source_is_normalized_to_external():
+    from agent.monitoring.cron_health import project_execution_event
+
+    event = project_execution_event(
+        {"job_id": "private", "source": "Chronos", "status": "claimed"}
+    )
+
+    assert event.source == "external"
+
+
+@pytest.mark.parametrize("message", ["oauth refresh failed", "tokenizer crashed", "HTTP 4015"])
+def test_error_classification_avoids_auth_substring_false_positives(message):
+    from agent.monitoring.cron_health import classify_cron_error
+
+    assert classify_cron_error(message) == "unknown"
+
+
+@pytest.mark.parametrize(
+    "message",
+    ["authentication failed", "not authorized", "access token expired", "HTTP 401"],
+)
+def test_error_classification_recognizes_auth_terms_and_status_tokens(message):
+    from agent.monitoring.cron_health import classify_cron_error
+
+    assert classify_cron_error(message) == "auth_failed"
+
+
+def test_cron_snapshot_exports_catch_up_occurrence_counter(monkeypatch):
+    from agent.monitoring import cron_health
+
+    monkeypatch.setattr(cron_health, "get_ticker_heartbeat_age", lambda: None)
+    monkeypatch.setattr(cron_health, "get_ticker_success_age", lambda: None)
+    monkeypatch.setattr(cron_health, "get_running_job_ids", lambda: frozenset())
+    monkeypatch.setattr(cron_health, "load_jobs", lambda: [])
+    monkeypatch.setattr(cron_health, "get_catch_up_occurrence_count", lambda: 3)
+
+    snapshot = cron_health.build_cron_health_snapshot()
+
+    assert _metric(snapshot, "hermes.cron.scheduler.catch_up_occurrences").value == 3
+
+
+def test_terminal_execution_emission_flushes_and_failures_are_fail_open(monkeypatch):
+    from agent.monitoring import cron_health, emitter
+
+    calls = []
+
+    class FakeEmitter:
+        def emit(self, event):
+            calls.append(("emit", event.to_dict()["status"]))
+
+        def flush(self, timeout):
+            calls.append(("flush", timeout))
+            raise RuntimeError("collector unavailable")
+
+    monkeypatch.setattr(emitter, "get_emitter", lambda: FakeEmitter())
+
+    cron_health.emit_execution_state(
+        {"job_id": "private", "source": "builtin", "status": "completed"}
+    )
+
+    assert calls == [("emit", "completed"), ("flush", 1.0)]
+
+
+def test_gateway_export_includes_cron_metrics_and_only_accepted_event_planes(monkeypatch):
+    from agent.monitoring import gateway_health_export
+
+    gateway_snapshot = type("Snapshot", (), {"metrics": []})()
+    cron_snapshot = type(
+        "Snapshot",
+        (),
+        {"metrics": [type("Metric", (), {"name": "hermes.cron.jobs.enabled", "value": 2, "attributes": {}})()]},
+    )()
+    monkeypatch.setattr(gateway_health_export, "_read_gateway_snapshot", lambda config: gateway_snapshot)
+    monkeypatch.setattr(gateway_health_export, "_read_cron_snapshot", lambda: cron_snapshot)
+
+    snapshot = gateway_health_export._read_runtime_snapshot({})
+
+    names = [metric.name for metric in snapshot.metrics]
+    # Cron metrics are folded into the gateway snapshot...
+    assert "hermes.cron.jobs.enabled" in names
+    # ...and the background/subagent-work gauges are appended (distinct from
+    # active_agents). Assert the relationship, not a frozen exact list.
+    assert "hermes.gateway.background_work" in names
+    assert "hermes.gateway.background_delegations" in names
+    assert gateway_health_export._gateway_health_event({"event": "cron_execution"}) is True
+    assert gateway_health_export._gateway_health_event({"event": "gateway_health"}) is True
+    assert gateway_health_export._gateway_health_event({"event": "run"}) is False
+
+
+def test_background_work_is_task_granular_and_delegations_is_unit_granular(monkeypatch):
+    """background_work expands batches to child tasks; background_delegations
+    counts dispatch units. A 3-task batch => work +3, delegations +1.
+    """
+    from agent.monitoring import gateway_health_export
+    from tools import async_delegation as ad
+
+    with ad._records_lock:
+        saved = dict(ad._records)
+        ad._records.clear()
+        ad._records["single"] = {"status": "running"}
+        ad._records["batch3"] = {"status": "running", "is_batch": True, "goals": ["a", "b", "c"]}
+    # Isolate from process_registry so we measure only the delegation contribution.
+    monkeypatch.setattr(
+        "tools.process_registry.process_registry.count_running", lambda: 0, raising=False
+    )
+    try:
+        # work = single(1) + batch(3) = 4 tasks; delegations = 2 units.
+        assert gateway_health_export._read_background_work_count() == 4
+        assert gateway_health_export._read_background_delegations_count() == 2
+    finally:
+        with ad._records_lock:
+            ad._records.clear()
+            ad._records.update(saved)
+
+
+def test_registered_observable_metric_names_cover_snapshot_metrics(monkeypatch):
+    """Every gauge emitted in the runtime snapshot must also be registered in the
+    observable-gauge metric_names list, or the OTLP exporter never observes it.
+
+    This asserts the vocabulary-registration invariant documented in
+    docs/observability/monitoring.md: an emitted-but-unregistered gauge is
+    silently dropped. Regression guard for background_work / cron additions.
+    """
+    import inspect
+    from agent.monitoring import gateway_health_export
+
+    # Build a representative snapshot (gateway + cron + background_work) without
+    # a live gateway by stubbing the gateway snapshot to the real metric names.
+    class _M:
+        def __init__(self, name):
+            self.name = name
+            self.value = 0
+            self.attributes = {}
+
+    gateway_snapshot = type("S", (), {"metrics": [
+        _M("hermes.gateway.up"), _M("hermes.gateway.active_agents"),
+        _M("hermes.gateway.busy"), _M("hermes.gateway.drainable"),
+        _M("hermes.gateway.restart_requested"),
+        _M("hermes.platform.up"), _M("hermes.platform.degraded"),
+    ]})()
+    cron_snapshot = type("S", (), {"metrics": [
+        _M("hermes.cron.scheduler.heartbeat_age_seconds"),
+        _M("hermes.cron.scheduler.last_success_age_seconds"),
+        _M("hermes.cron.scheduler.catch_up_occurrences"),
+        _M("hermes.cron.jobs.enabled"), _M("hermes.cron.jobs.running"),
+        _M("hermes.cron.jobs.overdue"),
+    ]})()
+    monkeypatch.setattr(gateway_health_export, "_read_gateway_snapshot", lambda config: gateway_snapshot)
+    monkeypatch.setattr(gateway_health_export, "_read_cron_snapshot", lambda: cron_snapshot)
+
+    snapshot_names = {m.name for m in gateway_health_export._read_runtime_snapshot({}).metrics}
+
+    # Extract the registered metric_names list literal from _start_metric_provider.
+    src = inspect.getsource(gateway_health_export._start_metric_provider)
+    registered = {n for n in snapshot_names if f'"{n}"' in src}
+
+    missing = snapshot_names - registered
+    assert not missing, f"gauges emitted but NOT registered in metric_names (will be silently dropped): {sorted(missing)}"
+
+
+def test_monitoring_docs_distinguish_relay_health_scope_and_terminal_flush():
+    from pathlib import Path
+
+    text = Path("docs/observability/monitoring.md").read_text(encoding="utf-8")
+
+    assert "Hermes Agent-owned Relay transport health" in text
+    assert "authoritative shared connector/platform state" in text
+    assert "up to one second" in text
+    assert "terminal" in text

tests/monitoring/test_emitter.py

@@ -0,0 +1,135 @@
+"""Tests for the monitoring emitter: hot-path invariant + subscriber fan-out."""
+
+from __future__ import annotations
+
+import time
+import threading
+
+from agent.monitoring.emitter import MonitoringEmitter
+from agent.monitoring.events import GatewayHealthEvent
+
+
+def test_emit_never_raises_when_disabled():
+    em = MonitoringEmitter(enabled=False)
+    em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"})
+    assert em.stats()["queued"] == 0
+    em.close()
+
+
+def test_process_singleton_stays_dormant_until_subscribed():
+    from agent.monitoring import emitter
+
+    emitter.reset_emitter_for_tests()
+    try:
+        emitter.emit({"event": "gateway_health", "name": "gateway.lifecycle"})
+        singleton = emitter.get_emitter()
+        assert singleton.stats()["queued"] == 0
+        assert singleton._started is False
+
+        subscriber = lambda _batch: None  # noqa: E731
+        singleton.subscribe(subscriber)
+        emitter.emit({"event": "gateway_health", "name": "gateway.lifecycle"})
+        assert singleton._started is True
+        singleton.unsubscribe(subscriber)
+    finally:
+        emitter.reset_emitter_for_tests()
+
+
+def test_emit_accepts_dataclass_and_dict(tmp_path):
+    em = MonitoringEmitter()
+    seen: list = []
+    em.subscribe(lambda batch: seen.extend(batch))
+    em.emit(GatewayHealthEvent(name="gateway.health_snapshot", active_agents=2))
+    em.emit({"event": "gateway_diagnostic", "name": "platform.fatal",
+             "subsystem": "platform.slack"})
+    em.flush()
+    em.close()
+    kinds = {ev.get("event") for ev in seen}
+    assert kinds == {"gateway_health", "gateway_diagnostic"}
+    health = next(ev for ev in seen if ev["event"] == "gateway_health")
+    assert health["active_agents"] == 2
+    assert "ts_ns" in health
+
+
+def test_subscriber_failure_is_isolated():
+    em = MonitoringEmitter()
+    good: list = []
+
+    def bad(batch):
+        raise RuntimeError("boom")
+
+    em.subscribe(bad)
+    em.subscribe(lambda batch: good.extend(batch))
+    em.emit({"event": "gateway_health", "name": "gateway.lifecycle"})
+    em.flush()
+    em.close()
+    assert len(good) == 1  # the raising subscriber did not break fan-out
+
+
+def test_flush_waits_for_in_flight_subscriber_delivery():
+    em = MonitoringEmitter()
+    subscriber_started = threading.Event()
+    release_subscriber = threading.Event()
+    flush_finished = threading.Event()
+
+    def blocking_subscriber(_batch):
+        subscriber_started.set()
+        release_subscriber.wait(timeout=2.0)
+
+    em.subscribe(blocking_subscriber)
+    em.emit({"event": "gateway_health", "name": "gateway.exit"})
+    assert subscriber_started.wait(timeout=1.0)
+
+    flush_thread = threading.Thread(
+        target=lambda: (em.flush(timeout=1.0), flush_finished.set()),
+        daemon=True,
+    )
+    flush_thread.start()
+
+    try:
+        assert not flush_finished.wait(timeout=0.1)
+    finally:
+        release_subscriber.set()
+
+    assert flush_finished.wait(timeout=1.0)
+    em.close()
+
+
+def test_unsubscribe_stops_delivery():
+    em = MonitoringEmitter()
+    seen: list = []
+    cb = lambda batch: seen.extend(batch)  # noqa: E731
+    em.subscribe(cb)
+    em.emit({"event": "gateway_health", "name": "a"})
+    em.flush()
+    em.unsubscribe(cb)
+    em.emit({"event": "gateway_health", "name": "b"})
+    em.flush()
+    em.close()
+    assert [ev["name"] for ev in seen] == ["a"]
+
+
+def test_queue_full_drops_oldest():
+    em = MonitoringEmitter()
+    # Fill the queue without a dispatcher running by not letting it start:
+    # emit() starts the thread, so instead assert drop accounting via stats
+    # after a burst larger than the queue.
+    for i in range(11_000):
+        em.emit({"event": "gateway_health", "name": f"e{i}"})
+    # Give the dispatcher a moment; total dispatched + queued + dropped == emitted.
+    em.flush(timeout=5.0)
+    stats = em.stats()
+    em.close()
+    assert stats["dropped"] >= 0
+    assert stats["dispatched"] + stats["queued"] + stats["dropped"] >= 10_000
+
+
+def test_hot_path_is_fast():
+    em = MonitoringEmitter()
+    start = time.perf_counter()
+    for _ in range(1_000):
+        em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"})
+    elapsed = time.perf_counter() - start
+    em.close()
+    # 1000 emits should be far under a second even on slow CI.
+    assert elapsed < 1.0

tests/monitoring/test_export_redaction.py

@@ -0,0 +1,69 @@
+"""Export redaction tests — the security-critical layer.
+
+Invariants:
+  * One unconditional scrub: secrets AND PII, no modes, no knobs.
+  * Fails CLOSED: if the redactor can't run, the raw string is never emitted.
+  * Structure (subsystem names, error codes) survives; free-text PII does not.
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+import agent.monitoring.redaction as R
+
+
+def test_secret_key_always_stripped():
+    fake_key = "sk-ant-api03-" + "A" * 24  # constructed to dodge literal-scrubbers
+    out = R.redact_for_export(f"calling with key {fake_key} and moving on")
+    assert out is not None
+    assert fake_key not in out
+
+
+def test_token_shapes_stripped():
+    ghp = "ghp_" + "0123456789abcdef" * 2 + "0123"
+    slack = "xoxb-" + "123456789012-abcdefABCDEF"
+    out = R.redact_for_export(f"token {ghp} and {slack} leaked")
+    assert out is not None
+    assert ghp not in out
+    assert slack not in out
+    assert "[redacted]" in out
+
+
+def test_bearer_header_stripped():
+    out = R.redact_for_export("Authorization: Bearer abc.def-ghi_jkl")
+    assert out is not None
+    assert "abc.def-ghi_jkl" not in out
+
+
+def test_none_passthrough():
+    assert R.redact_for_export(None) is None
+
+
+def test_pii_always_stripped():
+    text = ("reach alice@example.com or +1 415 555 0100, "
+            "install 123e4567-e89b-12d3-a456-426614174000")
+    out = R.redact_for_export(text)
+    assert out is not None
+    assert "alice@example.com" not in out
+    assert "426614174000" not in out
+    assert "[email]" in out
+    assert "[id]" in out
+    assert "[phone]" in out
+
+
+def test_ordinary_words_survive():
+    assert R.redact_for_export("just ordinary words") == "just ordinary words"
+
+
+def test_structure_preserved():
+    out = R.redact_for_export("platform.slack entered fatal after auth_failed")
+    assert out is not None
+    assert "platform.slack" in out
+    assert "auth_failed" in out
+
+
+def test_fails_closed_when_redactor_unavailable():
+    with mock.patch("agent.redact.redact_sensitive_text", side_effect=RuntimeError):
+        out = R.redact_for_export("secret sauce sk-live-key")
+    assert out == "[redaction-unavailable]"

tests/monitoring/test_gateway_health_export.py

@@ -0,0 +1,729 @@
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+
+def test_gateway_diagnostic_event_preserves_positional_error_class():
+    from agent.monitoring.events import GatewayDiagnosticEvent
+
+    event = GatewayDiagnosticEvent("gateway.log.warning", "gateway", "auth_failed")
+
+    assert event.error_class == "auth_failed"
+    assert event.source_logger is None
+
+
+def test_default_config_keeps_gateway_health_export_disabled():
+    from hermes_cli.config import DEFAULT_CONFIG
+
+    cfg = DEFAULT_CONFIG["monitoring"]["gateway_health_export"]
+
+    assert cfg["enabled"] is False
+    assert cfg["metrics_enabled"] is True
+    assert cfg["diagnostic_events_enabled"] is True
+    assert cfg["warning_error_events_enabled"] is True
+    assert cfg["export_interval_seconds"] == 60
+    assert cfg["logs_export_interval_seconds"] == 5
+    assert cfg["resource_attributes"]["deployment.environment.name"] == "production"
+    assert "deployment.environment" not in cfg["resource_attributes"]
+    # Redaction is always-on and deliberately NOT configurable.
+    assert "redaction" not in cfg
+
+
+def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics():
+    from agent.monitoring.gateway_health import build_gateway_health_snapshot
+
+    runtime = {
+        "gateway_state": "running",
+        "pid": 1234,
+        "active_agents": "2",
+        "restart_requested": False,
+        "platforms": {
+            "slack": {"state": "running"},
+            "telegram": {
+                "state": "fatal",
+                "error_code": "auth_failed",
+                "error_message": "token xoxb-secret rejected for user 123",
+            },
+        },
+    }
+
+    snapshot = build_gateway_health_snapshot(
+        runtime,
+        gateway_running=True,
+        profile="default",
+        install_id="install-1",
+        version="2026.7.test",
+        supervision_mode="manual",
+    )
+
+    metric_names = {m.name for m in snapshot.metrics}
+    assert {
+        "hermes.gateway.up",
+        "hermes.gateway.active_agents",
+        "hermes.gateway.busy",
+        "hermes.gateway.drainable",
+        "hermes.gateway.restart_requested",
+        "hermes.platform.up",
+        "hermes.platform.degraded",
+    } <= metric_names
+
+    active = next(m for m in snapshot.metrics if m.name == "hermes.gateway.active_agents")
+    assert active.value == 2
+    assert active.attributes == {
+        "service.instance.id": active.attributes["service.instance.id"],
+        "service.version": "2026.7.test",
+        "hermes.supervision_mode": "manual",
+    }
+    assert active.attributes["service.instance.id"].startswith("sha256:")
+    assert "install-1" not in active.attributes["service.instance.id"]
+
+    busy = next(m for m in snapshot.metrics if m.name == "hermes.gateway.busy")
+    drainable = next(m for m in snapshot.metrics if m.name == "hermes.gateway.drainable")
+    assert busy.value == 1
+    assert drainable.value == 1
+
+    degraded = next(
+        m for m in snapshot.metrics
+        if m.name == "hermes.platform.degraded" and m.attributes["hermes.platform"] == "telegram"
+    )
+    assert degraded.value == 1
+    assert degraded.attributes["hermes.error_code"] == "auth_failed"
+    assert all("secret" not in str(v).lower() for v in degraded.attributes.values())
+
+
+def test_gateway_health_snapshot_emits_content_free_diagnostic_event():
+    from agent.monitoring.gateway_health import build_gateway_health_snapshot
+
+    snapshot = build_gateway_health_snapshot(
+        {
+            "gateway_state": "running",
+            "active_agents": 1,
+            "platforms": {
+                "slack": {"state": "fatal", "error_code": "auth_failed", "error_message": "Bearer sk-live-secret"},
+            },
+        },
+        gateway_running=True,
+        profile="default",
+        install_id="install-1",
+        version="v-test",
+        supervision_mode="container",
+    )
+
+    events = [event.to_dict() for event in snapshot.events]
+    health = next(e for e in events if e["event"] == "gateway_health")
+    platform = next(e for e in events if e["event"] == "gateway_diagnostic" and e["name"] == "platform.fatal")
+
+    assert health["gateway_state"] == "running"
+    assert health["active_agents"] == 1
+    assert health["gateway_busy"] is True
+    assert health["gateway_drainable"] is True
+    assert health["fatal_platform_count"] == 1
+    assert platform["platform"] == "slack"
+    assert platform["error_code"] == "auth_failed"
+    assert "redacted_message" not in platform
+    assert "Bearer" not in str(platform)
+
+
+def test_gateway_health_snapshot_preserves_real_bounded_platform_states():
+    from agent.monitoring.gateway_health import build_gateway_health_snapshot
+
+    expected = {
+        "connecting",
+        "connected",
+        "disconnected",
+        "disabled",
+        "fatal",
+        "paused",
+        "retrying",
+    }
+    snapshot = build_gateway_health_snapshot(
+        {
+            "gateway_state": "running",
+            "platforms": {state: {"state": state} for state in expected},
+        },
+        gateway_running=True,
+        profile="default",
+        install_id="install-1",
+        version="v-test",
+        supervision_mode="container",
+    )
+
+    observed = {
+        metric.attributes["hermes.platform.state"]
+        for metric in snapshot.metrics
+        if metric.name == "hermes.platform.up"
+    }
+    assert observed == expected
+
+
+def test_gateway_diagnostic_log_handler_never_carries_rendered_message(caplog):
+    from agent.monitoring import emitter
+    from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
+
+    captured = []
+
+    class DummyEmitter:
+        def emit(self, event):
+            captured.append(event.to_dict())
+
+    old = emitter.get_emitter
+    emitter.get_emitter = lambda: DummyEmitter()  # type: ignore[assignment]
+    try:
+        handler = GatewayDiagnosticLogHandler(profile="default", version="v-test")
+        logger = logging.getLogger("gateway.platforms.slack")
+        logger.setLevel(logging.DEBUG)
+        logger.addHandler(handler)
+        try:
+            logger.info("ignore info token sk-live-secret")
+            logger.warning(
+                "Unauthorized user: acct_7f3a (Alice Smith) on slack; "
+                "token «redacted:sk-…»"
+            )
+        finally:
+            logger.removeHandler(handler)
+    finally:
+        emitter.get_emitter = old  # type: ignore[assignment]
+
+    assert len(captured) == 1
+    event = captured[0]
+    assert event["event"] == "gateway_diagnostic"
+    assert event["name"] == "gateway.log.warning"
+    assert event["subsystem"] == "platform.slack"
+    assert event["source_logger"] == "gateway.platforms.slack"
+    assert event["error_class"] == "auth_failed"
+    assert "redacted_message" not in event
+    assert "acct_7f3a" not in str(event)
+    assert "Alice Smith" not in str(event)
+
+
+@pytest.mark.parametrize(
+    "message",
+    [
+        "Connect call failed ('127.0.0.1', 9)",
+        "failed to connect to relay",
+        "connection refused",
+        "network is unreachable",
+        "temporary failure in name resolution",
+    ],
+)
+def test_gateway_error_classifier_recognizes_bounded_network_failures(message):
+    from agent.monitoring.gateway_health import classify_gateway_error
+
+    assert classify_gateway_error(message) == "network_error"
+
+
+def test_gateway_diagnostic_log_handler_enriches_relay_scope_without_message_content(
+    monkeypatch,
+):
+    from agent.monitoring import emitter
+    from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
+
+    captured = []
+
+    class DummyEmitter:
+        def emit(self, event):
+            captured.append(event.to_dict())
+
+    monkeypatch.setattr(emitter, "get_emitter", lambda: DummyEmitter())
+    handler = GatewayDiagnosticLogHandler(profile="default", version="v-test")
+    logger = logging.getLogger("gateway.relay.adapter")
+    logger.addHandler(handler)
+    try:
+        logger.warning(
+            "Connect call failed for ws://alice@example.com/private; "
+            "credential=«redacted:sk-…»"
+        )
+    finally:
+        logger.removeHandler(handler)
+
+    assert len(captured) == 1
+    event = captured[0]
+    assert event["subsystem"] == "platform.relay"
+    assert event["platform"] == "relay"
+    assert event["source_logger"] == "gateway.relay.adapter"
+    assert event["error_class"] == "network_error"
+    assert event["error_code"] == "network_error"
+    assert "alice@example.com" not in str(event)
+    assert "private" not in str(event)
+
+
+def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch):
+    from agent.monitoring import emitter
+    from agent.monitoring.gateway_health import emit_runtime_status_transition
+
+    captured = []
+
+    class DummyEmitter:
+        def emit(self, event):
+            captured.append(event.to_dict())
+
+    old = emitter.emit
+    monkeypatch.setattr(emitter, "emit", lambda event: captured.append(event.to_dict()))
+
+    previous = {"gateway_state": "starting", "platforms": {"slack": {"state": "running"}}}
+    current = {
+        "gateway_state": "running",
+        "pid": 123,
+        "active_agents": 1,
+        "platforms": {
+            "slack": {
+                "state": "fatal",
+                "error_code": "auth_failed",
+                "error_message": "Bearer *** failed",
+            }
+        },
+    }
+
+    emit_runtime_status_transition(previous, current)
+
+    names = [e["name"] for e in captured]
+    assert "gateway.lifecycle" in names
+    assert "platform.state_change" in names
+    assert "platform.fatal" in names
+    lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle")
+    assert lifecycle["old_state"] == "starting"
+    assert lifecycle["new_state"] == "running"
+    assert lifecycle["exit_reason"] is None
+    platform = next(e for e in captured if e["name"] == "platform.state_change")
+    assert platform["old_state"] == "running"
+    assert platform["new_state"] == "fatal"
+    assert platform["error_code"] == "auth_failed"
+    assert "redacted_message" not in platform
+
+
+def test_runtime_status_transition_emits_startup_failed_and_exit():
+    from agent.monitoring.gateway_health import emit_runtime_status_transition
+    from agent.monitoring import emitter
+
+    captured = []
+    old = emitter.emit
+    emitter.emit = lambda event: captured.append(event.to_dict())  # type: ignore[assignment]
+    try:
+        emit_runtime_status_transition(
+            {"gateway_state": "starting"},
+            {
+                "gateway_state": "startup_failed",
+                "exit_reason": "Bearer top-secret-token rejected for user@example.com",
+            },
+        )
+        emit_runtime_status_transition(
+            {"gateway_state": "running"},
+            {
+                "gateway_state": "stopped",
+                "exit_reason": "shutdown requested by user@example.com",
+                "restart_requested": True,
+            },
+        )
+    finally:
+        emitter.emit = old  # type: ignore[assignment]
+
+    names = [e["name"] for e in captured]
+    assert "gateway.startup_failed" in names
+    assert "gateway.exit" in names
+    failed = next(e for e in captured if e["name"] == "gateway.startup_failed")
+    assert "redacted_message" not in failed
+    lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle")
+    assert lifecycle["exit_reason"] == "auth_failed"
+    exit_event = next(e for e in captured if e["name"] == "gateway.exit")
+    assert exit_event["restart_requested"] is True
+    assert exit_event["exit_reason"] == "restart_requested"
+
+
+def test_otlp_attrs_include_gateway_transition_fields():
+    from agent.monitoring.otlp_exporter import _span_attrs
+
+    attrs = _span_attrs({
+        "event": "gateway_health",
+        "name": "gateway.lifecycle",
+        "old_state": "starting",
+        "new_state": "running",
+        "exit_reason": "restart",
+        "restart_requested": True,
+    })
+
+    assert attrs["hermes.old_state"] == "starting"
+    assert attrs["hermes.new_state"] == "running"
+    assert attrs["hermes.exit_reason"] == "restart"
+    assert attrs["hermes.restart_requested"] is True
+
+
+def test_otlp_attrs_redact_strings_and_never_export_profile():
+    from agent.monitoring.otlp_exporter import _span_attrs
+
+    attrs = _span_attrs({
+        "event": "gateway_health",
+        "name": "gateway.lifecycle",
+        "profile": "user@example.com",
+        "exit_reason": "Bearer top-secret-token for user@example.com",
+    })
+
+    assert "hermes.profile" not in attrs
+    assert "top-secret-token" not in str(attrs)
+    assert "user@example.com" not in str(attrs)
+
+
+def test_resource_attributes_are_allowlisted_and_sanitized():
+    from agent.monitoring.gateway_health_export import _safe_resource_attributes
+
+    attrs = _safe_resource_attributes({
+        "service.name": "hermes-gateway",
+        "service.instance.id": "install-1",
+        "deployment.environment.name": "staging",
+        "user.email": "user@example.com",
+        "authorization": "Bearer top-secret-token",
+        "custom.request.id": "unbounded",
+    })
+
+    assert attrs == {
+        "service.name": "hermes-gateway",
+        "service.instance.id": attrs["service.instance.id"],
+        "deployment.environment.name": "staging",
+    }
+    assert attrs["service.instance.id"].startswith("sha256:")
+    assert "install-1" not in attrs["service.instance.id"]
+
+
+def test_instance_id_hash_is_stable_and_distinguishes_instances():
+    from agent.monitoring.gateway_health import _safe_instance_id
+
+    first = _safe_instance_id("install-1")
+    repeat = _safe_instance_id("install-1")
+    second = _safe_instance_id("install-2")
+
+    assert first == repeat
+    assert first != second
+    assert first.startswith("sha256:")
+    assert "install-1" not in first
+
+
+def test_runtime_resource_attributes_include_stable_hashed_instance():
+    from agent.monitoring.gateway_health_export import _runtime_resource_attributes
+
+    config = {
+        "monitoring": {
+            "install_id": "private-install-id",
+            "gateway_health_export": {
+                "resource_attributes": {"deployment.environment.name": "staging"}
+            },
+        }
+    }
+
+    attrs = _runtime_resource_attributes(config, telemetry_scope="gateway_health")
+
+    assert attrs["service.name"] == "hermes-gateway"
+    assert attrs["service.instance.id"].startswith("sha256:")
+    assert len(attrs["service.instance.id"]) == len("sha256:") + 24
+    assert "private-install-id" not in str(attrs)
+    assert attrs["deployment.environment.name"] == "staging"
+    assert attrs["telemetry.scope"] == "gateway_health"
+
+
+def test_diagnostic_log_attributes_are_allowlisted_redacted_and_profile_free():
+    from agent.monitoring.gateway_health_export import _diagnostic_log_attributes
+
+    attrs = _diagnostic_log_attributes({
+        "event": "gateway_diagnostic",
+        "name": "platform.fatal",
+        "subsystem": "platform.slack",
+        "profile": "user@example.com",
+        "error_code": "Bearer top-secret-token",
+        "custom": "must-not-egress",
+    })
+
+    assert "hermes.profile" not in attrs
+    assert "hermes.custom" not in attrs
+    assert "top-secret-token" not in str(attrs)
+
+
+def test_diagnostic_log_streamer_uses_validated_source_as_otel_scope():
+    from types import SimpleNamespace
+
+    from agent.monitoring.gateway_health_export import GatewayDiagnosticLogStreamer
+
+    class FakeLogger:
+        def __init__(self):
+            self.records = []
+
+        def emit(self, record):
+            self.records.append(record)
+
+    class FakeProvider:
+        def __init__(self):
+            self.loggers = {}
+
+        def get_logger(self, name):
+            return self.loggers.setdefault(name, FakeLogger())
+
+    provider = FakeProvider()
+    streamer = object.__new__(GatewayDiagnosticLogStreamer)
+    streamer._provider = provider
+    streamer._logger = provider.get_logger("hermes.gateway.diagnostics")
+    streamer._LogRecord = lambda **kwargs: SimpleNamespace(**kwargs)
+    streamer._sdk = {
+        "INVALID_TRACE_ID": 0,
+        "INVALID_SPAN_ID": 0,
+        "TraceFlags": SimpleNamespace(DEFAULT=0),
+        "SeverityNumber": SimpleNamespace(
+            FATAL="fatal", ERROR="error", WARN="warn", INFO="info", DEBUG="debug"
+        ),
+    }
+    streamer.exported = 0
+
+    streamer([
+        {
+            "event": "gateway_diagnostic",
+            "name": "gateway.log.warning",
+            "subsystem": "platform.relay",
+            "platform": "relay",
+            "source_logger": "gateway.relay.adapter",
+            "error_class": "network_error",
+            "severity": "warning",
+        },
+        {
+            "event": "gateway_diagnostic",
+            "name": "gateway.log.warning",
+            "subsystem": "gateway",
+            "source_logger": "gateway.relay.adapter\nalice@example.com",
+            "error_class": "unknown",
+            "severity": "warning",
+        },
+    ])
+
+    precise = provider.loggers["gateway.relay.adapter"].records
+    fallback = provider.loggers["hermes.gateway.diagnostics"].records
+    assert len(precise) == 1
+    assert len(fallback) == 1
+    assert precise[0].body == "gateway diagnostic"
+    assert "source_logger" not in precise[0].attributes
+    assert "alice@example.com" not in str(provider.loggers)
+
+
+def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch):
+    from agent.monitoring import gateway_health_export
+    from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime
+
+    monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("missing sdk")))
+
+    runtime = gateway_health_export.start_gateway_health_export({
+        "monitoring": {
+            "gateway_health_export": {"enabled": True},
+            "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4317"}},
+        }
+    })
+
+    assert isinstance(runtime, GatewayHealthExportRuntime)
+    assert runtime.enabled is False
+    assert runtime.reason == "otlp_unavailable"
+
+
+def test_gateway_health_export_shutdown_flushes_before_unsubscribe(monkeypatch):
+    from agent.monitoring import emitter
+    from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime
+
+    calls = []
+
+    class FakeEmitter:
+        def flush(self, timeout):
+            calls.append(("flush", timeout))
+
+        def unsubscribe(self, subscriber):
+            calls.append(("unsubscribe", subscriber))
+
+    class Streamer:
+        def shutdown(self):
+            calls.append(("shutdown", self))
+
+    streamer = Streamer()
+    log_streamer = Streamer()
+    monkeypatch.setattr(emitter, "get_emitter", lambda: FakeEmitter())
+
+    runtime = GatewayHealthExportRuntime(
+        enabled=True,
+        streamer=streamer,
+        log_streamer=log_streamer,
+    )
+    runtime.shutdown()
+
+    assert calls[0][0] == "flush"
+    assert calls[1:3] == [
+        ("unsubscribe", streamer),
+        ("unsubscribe", log_streamer),
+    ]
+
+
+def test_gateway_health_export_streams_only_gateway_events(monkeypatch):
+    from agent.monitoring import gateway_health_export
+
+    captured = {}
+
+    def fake_start_streaming(config, *, event_filter=None):
+        captured["filter"] = event_filter
+        return object()
+
+    monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None)
+    monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {})
+    monkeypatch.setattr(gateway_health_export, "_start_diagnostic_log_streamer", lambda *a, **k: object())
+    monkeypatch.setattr(gateway_health_export, "_attach_log_handler", lambda *a, **k: None)
+    monkeypatch.setattr(gateway_health_export, "_emit_snapshot_events", lambda *a, **k: None)
+    monkeypatch.setattr(gateway_health_export, "_start_snapshot_thread", lambda *a, **k: None)
+    from agent.monitoring import otlp_exporter
+    monkeypatch.setattr(otlp_exporter, "start_streaming", fake_start_streaming)
+
+    runtime = gateway_health_export.start_gateway_health_export({
+        "monitoring": {
+            "gateway_health_export": {"enabled": True, "metrics_enabled": False},
+            "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}},
+        }
+    })
+
+    assert runtime.enabled is True
+    event_filter = captured["filter"]
+    assert event_filter({"event": "gateway_health"}) is True
+    assert event_filter({"event": "gateway_diagnostic"}) is False
+    assert event_filter({"event": "run"}) is False
+    assert event_filter({"event": "model_call"}) is False
+    assert event_filter({"event": "tool_call"}) is False
+
+
+def test_gateway_health_export_metric_failure_does_not_start_streamer(monkeypatch):
+    from agent.monitoring import gateway_health_export, otlp_exporter
+
+    started = []
+    monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {})
+    monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")))
+    monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: started.append(True))
+
+    runtime = gateway_health_export.start_gateway_health_export({
+        "monitoring": {
+            "gateway_health_export": {"enabled": True},
+            "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}},
+        }
+    })
+
+    assert runtime.enabled is False
+    assert runtime.reason == "metrics_start_failed"
+    assert started == []
+
+
+def test_gateway_health_export_diagnostic_partial_start_cleans_up(monkeypatch):
+    from agent.monitoring import emitter, gateway_health_export, otlp_exporter
+
+    class Streamer:
+        def __call__(self, _batch):
+            pass
+
+        def shutdown(self):
+            pass
+
+    streamer = Streamer()
+    monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {})
+    monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None)
+    monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: streamer)
+    monkeypatch.setattr(
+        gateway_health_export,
+        "_start_diagnostic_log_streamer",
+        lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")),
+    )
+    emitter.get_emitter().subscribe(streamer)
+
+    runtime = gateway_health_export.start_gateway_health_export({
+        "monitoring": {
+            "gateway_health_export": {"enabled": True, "metrics_enabled": False},
+            "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}},
+        }
+    })
+
+    assert runtime.enabled is False
+    assert runtime.reason == "diagnostics_start_failed"
+    assert streamer not in emitter.get_emitter()._subscribers
+
+
+def test_gateway_health_export_shutdown_is_bounded():
+    import threading
+    import time
+
+    from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime
+
+    release = threading.Event()
+
+    class Blocking:
+        def shutdown(self):
+            release.wait(10)
+
+    runtime = GatewayHealthExportRuntime(
+        enabled=True,
+        streamer=Blocking(),
+        log_streamer=Blocking(),
+        metric_provider=Blocking(),
+    )
+    started = time.monotonic()
+    runtime.shutdown()
+    elapsed = time.monotonic() - started
+    release.set()
+
+    assert elapsed < 2.5
+
+
+def test_otlp_streamer_shutdown_unsubscribes(monkeypatch):
+    from agent.monitoring import emitter
+    from agent.monitoring.otlp_exporter import OTLPStreamer
+
+    class Dummy:
+        def force_flush(self):
+            pass
+        def shutdown(self):
+            pass
+
+    e = emitter.get_emitter()
+    streamer = OTLPStreamer.__new__(OTLPStreamer)
+    streamer._processor = Dummy()
+    streamer._provider = Dummy()
+    streamer._event_filter = None
+    streamer.exported = 0
+    e.subscribe(streamer)
+    assert streamer in e._subscribers
+
+    streamer.shutdown()
+
+    assert streamer not in e._subscribers
+
+
+def test_gateway_diagnostic_log_handler_never_raises_on_malformed_record():
+    from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
+
+    handler = GatewayDiagnosticLogHandler(profile="default", version="v-test")
+    record = logging.LogRecord(
+        "gateway.platforms.slack",
+        logging.WARNING,
+        __file__,
+        1,
+        "broken %s %s",
+        ("one",),
+        None,
+    )
+
+    handler.emit(record)
+
+
+def test_install_id_persists_across_calls(tmp_path, monkeypatch):
+    """A minted install id must survive restarts (service.instance.id continuity)."""
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+    (tmp_path / "config.yaml").write_text("{}\n")
+
+    import hermes_cli.config as cfg_mod
+    from agent.monitoring.policy import ensure_install_id
+
+    first = ensure_install_id(cfg_mod.load_config())
+    assert first and first != "unknown"
+    # Persisted: a fresh load (simulating a new gateway process) returns the same id.
+    second = ensure_install_id(cfg_mod.load_config())
+    assert second == first
+    assert first in (tmp_path / "config.yaml").read_text()
+
+
+def test_install_id_existing_value_wins(monkeypatch):
+    from agent.monitoring.policy import ensure_install_id
+
+    assert ensure_install_id({"monitoring": {"install_id": "keep-me"}}) == "keep-me"

tests/monitoring/test_otlp_exporter.py

@@ -0,0 +1,140 @@
+"""OTLP exporter tests: config resolution, span mapping, streaming subscriber.
+
+No SQLite involved — monitoring is an egress path, so the exporter consumes
+emitter batches directly. Uses the in-memory OTel span exporter; skipped when
+the optional otlp extra is not installed.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed")
+
+import agent.monitoring.otlp_exporter as OE
+from agent.monitoring.emitter import MonitoringEmitter
+
+
+def _mem_provider():
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+    exporter = InMemorySpanExporter()
+    provider = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    return provider, exporter
+
+
+def test_gateway_health_event_maps_to_span_with_attrs():
+    provider, mem = _mem_provider()
+    n = OE.export_batch(provider, [{
+        "event": "gateway_health", "name": "gateway.lifecycle",
+        "old_state": "starting", "new_state": "running",
+        "active_agents": 2, "pid": 4242,
+    }])
+    assert n == 1
+    spans = mem.get_finished_spans()
+    assert spans[0].name == "hermes.gateway_health"
+    attrs = dict(spans[0].attributes or {})
+    assert attrs["hermes.old_state"] == "starting"
+    assert attrs["hermes.new_state"] == "running"
+    assert attrs["hermes.active_agents"] == 2
+
+
+def test_gateway_diagnostic_event_drops_arbitrary_message_content():
+    provider, mem = _mem_provider()
+    OE.export_batch(provider, [{
+        "event": "gateway_diagnostic", "name": "platform.fatal",
+        "subsystem": "platform.slack", "error_class": "auth_failed",
+        "redacted_message": "Unauthorized user: acct_7f3a (Alice Smith)",
+        "severity": "error",
+    }])
+    attrs = dict(mem.get_finished_spans()[0].attributes or {})
+    assert attrs["hermes.error_class"] == "auth_failed"
+    assert "hermes.redacted_message" not in attrs
+    assert "acct_7f3a" not in str(attrs)
+    assert "Alice Smith" not in str(attrs)
+
+
+def test_unknown_event_kind_exports_no_attrs_beyond_kind():
+    provider, mem = _mem_provider()
+    OE.export_batch(provider, [{"event": "model_call", "provider": "anthropic",
+                                "model": "claude-opus-4"}])
+    attrs = dict(mem.get_finished_spans()[0].attributes or {})
+    # Non-monitoring event kinds carry no attribute mapping on this plane.
+    assert attrs == {"hermes.event": "model_call"}
+
+
+def test_headers_resolve_from_env_not_value(monkeypatch):
+    monkeypatch.setenv("DD_KEY_ENV", "secret-value")
+    resolved = OE._resolve_headers({"DD-API-KEY": "DD_KEY_ENV", "X-Missing": "NOPE_ENV"})
+    assert resolved == {"DD-API-KEY": "secret-value"}
+
+
+def test_is_enabled_requires_endpoint_and_flag():
+    assert OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}})
+    assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True}}}})
+    assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"endpoint": "http://x"}}}})
+    assert not OE.is_enabled({})
+
+
+def test_trace_resource_includes_stable_hashed_instance():
+    attrs = OE._resource_attributes(
+        {"monitoring": {"install_id": "private-install-id"}}
+    )
+
+    assert attrs["service.name"] == "hermes-gateway"
+    assert attrs["service.instance.id"].startswith("sha256:")
+    assert len(attrs["service.instance.id"]) == len("sha256:") + 24
+    assert "private-install-id" not in str(attrs)
+    assert attrs["telemetry.scope"] == "gateway_monitoring"
+
+
+def test_export_otlp_feature_specs_match_pyproject():
+    from tools.lazy_deps import LAZY_DEPS
+    import re
+    from pathlib import Path
+
+    specs = set(LAZY_DEPS["export.otlp"])
+    pyproject = Path(__file__).resolve().parents[2] / "pyproject.toml"
+    m = re.search(r'^otlp = \[(.*?)\]', pyproject.read_text(), re.M | re.S)
+    assert m, "otlp extra missing from pyproject.toml"
+    extra = set(re.findall(r'"([^"]+)"', m.group(1)))
+    assert specs == extra
+
+
+def test_streamer_receives_events_and_respects_filter(monkeypatch):
+    provider, mem = _mem_provider()
+    monkeypatch.setattr(OE, "_make_provider", lambda cfg: (provider, None))
+    streamer = OE.OTLPStreamer(
+        {}, event_filter=lambda ev: ev.get("event") == "gateway_health")
+
+    em = MonitoringEmitter()
+    em.subscribe(streamer)
+    em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"})
+    em.emit({"event": "model_call", "provider": "anthropic"})  # filtered out
+    em.flush()
+    em.close()
+
+    spans = mem.get_finished_spans()
+    assert [s.name for s in spans] == ["hermes.gateway_health"]
+    assert streamer.exported == 1
+
+
+def test_failing_streamer_never_breaks_emitter(monkeypatch):
+    def boom(cfg):
+        raise RuntimeError("no provider")
+
+    em = MonitoringEmitter()
+
+    def bad_subscriber(batch):
+        raise RuntimeError("export down")
+
+    seen: list = []
+    em.subscribe(bad_subscriber)
+    em.subscribe(lambda batch: seen.extend(batch))
+    em.emit({"event": "gateway_health", "name": "gateway.lifecycle"})
+    em.flush()
+    em.close()
+    assert len(seen) == 1

tests/tools/test_async_delegation.py

@@ -632,6 +632,34 @@ def test_completed_records_pruned_to_cap():
     assert len(ad.list_async_delegations()) <= ad._MAX_RETAINED_COMPLETED
 
 
+def test_active_task_count_expands_batches_while_active_count_stays_unit(monkeypatch):
+    """active_count() counts dispatch UNITS (batch=1); active_task_count()
+    expands a batch to its child count. This is the batch-vs-single distinction
+    the background_work metric relies on so a 3-task fan-out isn't undercounted
+    as 1 running subagent.
+    """
+    # Deterministic: install synthetic running records directly, no real spawn.
+    with ad._records_lock:
+        saved = dict(ad._records)
+        ad._records.clear()
+        ad._records["single_a"] = {"status": "running"}  # single subagent
+        ad._records["batch_3"] = {"status": "running", "is_batch": True,
+                                   "goals": ["g1", "g2", "g3"]}  # 3-task batch
+        ad._records["batch_missing"] = {"status": "running", "is_batch": True}  # goals absent -> 1
+        ad._records["done"] = {"status": "completed", "is_batch": True,
+                                "goals": ["x", "y"]}  # not running -> ignored
+    try:
+        # 3 running UNITS (single + 2 batches); the completed one is excluded.
+        assert ad.active_count() == 3
+        # TASKS: single(1) + batch_3(3) + batch_missing(1, fallback) = 5.
+        assert ad.active_task_count() == 5
+    finally:
+        with ad._records_lock:
+            ad._records.clear()
+            ad._records.update(saved)
+
+
+
 def test_completion_is_persisted_and_delivery_can_be_acknowledged(tmp_path, monkeypatch):
     """A finished child remains pending on disk until its queue consumer acks it."""
     monkeypatch.setenv("HERMES_HOME", str(tmp_path))

tests/tools/test_dockerfile_pid1_reaping.py

@@ -172,6 +172,20 @@ def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text):
     )
 
 
+def test_dockerfile_preinstalls_gateway_monitoring_otlp_runtime(dockerfile_text):
+    sync_steps = [
+        step for step in _run_steps(dockerfile_text)
+        if "uv sync" in step and "--no-install-project" in step
+    ]
+
+    assert sync_steps, "Dockerfile must install Python dependencies with uv sync"
+    assert any("--extra otlp" in step for step in sync_steps), (
+        "Published Docker images must preload the Hermes [otlp] runtime extra "
+        "so enabled Gateway Health export does not depend on first-boot package "
+        "installation into the immutable container environment."
+    )
+
+
 def test_dockerfile_preinstalls_matrix_dependencies(dockerfile_text):
     sync_steps = [
         step for step in _run_steps(dockerfile_text)

tools/async_delegation.py

@@ -533,7 +533,14 @@ def _get_executor(max_workers: int) -> ThreadPoolExecutor:
 
 
 def active_count() -> int:
-    """Number of async delegations currently running."""
+    """Number of async delegation UNITS currently running.
+
+    A unit is one dispatch: a single subagent OR a whole fan-out batch. A batch
+    counts as ONE here because it occupies one async-pool slot (the capacity
+    semantics ``dispatch_async_delegation_batch`` relies on). For the count of
+    actual concurrent child subagents (batch expanded), use
+    ``active_task_count()``.
+    """
     with _records_lock:
         return sum(
             1 for r in _records.values()
@@ -541,6 +548,29 @@ def active_count() -> int:
         )
 
 
+def active_task_count() -> int:
+    """Number of async delegation TASKS (child subagents) currently running.
+
+    Unlike ``active_count()`` (units/slots), this expands a batch to its child
+    count: a running batch of N tasks contributes N, a single subagent
+    contributes 1. This is the truthful "how many subagents are actually
+    working right now" figure for observability, where a 3-task batch shown as
+    "1" undercounts real concurrent work. Falls back to counting a batch as 1
+    if its goal list is missing.
+    """
+    with _records_lock:
+        total = 0
+        for r in _records.values():
+            if r.get("status") not in {"running", "finalizing"}:
+                continue
+            if r.get("is_batch"):
+                goals = r.get("goals")
+                total += len(goals) if isinstance(goals, (list, tuple)) and goals else 1
+            else:
+                total += 1
+        return total
+
+
 def _new_delegation_id() -> str:
     return f"deleg_{uuid.uuid4().hex[:8]}"
 

tools/lazy_deps.py

@@ -116,6 +116,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = {
     "search.firecrawl": ("firecrawl-py==4.17.0",),
     "search.parallel": ("parallel-web==0.4.2",),
 
+    # ─── Monitoring ─────────────────────────────────────────────────────────
+    # OTLP gateway monitoring export. Lazily installed on first use of
+    # monitoring.gateway_health_export / monitoring.export.otlp. Tracks the
+    # `otlp` extra in pyproject.toml — bump both together.
+    "export.otlp": (
+        "opentelemetry-sdk==1.39.1",
+        "opentelemetry-exporter-otlp-proto-http==1.39.1",
+    ),
+
     # ─── TTS providers ─────────────────────────────────────────────────────
     # Pinned to exact versions to match pyproject.toml's no-ranges policy
     # (see comment at top of [project.dependencies]). When bumping, update

uv.lock

@@ -1444,7 +1444,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/51/58/5404031044f55afad7aad1aff8be3f22b1bed03e237cfeabbc7e5c8cfde0/greenlet-3.5.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:aca9b4ce85b152b5524ef7d88170efdff80dc0032aa8b75f9aaf7f3479ea95b4", size = 287424, upload-time = "2026-06-26T18:20:31.469Z" },
     { url = "https://files.pythonhosted.org/packages/b4/bf/1c65e9b94a54d547068fa5b5a8a06f221f3316b48908e08668d29c77cb50/greenlet-3.5.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f71be4920368fe1fabeeaa53d1e3548337e2b223d9565f8ad5e392a75ba23fc", size = 606523, upload-time = "2026-06-26T19:07:08.859Z" },
     { url = "https://files.pythonhosted.org/packages/b8/c7/b66baacc95775ad511287acb0137b95574a9ce5491902372b7564799d790/greenlet-3.5.3-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d77e67f65f98449e3fb83f795b5d0a8437aead2f874ca89c96576caf4be3af6", size = 618315, upload-time = "2026-06-26T19:10:06.055Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/a0/68afd1ebad40db87dac0a28ffa120726b98bf9c7c40c481b0f63c105d298/greenlet-3.5.3-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e18619ba655ac05d78d80fc83cac4ba892bd6927b99e3b8237aee861aaacc8bb", size = 626155, upload-time = "2026-06-26T19:24:14.44Z" },
     { url = "https://files.pythonhosted.org/packages/78/2b/28ed29463522fdbe4c15b1f63922041626a7478316b34ab4adda3f0a4aba/greenlet-3.5.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8540f1e6205bd13ca0ce685581037219ca54a1b41a0a15d228c6c9b8ad5903d7", size = 617381, upload-time = "2026-06-26T18:32:16.077Z" },
+    { url = "https://files.pythonhosted.org/packages/07/7f/e327d912239ec4b3b49999e3967389bcf1ee8722b9ee9194d2752ecd558a/greenlet-3.5.3-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d27c0c653a60d9535f690226474a5cc1036a8b0d7b57504d1c4f89c44a07a80c", size = 421083, upload-time = "2026-06-26T19:25:35.804Z" },
     { url = "https://files.pythonhosted.org/packages/2a/7b/ad04e9d1337fc04965dc9fc616b6a72cb65a24b800a014c011ec812f5489/greenlet-3.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ef56fe650f50575bf843acde967b9c567687f3c22340941a899b7bc56e956a8", size = 1577771, upload-time = "2026-06-26T19:09:01.537Z" },
     { url = "https://files.pythonhosted.org/packages/d8/33/6c87ab7ba663f70ca21f3022aad1ffe56d3f3e0521e836c2415e13abcc3c/greenlet-3.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5121af01cf911e70056c00d4b46d5e9b5d1415550038573d744138bacb59e6b8", size = 1644048, upload-time = "2026-06-26T18:31:42.996Z" },
     { url = "https://files.pythonhosted.org/packages/1c/35/f0d8ee998b422cf8693b270f098e55d8d4ec8006b061b333f54f177d28d9/greenlet-3.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:0f41e4a05a3c0cb31b17023eff28dd111e1d16bf7d7d00406cd7df23f31398a7", size = 239137, upload-time = "2026-06-26T18:23:21.664Z" },
@@ -1452,7 +1454,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5d/6e/4c37d51a2b7f82d2ff11bb6b5f7d766d9a011726624af255e843727627a3/greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2", size = 288685, upload-time = "2026-06-26T18:22:08.977Z" },
     { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" },
     { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/fb/d97dc261209c80744b7c8132693a30d70ec6e7315e632cb0a10b3fec94dd/greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23", size = 622351, upload-time = "2026-06-26T19:24:16.32Z" },
     { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/ac/e5fee13cbbd0e8de312d9a146584b8a51891c68847330ef9dc8b5109d23f/greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c", size = 425395, upload-time = "2026-06-26T19:25:37.144Z" },
     { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" },
     { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" },
     { url = "https://files.pythonhosted.org/packages/14/40/c57489acf8e37d74e2913d4eff63aa0dba17acccc4bdeef874dde2dbbec9/greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c", size = 239882, upload-time = "2026-06-26T18:23:27.518Z" },
@@ -1460,7 +1464,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" },
     { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" },
     { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" },
+    { url = "https://files.pythonhosted.org/packages/57/61/2f5b1adf256d039f5dab8005de8d3d7ad2b0070a3219c0e036b3fbfeb440/greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1", size = 625580, upload-time = "2026-06-26T19:24:18.344Z" },
     { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/d9/ab7fc9e543e44d6879b0a6ef9a4b2188940fd180cc65d6f646883ddf7201/greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda", size = 427933, upload-time = "2026-06-26T19:25:38.219Z" },
     { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" },
     { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" },
     { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" },
@@ -1693,6 +1699,10 @@ mistral = [
 modal = [
     { name = "modal" },
 ]
+otlp = [
+    { name = "opentelemetry-exporter-otlp-proto-http" },
+    { name = "opentelemetry-sdk" },
+]
 parallel-web = [
     { name = "parallel-web" },
 ]
@@ -1848,6 +1858,8 @@ requires-dist = [
     { name = "numpy", marker = "extra == 'wake'", specifier = "==2.4.3" },
     { name = "onnxruntime", marker = "extra == 'wake'", specifier = "==1.27.0" },
     { name = "openai", specifier = "==2.24.0" },
+    { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp'", specifier = "==1.39.1" },
+    { name = "opentelemetry-sdk", marker = "extra == 'otlp'", specifier = "==1.39.1" },
     { name = "openwakeword", marker = "extra == 'wake'", specifier = "==0.6.0" },
     { name = "packaging", specifier = "==26.0" },
     { name = "parallel-web", marker = "extra == 'parallel-web'", specifier = "==0.4.2" },
@@ -1900,7 +1912,7 @@ requires-dist = [
     { name = "websockets", specifier = "==15.0.1" },
     { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" },
 ]
-provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "wake", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"]
+provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "wake", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"]
 
 [[package]]
 name = "hf-xet"
@@ -3969,7 +3981,7 @@ resolution-markers = [
     "python_full_version < '3.12'",
 ]
 dependencies = [
-    { name = "numpy" },
+    { name = "numpy", marker = "python_full_version < '3.12'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
 wheels = [
@@ -4024,7 +4036,7 @@ resolution-markers = [
     "python_full_version == '3.12.*'",
 ]
 dependencies = [
-    { name = "numpy" },
+    { name = "numpy", marker = "python_full_version >= '3.12'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" }
 wheels = [