fix(monitoring): complete production OTLP runtime

This commit is contained in:
Victor Kyriazakos 2026-07-22 21:34:24 +00:00
parent 98a0260b0a
commit 73190cdbfe
4 changed files with 76 additions and 11 deletions

View file

@ -152,7 +152,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
# frontend stats the readme path during dep resolution, so we `touch` an
# empty placeholder — the real README is restored by `COPY . .` below.
#
# `uv sync --frozen --no-install-project --extra all --extra messaging`
# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp`
# installs the deps reachable through the composite `[all]` extra
# (handpicked set intended for the production image — excludes `[dev]`),
# plus gateway messaging adapters that should work in the published image
@ -165,6 +165,10 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
# so Docker users can use these providers without requiring runtime
# lazy-install access to PyPI (often blocked in containerized envs).
#
# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway
# Health export is enabled. Collector and observability-backend dependencies
# remain external and are not part of the Hermes production image.
#
# The hindsight memory provider's client (hindsight-client) is baked in
# for the same reason: it lazy-installs into /opt/hermes/.venv at first
# use, which lives inside the (immutable) image layer rather than the
@ -182,7 +186,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
# The editable link is created after the source copy below.
COPY pyproject.toml uv.lock ./
RUN touch ./README.md
RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix
RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix
# ---------- Frontend build (cached independently from Python source) ----------
# Copy only the frontend source trees first so that Python-only changes don't

View file

@ -31,8 +31,11 @@ class GatewayHealthSnapshot:
_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
_KNOWN_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
_KNOWN_GATEWAY_STATES = {
"starting", "draining", "stopping", "stopped", "startup_failed", "unknown"
} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES
_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
"connecting", "disconnected", "disabled", "paused", "retrying", "unknown"
}
_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
@ -95,9 +98,9 @@ def classify_exit_reason(
return classified
def _bounded_state(raw: Any) -> str:
def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
state = str(raw or "unknown").lower()
return state if state in _KNOWN_STATES else "unknown"
return state if state in allowed else "unknown"
def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
@ -192,7 +195,9 @@ def build_gateway_health_snapshot(
) -> GatewayHealthSnapshot:
"""Convert gateway_state.json-compatible runtime state into P0 signals."""
runtime = runtime or {}
gateway_state = _bounded_state(runtime.get("gateway_state"))
gateway_state = _bounded_state(
runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
)
active_agents = _parse_active_agents(runtime.get("active_agents", 0))
busy = _derive_busy(gateway_running, gateway_state, active_agents)
drainable = _derive_drainable(gateway_running, gateway_state)
@ -213,7 +218,9 @@ def build_gateway_health_snapshot(
events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
for platform, pdata in platforms.items():
pdata = pdata if isinstance(pdata, dict) else {}
state = _bounded_state(pdata.get("state"))
state = _bounded_state(
pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
)
raw_error = pdata.get("error_code") or pdata.get("error_message")
error_code = classify_gateway_error(raw_error)
is_up = state in _RUNNING_PLATFORM_STATES
@ -289,8 +296,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
profile = _safe_profile()
version = _safe_version()
old_gateway_state = _bounded_state((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None
new_gateway_state = _bounded_state(current.get("gateway_state")) if current.get("gateway_state") is not None else None
old_gateway_state = _bounded_state(
(previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
) if (previous or {}).get("gateway_state") is not None else None
new_gateway_state = _bounded_state(
current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
) if current.get("gateway_state") is not None else None
if old_gateway_state != new_gateway_state and new_gateway_state:
out.append(GatewayHealthEvent(
name="gateway.lifecycle",
@ -345,8 +356,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
pdata = pdata if isinstance(pdata, dict) else {}
prev_raw = old_platforms.get(platform, {})
prev = prev_raw if isinstance(prev_raw, dict) else {}
old_state = _bounded_state(prev.get("state")) if prev.get("state") is not None else None
new_state = _bounded_state(pdata.get("state")) if pdata.get("state") is not None else None
old_state = _bounded_state(
prev.get("state"), allowed=_KNOWN_PLATFORM_STATES
) if prev.get("state") is not None else None
new_state = _bounded_state(
pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
) if pdata.get("state") is not None else None
if old_state == new_state or not new_state:
continue
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))

View file

@ -115,6 +115,38 @@ def test_gateway_health_snapshot_emits_content_free_diagnostic_event():
assert "Bearer" not in platform["redacted_message"]
def test_gateway_health_snapshot_preserves_real_bounded_platform_states():
from agent.monitoring.gateway_health import build_gateway_health_snapshot
expected = {
"connecting",
"connected",
"disconnected",
"disabled",
"fatal",
"paused",
"retrying",
}
snapshot = build_gateway_health_snapshot(
{
"gateway_state": "running",
"platforms": {state: {"state": state} for state in expected},
},
gateway_running=True,
profile="default",
install_id="install-1",
version="v-test",
supervision_mode="container",
)
observed = {
metric.attributes["hermes.platform.state"]
for metric in snapshot.metrics
if metric.name == "hermes.platform.up"
}
assert observed == expected
def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog):
from agent.monitoring import emitter
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler

View file

@ -172,6 +172,20 @@ def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text):
)
def test_dockerfile_preinstalls_gateway_monitoring_otlp_runtime(dockerfile_text):
sync_steps = [
step for step in _run_steps(dockerfile_text)
if "uv sync" in step and "--no-install-project" in step
]
assert sync_steps, "Dockerfile must install Python dependencies with uv sync"
assert any("--extra otlp" in step for step in sync_steps), (
"Published Docker images must preload the Hermes [otlp] runtime extra "
"so enabled Gateway Health export does not depend on first-boot package "
"installation into the immutable container environment."
)
def test_dockerfile_preinstalls_matrix_dependencies(dockerfile_text):
sync_steps = [
step for step in _run_steps(dockerfile_text)