diff --git a/Dockerfile b/Dockerfile index 388056faacd..4d2c0f1972f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -152,7 +152,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # frontend stats the readme path during dep resolution, so we `touch` an # empty placeholder — the real README is restored by `COPY . .` below. # -# `uv sync --frozen --no-install-project --extra all --extra messaging` +# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp` # installs the deps reachable through the composite `[all]` extra # (handpicked set intended for the production image — excludes `[dev]`), # plus gateway messaging adapters that should work in the published image @@ -165,6 +165,10 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # so Docker users can use these providers without requiring runtime # lazy-install access to PyPI (often blocked in containerized envs). # +# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway +# Health export is enabled. Collector and observability-backend dependencies +# remain external and are not part of the Hermes production image. +# # The hindsight memory provider's client (hindsight-client) is baked in # for the same reason: it lazy-installs into /opt/hermes/.venv at first # use, which lives inside the (immutable) image layer rather than the @@ -182,7 +186,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # The editable link is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md -RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix +RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix # ---------- Frontend build (cached independently from Python source) ---------- # Copy only the frontend source trees first so that Python-only changes don't diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py index 937b57ba442..ece881bca4e 100644 --- a/agent/monitoring/gateway_health.py +++ b/agent/monitoring/gateway_health.py @@ -31,8 +31,11 @@ class GatewayHealthSnapshot: _RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"} _FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"} -_KNOWN_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { +_KNOWN_GATEWAY_STATES = { "starting", "draining", "stopping", "stopped", "startup_failed", "unknown" +} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES +_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { + "connecting", "disconnected", "disabled", "paused", "retrying", "unknown" } _SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"} @@ -95,9 +98,9 @@ def classify_exit_reason( return classified -def _bounded_state(raw: Any) -> str: +def _bounded_state(raw: Any, *, allowed: set[str]) -> str: state = str(raw or "unknown").lower() - return state if state in _KNOWN_STATES else "unknown" + return state if state in allowed else "unknown" def _safe_metric_value(raw: Any, *, limit: int = 128) -> str: @@ -192,7 +195,9 @@ def build_gateway_health_snapshot( ) -> GatewayHealthSnapshot: """Convert gateway_state.json-compatible runtime state into P0 signals.""" runtime = runtime or {} - gateway_state = _bounded_state(runtime.get("gateway_state")) + gateway_state = _bounded_state( + runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) active_agents = _parse_active_agents(runtime.get("active_agents", 0)) busy = _derive_busy(gateway_running, gateway_state, active_agents) drainable = _derive_drainable(gateway_running, gateway_state) @@ -213,7 +218,9 @@ def build_gateway_health_snapshot( events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] for platform, pdata in platforms.items(): pdata = pdata if isinstance(pdata, dict) else {} - state = _bounded_state(pdata.get("state")) + state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) raw_error = pdata.get("error_code") or pdata.get("error_message") error_code = classify_gateway_error(raw_error) is_up = state in _RUNNING_PLATFORM_STATES @@ -289,8 +296,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] profile = _safe_profile() version = _safe_version() - old_gateway_state = _bounded_state((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None - new_gateway_state = _bounded_state(current.get("gateway_state")) if current.get("gateway_state") is not None else None + old_gateway_state = _bounded_state( + (previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = _bounded_state( + current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if current.get("gateway_state") is not None else None if old_gateway_state != new_gateway_state and new_gateway_state: out.append(GatewayHealthEvent( name="gateway.lifecycle", @@ -345,8 +356,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: pdata = pdata if isinstance(pdata, dict) else {} prev_raw = old_platforms.get(platform, {}) prev = prev_raw if isinstance(prev_raw, dict) else {} - old_state = _bounded_state(prev.get("state")) if prev.get("state") is not None else None - new_state = _bounded_state(pdata.get("state")) if pdata.get("state") is not None else None + old_state = _bounded_state( + prev.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if prev.get("state") is not None else None + new_state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if pdata.get("state") is not None else None if old_state == new_state or not new_state: continue error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py index 363b86cdfc4..df50269b3f5 100644 --- a/tests/monitoring/test_gateway_health_export.py +++ b/tests/monitoring/test_gateway_health_export.py @@ -115,6 +115,38 @@ def test_gateway_health_snapshot_emits_content_free_diagnostic_event(): assert "Bearer" not in platform["redacted_message"] +def test_gateway_health_snapshot_preserves_real_bounded_platform_states(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + expected = { + "connecting", + "connected", + "disconnected", + "disabled", + "fatal", + "paused", + "retrying", + } + snapshot = build_gateway_health_snapshot( + { + "gateway_state": "running", + "platforms": {state: {"state": state} for state in expected}, + }, + gateway_running=True, + profile="default", + install_id="install-1", + version="v-test", + supervision_mode="container", + ) + + observed = { + metric.attributes["hermes.platform.state"] + for metric in snapshot.metrics + if metric.name == "hermes.platform.up" + } + assert observed == expected + + def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): from agent.monitoring import emitter from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler diff --git a/tests/tools/test_dockerfile_pid1_reaping.py b/tests/tools/test_dockerfile_pid1_reaping.py index 699fd5709a1..3b3e069c45f 100644 --- a/tests/tools/test_dockerfile_pid1_reaping.py +++ b/tests/tools/test_dockerfile_pid1_reaping.py @@ -172,6 +172,20 @@ def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text): ) +def test_dockerfile_preinstalls_gateway_monitoring_otlp_runtime(dockerfile_text): + sync_steps = [ + step for step in _run_steps(dockerfile_text) + if "uv sync" in step and "--no-install-project" in step + ] + + assert sync_steps, "Dockerfile must install Python dependencies with uv sync" + assert any("--extra otlp" in step for step in sync_steps), ( + "Published Docker images must preload the Hermes [otlp] runtime extra " + "so enabled Gateway Health export does not depend on first-boot package " + "installation into the immutable container environment." + ) + + def test_dockerfile_preinstalls_matrix_dependencies(dockerfile_text): sync_steps = [ step for step in _run_steps(dockerfile_text)