mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(monitoring): complete production OTLP runtime
This commit is contained in:
parent
98a0260b0a
commit
73190cdbfe
4 changed files with 76 additions and 11 deletions
|
|
@ -152,7 +152,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
|
|||
# frontend stats the readme path during dep resolution, so we `touch` an
|
||||
# empty placeholder — the real README is restored by `COPY . .` below.
|
||||
#
|
||||
# `uv sync --frozen --no-install-project --extra all --extra messaging`
|
||||
# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp`
|
||||
# installs the deps reachable through the composite `[all]` extra
|
||||
# (handpicked set intended for the production image — excludes `[dev]`),
|
||||
# plus gateway messaging adapters that should work in the published image
|
||||
|
|
@ -165,6 +165,10 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
|
|||
# so Docker users can use these providers without requiring runtime
|
||||
# lazy-install access to PyPI (often blocked in containerized envs).
|
||||
#
|
||||
# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway
|
||||
# Health export is enabled. Collector and observability-backend dependencies
|
||||
# remain external and are not part of the Hermes production image.
|
||||
#
|
||||
# The hindsight memory provider's client (hindsight-client) is baked in
|
||||
# for the same reason: it lazy-installs into /opt/hermes/.venv at first
|
||||
# use, which lives inside the (immutable) image layer rather than the
|
||||
|
|
@ -182,7 +186,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
|
|||
# The editable link is created after the source copy below.
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN touch ./README.md
|
||||
RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix
|
||||
RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix
|
||||
|
||||
# ---------- Frontend build (cached independently from Python source) ----------
|
||||
# Copy only the frontend source trees first so that Python-only changes don't
|
||||
|
|
|
|||
|
|
@ -31,8 +31,11 @@ class GatewayHealthSnapshot:
|
|||
|
||||
_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
|
||||
_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
|
||||
_KNOWN_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
|
||||
_KNOWN_GATEWAY_STATES = {
|
||||
"starting", "draining", "stopping", "stopped", "startup_failed", "unknown"
|
||||
} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES
|
||||
_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
|
||||
"connecting", "disconnected", "disabled", "paused", "retrying", "unknown"
|
||||
}
|
||||
_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
|
||||
|
||||
|
|
@ -95,9 +98,9 @@ def classify_exit_reason(
|
|||
return classified
|
||||
|
||||
|
||||
def _bounded_state(raw: Any) -> str:
|
||||
def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
|
||||
state = str(raw or "unknown").lower()
|
||||
return state if state in _KNOWN_STATES else "unknown"
|
||||
return state if state in allowed else "unknown"
|
||||
|
||||
|
||||
def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
|
||||
|
|
@ -192,7 +195,9 @@ def build_gateway_health_snapshot(
|
|||
) -> GatewayHealthSnapshot:
|
||||
"""Convert gateway_state.json-compatible runtime state into P0 signals."""
|
||||
runtime = runtime or {}
|
||||
gateway_state = _bounded_state(runtime.get("gateway_state"))
|
||||
gateway_state = _bounded_state(
|
||||
runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
|
||||
)
|
||||
active_agents = _parse_active_agents(runtime.get("active_agents", 0))
|
||||
busy = _derive_busy(gateway_running, gateway_state, active_agents)
|
||||
drainable = _derive_drainable(gateway_running, gateway_state)
|
||||
|
|
@ -213,7 +218,9 @@ def build_gateway_health_snapshot(
|
|||
events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
|
||||
for platform, pdata in platforms.items():
|
||||
pdata = pdata if isinstance(pdata, dict) else {}
|
||||
state = _bounded_state(pdata.get("state"))
|
||||
state = _bounded_state(
|
||||
pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
|
||||
)
|
||||
raw_error = pdata.get("error_code") or pdata.get("error_message")
|
||||
error_code = classify_gateway_error(raw_error)
|
||||
is_up = state in _RUNNING_PLATFORM_STATES
|
||||
|
|
@ -289,8 +296,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
|
|||
out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
|
||||
profile = _safe_profile()
|
||||
version = _safe_version()
|
||||
old_gateway_state = _bounded_state((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None
|
||||
new_gateway_state = _bounded_state(current.get("gateway_state")) if current.get("gateway_state") is not None else None
|
||||
old_gateway_state = _bounded_state(
|
||||
(previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
|
||||
) if (previous or {}).get("gateway_state") is not None else None
|
||||
new_gateway_state = _bounded_state(
|
||||
current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
|
||||
) if current.get("gateway_state") is not None else None
|
||||
if old_gateway_state != new_gateway_state and new_gateway_state:
|
||||
out.append(GatewayHealthEvent(
|
||||
name="gateway.lifecycle",
|
||||
|
|
@ -345,8 +356,12 @@ def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current:
|
|||
pdata = pdata if isinstance(pdata, dict) else {}
|
||||
prev_raw = old_platforms.get(platform, {})
|
||||
prev = prev_raw if isinstance(prev_raw, dict) else {}
|
||||
old_state = _bounded_state(prev.get("state")) if prev.get("state") is not None else None
|
||||
new_state = _bounded_state(pdata.get("state")) if pdata.get("state") is not None else None
|
||||
old_state = _bounded_state(
|
||||
prev.get("state"), allowed=_KNOWN_PLATFORM_STATES
|
||||
) if prev.get("state") is not None else None
|
||||
new_state = _bounded_state(
|
||||
pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
|
||||
) if pdata.get("state") is not None else None
|
||||
if old_state == new_state or not new_state:
|
||||
continue
|
||||
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
|
||||
|
|
|
|||
|
|
@ -115,6 +115,38 @@ def test_gateway_health_snapshot_emits_content_free_diagnostic_event():
|
|||
assert "Bearer" not in platform["redacted_message"]
|
||||
|
||||
|
||||
def test_gateway_health_snapshot_preserves_real_bounded_platform_states():
|
||||
from agent.monitoring.gateway_health import build_gateway_health_snapshot
|
||||
|
||||
expected = {
|
||||
"connecting",
|
||||
"connected",
|
||||
"disconnected",
|
||||
"disabled",
|
||||
"fatal",
|
||||
"paused",
|
||||
"retrying",
|
||||
}
|
||||
snapshot = build_gateway_health_snapshot(
|
||||
{
|
||||
"gateway_state": "running",
|
||||
"platforms": {state: {"state": state} for state in expected},
|
||||
},
|
||||
gateway_running=True,
|
||||
profile="default",
|
||||
install_id="install-1",
|
||||
version="v-test",
|
||||
supervision_mode="container",
|
||||
)
|
||||
|
||||
observed = {
|
||||
metric.attributes["hermes.platform.state"]
|
||||
for metric in snapshot.metrics
|
||||
if metric.name == "hermes.platform.up"
|
||||
}
|
||||
assert observed == expected
|
||||
|
||||
|
||||
def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog):
|
||||
from agent.monitoring import emitter
|
||||
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
|
||||
|
|
|
|||
|
|
@ -172,6 +172,20 @@ def test_dockerfile_preinstalls_gateway_messaging_dependencies(dockerfile_text):
|
|||
)
|
||||
|
||||
|
||||
def test_dockerfile_preinstalls_gateway_monitoring_otlp_runtime(dockerfile_text):
|
||||
sync_steps = [
|
||||
step for step in _run_steps(dockerfile_text)
|
||||
if "uv sync" in step and "--no-install-project" in step
|
||||
]
|
||||
|
||||
assert sync_steps, "Dockerfile must install Python dependencies with uv sync"
|
||||
assert any("--extra otlp" in step for step in sync_steps), (
|
||||
"Published Docker images must preload the Hermes [otlp] runtime extra "
|
||||
"so enabled Gateway Health export does not depend on first-boot package "
|
||||
"installation into the immutable container environment."
|
||||
)
|
||||
|
||||
|
||||
def test_dockerfile_preinstalls_matrix_dependencies(dockerfile_text):
|
||||
sync_steps = [
|
||||
step for step in _run_steps(dockerfile_text)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue