mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(dashboard): cap gateway health probe timeout
This commit is contained in:
parent
f218474552
commit
87e31cb7d1
2 changed files with 73 additions and 6 deletions
|
|
@ -1372,14 +1372,29 @@ def _apply_main_model_assignment(
|
|||
|
||||
|
||||
_GATEWAY_HEALTH_URL = os.getenv("GATEWAY_HEALTH_URL")
|
||||
_GATEWAY_HEALTH_TIMEOUT_MAX = 1.0
|
||||
_GATEWAY_HEALTH_ROUTE_TIMEOUT = 1.0
|
||||
try:
|
||||
_GATEWAY_HEALTH_TIMEOUT = float(os.getenv("GATEWAY_HEALTH_TIMEOUT", "3"))
|
||||
_GATEWAY_HEALTH_TIMEOUT = float(os.getenv("GATEWAY_HEALTH_TIMEOUT", "1"))
|
||||
except (ValueError, TypeError):
|
||||
_log.warning(
|
||||
"Invalid GATEWAY_HEALTH_TIMEOUT value %r — using default 3.0s",
|
||||
"Invalid GATEWAY_HEALTH_TIMEOUT value %r — using default 1.0s",
|
||||
os.getenv("GATEWAY_HEALTH_TIMEOUT"),
|
||||
)
|
||||
_GATEWAY_HEALTH_TIMEOUT = 3.0
|
||||
_GATEWAY_HEALTH_TIMEOUT = 1.0
|
||||
if _GATEWAY_HEALTH_TIMEOUT <= 0:
|
||||
_log.warning(
|
||||
"Invalid non-positive GATEWAY_HEALTH_TIMEOUT value %.3fs — using default 1.0s",
|
||||
_GATEWAY_HEALTH_TIMEOUT,
|
||||
)
|
||||
_GATEWAY_HEALTH_TIMEOUT = 1.0
|
||||
elif _GATEWAY_HEALTH_TIMEOUT > _GATEWAY_HEALTH_TIMEOUT_MAX:
|
||||
_log.warning(
|
||||
"Capping GATEWAY_HEALTH_TIMEOUT %.3fs to %.3fs for dashboard liveness probes",
|
||||
_GATEWAY_HEALTH_TIMEOUT,
|
||||
_GATEWAY_HEALTH_TIMEOUT_MAX,
|
||||
)
|
||||
_GATEWAY_HEALTH_TIMEOUT = _GATEWAY_HEALTH_TIMEOUT_MAX
|
||||
|
||||
_STATUS_ACTIVE_SESSIONS_TIMEOUT = 0.75
|
||||
|
||||
|
|
@ -2823,9 +2838,17 @@ async def get_status(profile: Optional[str] = None):
|
|||
|
||||
if not gateway_running and _GATEWAY_HEALTH_URL:
|
||||
loop = asyncio.get_running_loop()
|
||||
alive, remote_health_body = await loop.run_in_executor(
|
||||
None, _probe_gateway_health
|
||||
)
|
||||
try:
|
||||
alive, remote_health_body = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, _probe_gateway_health),
|
||||
timeout=_GATEWAY_HEALTH_ROUTE_TIMEOUT,
|
||||
)
|
||||
except TimeoutError:
|
||||
_log.warning(
|
||||
"/api/status gateway health probe exceeded %.2fs; using local status",
|
||||
_GATEWAY_HEALTH_ROUTE_TIMEOUT,
|
||||
)
|
||||
alive, remote_health_body = False, None
|
||||
if alive:
|
||||
gateway_running = True
|
||||
# PID from the remote container (display only — not locally valid)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import os
|
|||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
|
@ -6851,6 +6852,26 @@ class TestProbeGatewayHealth:
|
|||
assert alive is False
|
||||
assert body is None
|
||||
|
||||
def test_probe_uses_configured_short_timeout(self, monkeypatch):
|
||||
"""The HTTP probe must not fall through to the OS TCP timeout."""
|
||||
import hermes_cli.web_server as ws
|
||||
|
||||
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_URL", "http://gw:8642")
|
||||
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_TIMEOUT", 0.75)
|
||||
timeouts = []
|
||||
|
||||
def mock_urlopen(req, **kwargs):
|
||||
timeouts.append(kwargs.get("timeout"))
|
||||
raise TimeoutError("mock timeout")
|
||||
|
||||
monkeypatch.setattr(ws.urllib.request, "urlopen", mock_urlopen)
|
||||
|
||||
alive, body = ws._probe_gateway_health()
|
||||
|
||||
assert alive is False
|
||||
assert body is None
|
||||
assert timeouts == [0.75, 0.75]
|
||||
|
||||
def test_normalizes_url_with_health_suffix(self, monkeypatch):
|
||||
"""If the user sets the URL to include /health, it's stripped to base."""
|
||||
import hermes_cli.web_server as ws
|
||||
|
|
@ -6973,6 +6994,29 @@ class TestStatusRemoteGateway:
|
|||
assert data["gateway_state"] == "running"
|
||||
assert data["gateway_health_url"] == "http://gw:8642"
|
||||
|
||||
def test_status_bounds_the_complete_remote_probe(self, monkeypatch):
|
||||
"""Two serial HTTP attempts cannot consume more than the route budget."""
|
||||
import hermes_cli.web_server as ws
|
||||
|
||||
probe_started = threading.Event()
|
||||
|
||||
def slow_probe():
|
||||
probe_started.set()
|
||||
threading.Event().wait(timeout=0.1)
|
||||
return True, {"status": "ok", "pid": 999}
|
||||
|
||||
monkeypatch.setattr(ws, "get_running_pid_cached", lambda: None)
|
||||
monkeypatch.setattr(ws, "read_runtime_status", lambda: None)
|
||||
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_URL", "http://gw:8642")
|
||||
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_ROUTE_TIMEOUT", 0.02)
|
||||
monkeypatch.setattr(ws, "_probe_gateway_health", slow_probe)
|
||||
|
||||
resp = self.client.get("/api/status")
|
||||
|
||||
assert probe_started.is_set()
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["gateway_running"] is False
|
||||
|
||||
def test_status_remote_probe_not_attempted_when_local_pid_found(self, monkeypatch):
|
||||
"""When local PID check succeeds, the remote probe is never called."""
|
||||
import hermes_cli.web_server as ws
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue