fix(dashboard): cap gateway health probe timeout

This commit is contained in:
yoma 2026-07-03 01:14:51 +08:00 committed by Teknium
parent f218474552
commit 87e31cb7d1
2 changed files with 73 additions and 6 deletions

View file

@ -1372,14 +1372,29 @@ def _apply_main_model_assignment(
_GATEWAY_HEALTH_URL = os.getenv("GATEWAY_HEALTH_URL")
_GATEWAY_HEALTH_TIMEOUT_MAX = 1.0
_GATEWAY_HEALTH_ROUTE_TIMEOUT = 1.0
try:
_GATEWAY_HEALTH_TIMEOUT = float(os.getenv("GATEWAY_HEALTH_TIMEOUT", "3"))
_GATEWAY_HEALTH_TIMEOUT = float(os.getenv("GATEWAY_HEALTH_TIMEOUT", "1"))
except (ValueError, TypeError):
_log.warning(
"Invalid GATEWAY_HEALTH_TIMEOUT value %r — using default 3.0s",
"Invalid GATEWAY_HEALTH_TIMEOUT value %r — using default 1.0s",
os.getenv("GATEWAY_HEALTH_TIMEOUT"),
)
_GATEWAY_HEALTH_TIMEOUT = 3.0
_GATEWAY_HEALTH_TIMEOUT = 1.0
if _GATEWAY_HEALTH_TIMEOUT <= 0:
_log.warning(
"Invalid non-positive GATEWAY_HEALTH_TIMEOUT value %.3fs — using default 1.0s",
_GATEWAY_HEALTH_TIMEOUT,
)
_GATEWAY_HEALTH_TIMEOUT = 1.0
elif _GATEWAY_HEALTH_TIMEOUT > _GATEWAY_HEALTH_TIMEOUT_MAX:
_log.warning(
"Capping GATEWAY_HEALTH_TIMEOUT %.3fs to %.3fs for dashboard liveness probes",
_GATEWAY_HEALTH_TIMEOUT,
_GATEWAY_HEALTH_TIMEOUT_MAX,
)
_GATEWAY_HEALTH_TIMEOUT = _GATEWAY_HEALTH_TIMEOUT_MAX
_STATUS_ACTIVE_SESSIONS_TIMEOUT = 0.75
@ -2823,9 +2838,17 @@ async def get_status(profile: Optional[str] = None):
if not gateway_running and _GATEWAY_HEALTH_URL:
loop = asyncio.get_running_loop()
alive, remote_health_body = await loop.run_in_executor(
None, _probe_gateway_health
)
try:
alive, remote_health_body = await asyncio.wait_for(
loop.run_in_executor(None, _probe_gateway_health),
timeout=_GATEWAY_HEALTH_ROUTE_TIMEOUT,
)
except TimeoutError:
_log.warning(
"/api/status gateway health probe exceeded %.2fs; using local status",
_GATEWAY_HEALTH_ROUTE_TIMEOUT,
)
alive, remote_health_body = False, None
if alive:
gateway_running = True
# PID from the remote container (display only — not locally valid)

View file

@ -5,6 +5,7 @@ import os
import json
import shutil
import sys
import threading
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch, MagicMock
@ -6851,6 +6852,26 @@ class TestProbeGatewayHealth:
assert alive is False
assert body is None
def test_probe_uses_configured_short_timeout(self, monkeypatch):
"""The HTTP probe must not fall through to the OS TCP timeout."""
import hermes_cli.web_server as ws
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_URL", "http://gw:8642")
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_TIMEOUT", 0.75)
timeouts = []
def mock_urlopen(req, **kwargs):
timeouts.append(kwargs.get("timeout"))
raise TimeoutError("mock timeout")
monkeypatch.setattr(ws.urllib.request, "urlopen", mock_urlopen)
alive, body = ws._probe_gateway_health()
assert alive is False
assert body is None
assert timeouts == [0.75, 0.75]
def test_normalizes_url_with_health_suffix(self, monkeypatch):
"""If the user sets the URL to include /health, it's stripped to base."""
import hermes_cli.web_server as ws
@ -6973,6 +6994,29 @@ class TestStatusRemoteGateway:
assert data["gateway_state"] == "running"
assert data["gateway_health_url"] == "http://gw:8642"
def test_status_bounds_the_complete_remote_probe(self, monkeypatch):
"""Two serial HTTP attempts cannot consume more than the route budget."""
import hermes_cli.web_server as ws
probe_started = threading.Event()
def slow_probe():
probe_started.set()
threading.Event().wait(timeout=0.1)
return True, {"status": "ok", "pid": 999}
monkeypatch.setattr(ws, "get_running_pid_cached", lambda: None)
monkeypatch.setattr(ws, "read_runtime_status", lambda: None)
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_URL", "http://gw:8642")
monkeypatch.setattr(ws, "_GATEWAY_HEALTH_ROUTE_TIMEOUT", 0.02)
monkeypatch.setattr(ws, "_probe_gateway_health", slow_probe)
resp = self.client.get("/api/status")
assert probe_started.is_set()
assert resp.status_code == 200
assert resp.json()["gateway_running"] is False
def test_status_remote_probe_not_attempted_when_local_pid_found(self, monkeypatch):
"""When local PID check succeeds, the remote probe is never called."""
import hermes_cli.web_server as ws