fix(dashboard): keep status responsive when session db locks

This commit is contained in:
yoma 2026-07-04 19:30:10 +08:00
parent 09693cd3a3
commit a3454dd15d
2 changed files with 80 additions and 16 deletions

View file

@ -1100,6 +1100,8 @@ except (ValueError, TypeError):
)
_GATEWAY_HEALTH_TIMEOUT = 3.0
_STATUS_ACTIVE_SESSIONS_TIMEOUT = 0.75
# DEPRECATED (scheduled for removal): GATEWAY_HEALTH_URL / GATEWAY_HEALTH_TIMEOUT.
# Cross-container / cross-host gateway liveness detection will be folded into a
# first-class dashboard config key so it's no longer Docker-adjacent lore buried
@ -1150,6 +1152,45 @@ def _probe_gateway_health() -> tuple[bool, dict | None]:
return False, None
def _count_status_active_sessions() -> int:
"""Return the dashboard status active-session count.
This is best-effort status garnish, not a critical path. Use a read-only
connection so /api/status never tries to initialise or migrate state.db
while another Hermes process is writing to it.
"""
from hermes_state import SessionDB
db = SessionDB(read_only=True)
try:
sessions = db.list_sessions_rich(limit=50)
now = time.time()
return sum(
1 for s in sessions
if s.get("ended_at") is None
and (now - s.get("last_active", s.get("started_at", 0))) < 300
)
finally:
db.close()
async def _status_active_sessions() -> int:
loop = asyncio.get_running_loop()
try:
return await asyncio.wait_for(
loop.run_in_executor(None, _count_status_active_sessions),
timeout=_STATUS_ACTIVE_SESSIONS_TIMEOUT,
)
except asyncio.TimeoutError:
_log.debug(
"/api/status active session count exceeded %.2fs; returning 0",
_STATUS_ACTIVE_SESSIONS_TIMEOUT,
)
except Exception as exc:
_log.debug("/api/status active session count unavailable: %s", exc)
return 0
# Image MIME types this endpoint will serve. Extension-allowlisted so an
# authenticated caller can't pull non-image files through it.
_MEDIA_CONTENT_TYPES = {
@ -2264,22 +2305,7 @@ async def get_status(profile: Optional[str] = None):
if gateway_running and gateway_state is None and remote_health_body is not None:
gateway_state = "running"
active_sessions = 0
try:
from hermes_state import SessionDB
db = SessionDB()
try:
sessions = db.list_sessions_rich(limit=50)
now = time.time()
active_sessions = sum(
1 for s in sessions
if s.get("ended_at") is None
and (now - s.get("last_active", s.get("started_at", 0))) < 300
)
finally:
db.close()
except Exception:
pass
active_sessions = await _status_active_sessions()
# Busy/drainable readout (NAS lifecycle-safety gate). active_agents is
# the in-flight gateway-turn count the gateway now persists at every

View file

@ -267,6 +267,44 @@ class TestWebServerEndpoints:
assert "active_sessions" in data
assert data["can_update_hermes"] is True
def test_status_active_session_count_uses_read_only_db(self, monkeypatch):
import hermes_cli.web_server as web_server
captured = {}
class _FakeDB:
def __init__(self, *args, **kwargs):
captured["read_only"] = kwargs.get("read_only")
def list_sessions_rich(self, limit):
captured["limit"] = limit
return [
{"ended_at": None, "last_active": 95},
{"ended_at": 99, "last_active": 99},
{"ended_at": None, "last_active": -300},
]
def close(self):
captured["closed"] = True
monkeypatch.setattr("hermes_state.SessionDB", _FakeDB)
monkeypatch.setattr(web_server.time, "time", lambda: 100)
assert web_server._count_status_active_sessions() == 1
assert captured == {"read_only": True, "limit": 50, "closed": True}
def test_get_status_degrades_when_active_session_count_fails(self, monkeypatch):
import hermes_cli.web_server as web_server
def _locked_count():
raise TimeoutError("database is locked")
monkeypatch.setattr(web_server, "_count_status_active_sessions", _locked_count)
resp = self.client.get("/api/status")
assert resp.status_code == 200
assert resp.json()["active_sessions"] == 0
def test_gateway_drain_begin_writes_marker(self):
from gateway import drain_control