mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-22 16:25:58 +00:00
f57157a128
Replace REST-based Discord liveness probe with local WebSocket/heartbeat state detection. REST success doesn't prove Gateway event delivery — a half-closed WebSocket can leave Bot.start() alive while REST returns 200. Now samples ready/open/ACK state and heartbeat latency; consecutive unhealthy samples emit one retryable fatal code so GatewayRunner rebuilds the adapter through the existing reconnect path. Also fixes three lifecycle gaps in the recovery path: 1. asyncio.wait_for() can remain blocked if adapter cleanup swallows cancellation — now uses bounded asyncio.wait() with task detachment. 2. Multiplexed secondary-profile adapters had no profile-scoped reconnect owner — now uses one runner-owned reconnect slot per profile. 3. An in-flight turn could send its final text through the disconnected adapter after a replacement was registered — now resolves the live same-profile replacement for unsent final responses only (message IDs never migrate, edits/deletes stay on the old transport). Adds an opt-in Linux/systemd event-loop watchdog (gateway.systemd_watchdog_seconds, default 0) for the failure mode where the whole asyncio loop stops making progress and no in-process liveness task can run. stdlib-only sd_notify, Type=notify/WatchdogSec generation, READY/STOPPING lifecycle. Co-authored-by: 王鑫 <wx.xw@bytedance.com>
98 lines
3 KiB
Python
98 lines
3 KiB
Python
"""Gateway lifecycle contract for the opt-in systemd watchdog."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from gateway.config import GatewayConfig
|
|
from gateway.run import GatewayRunner, start_gateway
|
|
from tests.gateway.restart_test_helpers import make_restart_runner
|
|
|
|
|
|
class _FakeWatchdog:
|
|
instances: list["_FakeWatchdog"] = []
|
|
|
|
def __init__(self, *, config_enabled: bool = True):
|
|
self.config_enabled = config_enabled
|
|
self.calls: list[str] = []
|
|
self.__class__.instances.append(self)
|
|
|
|
def start(self) -> bool:
|
|
self.calls.append("start")
|
|
return self.config_enabled
|
|
|
|
def ready(self, status: str) -> bool:
|
|
self.calls.append(f"ready:{status}")
|
|
return True
|
|
|
|
async def stop(self) -> None:
|
|
self.calls.append("stop")
|
|
|
|
|
|
def _bare_runner(*, seconds: int, running: bool = True) -> GatewayRunner:
|
|
runner = object.__new__(GatewayRunner)
|
|
runner.config = GatewayConfig(systemd_watchdog_seconds=seconds)
|
|
runner._running = running
|
|
runner._systemd_watchdog = None
|
|
return runner
|
|
|
|
|
|
def test_runner_starts_watchdog_only_after_running(monkeypatch):
|
|
_FakeWatchdog.instances.clear()
|
|
monkeypatch.setattr("gateway.systemd_notify.SystemdWatchdog", _FakeWatchdog)
|
|
runner = _bare_runner(seconds=120, running=True)
|
|
|
|
assert runner._start_systemd_watchdog() is True
|
|
|
|
watchdog = _FakeWatchdog.instances[-1]
|
|
assert watchdog.config_enabled is True
|
|
assert watchdog.calls == ["start", "ready:Hermes Gateway running"]
|
|
|
|
|
|
def test_runner_does_not_start_watchdog_when_disabled_or_not_running(monkeypatch):
|
|
_FakeWatchdog.instances.clear()
|
|
monkeypatch.setattr("gateway.systemd_notify.SystemdWatchdog", _FakeWatchdog)
|
|
|
|
assert _bare_runner(seconds=0)._start_systemd_watchdog() is False
|
|
assert _bare_runner(seconds=120, running=False)._start_systemd_watchdog() is False
|
|
assert _FakeWatchdog.instances == []
|
|
|
|
|
|
def test_gateway_ready_follows_background_service_startup():
|
|
source = inspect.getsource(start_gateway)
|
|
|
|
housekeeping_started = source.index("housekeeping_thread.start()")
|
|
watchdog_started = source.index("start_watchdog()")
|
|
shutdown_wait = source.index("await runner.wait_for_shutdown()", watchdog_started)
|
|
|
|
assert housekeeping_started < watchdog_started < shutdown_wait
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_gateway_stop_stops_watchdog_before_session_drain():
|
|
runner, _adapter = make_restart_runner()
|
|
order: list[str] = []
|
|
|
|
class _OrderingWatchdog:
|
|
async def stop(self) -> None:
|
|
order.append("watchdog_stop")
|
|
|
|
async def _notify_sessions() -> None:
|
|
order.append("notify_sessions")
|
|
|
|
runner._systemd_watchdog = _OrderingWatchdog()
|
|
runner._notify_active_sessions_of_shutdown = _notify_sessions
|
|
|
|
with (
|
|
patch("gateway.status.remove_pid_file"),
|
|
patch("gateway.status.write_runtime_status"),
|
|
):
|
|
await runner.stop()
|
|
|
|
assert order[:2] == [
|
|
"watchdog_stop",
|
|
"notify_sessions",
|
|
]
|