mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
f57157a128
Replace REST-based Discord liveness probe with local WebSocket/heartbeat state detection. REST success doesn't prove Gateway event delivery — a half-closed WebSocket can leave Bot.start() alive while REST returns 200. Now samples ready/open/ACK state and heartbeat latency; consecutive unhealthy samples emit one retryable fatal code so GatewayRunner rebuilds the adapter through the existing reconnect path. Also fixes three lifecycle gaps in the recovery path: 1. asyncio.wait_for() can remain blocked if adapter cleanup swallows cancellation — now uses bounded asyncio.wait() with task detachment. 2. Multiplexed secondary-profile adapters had no profile-scoped reconnect owner — now uses one runner-owned reconnect slot per profile. 3. An in-flight turn could send its final text through the disconnected adapter after a replacement was registered — now resolves the live same-profile replacement for unsent final responses only (message IDs never migrate, edits/deletes stay on the old transport). Adds an opt-in Linux/systemd event-loop watchdog (gateway.systemd_watchdog_seconds, default 0) for the failure mode where the whole asyncio loop stops making progress and no in-process liveness task can run. stdlib-only sd_notify, Type=notify/WatchdogSec generation, READY/STOPPING lifecycle. Co-authored-by: 王鑫 <wx.xw@bytedance.com>
123 lines
3.7 KiB
Python
123 lines
3.7 KiB
Python
"""Generated service behavior for the opt-in systemd watchdog."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from gateway.config import GatewayConfig
|
|
from hermes_cli import gateway as gateway_cli
|
|
|
|
|
|
def test_default_user_unit_keeps_simple_service_without_watchdog(monkeypatch):
|
|
monkeypatch.setattr(gateway_cli, "load_gateway_config", GatewayConfig)
|
|
|
|
unit = gateway_cli.generate_systemd_unit(system=False)
|
|
|
|
assert "Type=simple" in unit
|
|
assert "Type=notify" not in unit
|
|
assert "NotifyAccess=" not in unit
|
|
assert "WatchdogSec=" not in unit
|
|
|
|
|
|
def test_positive_watchdog_config_generates_notify_unit(monkeypatch):
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"load_gateway_config",
|
|
lambda: GatewayConfig.from_dict({"systemd_watchdog_seconds": 120}),
|
|
raising=False,
|
|
)
|
|
|
|
unit = gateway_cli.generate_systemd_unit(system=False)
|
|
|
|
assert "Type=notify" in unit
|
|
assert "NotifyAccess=main" in unit
|
|
assert "WatchdogSec=120s" in unit
|
|
|
|
|
|
def test_positive_watchdog_config_generates_notify_system_unit(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"load_gateway_config",
|
|
lambda: GatewayConfig(systemd_watchdog_seconds=30),
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_system_service_identity",
|
|
lambda _user: ("hermes", "hermes", str(tmp_path)),
|
|
)
|
|
|
|
unit = gateway_cli.generate_systemd_unit(system=True, run_as_user="hermes")
|
|
|
|
assert "Type=notify" in unit
|
|
assert "NotifyAccess=main" in unit
|
|
assert "WatchdogSec=30s" in unit
|
|
|
|
|
|
def test_user_unit_reads_watchdog_from_config_yaml(tmp_path, monkeypatch):
|
|
hermes_home = tmp_path / "home"
|
|
hermes_home.mkdir()
|
|
(hermes_home / "config.yaml").write_text(
|
|
"gateway:\n systemd_watchdog_seconds: 45\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
|
|
|
unit = gateway_cli.generate_systemd_unit(system=False)
|
|
|
|
assert "Type=notify" in unit
|
|
assert "NotifyAccess=main" in unit
|
|
assert "WatchdogSec=45s" in unit
|
|
|
|
|
|
def test_system_unit_reads_watchdog_from_target_home(tmp_path, monkeypatch):
|
|
caller_home = tmp_path / "caller"
|
|
target_home = tmp_path / "target"
|
|
caller_home.mkdir()
|
|
target_home.mkdir()
|
|
(caller_home / "config.yaml").write_text(
|
|
"gateway:\n systemd_watchdog_seconds: 0\n",
|
|
encoding="utf-8",
|
|
)
|
|
(target_home / "config.yaml").write_text(
|
|
"gateway:\n systemd_watchdog_seconds: 75\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("HERMES_HOME", str(caller_home))
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_system_service_identity",
|
|
lambda _user: ("service", "service", str(tmp_path / "account")),
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_hermes_home_for_target_user",
|
|
lambda _home: str(target_home),
|
|
)
|
|
|
|
unit = gateway_cli.generate_systemd_unit(system=True, run_as_user="service")
|
|
|
|
assert "Type=notify" in unit
|
|
assert "WatchdogSec=75s" in unit
|
|
|
|
|
|
def test_managed_watchdog_override_controls_generated_unit(tmp_path, monkeypatch):
|
|
hermes_home = tmp_path / "home"
|
|
managed_home = tmp_path / "managed"
|
|
hermes_home.mkdir()
|
|
managed_home.mkdir()
|
|
(hermes_home / "config.yaml").write_text(
|
|
"gateway:\n systemd_watchdog_seconds: 120\n",
|
|
encoding="utf-8",
|
|
)
|
|
(managed_home / "config.yaml").write_text(
|
|
"gateway:\n systemd_watchdog_seconds: 0\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
|
monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed_home))
|
|
|
|
from hermes_cli import managed_scope
|
|
|
|
managed_scope.invalidate_managed_cache()
|
|
unit = gateway_cli.generate_systemd_unit(system=False)
|
|
|
|
assert "Type=simple" in unit
|
|
assert "WatchdogSec=" not in unit
|