mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-25 17:18:11 +00:00
fix(gateway): stay alive on mixed retryable + non-retryable startup failures
When connected_count == 0 and at least one platform failed with a non-retryable error, the runner exited with GATEWAY_FATAL_CONFIG_EXIT_CODE (78) even if OTHER platforms failed for merely transient reasons. Real-world shape (NS-609, hosted instance): WhatsApp enabled but never paired (non-retryable whatsapp_not_paired) + Telegram TimedOut during polling startup (retryable) => exit 78 => the gateway either goes permanently down (supervisors honoring the exit-78 contract via RestartPreventExitStatus / the s6 finish->125 translation from #51228) or crash-loops (anything else). Either way Telegram never gets its retry and the dashboard drops with every exit, so a single unpaired platform plus one network blip disconnected every channel on the instance. Now exit 78 is reserved for the case where ALL startup failures are non-retryable (true config error, nothing to wait for). With mixed failures the gateway stays alive in degraded state: the reconnect watcher recovers the retryable platforms and the misconfigured ones stay fatal-parked and visible in runtime status.
This commit is contained in:
parent
7a62f62197
commit
eb2f5b5723
2 changed files with 76 additions and 1 deletions
|
|
@ -8302,7 +8302,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
)
|
||||
|
||||
if connected_count == 0:
|
||||
if startup_nonretryable_errors:
|
||||
if startup_nonretryable_errors and not startup_retryable_errors:
|
||||
reason = "; ".join(startup_nonretryable_errors)
|
||||
logger.error("Gateway hit a non-retryable startup conflict: %s", reason)
|
||||
try:
|
||||
|
|
@ -8314,6 +8314,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
|||
self._request_clean_exit(reason)
|
||||
self._startup_restore_in_progress = False
|
||||
return True
|
||||
if startup_nonretryable_errors:
|
||||
# Mixed failure mode (NS-609): some platforms are fatally
|
||||
# misconfigured (e.g. WhatsApp enabled but never paired) while
|
||||
# others hit merely transient errors (e.g. Telegram TimedOut
|
||||
# during polling startup). Exiting with
|
||||
# GATEWAY_FATAL_CONFIG_EXIT_CODE here is wrong in both
|
||||
# supervision worlds: under supervisors that honor the
|
||||
# exit-78 contract (systemd RestartPreventExitStatus, s6
|
||||
# finish→125 since #51228) the gateway goes PERMANENTLY down
|
||||
# over a network blip; under anything else it crash-loops.
|
||||
# Either way the retryable platforms never get their retry.
|
||||
# Log the fatal side loudly, then fall through to the
|
||||
# degraded/retry path below: the reconnect watcher recovers
|
||||
# the retryable platforms; the non-retryable ones remain
|
||||
# fatal-parked and visible in runtime status.
|
||||
logger.error(
|
||||
"%d platform(s) fatally misconfigured and parked: %s. "
|
||||
"Staying alive so retryable platforms can recover.",
|
||||
len(startup_nonretryable_errors),
|
||||
"; ".join(startup_nonretryable_errors),
|
||||
)
|
||||
if enabled_platform_count > 0:
|
||||
if startup_retryable_errors:
|
||||
# All enabled platforms hit retryable failures (network
|
||||
|
|
|
|||
|
|
@ -512,6 +512,60 @@ async def test_runner_exits_with_ex_config_on_nonretryable_startup_error(monkeyp
|
|||
assert state["gateway_state"] == "startup_failed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_stays_alive_on_mixed_retryable_and_nonretryable_errors(
|
||||
monkeypatch, tmp_path, caplog
|
||||
):
|
||||
"""Mixed startup failures — one platform fatally misconfigured, another
|
||||
merely transiently failing — must NOT exit with EX_CONFIG (NS-609).
|
||||
|
||||
Real-world shape: WhatsApp enabled but never paired (non-retryable
|
||||
``whatsapp_not_paired``) while Telegram hits a startup TimedOut
|
||||
(retryable). Exiting 78 here either takes the gateway permanently down
|
||||
(supervisors honoring the exit-78 contract) or crash-loops it (anything
|
||||
else) — and in both cases Telegram never gets its retry even though
|
||||
nothing is wrong with its config. The gateway must stay alive in
|
||||
degraded mode, park the fatal platform, and queue the retryable one."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
config = GatewayConfig(
|
||||
platforms={
|
||||
Platform.DISCORD: PlatformConfig(enabled=True, token="***"),
|
||||
Platform.TELEGRAM: PlatformConfig(enabled=True, token="***"),
|
||||
},
|
||||
sessions_dir=tmp_path / "sessions",
|
||||
)
|
||||
runner = GatewayRunner(config)
|
||||
|
||||
def _make_adapter(platform, platform_config):
|
||||
if platform == Platform.DISCORD:
|
||||
return _NonRetryableFailureAdapter()
|
||||
return _RetryableFailureAdapter()
|
||||
|
||||
monkeypatch.setattr(runner, "_create_adapter", _make_adapter)
|
||||
|
||||
import logging
|
||||
with caplog.at_level(logging.ERROR):
|
||||
ok = await runner.start()
|
||||
|
||||
# Gateway stays alive — no clean-exit request, no EX_CONFIG.
|
||||
assert ok is True
|
||||
assert runner.should_exit_cleanly is False
|
||||
assert runner.exit_code is None
|
||||
state = read_runtime_status()
|
||||
assert state["gateway_state"] in {"degraded", "running"}
|
||||
# The retryable platform is queued for reconnection…
|
||||
assert Platform.TELEGRAM in runner._failed_platforms
|
||||
assert state["platforms"]["telegram"]["state"] == "retrying"
|
||||
# …while the misconfigured one is parked as fatal, not retried.
|
||||
assert Platform.DISCORD not in runner._failed_platforms
|
||||
assert state["platforms"]["discord"]["state"] == "fatal"
|
||||
# The fatal side is still surfaced loudly for the operator.
|
||||
assert any(
|
||||
"fatally misconfigured" in record.message
|
||||
for record in caplog.records
|
||||
), "Expected an error log calling out the parked platform(s)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_path):
|
||||
"""A clean exit carrying GATEWAY_FATAL_CONFIG_EXIT_CODE must surface as a
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue