fix(gateway): stay alive on mixed retryable + non-retryable startup failures

When connected_count == 0 and at least one platform failed with a
non-retryable error, the runner exited with GATEWAY_FATAL_CONFIG_EXIT_CODE
(78) even if OTHER platforms failed for merely transient reasons.

Real-world shape (NS-609, hosted instance): WhatsApp enabled but never
paired (non-retryable whatsapp_not_paired) + Telegram TimedOut during
polling startup (retryable) => exit 78 => the gateway either goes
permanently down (supervisors honoring the exit-78 contract via
RestartPreventExitStatus / the s6 finish->125 translation from #51228) or
crash-loops (anything else). Either way Telegram never gets its retry and
the dashboard drops with every exit, so a single unpaired platform plus
one network blip disconnected every channel on the instance.

Now exit 78 is reserved for the case where ALL startup failures are
non-retryable (true config error, nothing to wait for). With mixed
failures the gateway stays alive in degraded state: the reconnect watcher
recovers the retryable platforms and the misconfigured ones stay
fatal-parked and visible in runtime status.
This commit is contained in:
Shannon Sands 2026-07-23 15:07:39 +10:00 committed by Teknium
parent 7a62f62197
commit eb2f5b5723
2 changed files with 76 additions and 1 deletions

View file

@ -8302,7 +8302,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
)
if connected_count == 0:
if startup_nonretryable_errors:
if startup_nonretryable_errors and not startup_retryable_errors:
reason = "; ".join(startup_nonretryable_errors)
logger.error("Gateway hit a non-retryable startup conflict: %s", reason)
try:
@ -8314,6 +8314,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
self._request_clean_exit(reason)
self._startup_restore_in_progress = False
return True
if startup_nonretryable_errors:
# Mixed failure mode (NS-609): some platforms are fatally
# misconfigured (e.g. WhatsApp enabled but never paired) while
# others hit merely transient errors (e.g. Telegram TimedOut
# during polling startup). Exiting with
# GATEWAY_FATAL_CONFIG_EXIT_CODE here is wrong in both
# supervision worlds: under supervisors that honor the
# exit-78 contract (systemd RestartPreventExitStatus, s6
# finish→125 since #51228) the gateway goes PERMANENTLY down
# over a network blip; under anything else it crash-loops.
# Either way the retryable platforms never get their retry.
# Log the fatal side loudly, then fall through to the
# degraded/retry path below: the reconnect watcher recovers
# the retryable platforms; the non-retryable ones remain
# fatal-parked and visible in runtime status.
logger.error(
"%d platform(s) fatally misconfigured and parked: %s. "
"Staying alive so retryable platforms can recover.",
len(startup_nonretryable_errors),
"; ".join(startup_nonretryable_errors),
)
if enabled_platform_count > 0:
if startup_retryable_errors:
# All enabled platforms hit retryable failures (network

View file

@ -512,6 +512,60 @@ async def test_runner_exits_with_ex_config_on_nonretryable_startup_error(monkeyp
assert state["gateway_state"] == "startup_failed"
@pytest.mark.asyncio
async def test_runner_stays_alive_on_mixed_retryable_and_nonretryable_errors(
monkeypatch, tmp_path, caplog
):
"""Mixed startup failures — one platform fatally misconfigured, another
merely transiently failing must NOT exit with EX_CONFIG (NS-609).
Real-world shape: WhatsApp enabled but never paired (non-retryable
``whatsapp_not_paired``) while Telegram hits a startup TimedOut
(retryable). Exiting 78 here either takes the gateway permanently down
(supervisors honoring the exit-78 contract) or crash-loops it (anything
else) and in both cases Telegram never gets its retry even though
nothing is wrong with its config. The gateway must stay alive in
degraded mode, park the fatal platform, and queue the retryable one."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
config = GatewayConfig(
platforms={
Platform.DISCORD: PlatformConfig(enabled=True, token="***"),
Platform.TELEGRAM: PlatformConfig(enabled=True, token="***"),
},
sessions_dir=tmp_path / "sessions",
)
runner = GatewayRunner(config)
def _make_adapter(platform, platform_config):
if platform == Platform.DISCORD:
return _NonRetryableFailureAdapter()
return _RetryableFailureAdapter()
monkeypatch.setattr(runner, "_create_adapter", _make_adapter)
import logging
with caplog.at_level(logging.ERROR):
ok = await runner.start()
# Gateway stays alive — no clean-exit request, no EX_CONFIG.
assert ok is True
assert runner.should_exit_cleanly is False
assert runner.exit_code is None
state = read_runtime_status()
assert state["gateway_state"] in {"degraded", "running"}
# The retryable platform is queued for reconnection…
assert Platform.TELEGRAM in runner._failed_platforms
assert state["platforms"]["telegram"]["state"] == "retrying"
# …while the misconfigured one is parked as fatal, not retried.
assert Platform.DISCORD not in runner._failed_platforms
assert state["platforms"]["discord"]["state"] == "fatal"
# The fatal side is still surfaced loudly for the operator.
assert any(
"fatally misconfigured" in record.message
for record in caplog.records
), "Expected an error log calling out the parked platform(s)"
@pytest.mark.asyncio
async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_path):
"""A clean exit carrying GATEWAY_FATAL_CONFIG_EXIT_CODE must surface as a