mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-25 17:18:11 +00:00
eb2f5b5723
When connected_count == 0 and at least one platform failed with a non-retryable error, the runner exited with GATEWAY_FATAL_CONFIG_EXIT_CODE (78) even if OTHER platforms failed for merely transient reasons. Real-world shape (NS-609, hosted instance): WhatsApp enabled but never paired (non-retryable whatsapp_not_paired) + Telegram TimedOut during polling startup (retryable) => exit 78 => the gateway either goes permanently down (supervisors honoring the exit-78 contract via RestartPreventExitStatus / the s6 finish->125 translation from #51228) or crash-loops (anything else). Either way Telegram never gets its retry and the dashboard drops with every exit, so a single unpaired platform plus one network blip disconnected every channel on the instance. Now exit 78 is reserved for the case where ALL startup failures are non-retryable (true config error, nothing to wait for). With mixed failures the gateway stays alive in degraded state: the reconnect watcher recovers the retryable platforms and the misconfigured ones stay fatal-parked and visible in runtime status.
626 lines
24 KiB
Python
626 lines
24 KiB
Python
import pytest
|
|
from unittest.mock import AsyncMock
|
|
|
|
from gateway.config import GatewayConfig, Platform, PlatformConfig
|
|
from gateway.platforms.base import BasePlatformAdapter
|
|
from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
|
|
from gateway.run import GatewayRunner
|
|
from gateway.status import read_runtime_status
|
|
|
|
|
|
class _RetryableFailureAdapter(BasePlatformAdapter):
|
|
def __init__(self):
|
|
super().__init__(PlatformConfig(enabled=True, token="***"), Platform.TELEGRAM)
|
|
|
|
async def connect(self, *, is_reconnect: bool = False) -> bool:
|
|
self._set_fatal_error(
|
|
"telegram_connect_error",
|
|
"Telegram startup failed: temporary DNS resolution failure.",
|
|
retryable=True,
|
|
)
|
|
return False
|
|
|
|
async def disconnect(self) -> None:
|
|
self._mark_disconnected()
|
|
|
|
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
|
raise NotImplementedError
|
|
|
|
async def get_chat_info(self, chat_id):
|
|
return {"id": chat_id}
|
|
|
|
|
|
class _DisabledAdapter(BasePlatformAdapter):
|
|
def __init__(self):
|
|
super().__init__(PlatformConfig(enabled=False, token="***"), Platform.TELEGRAM)
|
|
|
|
async def connect(self, *, is_reconnect: bool = False) -> bool:
|
|
raise AssertionError("connect should not be called for disabled platforms")
|
|
|
|
async def disconnect(self) -> None:
|
|
self._mark_disconnected()
|
|
|
|
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
|
raise NotImplementedError
|
|
|
|
async def get_chat_info(self, chat_id):
|
|
return {"id": chat_id}
|
|
|
|
|
|
class _SuccessfulAdapter(BasePlatformAdapter):
|
|
def __init__(self):
|
|
super().__init__(PlatformConfig(enabled=True, token="***"), Platform.DISCORD)
|
|
|
|
async def connect(self, *, is_reconnect: bool = False) -> bool:
|
|
return True
|
|
|
|
async def disconnect(self) -> None:
|
|
self._mark_disconnected()
|
|
|
|
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
|
raise NotImplementedError
|
|
|
|
async def get_chat_info(self, chat_id):
|
|
return {"id": chat_id}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_stays_alive_for_retryable_startup_errors(monkeypatch, tmp_path):
|
|
"""Retryable startup errors should leave the gateway running in
|
|
degraded mode so the reconnect watcher can recover the platform when
|
|
the underlying problem clears. Previously this returned False from
|
|
``start()`` and exited the process, which converted a single broken
|
|
platform (e.g. unpaired WhatsApp, DNS blip on Telegram) into a
|
|
systemd restart loop and killed cron jobs in the meantime.
|
|
"""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.TELEGRAM: PlatformConfig(enabled=True, token="***")
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
runner = GatewayRunner(config)
|
|
|
|
monkeypatch.setattr(runner, "_create_adapter", lambda platform, platform_config: _RetryableFailureAdapter())
|
|
|
|
ok = await runner.start()
|
|
|
|
# Gateway stays alive in degraded mode; reconnect watcher takes over.
|
|
assert ok is True
|
|
assert runner.should_exit_cleanly is False
|
|
state = read_runtime_status()
|
|
assert state["gateway_state"] in {"degraded", "running"}
|
|
# Telegram was queued for retry, not given up on.
|
|
assert Platform.TELEGRAM in runner._failed_platforms
|
|
assert state["platforms"]["telegram"]["state"] == "retrying"
|
|
assert state["platforms"]["telegram"]["error_code"] == "telegram_connect_error"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_allows_cron_only_mode_when_no_platforms_are_enabled(monkeypatch, tmp_path):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.TELEGRAM: PlatformConfig(enabled=False, token="***")
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
runner = GatewayRunner(config)
|
|
|
|
ok = await runner.start()
|
|
|
|
assert ok is True
|
|
assert runner.should_exit_cleanly is False
|
|
assert runner.adapters == {}
|
|
state = read_runtime_status()
|
|
assert state["gateway_state"] == "running"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_records_connected_platform_state_on_success(monkeypatch, tmp_path):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.DISCORD: PlatformConfig(enabled=True, token="***")
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
runner = GatewayRunner(config)
|
|
|
|
monkeypatch.setattr(runner, "_create_adapter", lambda platform, platform_config: _SuccessfulAdapter())
|
|
monkeypatch.setattr(runner.hooks, "discover_and_load", lambda: None)
|
|
monkeypatch.setattr(runner.hooks, "emit", AsyncMock())
|
|
|
|
ok = await runner.start()
|
|
|
|
assert ok is True
|
|
state = read_runtime_status()
|
|
assert state["gateway_state"] == "running"
|
|
assert state["platforms"]["discord"]["state"] == "connected"
|
|
assert state["platforms"]["discord"]["error_code"] is None
|
|
assert state["platforms"]["discord"]["error_message"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_gateway_verbosity_imports_redacting_formatter(monkeypatch, tmp_path):
|
|
"""Verbosity != None must not crash with NameError on RedactingFormatter (#8044)."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
class _CleanExitRunner:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.should_exit_cleanly = True
|
|
self.exit_reason = None
|
|
self.exit_code = None
|
|
self.adapters = {}
|
|
|
|
async def start(self):
|
|
assert self._platform_lock_takeover_on_start is False
|
|
return True
|
|
|
|
async def stop(self):
|
|
return None
|
|
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
|
|
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
|
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
|
|
monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr("gateway.run.GatewayRunner", _CleanExitRunner)
|
|
|
|
from gateway.run import start_gateway
|
|
|
|
# verbosity=1 triggers the code path that uses RedactingFormatter.
|
|
# Before the fix this raised NameError.
|
|
ok = await start_gateway(config=GatewayConfig(), replace=False, verbosity=1)
|
|
|
|
assert ok is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_gateway_replace_force_uses_terminate_pid(monkeypatch, tmp_path):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
calls = []
|
|
|
|
class _CleanExitRunner:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.should_exit_cleanly = True
|
|
self.exit_reason = None
|
|
self.exit_code = None
|
|
self.adapters = {}
|
|
|
|
async def start(self):
|
|
assert self._platform_lock_takeover_on_start is True
|
|
return True
|
|
|
|
async def stop(self):
|
|
return None
|
|
|
|
# get_running_pid returns 42 before we kill the old gateway, then None
|
|
# after remove_pid_file() clears the record (reflects real behavior).
|
|
_pid_state = {"alive": True}
|
|
def _mock_get_running_pid():
|
|
return 42 if _pid_state["alive"] else None
|
|
def _mock_remove_pid_file():
|
|
_pid_state["alive"] = False
|
|
monkeypatch.setattr("gateway.status.get_running_pid", _mock_get_running_pid)
|
|
monkeypatch.setattr("gateway.status.remove_pid_file", _mock_remove_pid_file)
|
|
monkeypatch.setattr(
|
|
"gateway.status.release_all_scoped_locks",
|
|
lambda **kwargs: 0,
|
|
)
|
|
# force-kill reaps the process: terminate_pid(force=True) flips it dead,
|
|
# and the post-kill re-poll via _pid_exists then sees it gone so the
|
|
# replacement proceeds.
|
|
def _mock_terminate_pid(pid, force=False):
|
|
calls.append((pid, force))
|
|
if force:
|
|
_pid_state["alive"] = False
|
|
monkeypatch.setattr("gateway.status.terminate_pid", _mock_terminate_pid)
|
|
monkeypatch.setattr(
|
|
"gateway.status._pid_exists", lambda pid: _pid_state["alive"]
|
|
)
|
|
monkeypatch.setattr("gateway.run.os.getpid", lambda: 100)
|
|
monkeypatch.setattr("gateway.run.os.kill", lambda pid, sig: None)
|
|
monkeypatch.setattr("time.sleep", lambda _: None)
|
|
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
|
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
|
|
monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr("gateway.run.GatewayRunner", _CleanExitRunner)
|
|
|
|
from gateway.run import start_gateway
|
|
|
|
ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None)
|
|
|
|
assert ok is True
|
|
assert calls == [(42, False), (42, True)]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_gateway_replace_aborts_when_force_killed_pid_still_alive(
|
|
monkeypatch, tmp_path
|
|
):
|
|
"""Regression for #19471 (duplicate-gateway half).
|
|
|
|
If SIGKILL fails to reap the old gateway, --replace must NOT clear the PID
|
|
file / scoped locks and start a fresh instance — that leaves two live
|
|
gateways fighting over the same token. It should abort instead.
|
|
"""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
calls = []
|
|
removed_pid = False
|
|
released_locks = False
|
|
|
|
class _RunnerShouldNotStart:
|
|
def __init__(self, config):
|
|
raise AssertionError("replacement must not start while old PID is alive")
|
|
|
|
def _mock_remove_pid_file():
|
|
nonlocal removed_pid
|
|
removed_pid = True
|
|
|
|
def _mock_release_all_scoped_locks(**kwargs):
|
|
nonlocal released_locks
|
|
released_locks = True
|
|
return 0
|
|
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 42)
|
|
monkeypatch.setattr("gateway.status.remove_pid_file", _mock_remove_pid_file)
|
|
monkeypatch.setattr(
|
|
"gateway.status.release_all_scoped_locks",
|
|
_mock_release_all_scoped_locks,
|
|
)
|
|
monkeypatch.setattr(
|
|
"gateway.status.terminate_pid",
|
|
lambda pid, force=False: calls.append((pid, force)),
|
|
)
|
|
# _pid_exists never goes False — the force-kill did not take.
|
|
monkeypatch.setattr("gateway.status._pid_exists", lambda pid: True)
|
|
monkeypatch.setattr("gateway.run.os.getpid", lambda: 100)
|
|
monkeypatch.setattr("gateway.run.os.kill", lambda pid, sig: None)
|
|
monkeypatch.setattr("time.sleep", lambda _: None)
|
|
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
|
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
|
|
monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr("gateway.run.GatewayRunner", _RunnerShouldNotStart)
|
|
|
|
from gateway.run import start_gateway
|
|
|
|
ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None)
|
|
|
|
assert ok is False
|
|
assert calls == [(42, False), (42, True)]
|
|
assert removed_pid is False
|
|
assert released_locks is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_gateway_replace_writes_takeover_marker_before_sigterm(
|
|
monkeypatch, tmp_path
|
|
):
|
|
"""--replace must write a takeover marker BEFORE sending SIGTERM.
|
|
|
|
The marker lets the target's shutdown handler identify the signal as a
|
|
planned takeover (→ exit 0) rather than an unexpected kill (→ exit 1).
|
|
Without the marker, PR #5646's signal-recovery path would revive the
|
|
target via systemd Restart=on-failure, starting a flap loop.
|
|
"""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
# Record the ORDER of marker-write + terminate_pid calls
|
|
events: list[str] = []
|
|
marker_paths_seen: list = []
|
|
|
|
def record_write_marker(target_pid: int) -> bool:
|
|
events.append(f"write_marker(target_pid={target_pid})")
|
|
# Also check that the marker file actually exists after this call
|
|
marker_paths_seen.append(
|
|
(tmp_path / ".gateway-takeover.json").exists() is False # not yet
|
|
)
|
|
# Actually write the marker so we can verify cleanup later
|
|
from gateway.status import _get_takeover_marker_path, _write_json_file
|
|
_write_json_file(_get_takeover_marker_path(), {
|
|
"target_pid": target_pid,
|
|
"target_start_time": 0,
|
|
"replacer_pid": 100,
|
|
"written_at": "2026-04-17T00:00:00+00:00",
|
|
})
|
|
return True
|
|
|
|
def record_terminate(pid, force=False):
|
|
events.append(f"terminate_pid(pid={pid}, force={force})")
|
|
|
|
class _CleanExitRunner:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.should_exit_cleanly = True
|
|
self.exit_reason = None
|
|
self.exit_code = None
|
|
self.adapters = {}
|
|
|
|
async def start(self):
|
|
return True
|
|
|
|
async def stop(self):
|
|
return None
|
|
|
|
_pid_state = {"alive": True}
|
|
def _mock_get_running_pid():
|
|
return 42 if _pid_state["alive"] else None
|
|
def _mock_remove_pid_file():
|
|
_pid_state["alive"] = False
|
|
monkeypatch.setattr("gateway.status.get_running_pid", _mock_get_running_pid)
|
|
monkeypatch.setattr("gateway.status.remove_pid_file", _mock_remove_pid_file)
|
|
monkeypatch.setattr(
|
|
"gateway.status.release_all_scoped_locks",
|
|
lambda **kwargs: 0,
|
|
)
|
|
monkeypatch.setattr("gateway.status.write_takeover_marker", record_write_marker)
|
|
monkeypatch.setattr("gateway.status.terminate_pid", record_terminate)
|
|
monkeypatch.setattr("gateway.run.os.getpid", lambda: 100)
|
|
# Simulate old process exiting on first check so we don't loop into force-kill
|
|
monkeypatch.setattr(
|
|
"gateway.run.os.kill",
|
|
lambda pid, sig: (_ for _ in ()).throw(ProcessLookupError()),
|
|
)
|
|
monkeypatch.setattr("time.sleep", lambda _: None)
|
|
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
|
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
|
|
monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr("gateway.run.GatewayRunner", _CleanExitRunner)
|
|
|
|
from gateway.run import start_gateway
|
|
|
|
ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None)
|
|
|
|
assert ok is True
|
|
# Ordering: marker written BEFORE SIGTERM
|
|
assert events[0] == "write_marker(target_pid=42)"
|
|
assert any(e.startswith("terminate_pid(pid=42") for e in events[1:])
|
|
# Marker file cleanup: replacer cleans it after loop completes
|
|
assert not (tmp_path / ".gateway-takeover.json").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_gateway_replace_clears_marker_on_permission_denied(
|
|
monkeypatch, tmp_path
|
|
):
|
|
"""If we fail to kill the existing PID (permission denied), clean up the
|
|
marker so it doesn't grief an unrelated future shutdown."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
def write_marker(target_pid: int) -> bool:
|
|
from gateway.status import _get_takeover_marker_path, _write_json_file
|
|
_write_json_file(_get_takeover_marker_path(), {
|
|
"target_pid": target_pid,
|
|
"target_start_time": 0,
|
|
"replacer_pid": 100,
|
|
"written_at": "2026-04-17T00:00:00+00:00",
|
|
})
|
|
return True
|
|
|
|
def raise_permission(pid, force=False):
|
|
raise PermissionError("simulated EPERM")
|
|
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 42)
|
|
monkeypatch.setattr("gateway.status.write_takeover_marker", write_marker)
|
|
monkeypatch.setattr("gateway.status.terminate_pid", raise_permission)
|
|
monkeypatch.setattr("gateway.run.os.getpid", lambda: 100)
|
|
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
|
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
|
|
monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
|
|
|
|
from gateway.run import start_gateway
|
|
|
|
# Should return False due to permission error
|
|
ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None)
|
|
|
|
assert ok is False
|
|
# Marker must NOT be left behind
|
|
assert not (tmp_path / ".gateway-takeover.json").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_degrades_gracefully_when_all_adapters_missing(monkeypatch, tmp_path, caplog):
|
|
"""When all enabled platforms have no adapter (missing library or credentials),
|
|
the gateway should NOT return failure — it should warn and continue running for
|
|
cron job execution, matching the behaviour of 'no platforms enabled' (#5196).
|
|
|
|
In fleet deployments the same config.yaml is shared across nodes that may only
|
|
have credentials for a subset of platforms. Requiring perfect credentials on
|
|
every node makes fleet operation impossible."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.TELEGRAM: PlatformConfig(enabled=True, token="***"),
|
|
Platform.DISCORD: PlatformConfig(enabled=True, token="***"),
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
runner = GatewayRunner(config)
|
|
|
|
# Simulate _create_adapter returning None for ALL platforms (missing library /
|
|
# missing credentials — no connection attempt ever made).
|
|
monkeypatch.setattr(runner, "_create_adapter", lambda platform, cfg: None)
|
|
|
|
import logging
|
|
with caplog.at_level(logging.WARNING):
|
|
ok = await runner.start()
|
|
|
|
# Must NOT return False — gateway should keep running for cron.
|
|
assert ok is True
|
|
assert runner.should_exit_cleanly is False
|
|
assert runner.adapters == {}
|
|
# Runtime state must remain "running", not "startup_failed".
|
|
state = read_runtime_status()
|
|
assert state["gateway_state"] == "running"
|
|
# A warning must be emitted explaining why no platforms connected.
|
|
assert any(
|
|
"No adapter could be created" in record.message
|
|
for record in caplog.records
|
|
), "Expected degraded-mode warning when all adapters are missing"
|
|
|
|
|
|
class _NonRetryableFailureAdapter(BasePlatformAdapter):
|
|
"""Simulates a fatal config error like token collision."""
|
|
def __init__(self):
|
|
super().__init__(PlatformConfig(enabled=True, token="***"), Platform.DISCORD)
|
|
|
|
async def connect(self, *, is_reconnect: bool = False) -> bool:
|
|
self._set_fatal_error(
|
|
"discord-bot-token_lock",
|
|
"Discord bot token already in use (PID 999). Stop the other gateway first.",
|
|
retryable=False,
|
|
)
|
|
return False
|
|
|
|
async def disconnect(self) -> None:
|
|
self._mark_disconnected()
|
|
|
|
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
|
raise NotImplementedError
|
|
|
|
async def get_chat_info(self, chat_id):
|
|
return {"id": chat_id}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_exits_with_ex_config_on_nonretryable_startup_error(monkeypatch, tmp_path):
|
|
"""Non-retryable startup errors (token collision, no platforms) must
|
|
set exit_code to 78 (EX_CONFIG) so the s6 finish script can translate
|
|
it to exit 125 (permanent failure). See #51228."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.DISCORD: PlatformConfig(enabled=True, token="***")
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
runner = GatewayRunner(config)
|
|
|
|
monkeypatch.setattr(runner, "_create_adapter", lambda platform, platform_config: _NonRetryableFailureAdapter())
|
|
|
|
ok = await runner.start()
|
|
|
|
assert ok is True # start() returns True (clean exit requested)
|
|
assert runner.should_exit_cleanly is True
|
|
assert runner.exit_code == GATEWAY_FATAL_CONFIG_EXIT_CODE
|
|
state = read_runtime_status()
|
|
assert state["gateway_state"] == "startup_failed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_stays_alive_on_mixed_retryable_and_nonretryable_errors(
|
|
monkeypatch, tmp_path, caplog
|
|
):
|
|
"""Mixed startup failures — one platform fatally misconfigured, another
|
|
merely transiently failing — must NOT exit with EX_CONFIG (NS-609).
|
|
|
|
Real-world shape: WhatsApp enabled but never paired (non-retryable
|
|
``whatsapp_not_paired``) while Telegram hits a startup TimedOut
|
|
(retryable). Exiting 78 here either takes the gateway permanently down
|
|
(supervisors honoring the exit-78 contract) or crash-loops it (anything
|
|
else) — and in both cases Telegram never gets its retry even though
|
|
nothing is wrong with its config. The gateway must stay alive in
|
|
degraded mode, park the fatal platform, and queue the retryable one."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.DISCORD: PlatformConfig(enabled=True, token="***"),
|
|
Platform.TELEGRAM: PlatformConfig(enabled=True, token="***"),
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
runner = GatewayRunner(config)
|
|
|
|
def _make_adapter(platform, platform_config):
|
|
if platform == Platform.DISCORD:
|
|
return _NonRetryableFailureAdapter()
|
|
return _RetryableFailureAdapter()
|
|
|
|
monkeypatch.setattr(runner, "_create_adapter", _make_adapter)
|
|
|
|
import logging
|
|
with caplog.at_level(logging.ERROR):
|
|
ok = await runner.start()
|
|
|
|
# Gateway stays alive — no clean-exit request, no EX_CONFIG.
|
|
assert ok is True
|
|
assert runner.should_exit_cleanly is False
|
|
assert runner.exit_code is None
|
|
state = read_runtime_status()
|
|
assert state["gateway_state"] in {"degraded", "running"}
|
|
# The retryable platform is queued for reconnection…
|
|
assert Platform.TELEGRAM in runner._failed_platforms
|
|
assert state["platforms"]["telegram"]["state"] == "retrying"
|
|
# …while the misconfigured one is parked as fatal, not retried.
|
|
assert Platform.DISCORD not in runner._failed_platforms
|
|
assert state["platforms"]["discord"]["state"] == "fatal"
|
|
# The fatal side is still surfaced loudly for the operator.
|
|
assert any(
|
|
"fatally misconfigured" in record.message
|
|
for record in caplog.records
|
|
), "Expected an error log calling out the parked platform(s)"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_path):
|
|
"""A clean exit carrying GATEWAY_FATAL_CONFIG_EXIT_CODE must surface as a
|
|
process-level SystemExit(78) — NOT a truthy return — so main() exits 78
|
|
and the s6 finish script can translate it to 125 (no restart).
|
|
|
|
This guards the propagation gap: runner.start() stamps exit_code=78 and
|
|
requests a clean exit, but start_gateway()'s clean-exit branch used to
|
|
`return True` before the SystemExit(exit_code) site, so main() exited 0
|
|
and s6 crash-looped anyway (#51228)."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
class _FatalConfigRunner:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.should_exit_cleanly = True
|
|
self.exit_reason = "discord: Discord bot token already in use"
|
|
self.exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE
|
|
self.adapters = {}
|
|
|
|
async def start(self):
|
|
return True
|
|
|
|
async def stop(self):
|
|
return None
|
|
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
|
|
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
|
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
|
|
monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr("gateway.run.GatewayRunner", _FatalConfigRunner)
|
|
|
|
from gateway.run import start_gateway
|
|
|
|
with pytest.raises(SystemExit) as exc_info:
|
|
await start_gateway(config=GatewayConfig(), replace=False, verbosity=0)
|
|
|
|
assert exc_info.value.code == GATEWAY_FATAL_CONFIG_EXIT_CODE
|
|
|
|
|
|
def test_runner_warns_when_docker_gateway_lacks_explicit_output_mount(monkeypatch, tmp_path, caplog):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
monkeypatch.setenv("TERMINAL_ENV", "docker")
|
|
monkeypatch.setenv("TERMINAL_DOCKER_VOLUMES", '["/etc/localtime:/etc/localtime:ro"]')
|
|
config = GatewayConfig(
|
|
platforms={
|
|
Platform.TELEGRAM: PlatformConfig(enabled=True, token="***")
|
|
},
|
|
sessions_dir=tmp_path / "sessions",
|
|
)
|
|
|
|
with caplog.at_level("WARNING"):
|
|
GatewayRunner(config)
|
|
|
|
assert any(
|
|
"host-visible output mount" in record.message
|
|
for record in caplog.records
|
|
)
|