From fd96e138b64aa9f2266e971700df7bfd63cc41d5 Mon Sep 17 00:00:00 2001 From: web3blind <264741654+web3blind@users.noreply.github.com> Date: Tue, 21 Jul 2026 07:29:52 +0000 Subject: [PATCH] fix(gateway): hard-exit CLI runner after graceful teardown --- hermes_cli/gateway.py | 22 ++++- .../hermes_cli/test_gateway_run_hard_exit.py | 87 +++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tests/hermes_cli/test_gateway_run_hard_exit.py diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 69f6464e806b..7f81072515f7 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -4975,6 +4975,17 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo except Exception as _be: logger.debug("respawn-storm breaker check failed (non-fatal): %s", _be) + def _hard_exit_after_gateway_teardown(code: int) -> None: + # ``hermes gateway run`` enters through this CLI wrapper, not through + # ``gateway.run.main()``. Mirror that module's wedge-proof exit path: + # once start_gateway() has completed graceful teardown, bypass Python + # finalization so non-daemon worker threads (notably in-flight cron + # ThreadPoolExecutor jobs) cannot keep the old gateway alive and delay a + # service-managed /restart by minutes. + from gateway.run import _exit_after_graceful_shutdown + + _exit_after_graceful_shutdown(code) + success = False try: success = asyncio.run(start_gateway(replace=replace, verbosity=verbosity)) @@ -4994,7 +5005,13 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo code=getattr(e, "code", None), traceback=_traceback.format_exc(), ) - raise + if e.code is None: + _code = 0 + elif isinstance(e.code, int): + _code = e.code + else: + _code = 1 + _hard_exit_after_gateway_teardown(_code) except BaseException as e: # Absolutely everything else: Exception, asyncio.CancelledError, # even exotic BaseException subclasses. We want the cause logged. @@ -5007,8 +5024,9 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo raise if not success: _exit_diag("gateway.exit_nonzero") - sys.exit(1) + _hard_exit_after_gateway_teardown(1) _exit_diag("gateway.exit_clean") + _hard_exit_after_gateway_teardown(0) # ============================================================================= diff --git a/tests/hermes_cli/test_gateway_run_hard_exit.py b/tests/hermes_cli/test_gateway_run_hard_exit.py new file mode 100644 index 000000000000..be69b0915e77 --- /dev/null +++ b/tests/hermes_cli/test_gateway_run_hard_exit.py @@ -0,0 +1,87 @@ +"""Regression tests for CLI gateway run exit behavior. + +``hermes gateway run`` enters through hermes_cli.gateway, not gateway.run.main(). +After graceful teardown it must use the same hard-exit backstop as gateway.run.main() +so Python finalization does not wait on non-daemon worker threads (for example +in-flight cron ThreadPoolExecutor jobs) and delay service-managed restarts. +""" + +from __future__ import annotations + +import types + +import pytest + + +class _HardExitObserved(BaseException): + def __init__(self, code: int): + super().__init__(code) + self.code = code + + +def _prepare(monkeypatch): + import hermes_cli.gateway as gateway_cli + import gateway.run as gateway_run + + monkeypatch.setattr(gateway_cli, "_guard_official_docker_root_gateway", lambda: None) + monkeypatch.setattr(gateway_cli, "_guard_named_profile_under_multiplexer", lambda force=False: None) + monkeypatch.setattr(gateway_cli, "_guard_supervised_gateway_conflict", lambda force=False: None) + monkeypatch.setattr(gateway_cli, "_guard_existing_gateway_process_conflict", lambda replace=False: None) + monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: False) + monkeypatch.setattr(gateway_cli.sys, "stdin", types.SimpleNamespace(isatty=lambda: False)) + monkeypatch.setenv("HERMES_GATEWAY_EXIT_DIAG", "0") + + async def _start_gateway(*args, **kwargs): # pragma: no cover - never awaited by fake run + return True + + def _hard_exit(code: int) -> None: + raise _HardExitObserved(code) + + monkeypatch.setattr(gateway_run, "start_gateway", _start_gateway) + monkeypatch.setattr(gateway_run, "_exit_after_graceful_shutdown", _hard_exit) + return gateway_cli + + +def test_run_gateway_hard_exits_after_clean_return(monkeypatch): + gateway_cli = _prepare(monkeypatch) + + def _fake_run(coro): + coro.close() + return True + + monkeypatch.setattr(gateway_cli.asyncio, "run", _fake_run) + + with pytest.raises(_HardExitObserved) as excinfo: + gateway_cli.run_gateway() + + assert excinfo.value.code == 0 + + +def test_run_gateway_hard_exits_after_service_restart_systemexit(monkeypatch): + gateway_cli = _prepare(monkeypatch) + + def _fake_run(coro): + coro.close() + raise SystemExit(75) + + monkeypatch.setattr(gateway_cli.asyncio, "run", _fake_run) + + with pytest.raises(_HardExitObserved) as excinfo: + gateway_cli.run_gateway() + + assert excinfo.value.code == 75 + + +def test_run_gateway_hard_exits_after_failed_return(monkeypatch): + gateway_cli = _prepare(monkeypatch) + + def _fake_run(coro): + coro.close() + return False + + monkeypatch.setattr(gateway_cli.asyncio, "run", _fake_run) + + with pytest.raises(_HardExitObserved) as excinfo: + gateway_cli.run_gateway() + + assert excinfo.value.code == 1