mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-09 13:21:42 +00:00
fix(gateway): drain in-flight cron delivery on restart instead of dropping it
A cron delivery uses the live adapter by scheduling the send coroutine onto the gateway event loop (safe_schedule_threadsafe) and blocking the ticker thread on future.result(). On shutdown/restart the cleanup ran a synchronous cron_thread.join(timeout=5), which blocks the event loop — so the pending delivery coroutine could never execute, the join always timed out, and the message was silently dropped (#58818). The default agent.restart_drain_timeout is 0, so this fired on every restart with an in-flight delivery. Replace the blocking joins with _await_thread_exit(), which polls is_alive() via await asyncio.sleep so the loop keeps running and finishes the queued delivery before teardown. The cron wait is bounded by the delivery future's own 60s ceiling (plus margin); housekeeping keeps a short bound. When no delivery is in flight the ticker exits on stop_event immediately, so shutdown stays snappy.
This commit is contained in:
parent
beaa1a08e6
commit
dcd70c5823
1 changed files with 46 additions and 3 deletions
|
|
@ -19737,6 +19737,37 @@ def _start_cron_ticker(stop_event: threading.Event, adapters=None, loop=None, in
|
|||
InProcessCronScheduler().start(stop_event, adapters=adapters, loop=loop, interval=interval)
|
||||
|
||||
|
||||
# Upper bound for cooperatively draining the cron ticker on shutdown. The cron
|
||||
# thread delivers via ``safe_schedule_threadsafe`` and blocks on
|
||||
# ``future.result(timeout=60)`` (see cron/scheduler.py::_deliver_result), so a
|
||||
# single in-flight delivery unblocks within ~60s. The extra margin covers the
|
||||
# hop back through run_one_job's bookkeeping.
|
||||
_CRON_SHUTDOWN_DRAIN_TIMEOUT = 65.0
|
||||
|
||||
|
||||
async def _await_thread_exit(
|
||||
thread: Optional[threading.Thread], timeout: float, poll: float = 0.1
|
||||
) -> bool:
|
||||
"""Wait for a daemon thread to exit WITHOUT blocking the event loop.
|
||||
|
||||
A synchronous ``thread.join()`` here would freeze the event loop — fatal
|
||||
for the cron ticker, whose in-flight delivery is a coroutine scheduled onto
|
||||
*this* loop via ``safe_schedule_threadsafe``. Blocking the loop deadlocks
|
||||
that delivery (the loop can never run it), so ``join(timeout=5)`` always
|
||||
times out and the message is silently dropped on restart (#58818).
|
||||
|
||||
Polling ``is_alive()`` with ``await asyncio.sleep`` keeps the loop running
|
||||
so the pending delivery completes, then the ticker sees ``stop_event`` and
|
||||
exits. Returns True if the thread exited within ``timeout``.
|
||||
"""
|
||||
if thread is None:
|
||||
return True
|
||||
deadline = asyncio.get_running_loop().time() + max(0.0, timeout)
|
||||
while thread.is_alive() and asyncio.get_running_loop().time() < deadline:
|
||||
await asyncio.sleep(poll)
|
||||
return not thread.is_alive()
|
||||
|
||||
|
||||
async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool:
|
||||
"""
|
||||
Start the gateway and run until interrupted.
|
||||
|
|
@ -20231,14 +20262,26 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
|
|||
logger.error("Gateway exiting with failure: %s", runner.exit_reason)
|
||||
return False
|
||||
|
||||
# Stop cron scheduler + housekeeping cleanly
|
||||
# Stop cron scheduler + housekeeping cleanly.
|
||||
#
|
||||
# These MUST be awaited cooperatively, not join()ed. A cron delivery in
|
||||
# flight when the gateway restarts is a coroutine scheduled onto THIS event
|
||||
# loop (safe_schedule_threadsafe); the ticker thread is blocked on its
|
||||
# future.result(). A synchronous cron_thread.join() would block the loop,
|
||||
# so that delivery could never run — it timed out and the message was
|
||||
# silently dropped (#58818). Awaiting keeps the loop alive so the in-flight
|
||||
# delivery finishes before we tear down.
|
||||
cron_stop.set()
|
||||
try:
|
||||
cron_provider.stop()
|
||||
except Exception as e:
|
||||
logger.debug("Cron provider stop() error: %s", e)
|
||||
cron_thread.join(timeout=5)
|
||||
housekeeping_thread.join(timeout=5)
|
||||
if not await _await_thread_exit(cron_thread, timeout=_CRON_SHUTDOWN_DRAIN_TIMEOUT):
|
||||
logger.warning(
|
||||
"Cron ticker did not exit within %.0fs of shutdown — an in-flight "
|
||||
"delivery may have been dropped.", _CRON_SHUTDOWN_DRAIN_TIMEOUT,
|
||||
)
|
||||
await _await_thread_exit(housekeeping_thread, timeout=5)
|
||||
|
||||
# Stop the planned-stop watcher (daemon=True so this is belt-and-suspenders).
|
||||
_planned_stop_watcher_stop.set()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue