mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-09 13:21:42 +00:00
fix(telegram): bound updater.stop() with timeout to prevent CLOSE-WAIT reconnect hang
When the TCP connection enters CLOSE-WAIT the PTB polling task is blocked on epoll on a dead socket and never wakes. updater.stop() awaits that task and therefore hangs indefinitely. Consequence: _polling_error_task stays alive-but-blocked forever; every subsequent heartbeat probe sees it as "in-flight" and skips triggering a new reconnect; the gateway silently drops messages for hours until a manual restart. Field incident: 11-hour outage on 2026-07-04 UTC despite the heartbeat loop firing a reconnect at 01:11 — stop() blocked the entire ladder. Fix: wrap the updater.stop() call inside asyncio.wait_for(timeout=15). On TimeoutError log a warning and continue to _drain_polling_connections() + start_polling() — same recovery path, just unblocked. The heartbeat loop (PR #48496) correctly detects the dead socket and fires _handle_polling_network_error. This commit is the missing second half: ensuring the reconnect itself always completes. Test: test_handle_polling_network_error_updater_stop_timeout() simulates a hang by making stop() sleep forever and verifies that drain + start_polling are still reached after the timeout. Fixes #58270
This commit is contained in:
parent
60906be3fc
commit
8645b34303
2 changed files with 78 additions and 1 deletions
|
|
@ -1990,7 +1990,25 @@ class TelegramAdapter(BasePlatformAdapter):
|
|||
|
||||
try:
|
||||
if app and app.updater and app.updater.running:
|
||||
await app.updater.stop()
|
||||
try:
|
||||
# Guard stop() with a timeout: when the underlying TCP
|
||||
# connection is in CLOSE-WAIT the PTB polling task is
|
||||
# blocked on epoll on the dead socket and never wakes up,
|
||||
# so an unguarded stop() hangs indefinitely. The result
|
||||
# is that _polling_error_task stays alive-but-blocked
|
||||
# forever, every subsequent heartbeat probe sees it as
|
||||
# "in-flight" and skips triggering a new reconnect, and
|
||||
# the gateway silently drops messages for hours.
|
||||
# Bounding stop() lets the reconnect ladder always advance.
|
||||
# Refs: NousResearch/hermes-agent#58270
|
||||
await asyncio.wait_for(app.updater.stop(), timeout=15.0)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"[%s] updater.stop() timed out during network-error "
|
||||
"reconnect (likely CLOSE-WAIT socket); forcing drain "
|
||||
"and restart without clean stop",
|
||||
self.name,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -812,3 +812,62 @@ async def test_schedule_polling_recovery_tracks_background_task():
|
|||
await adapter._polling_error_task
|
||||
adapter._handle_polling_network_error.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_polling_network_error_updater_stop_timeout():
|
||||
"""updater.stop() hanging (CLOSE-WAIT) must not block the reconnect ladder.
|
||||
|
||||
When the underlying TCP connection is in CLOSE-WAIT, PTB's polling task is
|
||||
blocked on epoll on the dead socket. updater.stop() awaits that task and
|
||||
therefore hangs indefinitely. The fix wraps stop() in asyncio.wait_for()
|
||||
with a 15-second timeout so the reconnect always advances.
|
||||
|
||||
This test simulates the hang by making stop() sleep forever and verifies
|
||||
that _drain_polling_connections() and start_polling() are still called
|
||||
after the timeout fires.
|
||||
Refs: NousResearch/hermes-agent#58270
|
||||
"""
|
||||
adapter = _make_adapter()
|
||||
adapter._polling_network_error_count = 0
|
||||
|
||||
# Build a fake app whose updater.stop() hangs forever.
|
||||
app = MagicMock()
|
||||
app.updater = MagicMock()
|
||||
app.updater.running = True
|
||||
|
||||
async def _hanging_stop():
|
||||
await asyncio.sleep(9999) # simulate CLOSE-WAIT block
|
||||
|
||||
app.updater.stop = _hanging_stop
|
||||
app.updater.start_polling = AsyncMock()
|
||||
adapter._app = app
|
||||
|
||||
drain_called = []
|
||||
|
||||
async def _fake_drain():
|
||||
drain_called.append(True)
|
||||
|
||||
adapter._drain_polling_connections = _fake_drain
|
||||
|
||||
start_polling_called = []
|
||||
|
||||
async def _fake_start_polling(**kwargs):
|
||||
start_polling_called.append(True)
|
||||
|
||||
app.updater.start_polling = AsyncMock(side_effect=_fake_start_polling)
|
||||
|
||||
# Patch the timeout constant so the test completes fast.
|
||||
import plugins.platforms.telegram.adapter as _mod
|
||||
original_wait_for = asyncio.wait_for
|
||||
|
||||
async def _fast_wait_for(coro, timeout):
|
||||
# Substitute a 0.05s timeout so the test doesn't take 15s.
|
||||
return await original_wait_for(coro, 0.05)
|
||||
|
||||
with patch("asyncio.wait_for", side_effect=_fast_wait_for):
|
||||
await adapter._handle_polling_network_error(OSError("CLOSE-WAIT test"))
|
||||
|
||||
# The reconnect ladder must have advanced past the hung stop().
|
||||
assert drain_called, "_drain_polling_connections was not called after stop() timeout"
|
||||
assert start_polling_called, "start_polling was not called after stop() timeout"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue