fix(telegram): bound updater.stop() with timeout to prevent CLOSE-WAIT reconnect hang

When the TCP connection enters CLOSE-WAIT the PTB polling task is blocked
on epoll on a dead socket and never wakes.  updater.stop() awaits that task
and therefore hangs indefinitely.

Consequence: _polling_error_task stays alive-but-blocked forever; every
subsequent heartbeat probe sees it as "in-flight" and skips triggering a
new reconnect; the gateway silently drops messages for hours until a manual
restart.  Field incident: 11-hour outage on 2026-07-04 UTC despite the
heartbeat loop firing a reconnect at 01:11 — stop() blocked the entire
ladder.

Fix: wrap the updater.stop() call inside asyncio.wait_for(timeout=15).
On TimeoutError log a warning and continue to _drain_polling_connections()
+ start_polling() — same recovery path, just unblocked.

The heartbeat loop (PR #48496) correctly detects the dead socket and fires
_handle_polling_network_error.  This commit is the missing second half:
ensuring the reconnect itself always completes.

Test: test_handle_polling_network_error_updater_stop_timeout() simulates
a hang by making stop() sleep forever and verifies that drain + start_polling
are still reached after the timeout.

Fixes #58270
This commit is contained in:
terry197913 2026-07-04 12:49:24 +00:00 committed by kshitij
parent 60906be3fc
commit 8645b34303
2 changed files with 78 additions and 1 deletions

View file

@ -1990,7 +1990,25 @@ class TelegramAdapter(BasePlatformAdapter):
try:
if app and app.updater and app.updater.running:
await app.updater.stop()
try:
# Guard stop() with a timeout: when the underlying TCP
# connection is in CLOSE-WAIT the PTB polling task is
# blocked on epoll on the dead socket and never wakes up,
# so an unguarded stop() hangs indefinitely. The result
# is that _polling_error_task stays alive-but-blocked
# forever, every subsequent heartbeat probe sees it as
# "in-flight" and skips triggering a new reconnect, and
# the gateway silently drops messages for hours.
# Bounding stop() lets the reconnect ladder always advance.
# Refs: NousResearch/hermes-agent#58270
await asyncio.wait_for(app.updater.stop(), timeout=15.0)
except asyncio.TimeoutError:
logger.warning(
"[%s] updater.stop() timed out during network-error "
"reconnect (likely CLOSE-WAIT socket); forcing drain "
"and restart without clean stop",
self.name,
)
except Exception:
pass

View file

@ -812,3 +812,62 @@ async def test_schedule_polling_recovery_tracks_background_task():
await adapter._polling_error_task
adapter._handle_polling_network_error.assert_awaited_once()
@pytest.mark.asyncio
async def test_handle_polling_network_error_updater_stop_timeout():
"""updater.stop() hanging (CLOSE-WAIT) must not block the reconnect ladder.
When the underlying TCP connection is in CLOSE-WAIT, PTB's polling task is
blocked on epoll on the dead socket. updater.stop() awaits that task and
therefore hangs indefinitely. The fix wraps stop() in asyncio.wait_for()
with a 15-second timeout so the reconnect always advances.
This test simulates the hang by making stop() sleep forever and verifies
that _drain_polling_connections() and start_polling() are still called
after the timeout fires.
Refs: NousResearch/hermes-agent#58270
"""
adapter = _make_adapter()
adapter._polling_network_error_count = 0
# Build a fake app whose updater.stop() hangs forever.
app = MagicMock()
app.updater = MagicMock()
app.updater.running = True
async def _hanging_stop():
await asyncio.sleep(9999) # simulate CLOSE-WAIT block
app.updater.stop = _hanging_stop
app.updater.start_polling = AsyncMock()
adapter._app = app
drain_called = []
async def _fake_drain():
drain_called.append(True)
adapter._drain_polling_connections = _fake_drain
start_polling_called = []
async def _fake_start_polling(**kwargs):
start_polling_called.append(True)
app.updater.start_polling = AsyncMock(side_effect=_fake_start_polling)
# Patch the timeout constant so the test completes fast.
import plugins.platforms.telegram.adapter as _mod
original_wait_for = asyncio.wait_for
async def _fast_wait_for(coro, timeout):
# Substitute a 0.05s timeout so the test doesn't take 15s.
return await original_wait_for(coro, 0.05)
with patch("asyncio.wait_for", side_effect=_fast_wait_for):
await adapter._handle_polling_network_error(OSError("CLOSE-WAIT test"))
# The reconnect ladder must have advanced past the hung stop().
assert drain_called, "_drain_polling_connections was not called after stop() timeout"
assert start_polling_called, "start_polling was not called after stop() timeout"