mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-09 13:21:42 +00:00
fix(telegram): add timeout to start_polling() in network error handler
When the connection pool is in a degraded state after _drain_polling_connections(), start_polling() can hang indefinitely when both primary and fallback Telegram endpoints are unreachable. The httpx client may hold a stale socket that neither connects nor times out within PTB's internal flow, causing the reconnect ladder to stall at attempt 1/10 forever. Wrap start_polling() in asyncio.wait_for() with a 30-second timeout so a hung call raises asyncio.TimeoutError and feeds back into the existing retry ladder. This unblocks: - The 10-retry ladder advances to attempt 2, 3, ... - The heartbeat loop sees _polling_error_task.done() and can trigger recovery - The reconnect watcher gets the adapter in _failed_platforms Fixes #59614
This commit is contained in:
parent
ce038a0e05
commit
4aaaa206aa
2 changed files with 122 additions and 5 deletions
|
|
@ -416,6 +416,12 @@ def _rich_normalize_linebreaks(text: str) -> str:
|
|||
# reconnect/teardown ladder. This is an internal safety bound (not a user knob),
|
||||
# applied identically at every stop() site so no path can hang on a dead socket.
|
||||
_UPDATER_STOP_TIMEOUT = 15.0
|
||||
# start_polling() can also hang when the connection pool is in a degraded state
|
||||
# after _drain_polling_connections(), particularly when both primary and fallback
|
||||
# Telegram endpoints are unreachable. Bounding start_polling() prevents the
|
||||
# reconnect ladder from stalling indefinitely and allows the heartbeat loop to
|
||||
# trigger its own recovery path. Refs: NousResearch/hermes-agent#59614
|
||||
_UPDATER_START_TIMEOUT = 30.0
|
||||
|
||||
|
||||
class TelegramAdapter(BasePlatformAdapter):
|
||||
|
|
@ -2072,11 +2078,26 @@ class TelegramAdapter(BasePlatformAdapter):
|
|||
try:
|
||||
if not app:
|
||||
raise RuntimeError("Telegram application was torn down during reconnect")
|
||||
await app.updater.start_polling(
|
||||
allowed_updates=Update.ALL_TYPES,
|
||||
drop_pending_updates=False,
|
||||
error_callback=self._polling_error_callback_ref,
|
||||
)
|
||||
# Guard start_polling() with a timeout: when the connection pool is
|
||||
# in a degraded state (e.g., after _drain_polling_connections()), the
|
||||
# httpx client may hold a stale socket that neither connects nor times
|
||||
# out within PTB's internal flow. Bounding start_polling() prevents
|
||||
# the reconnect ladder from stalling indefinitely and allows the
|
||||
# heartbeat loop to trigger its own recovery path.
|
||||
# Refs: NousResearch/hermes-agent#59614
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
app.updater.start_polling(
|
||||
allowed_updates=Update.ALL_TYPES,
|
||||
drop_pending_updates=False,
|
||||
error_callback=self._polling_error_callback_ref,
|
||||
),
|
||||
timeout=_UPDATER_START_TIMEOUT,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise RuntimeError(
|
||||
"start_polling() timed out — connection pool may be wedged"
|
||||
)
|
||||
logger.info(
|
||||
"[%s] Telegram polling resumed after network error (attempt %d)",
|
||||
self.name, attempt,
|
||||
|
|
|
|||
96
tests/gateway/test_telegram_start_polling_timeout.py
Normal file
96
tests/gateway/test_telegram_start_polling_timeout.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
"""Test that start_polling() timeout prevents indefinite hanging.
|
||||
|
||||
This is a regression test for issue #59614 where start_polling() could hang
|
||||
indefinitely when the connection pool is in a degraded state.
|
||||
"""
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
|
||||
# Only import TelegramAdapter if it exists
|
||||
try:
|
||||
from hermes.plugins.platforms.telegram import adapter
|
||||
_UPDATER_START_TIMEOUT = adapter._UPDATER_START_TIMEOUT
|
||||
except (ImportError, AttributeError):
|
||||
pytest.skip("Telegram adapter not available", allow_module_level=True)
|
||||
|
||||
|
||||
class TestStartPollingTimeout:
|
||||
"""Test that start_polling() timeout prevents indefinite hanging."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_polling_timeout_raises_runtime_error(self):
|
||||
"""When start_polling() times out, it should raise RuntimeError."""
|
||||
from hermes.plugins.platforms.telegram.adapter import TelegramAdapter
|
||||
|
||||
# Mock the adapter's internal state
|
||||
adapter = TelegramAdapter.__new__(TelegramAdapter)
|
||||
adapter.name = "test_bot"
|
||||
adapter.has_fatal_error = False
|
||||
adapter._polling_network_error_count = 1
|
||||
adapter._polling_error_callback_ref = None
|
||||
adapter._background_tasks = set()
|
||||
adapter._send_path_degraded = True
|
||||
|
||||
# Mock the app and updater
|
||||
mock_app = MagicMock()
|
||||
mock_updater = AsyncMock()
|
||||
mock_app.updater = mock_updater
|
||||
adapter._app = mock_app
|
||||
|
||||
# Make start_polling() hang indefinitely (simulate the bug)
|
||||
async def hanging_start_polling(**kwargs):
|
||||
await asyncio.sleep(1000) # Hang for a long time
|
||||
return None
|
||||
|
||||
mock_updater.start_polling = hanging_start_polling
|
||||
mock_updater.running = True
|
||||
|
||||
# Mock _drain_polling_connections to avoid actual connection cleanup
|
||||
with patch.object(adapter, '_drain_polling_connections', new=AsyncMock()):
|
||||
# Trigger the network error handler
|
||||
task = asyncio.create_task(adapter._handle_polling_network_error(Exception("test")))
|
||||
|
||||
# Wait for the timeout to trigger (start_polling_timeout is 30s)
|
||||
try:
|
||||
await asyncio.wait_for(task, timeout=_UPDATER_START_TIMEOUT + 5)
|
||||
except asyncio.TimeoutError:
|
||||
task.cancel()
|
||||
pytest.fail("Network error handler did not complete within timeout")
|
||||
|
||||
# The task should have completed (either with success or error)
|
||||
# The important part is that it didn't hang forever
|
||||
assert task.done()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_polling_success_returns_normally(self):
|
||||
"""When start_polling() succeeds quickly, it should return normally."""
|
||||
from hermes.plugins.platforms.telegram.adapter import TelegramAdapter
|
||||
|
||||
# Mock the adapter's internal state
|
||||
adapter = TelegramAdapter.__new__(TelegramAdapter)
|
||||
adapter.name = "test_bot"
|
||||
adapter.has_fatal_error = False
|
||||
adapter._polling_network_error_count = 1
|
||||
adapter._polling_error_callback_ref = None
|
||||
adapter._background_tasks = set()
|
||||
adapter._send_path_degraded = True
|
||||
|
||||
# Mock the app and updater
|
||||
mock_app = MagicMock()
|
||||
mock_updater = AsyncMock()
|
||||
mock_app.updater = mock_updater
|
||||
adapter._app = mock_app
|
||||
|
||||
# Make start_polling() succeed immediately
|
||||
mock_updater.start_polling = AsyncMock(return_value=None)
|
||||
mock_updater.running = True
|
||||
|
||||
# Mock _drain_polling_connections and _verify_polling_after_reconnect
|
||||
with patch.object(adapter, '_drain_polling_connections', new=AsyncMock()), \
|
||||
patch.object(adapter, '_verify_polling_after_reconnect', new=AsyncMock()):
|
||||
# Trigger the network error handler
|
||||
await adapter._handle_polling_network_error(Exception("test"))
|
||||
|
||||
# Verify that start_polling was called
|
||||
mock_updater.start_polling.assert_called_once()
|
||||
Loading…
Add table
Add a link
Reference in a new issue