hermes-agent/tests/gateway/test_telegram_network_reconnect.py

"""
Tests for Telegram polling network error recovery.

Specifically tests the fix for #3173 — when start_polling() fails after a
network error, the adapter must self-reschedule the next reconnect attempt
rather than silently leaving polling dead.
"""

import ast
import asyncio
from pathlib import Path
import sys
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from gateway.config import PlatformConfig


def _ensure_telegram_mock():
    if "telegram" in sys.modules and hasattr(sys.modules["telegram"], "__file__"):
        return

    telegram_mod = MagicMock()
    telegram_mod.ext.ContextTypes.DEFAULT_TYPE = type(None)
    telegram_mod.constants.ParseMode.MARKDOWN_V2 = "MarkdownV2"
    telegram_mod.constants.ChatType.GROUP = "group"
    telegram_mod.constants.ChatType.SUPERGROUP = "supergroup"
    telegram_mod.constants.ChatType.CHANNEL = "channel"
    telegram_mod.constants.ChatType.PRIVATE = "private"

    for name in ("telegram", "telegram.ext", "telegram.constants", "telegram.request"):
        sys.modules.setdefault(name, telegram_mod)


_ensure_telegram_mock()

from plugins.platforms.telegram.adapter import TelegramAdapter  # noqa: E402


@pytest.fixture(autouse=True)
def _no_auto_discovery(monkeypatch):
    """Disable DoH auto-discovery so connect() uses the plain builder chain."""
    async def _noop():
        return []
    monkeypatch.setattr("plugins.platforms.telegram.adapter.discover_fallback_ips", _noop)


def _make_adapter() -> TelegramAdapter:
    return TelegramAdapter(PlatformConfig(enabled=True, token="test-token"))


@pytest.mark.asyncio
async def test_reconnect_self_schedules_on_start_polling_failure():
    """
    When start_polling() raises during a network error retry, the adapter must
    schedule a new _handle_polling_network_error task — otherwise polling stays
    dead with no further error callbacks to trigger recovery.

    Regression test for #3173: gateway becomes unresponsive after Telegram 502.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1

    mock_updater = MagicMock()
    mock_updater.running = True
    mock_updater.stop = AsyncMock()
    mock_updater.start_polling = AsyncMock(side_effect=Exception("Timed out"))

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    adapter._app = mock_app

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    # A retry task must have been added to _background_tasks
    pending = [t for t in adapter._background_tasks if not t.done()]
    assert len(pending) >= 1, (
        "Expected at least one self-rescheduled retry task in _background_tasks "
        f"after start_polling failure, got {len(pending)}"
    )

    # Clean up — cancel the pending retry so it doesn't run after the test
    for t in pending:
        t.cancel()
        try:
            await t
        except (asyncio.CancelledError, Exception):
            pass


@pytest.mark.asyncio
async def test_reconnect_does_not_self_schedule_when_fatal_error_set():
    """
    When a fatal error is already set, the failed reconnect should NOT create
    another retry task — the gateway is already shutting down this adapter.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1
    adapter._set_fatal_error("telegram_network_error", "already fatal", retryable=True)

    mock_updater = MagicMock()
    mock_updater.running = True
    mock_updater.stop = AsyncMock()
    mock_updater.start_polling = AsyncMock(side_effect=Exception("Timed out"))

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    adapter._app = mock_app

    initial_count = len(adapter._background_tasks)

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Timed out"))

    assert len(adapter._background_tasks) == initial_count, (
        "Should not schedule a retry when a fatal error is already set"
    )


@pytest.mark.asyncio
async def test_reconnect_chained_retry_updates_polling_error_task():
    """
    When start_polling() fails and the handler self-schedules a retry, that
    retry task must become the new `_polling_error_task` — otherwise the
    reentrancy guard used by the heartbeat loop, the pending-updates probe,
    and the PTB error callback goes stale while a recovery is still in
    flight, letting a second concurrent recovery start for the same outage.

    Regression test for the race behind the "half-destroyed adapter" bug
    (gateway reports connected but silently stops processing messages).
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1

    mock_updater = MagicMock()
    mock_updater.running = True
    mock_updater.stop = AsyncMock()
    mock_updater.start_polling = AsyncMock(side_effect=Exception("Timed out"))

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    adapter._app = mock_app

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    assert adapter._polling_error_task is not None
    assert not adapter._polling_error_task.done()

    adapter._polling_error_task.cancel()
    try:
        await adapter._polling_error_task
    except (asyncio.CancelledError, Exception):
        pass


@pytest.mark.asyncio
async def test_reconnect_success_resets_error_count():
    """
    When start_polling() succeeds, _polling_network_error_count should reset to 0.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 3

    mock_updater = MagicMock()
    mock_updater.running = True
    mock_updater.stop = AsyncMock()
    mock_updater.start_polling = AsyncMock()  # succeeds

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())  # heartbeat probe path
    adapter._app = mock_app

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    assert adapter._polling_network_error_count == 0

    # Clean up the heartbeat-probe task scheduled after a successful reconnect.
    pending = [t for t in adapter._background_tasks if not t.done()]
    for t in pending:
        t.cancel()
        try:
            await t
        except (asyncio.CancelledError, Exception):
            pass


@pytest.mark.asyncio
async def test_reconnect_triggers_fatal_after_max_retries():
    """
    After MAX_NETWORK_RETRIES attempts, the adapter should set a fatal error
    rather than retrying forever.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 10  # MAX_NETWORK_RETRIES

    fatal_handler = AsyncMock()
    adapter.set_fatal_error_handler(fatal_handler)

    mock_app = MagicMock()
    adapter._app = mock_app

    await adapter._handle_polling_network_error(Exception("still failing"))

    assert adapter.has_fatal_error
    assert adapter.fatal_error_code == "telegram_network_error"
    fatal_handler.assert_called_once()


# ---------------------------------------------------------------------------
# Connection pool drain tests (PR #16466 salvage)
# ---------------------------------------------------------------------------

def _make_mock_app():
    """Build a mock Application with an explicit polling request object."""
    mock_polling_req = AsyncMock()
    mock_polling_req.shutdown = AsyncMock()
    mock_polling_req.initialize = AsyncMock()

    mock_bot = MagicMock()
    mock_bot._request = (mock_polling_req, MagicMock())  # (getUpdates, general)

    mock_updater = MagicMock()
    mock_updater.running = True
    mock_updater.stop = AsyncMock()
    mock_updater.start_polling = AsyncMock()

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot = mock_bot
    return mock_app, mock_polling_req


@pytest.mark.asyncio
async def test_reconnect_drains_polling_request_only():
    """During reconnect, only the polling request (_request[0]) must be cycled.

    The general request (_request[1]) must NOT be touched — doing so would
    break concurrent send_message / edit_message calls.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1

    mock_app, mock_polling_req = _make_mock_app()
    adapter._app = mock_app

    general_req = mock_app.bot._request[1]

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    # Polling request must be shut down and re-initialized
    mock_polling_req.shutdown.assert_called_once()
    mock_polling_req.initialize.assert_called_once()

    # General request must NOT be touched
    general_req.shutdown.assert_not_called()
    general_req.initialize.assert_not_called()

    # Reconnect must still succeed
    mock_app.updater.start_polling.assert_called_once()
    assert adapter._polling_network_error_count == 0


@pytest.mark.asyncio
async def test_reconnect_continues_if_drain_fails():
    """If the polling request drain raises, start_polling must still proceed."""
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1

    mock_app, mock_polling_req = _make_mock_app()
    # Both shutdown and initialize fail
    mock_polling_req.shutdown = AsyncMock(side_effect=Exception("shutdown boom"))
    mock_polling_req.initialize = AsyncMock(side_effect=Exception("init boom"))
    adapter._app = mock_app

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    # start_polling must still be called despite drain failure
    mock_app.updater.start_polling.assert_called_once()
    assert adapter._polling_network_error_count == 0


@pytest.mark.asyncio
async def test_initialize_still_runs_when_shutdown_fails():
    """If shutdown() raises, initialize() must still be attempted.

    This prevents a failed shutdown from leaving the request pool in a
    permanently closed state.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1

    mock_app, mock_polling_req = _make_mock_app()
    mock_polling_req.shutdown = AsyncMock(side_effect=Exception("shutdown boom"))
    adapter._app = mock_app

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    # initialize MUST be called even though shutdown raised
    mock_polling_req.initialize.assert_called_once()
    mock_app.updater.start_polling.assert_called_once()


@pytest.mark.asyncio
async def test_conflict_retry_also_drains_polling_connections():
    """_handle_polling_conflict must also drain the polling pool on retry."""
    adapter = _make_adapter()
    adapter._polling_conflict_count = 0

    mock_app, mock_polling_req = _make_mock_app()
    adapter._app = mock_app

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_conflict(Exception("Conflict: terminated by other getUpdates"))

    # Polling request must be drained during conflict retry too
    mock_polling_req.shutdown.assert_called_once()
    mock_polling_req.initialize.assert_called_once()
    mock_app.updater.start_polling.assert_called_once()


@pytest.mark.asyncio
async def test_drain_helper_noop_without_app():
    """_drain_polling_connections must be a no-op when _app is None."""
    adapter = _make_adapter()
    adapter._app = None
    # Should not raise
    await adapter._drain_polling_connections()


# ── Heartbeat probe ──────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_heartbeat_probe_no_op_when_polling_healthy():
    """
    Probe scheduled after a successful reconnect: Updater.running=True and
    bot.get_me() returns quickly → recovery confirmed, no further action.
    """
    adapter = _make_adapter()

    mock_updater = MagicMock()
    mock_updater.running = True

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())
    adapter._app = mock_app

    adapter._handle_polling_network_error = AsyncMock()

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._verify_polling_after_reconnect()

    mock_app.bot.get_me.assert_awaited_once()
    adapter._handle_polling_network_error.assert_not_awaited()


@pytest.mark.asyncio
async def test_heartbeat_probe_reenters_ladder_when_updater_not_running():
    """
    If Updater.running has flipped to False by the heartbeat delay, treat
    as wedged: re-enter the reconnect ladder.
    """
    adapter = _make_adapter()

    mock_updater = MagicMock()
    mock_updater.running = False

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot.get_me = AsyncMock()
    adapter._app = mock_app

    adapter._handle_polling_network_error = AsyncMock()

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._verify_polling_after_reconnect()

    mock_app.bot.get_me.assert_not_called()
    adapter._handle_polling_network_error.assert_awaited_once()
    err = adapter._handle_polling_network_error.await_args.args[0]
    assert isinstance(err, RuntimeError)
    assert "not running" in str(err).lower()


@pytest.mark.asyncio
async def test_heartbeat_probe_reenters_ladder_when_get_me_times_out():
    """
    If bot.get_me() hangs longer than PROBE_TIMEOUT, treat as wedged.
    Simulates the connection-pool wedge that motivated this fix.
    """
    adapter = _make_adapter()

    mock_updater = MagicMock()
    mock_updater.running = True

    async def hang_forever(*args, **kwargs):
        await asyncio.sleep(3600)

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot.get_me = AsyncMock(side_effect=hang_forever)
    adapter._app = mock_app

    adapter._handle_polling_network_error = AsyncMock()

    async def fast_wait_for(coro, timeout):
        if asyncio.iscoroutine(coro):
            coro.close()
        raise asyncio.TimeoutError()

    with patch("asyncio.sleep", new_callable=AsyncMock):
        with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", new=fast_wait_for):
            await adapter._verify_polling_after_reconnect()

    adapter._handle_polling_network_error.assert_awaited_once()


@pytest.mark.asyncio
async def test_heartbeat_probe_reenters_ladder_on_get_me_network_error():
    """
    Any exception raised by bot.get_me() (NetworkError, ConnectionError, etc.)
    should re-enter the reconnect ladder with the original exception.
    """
    adapter = _make_adapter()

    mock_updater = MagicMock()
    mock_updater.running = True

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot.get_me = AsyncMock(side_effect=ConnectionError("pool wedged"))
    adapter._app = mock_app

    adapter._handle_polling_network_error = AsyncMock()

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._verify_polling_after_reconnect()

    adapter._handle_polling_network_error.assert_awaited_once()
    assert isinstance(
        adapter._handle_polling_network_error.await_args.args[0], ConnectionError
    )


@pytest.mark.asyncio
async def test_heartbeat_probe_skips_when_already_fatal():
    """
    If the adapter is already in fatal-error state by the time the probe
    delay elapses, the probe should bail without further action.
    """
    adapter = _make_adapter()
    adapter._set_fatal_error("telegram_polling_conflict", "already fatal", retryable=False)

    mock_app = MagicMock()
    mock_app.bot.get_me = AsyncMock()
    adapter._app = mock_app

    adapter._handle_polling_network_error = AsyncMock()

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._verify_polling_after_reconnect()

    mock_app.bot.get_me.assert_not_called()
    adapter._handle_polling_network_error.assert_not_awaited()


@pytest.mark.asyncio
async def test_reconnect_schedules_heartbeat_probe_on_success():
    """
    After a successful start_polling() in the reconnect path, a probe task
    must be added to _background_tasks. Without it, a wedged Updater would
    sit silent indefinitely with no further error_callback to advance the
    reconnect ladder.
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 1

    mock_updater = MagicMock()
    mock_updater.running = True
    mock_updater.stop = AsyncMock()
    mock_updater.start_polling = AsyncMock()  # succeeds

    mock_app = MagicMock()
    mock_app.updater = mock_updater
    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())
    adapter._app = mock_app

    initial_count = len(adapter._background_tasks)

    with patch("asyncio.sleep", new_callable=AsyncMock):
        await adapter._handle_polling_network_error(Exception("Bad Gateway"))

    assert len(adapter._background_tasks) > initial_count, (
        "Expected a heartbeat probe task to be scheduled after a successful "
        "reconnect's start_polling()"
    )

    # Clean up.
    pending = [t for t in adapter._background_tasks if not t.done()]
    for t in pending:
        t.cancel()
        try:
            await t
        except (asyncio.CancelledError, Exception):
            pass


# ── Persistent heartbeat loop (_polling_heartbeat_loop) ──────────────────────
#
# These tests cover the continuous CLOSE-WAIT detection loop that fixes the bug
# (#48495) where a dead Telegram TCP socket caused the gateway to stop receiving
# messages silently. The _verify_polling_after_reconnect tests above cover the
# one-shot post-reconnect probe; these cover the background loop that runs for
# the gateway's full lifetime in polling mode.
#
# Loop structure: while True: sleep(INTERVAL) → fatal/app checks → get_me().
# So with cancel raised on the Nth patched sleep, get_me() fires (N-1) times.


@pytest.mark.asyncio
async def test_heartbeat_loop_exits_cleanly_on_cancel():
    """The heartbeat loop must exit without raising when cancelled (normal shutdown)."""
    adapter = _make_adapter()

    mock_app = MagicMock()
    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())
    adapter._app = mock_app

    sleep_count = 0

    async def fast_sleep(seconds):
        nonlocal sleep_count
        sleep_count += 1
        # sleep #1 → get_me, sleep #2 → get_me, sleep #3 → cancel.
        if sleep_count >= 3:
            raise asyncio.CancelledError()

    with patch("asyncio.sleep", side_effect=fast_sleep):
        # Should not raise — CancelledError is swallowed internally.
        await adapter._polling_heartbeat_loop()

    assert mock_app.bot.get_me.await_count == 2


@pytest.mark.asyncio
async def test_heartbeat_loop_triggers_reconnect_on_timeout():
    """A TimeoutError from get_me() must schedule a reconnect via _handle_polling_network_error."""
    adapter = _make_adapter()
    adapter._handle_polling_network_error = AsyncMock()

    mock_app = MagicMock()
    adapter._app = mock_app

    sleep_call = 0

    async def fast_sleep(seconds):
        nonlocal sleep_call
        sleep_call += 1
        if sleep_call >= 3:
            raise asyncio.CancelledError()

    async def fast_wait_for(coro, timeout):
        if asyncio.iscoroutine(coro):
            coro.close()
        raise asyncio.TimeoutError()

    with patch("asyncio.sleep", side_effect=fast_sleep):
        with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", side_effect=fast_wait_for):
            await adapter._polling_heartbeat_loop()

    # A reconnect task must have been created.
    assert adapter._polling_error_task is not None


@pytest.mark.asyncio
async def test_heartbeat_loop_triggers_reconnect_on_os_error():
    """An OSError (e.g. connection reset) from get_me() must trigger a reconnect."""
    adapter = _make_adapter()
    adapter._handle_polling_network_error = AsyncMock()

    mock_app = MagicMock()
    adapter._app = mock_app

    sleep_call = 0

    async def fast_sleep(seconds):
        nonlocal sleep_call
        sleep_call += 1
        if sleep_call >= 3:
            raise asyncio.CancelledError()

    async def os_error_wait_for(coro, timeout):
        if asyncio.iscoroutine(coro):
            coro.close()
        raise OSError("Connection reset by peer")

    with patch("asyncio.sleep", side_effect=fast_sleep):
        with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", side_effect=os_error_wait_for):
            await adapter._polling_heartbeat_loop()

    assert adapter._polling_error_task is not None


@pytest.mark.asyncio
async def test_heartbeat_loop_skips_reconnect_if_already_in_progress():
    """If a reconnect task is already running, the heartbeat must not spawn another."""
    adapter = _make_adapter()

    # Simulate an already-running reconnect task.
    existing_task = asyncio.get_event_loop().create_task(asyncio.sleep(3600))
    adapter._polling_error_task = existing_task
    adapter._handle_polling_network_error = AsyncMock()

    mock_app = MagicMock()
    adapter._app = mock_app

    sleep_call = 0

    async def fast_sleep(seconds):
        nonlocal sleep_call
        sleep_call += 1
        if sleep_call >= 3:
            raise asyncio.CancelledError()

    async def timeout_wait_for(coro, timeout):
        if asyncio.iscoroutine(coro):
            coro.close()
        raise asyncio.TimeoutError()

    with patch("asyncio.sleep", side_effect=fast_sleep):
        with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", side_effect=timeout_wait_for):
            await adapter._polling_heartbeat_loop()

    # _handle_polling_network_error must NOT have been called — existing task still running.
    adapter._handle_polling_network_error.assert_not_awaited()

    existing_task.cancel()
    try:
        await existing_task
    except (asyncio.CancelledError, Exception):
        pass


@pytest.mark.asyncio
async def test_heartbeat_loop_ignores_non_connectivity_errors():
    """Errors that are not connectivity failures (e.g. TelegramError) must be swallowed."""
    adapter = _make_adapter()
    adapter._handle_polling_network_error = AsyncMock()

    mock_app = MagicMock()
    adapter._app = mock_app

    sleep_call = 0

    async def fast_sleep(seconds):
        nonlocal sleep_call
        sleep_call += 1
        if sleep_call >= 3:
            raise asyncio.CancelledError()

    async def telegram_error_wait_for(coro, timeout):
        if asyncio.iscoroutine(coro):
            coro.close()
        raise RuntimeError("TelegramError: Unauthorized")  # non-OSError, non-TimeoutError

    with patch("asyncio.sleep", side_effect=fast_sleep):
        with patch("plugins.platforms.telegram.adapter.asyncio.wait_for", side_effect=telegram_error_wait_for):
            await adapter._polling_heartbeat_loop()

    # No reconnect should have been triggered for a non-connectivity error.
    adapter._handle_polling_network_error.assert_not_awaited()


async def _heartbeat_exception_case(exc, *, pending_probe=False):
    adapter = _make_adapter()
    reconnect_handler = AsyncMock()
    adapter._handle_polling_network_error = reconnect_handler  # type: ignore[method-assign]
    mock_app = MagicMock()
    mock_app.updater.running = True
    if pending_probe:
        mock_app.bot.get_me = AsyncMock(return_value=MagicMock())
        mock_app.bot.get_webhook_info = AsyncMock(side_effect=exc)
    else:
        mock_app.bot.get_me = AsyncMock(side_effect=exc)
    adapter._app = mock_app

    sleep_calls = 0

    async def fast_sleep(_seconds):
        nonlocal sleep_calls
        sleep_calls += 1
        if sleep_calls >= 2:
            raise asyncio.CancelledError()

    with patch("asyncio.sleep", side_effect=fast_sleep):
        await adapter._polling_heartbeat_loop()
    await asyncio.sleep(0)
    return adapter


@pytest.mark.asyncio
@pytest.mark.parametrize("pending_probe", [False, True])
async def test_heartbeat_routes_ptb_transport_errors_to_reconnect(pending_probe):
    from telegram.error import NetworkError, TimedOut

    for exc in (NetworkError("network"), TimedOut("timeout")):
        adapter = await _heartbeat_exception_case(exc, pending_probe=pending_probe)
        reconnect_handler = adapter._handle_polling_network_error
        assert isinstance(reconnect_handler, AsyncMock)
        reconnect_handler.assert_awaited_once_with(exc)
        assert adapter._polling_error_task is not None


@pytest.mark.asyncio
@pytest.mark.parametrize("pending_probe", [False, True])
async def test_heartbeat_ignores_ptb_semantic_errors(pending_probe):
    from telegram.error import BadRequest, Forbidden, InvalidToken, RetryAfter

    for exc in (
        BadRequest("bad request"),
        Forbidden("forbidden"),
        InvalidToken("invalid token"),
        RetryAfter(1),
    ):
        adapter = await _heartbeat_exception_case(exc, pending_probe=pending_probe)
        reconnect_handler = adapter._handle_polling_network_error
        assert isinstance(reconnect_handler, AsyncMock)
        reconnect_handler.assert_not_awaited()
        assert adapter._polling_error_task is None


@pytest.mark.parametrize(
    ("error_name", "expected"),
    [
        ("NetworkError", True),
        ("TimedOut", True),
        ("BadRequest", False),
        ("Forbidden", False),
        ("InvalidToken", False),
        ("RetryAfter", False),
    ],
)
def test_network_error_classifier_matches_ptb_semantics(error_name, expected):
    import telegram.error as telegram_error

    error_type = getattr(telegram_error, error_name)
    error = error_type(1) if error_name == "RetryAfter" else error_type(error_name)
    assert TelegramAdapter._looks_like_network_error(error) is expected


def _calls_shared_network_classifier(node):
    return any(
        isinstance(child, ast.Call)
        and isinstance(child.func, ast.Attribute)
        and child.func.attr == "_looks_like_network_error"
        for child in ast.walk(node)
    )


def test_polling_error_callback_uses_shared_network_classifier():
    source = Path(TelegramAdapter.connect.__code__.co_filename).read_text(encoding="utf-8")
    tree = ast.parse(source)
    callbacks = [
        node
        for node in ast.walk(tree)
        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
        and node.name == "_polling_error_callback"
    ]
    assert len(callbacks) == 1
    assert _calls_shared_network_classifier(callbacks[0])


def test_connect_initialize_retry_uses_shared_network_classifier():
    source = Path(TelegramAdapter.connect.__code__.co_filename).read_text(encoding="utf-8")
    tree = ast.parse(source)
    connect = next(
        node
        for node in ast.walk(tree)
        if isinstance(node, ast.AsyncFunctionDef) and node.name == "connect"
    )
    exception_handlers = [
        node
        for node in ast.walk(connect)
        if isinstance(node, ast.ExceptHandler)
        and isinstance(node.type, ast.Name)
        and node.type.id == "Exception"
    ]
    assert any(_calls_shared_network_classifier(handler) for handler in exception_handlers)


@pytest.mark.asyncio
async def test_heartbeat_loop_exits_on_fatal_error():
    """A fatal error short-circuits the loop before probing get_me()."""
    adapter = _make_adapter()
    adapter._set_fatal_error("telegram_network_error", "boom", retryable=True)

    mock_app = MagicMock()
    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())
    adapter._app = mock_app

    async def fast_sleep(seconds):
        return None

    with patch("asyncio.sleep", side_effect=fast_sleep):
        await adapter._polling_heartbeat_loop()

    # Fatal error returns before the get_me() probe.
    mock_app.bot.get_me.assert_not_awaited()


@pytest.mark.asyncio
async def test_disconnect_cancels_heartbeat_task():
    """disconnect() must cancel the heartbeat task before shutting down the app."""
    adapter = _make_adapter()

    # Simulate a running heartbeat.
    heartbeat_task = asyncio.get_event_loop().create_task(asyncio.sleep(3600))
    adapter._polling_heartbeat_task = heartbeat_task

    mock_app = MagicMock()
    mock_app.updater = MagicMock()
    mock_app.updater.running = False
    mock_app.running = False
    mock_app.shutdown = AsyncMock()
    adapter._app = mock_app

    await adapter.disconnect()

    assert heartbeat_task.cancelled(), "Heartbeat task must be cancelled by disconnect()"
    assert adapter._polling_heartbeat_task is None


# ── Bootstrap degradation: keep polling alive during outages (#47508) ────


@pytest.mark.asyncio
async def test_delete_webhook_network_error_is_recoverable():
    """deleteWebhook timeouts must not fail gateway startup.

    A transient Bot API outage during bootstrap should be treated as
    recoverable and continue toward polling, so it never becomes a systemd
    service failure.
    """
    adapter = _make_adapter()
    mock_bot = MagicMock()
    mock_bot.delete_webhook = AsyncMock(side_effect=ConnectionError("api.telegram.org timeout"))
    adapter._bot = mock_bot

    result = await adapter._delete_webhook_best_effort()

    assert result is False
    assert adapter._send_path_degraded is True
    mock_bot.delete_webhook.assert_awaited_once_with(drop_pending_updates=False)
    assert not adapter.has_fatal_error


@pytest.mark.asyncio
async def test_polling_bootstrap_network_error_schedules_background_recovery():
    """Initial start_polling() network failure should degrade, not raise."""
    adapter = _make_adapter()
    mock_updater = MagicMock()
    mock_updater.start_polling = AsyncMock(side_effect=ConnectionError("bootstrap timeout"))
    mock_app = MagicMock()
    mock_app.updater = mock_updater
    adapter._app = mock_app
    adapter._schedule_polling_recovery = MagicMock()

    result = await adapter._start_polling_resilient(
        drop_pending_updates=True,
        error_callback=lambda error: None,
    )

    assert result is False
    adapter._schedule_polling_recovery.assert_called_once()
    err = adapter._schedule_polling_recovery.call_args.args[0]
    assert isinstance(err, ConnectionError)
    assert adapter._schedule_polling_recovery.call_args.kwargs["reason"] == "polling bootstrap"
    assert not adapter.has_fatal_error


@pytest.mark.asyncio
async def test_polling_bootstrap_conflict_schedules_conflict_recovery_task():
    """Initial 409 polling conflict should also be recovered in background."""
    adapter = _make_adapter()
    mock_updater = MagicMock()
    mock_updater.start_polling = AsyncMock(
        side_effect=Exception("Conflict: terminated by other getUpdates request")
    )
    mock_app = MagicMock()
    mock_app.updater = mock_updater
    adapter._app = mock_app
    adapter._handle_polling_conflict = AsyncMock()

    result = await adapter._start_polling_resilient(
        drop_pending_updates=True,
        error_callback=lambda error: None,
    )

    assert result is False
    pending = [t for t in adapter._background_tasks if not t.done()]
    assert pending, "expected background conflict recovery task"
    for task in pending:
        task.cancel()
        try:
            await task
        except (asyncio.CancelledError, Exception):
            pass
    assert not adapter.has_fatal_error


@pytest.mark.asyncio
async def test_schedule_polling_recovery_tracks_background_task():
    """Background recovery task is registered so it isn't GC'd mid-flight."""
    adapter = _make_adapter()
    adapter._handle_polling_network_error = AsyncMock()

    adapter._schedule_polling_recovery(ConnectionError("boom"), reason="unit test")

    assert adapter._send_path_degraded is True
    assert adapter._polling_error_task is not None
    assert adapter._polling_error_task in adapter._background_tasks
    await adapter._polling_error_task
    adapter._handle_polling_network_error.assert_awaited_once()


@pytest.mark.asyncio
async def test_handle_polling_network_error_updater_stop_timeout():
    """updater.stop() hanging (CLOSE-WAIT) must not block the reconnect ladder.

    When the underlying TCP connection is in CLOSE-WAIT, PTB's polling task is
    blocked on epoll on the dead socket.  updater.stop() awaits that task and
    therefore hangs indefinitely.  The fix wraps stop() in asyncio.wait_for()
    with a 15-second timeout so the reconnect always advances.

    This test simulates the hang by making stop() sleep forever and verifies
    that _drain_polling_connections() and start_polling() are still called
    after the timeout fires.
    Refs: NousResearch/hermes-agent#58270
    """
    adapter = _make_adapter()
    adapter._polling_network_error_count = 0

    # Build a fake app whose updater.stop() hangs forever.
    app = MagicMock()
    app.updater = MagicMock()
    app.updater.running = True

    async def _hanging_stop():
        await asyncio.sleep(9999)  # simulate CLOSE-WAIT block

    app.updater.stop = _hanging_stop
    app.updater.start_polling = AsyncMock()
    adapter._app = app

    drain_called = []

    async def _fake_drain():
        drain_called.append(True)

    adapter._drain_polling_connections = _fake_drain

    start_polling_called = []

    async def _fake_start_polling(**kwargs):
        start_polling_called.append(True)

    app.updater.start_polling = AsyncMock(side_effect=_fake_start_polling)

    # Shrink the stop() watchdog bound so the test completes fast instead of
    # waiting the full _UPDATER_STOP_TIMEOUT. Patching the named constant is
    # cleaner than monkeypatching asyncio.wait_for process-wide.
    import plugins.platforms.telegram.adapter as _mod

    with patch.object(_mod, "_UPDATER_STOP_TIMEOUT", 0.05):
        await adapter._handle_polling_network_error(OSError("CLOSE-WAIT test"))

    # The reconnect ladder must have advanced past the hung stop().
    assert drain_called, "_drain_polling_connections was not called after stop() timeout"
    assert start_polling_called, "start_polling was not called after stop() timeout"