hermes-agent/tests/tools/test_mcp_reconnect_retry_reset.py

"""Tests for MCP reconnect retry counter reset (#57604).

Verifies that the reconnect retry counter resets after each successful
reconnection, so transient blips do not accumulate toward permanent parking.
"""

import asyncio

import pytest


@pytest.mark.no_isolate
def test_reconnect_counter_resets_after_successful_session(monkeypatch, tmp_path):
    """Transient disconnections must not accumulate toward permanent parking.

    Before the fix, ``retries`` was a local variable in ``run()`` that only
    reset on clean transport return (line 2367) or park-wake (line 2468).
    Each exception from ``_run_stdio`` incremented it without reset, so 5
    transient blips over a long-uptime gateway would permanently park the
    server.

    After the fix, ``_reconnect_retries`` is an instance variable that resets
    to 0 whenever a session is successfully established (``_reset_server_error``
    call sites in ``_run_stdio`` / ``_run_http``).
    """
    monkeypatch.setenv("HERMES_HOME", str(tmp_path))

    from tools import mcp_tool
    from tools.mcp_tool import MCPServerTask

    # Shrink budget so the test can exhaust it quickly if the counter
    # does NOT reset (the bug scenario).
    monkeypatch.setattr(mcp_tool, "_MAX_RECONNECT_RETRIES", 3)

    _real_sleep = asyncio.sleep

    async def _fast_sleep(_delay, *a, **kw):
        await _real_sleep(0)

    monkeypatch.setattr(mcp_tool.asyncio, "sleep", _fast_sleep)

    state = {
        "transport_calls": 0,
        "parked": False,
        "max_retries_seen": 0,
    }

    async def _scenario():
        class _Task(MCPServerTask):
            def _is_http(self):
                return False

            def _deregister_tools(self):
                state["parked"] = True
                self._registered_tool_names = []

            async def _run_stdio(self, config):
                state["transport_calls"] += 1
                call = state["transport_calls"]

                if call == 1:
                    # First connect: succeed (sets _ready), then fail.
                    self.session = object()
                    self._ready.set()
                    self.session = None
                    raise RuntimeError("blip 1")

                # Subsequent calls: succeed (session established, which
                # triggers _reset_server_error and should reset retries),
                # then immediately fail again — simulating a new transient
                # blip.  If retries accumulate, call 4 would exceed the
                # budget of 3 and park.  If retries reset correctly,
                # this loop can continue indefinitely.
                if call <= 8:
                    self.session = object()
                    # _run_stdio calls _reset_server_error and sets
                    # _reconnect_retries = 0 after session establishment.
                    # We simulate that by calling the real method.
                    mcp_tool._reset_server_error(self.name)
                    self._reconnect_retries = 0
                    self.session = None
                    raise RuntimeError(f"blip {call}")

                # If we reach here without parking, the fix works.
                self.session = object()
                mcp_tool._reset_server_error(self.name)
                self._reconnect_retries = 0
                await self._wait_for_lifecycle_event()
                return

        task = _Task("srv")
        task._registered_tool_names = ["srv__tool"]

        run_task = asyncio.ensure_future(task.run({"command": "x"}))

        # Let the scenario run.
        for _ in range(2000):
            await _real_sleep(0)
            if state["transport_calls"] >= 8 or state["parked"]:
                break

        # The fix: the server should NOT have parked despite 8 transient
        # disconnections, because each successful reconnection reset the
        # retry counter.
        assert not state["parked"], (
            f"server parked after {state['transport_calls']} transport calls "
            f"— retry counter accumulated instead of resetting"
        )
        assert state["transport_calls"] >= 8, (
            f"only {state['transport_calls']} transport calls reached "
            f"(expected >= 8)"
        )

        # Verify the counter is an instance variable, not a local.
        assert hasattr(task, "_reconnect_retries"), (
            "_reconnect_retries should be an instance variable"
        )

        # Clean shutdown.
        task._shutdown_event.set()
        task._reconnect_event.set()
        try:
            await asyncio.wait_for(run_task, timeout=2)
        except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
            run_task.cancel()

    asyncio.run(_scenario())


@pytest.mark.no_isolate
def test_reconnect_counter_still_parks_on_consecutive_failures(monkeypatch, tmp_path):
    """The server must still park when failures are genuinely consecutive
    (no successful reconnection in between).
    """
    monkeypatch.setenv("HERMES_HOME", str(tmp_path))

    from tools import mcp_tool
    from tools.mcp_tool import MCPServerTask

    monkeypatch.setattr(mcp_tool, "_MAX_RECONNECT_RETRIES", 2)

    _real_sleep = asyncio.sleep

    async def _fast_sleep(_delay, *a, **kw):
        await _real_sleep(0)

    monkeypatch.setattr(mcp_tool.asyncio, "sleep", _fast_sleep)

    state = {"transport_calls": 0, "parked": False}

    async def _scenario():
        class _Task(MCPServerTask):
            def _is_http(self):
                return False

            def _deregister_tools(self):
                state["parked"] = True
                self._registered_tool_names = []

            async def _run_stdio(self, config):
                state["transport_calls"] += 1
                call = state["transport_calls"]

                if call == 1:
                    self.session = object()
                    self._ready.set()
                    self.session = None
                    raise RuntimeError("first failure")

                # All subsequent calls fail WITHOUT establishing a session
                # (no _reset_server_error, no retry reset). This simulates
                # genuinely consecutive failures.
                raise RuntimeError(f"failure {call}")

        task = _Task("srv")
        task._registered_tool_names = ["srv__tool"]

        run_task = asyncio.ensure_future(task.run({"command": "x"}))

        for _ in range(500):
            await _real_sleep(0)
            if state["parked"] or run_task.done():
                break

        assert state["parked"], (
            "server should park on consecutive failures without successful reconnect"
        )

        task._shutdown_event.set()
        task._reconnect_event.set()
        try:
            await asyncio.wait_for(run_task, timeout=2)
        except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
            run_task.cancel()

    asyncio.run(_scenario())