hermes-agent/tests/agent/test_cascading_interrupt_6600.py

"""Regression guard for the cascading-interrupt hang (PR #6600).

Original diagnosis and fix by Kristian Vastveit (@kristianvast) in PR #6600,
against the then-inline ``_interruptible_api_call`` /
``_interruptible_streaming_api_call`` methods in run_agent.py. Those methods
have since been extracted into ``agent/chat_completion_helpers.py``, so the
fix is reapplied there and these tests target the extracted functions.

The bug: when ``agent.interrupt()`` fires during an active LLM call, the main
poll loop force-closes the worker-local httpx client to stop token generation.
That raises a transport error (RemoteProtocolError) on the worker — the
EXPECTED consequence of our own close, not a network bug. The streaming retry
loop misclassified it as a transient connection error and retried, each doomed
retry stalling for the full stream-stale timeout (up to 300s). Because the
gateway caches AIAgent instances per session, the stale worker outlived the
turn and raced the next turn's request — the root of the multi-minute
cascading-interrupt hang.

The fix: a request-local ``_request_cancelled`` token set by the poll loop
right before the force-close. The worker's exception handler checks it and
exits cleanly (no retry, no fallback, no "reconnecting" status) instead of
treating the forced error as transient.
"""
import threading
import time
import types
from unittest.mock import MagicMock

import httpx
import pytest

from agent import chat_completion_helpers as cch




def _make_agent():
    """A MagicMock agent wired with just enough surface for the helpers."""
    agent = MagicMock()
    agent.api_mode = "chat_completions"
    agent._interrupt_requested = False
    agent.verbose_logging = False
    # _compute_non_stream_stale_timeout / streaming setup helpers return
    # benign values; the real call path is mocked per-test.
    agent._compute_non_stream_stale_timeout.return_value = 5.0
    return agent


def test_non_streaming_cancel_does_not_surface_network_error():
    """A force-close during a non-streaming call must raise InterruptedError,
    not the swallowed transport error."""
    agent = _make_agent()

    create_calls = {"n": 0}
    fake_client = MagicMock()

    def _create(**kwargs):
        create_calls["n"] += 1
        # Simulate the main thread firing an interrupt mid-call, then the
        # force-close raising a transport error on this worker.
        agent._interrupt_requested = True
        time.sleep(0.3)  # let the poll loop observe the interrupt + force-close
        raise httpx.RemoteProtocolError("peer closed connection")

    fake_client.chat.completions.create.side_effect = _create
    agent._create_request_openai_client.return_value = fake_client
    agent._close_request_openai_client = MagicMock()
    agent._abort_request_openai_client = MagicMock()

    t0 = time.time()
    with pytest.raises(InterruptedError):
        cch.interruptible_api_call(agent, {"model": "x", "messages": []})
    elapsed = time.time() - t0

    # The forced RemoteProtocolError must NOT surface as the raised error.
    assert create_calls["n"] == 1
    assert elapsed < 10.0, f"interrupt took {elapsed:.1f}s — should be near-instant (guarding the 30s+ hang)"






# ---------------------------------------------------------------------------
# #67142: direct-Anthropic stale/interrupt watchdog must abort the request-local
# client from the poll (stranger) thread and NEVER close/rebuild the shared
# _anthropic_client — closing it there released a live TLS FD that the kernel
# recycled into a SQLite handle, writing a TLS record over a DB header.
# ---------------------------------------------------------------------------


def _make_anthropic_agent():
    agent = _make_agent()
    agent.api_mode = "anthropic_messages"
    return agent


def _wait_for_mock_call(mock, timeout=3.0):
    deadline = time.time() + timeout
    while time.time() < deadline:
        if mock.called:
            return
        time.sleep(0.02)
    raise AssertionError(f"{mock!r} was not called within {timeout}s")


def test_anthropic_non_streaming_stale_aborts_request_client_not_shared():
    """Stale non-streaming Anthropic call: the poll thread aborts the
    request-local client's socket; the shared client is never closed/rebuilt,
    and the worker still unblocks and closes its own client (no #28161 hang)."""
    agent = _make_anthropic_agent()
    agent._compute_non_stream_stale_timeout.return_value = 0.05
    agent._codex_silent_hang_hint = MagicMock(return_value=None)

    request_client = MagicMock()
    agent._create_request_anthropic_client = MagicMock(return_value=request_client)
    agent._abort_request_anthropic_client = MagicMock()
    agent._close_request_anthropic_client = MagicMock()

    def _create(_api_kwargs, *, client):
        assert client is request_client
        # Outlive the 0.05s stale timeout AND the worker join (2.0s) so the
        # stale detector surfaces its TimeoutError.
        time.sleep(2.5)
        return object()

    agent._anthropic_messages_create = MagicMock(side_effect=_create)

    with pytest.raises(TimeoutError):
        cch.interruptible_api_call(agent, {"model": "x", "messages": []})

    # Shared client untouched from the poll thread.
    agent._anthropic_client.close.assert_not_called()
    agent._rebuild_anthropic_client.assert_not_called()
    # Poll (stranger) thread aborts the request-local client's socket only.
    agent._abort_request_anthropic_client.assert_called_once_with(
        request_client, reason="stale_call_kill"
    )
    # Worker unblocks and closes its own request client from its own thread.
    _wait_for_mock_call(agent._close_request_anthropic_client)