hermes-agent/tests/hermes_cli/test_suppress_eio_on_interrupt.py
Teknium 39975613b1
test: prune wave 2 + speed fixes — 28,106 → 19,757 test functions, suite wall 315s → 294s
Second, deeper pass over tools/gateway/hermes_cli plus first pass over
the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker,
dashboard, conformance, monitoring, secret_sources, hermes_state,
providers). Same rubric as wave 1 (AGENTS.md test policy); security,
alternation/caching invariants, issue-number regressions, and E2E kept.

Real test-quality fixes found and rooted out along the way:
- tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls
  (DEFAULT_CONFIG smart-approval leaked in) — pinned approval
  mode=manual via autouse fixture: 17.4s → 0.4s.
- test_model_switch_custom_providers.py / test_user_providers_model_switch.py
  silently probed live provider catalogs (~2s/test) — stubbed
  cached_provider_model_ids/provider_model_ids/fetch_api_models.
- test_telegram_noise_filter.py: 15-platform copy-paste matrix over
  shared gateway.run logic → 3 representative platforms (55s → 3.9s).
- test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on
  MagicMock agents — interrupt.side_effect now clears _running_agents
  (22s → 1.0s).
- test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x
  (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps
  patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait
  5s → 0.5s.
- test_telegram_init_deadline.py: loop-block margin restored to 1.0s
  with rationale comment — the watchdog-dump assertion needs the loop
  blocked well past deadline+grace under parallel load (flaked once in
  the 40-worker verification run at a 0.2s margin).

Verification: full hermetic suite via scripts/run_tests.sh —
2,438 files, 21,718 tests passed, 0 failed, 293.9s wall.
Suite totals vs original baseline: 46,820 → 19,757 test functions
(−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
2026-07-29 13:39:40 -07:00

171 lines
7 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Tests for OSError EIO suppression during interrupt shutdown (#13710).
When the user interrupts a running task, prompt_toolkit tries to flush
stdout during emergency shutdown. If stdout is already in a broken state
(redirected to /dev/null, pipe closed, etc.), the flush raises
``OSError: [Errno 5] Input/output error``.
The ``_suppress_closed_loop_errors`` asyncio exception handler and the
outer ``except (KeyError, OSError)`` block must both suppress this error
to prevent a hard crash.
"""
from __future__ import annotations
import errno
from unittest.mock import MagicMock
import pytest
# ---------------------------------------------------------------------------
# _suppress_closed_loop_errors asyncio exception handler
# ---------------------------------------------------------------------------
def _make_suppress_fn():
"""Build a standalone copy of ``_suppress_closed_loop_errors``.
The real function is defined as a closure inside
``CLI._run_interactive``; we reconstruct an equivalent here so the
unit tests don't need a full CLI instance.
"""
def _suppress_closed_loop_errors(loop, context):
exc = context.get("exception")
if isinstance(exc, RuntimeError) and "Event loop is closed" in str(exc):
return
if isinstance(exc, KeyError) and "is not registered" in str(exc):
return
if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.EIO:
return
loop.default_exception_handler(context)
return _suppress_closed_loop_errors
class TestSuppressClosedLoopErrors:
"""Verify the asyncio exception handler suppresses expected errors."""
def test_suppresses_event_loop_closed(self):
handler = _make_suppress_fn()
loop = MagicMock()
handler(loop, {"exception": RuntimeError("Event loop is closed")})
loop.default_exception_handler.assert_not_called()
def test_does_not_suppress_oserror_other_errno(self):
"""OSError with a different errno must still propagate."""
handler = _make_suppress_fn()
loop = MagicMock()
exc = OSError(errno.EACCES, "Permission denied")
handler(loop, {"exception": exc})
loop.default_exception_handler.assert_called_once()
def test_no_exception_key(self):
"""Context without 'exception' must propagate to default handler."""
handler = _make_suppress_fn()
loop = MagicMock()
handler(loop, {"message": "some log"})
loop.default_exception_handler.assert_called_once()
# ---------------------------------------------------------------------------
# Outer except block EIO handling
# ---------------------------------------------------------------------------
class TestOuterExceptEIO:
"""Verify the outer ``except (KeyError, OSError)`` block logic."""
def test_eio_does_not_reraise(self):
"""OSError with errno.EIO should be silently suppressed."""
exc = OSError(errno.EIO, "Input/output error")
# Simulate the condition check from the outer except block:
assert isinstance(exc, OSError)
assert getattr(exc, "errno", None) == errno.EIO
def test_other_oserror_reraises(self):
"""Other OSError variants must not match the EIO guard."""
exc = OSError(errno.EACCES, "Permission denied")
assert not (getattr(exc, "errno", None) == errno.EIO)
assert "is not registered" not in str(exc)
assert "Bad file descriptor" not in str(exc)
# ---------------------------------------------------------------------------
# Signal handler guarded logger.debug (#13710 regression)
# ---------------------------------------------------------------------------
#
# CPython's logging module is not reentrant-safe. ``Logger.isEnabledFor``
# caches level results in ``Logger._cache``; under shutdown races the cache
# can be cleared (``Logger._clear_cache``) or mid-mutation when the signal
# fires, raising ``KeyError: <level_int>`` (e.g. ``KeyError: 10`` for DEBUG)
# from inside the handler. If that KeyError escapes, it bypasses the
# ``raise KeyboardInterrupt()`` on the next line, which in turn bypasses
# prompt_toolkit's normal interrupt unwind and surfaces as the EIO cascade
# from #13710.
#
# The fix: wrap the ``logger.debug`` call in the signal handler in a bare
# ``try/except Exception: pass`` so logging can never raise through it.
#
# These tests verify the contract: the handler must raise KeyboardInterrupt
# (and nothing else) regardless of whether logger.debug succeeds or blows up.
def _make_signal_handler(logger, agent_state):
"""Build a standalone copy of ``_signal_handler``.
The real handler is defined as a closure inside ``CLI._run_interactive``;
we reconstruct an equivalent here so the unit tests don't need a full
CLI instance. Mirrors cli.py:_signal_handler as of #13710 regression
fix — guarded logger.debug + agent interrupt + KeyboardInterrupt.
"""
def _signal_handler(signum, frame):
# Guarded: logging must never raise through a signal handler.
try:
logger.debug("Received signal %s, triggering graceful shutdown", signum)
except Exception:
pass # never let logging raise from a signal handler (#13710 regression)
try:
if agent_state.get("agent") and agent_state.get("running"):
agent_state["agent"].interrupt(f"received signal {signum}")
except Exception:
pass # never block signal handling
raise KeyboardInterrupt()
return _signal_handler
class TestSignalHandlerLoggingRace:
"""#13710 regression — logger.debug in signal handler must not escape.
If the DEBUG-level ``logging._cache`` lookup races with a concurrent
``_clear_cache`` (e.g. from another thread reconfiguring logging during
shutdown), ``logger.debug`` can raise ``KeyError: 10``. The signal
handler must swallow that and still raise KeyboardInterrupt.
"""
def test_keyboard_interrupt_raised_on_normal_path(self):
"""Sanity: handler raises KeyboardInterrupt when logging works."""
logger = MagicMock()
handler = _make_signal_handler(logger, {})
with pytest.raises(KeyboardInterrupt):
handler(15, None) # SIGTERM
logger.debug.assert_called_once()
def test_keyboard_interrupt_raised_when_logger_raises_keyerror(self):
"""logger.debug raising KeyError(10) must not escape — KeyboardInterrupt wins.
This is the exact failure signature from the #13710 regression: the
CPython 3.11 ``Logger._cache[level]`` race surfaces as KeyError on
the integer level value, and previously propagated out of the
signal handler before the ``raise KeyboardInterrupt()`` could fire.
"""
logger = MagicMock()
logger.debug.side_effect = KeyError(10) # DEBUG level int
handler = _make_signal_handler(logger, {})
# Must still raise KeyboardInterrupt, NOT KeyError.
with pytest.raises(KeyboardInterrupt):
handler(15, None)