mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
171 lines
7 KiB
Python
171 lines
7 KiB
Python
"""Tests for OSError EIO suppression during interrupt shutdown (#13710).
|
||
|
||
When the user interrupts a running task, prompt_toolkit tries to flush
|
||
stdout during emergency shutdown. If stdout is already in a broken state
|
||
(redirected to /dev/null, pipe closed, etc.), the flush raises
|
||
``OSError: [Errno 5] Input/output error``.
|
||
|
||
The ``_suppress_closed_loop_errors`` asyncio exception handler and the
|
||
outer ``except (KeyError, OSError)`` block must both suppress this error
|
||
to prevent a hard crash.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import errno
|
||
from unittest.mock import MagicMock
|
||
|
||
import pytest
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _suppress_closed_loop_errors – asyncio exception handler
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _make_suppress_fn():
|
||
"""Build a standalone copy of ``_suppress_closed_loop_errors``.
|
||
|
||
The real function is defined as a closure inside
|
||
``CLI._run_interactive``; we reconstruct an equivalent here so the
|
||
unit tests don't need a full CLI instance.
|
||
"""
|
||
def _suppress_closed_loop_errors(loop, context):
|
||
exc = context.get("exception")
|
||
if isinstance(exc, RuntimeError) and "Event loop is closed" in str(exc):
|
||
return
|
||
if isinstance(exc, KeyError) and "is not registered" in str(exc):
|
||
return
|
||
if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.EIO:
|
||
return
|
||
loop.default_exception_handler(context)
|
||
return _suppress_closed_loop_errors
|
||
|
||
|
||
class TestSuppressClosedLoopErrors:
|
||
"""Verify the asyncio exception handler suppresses expected errors."""
|
||
|
||
def test_suppresses_event_loop_closed(self):
|
||
handler = _make_suppress_fn()
|
||
loop = MagicMock()
|
||
handler(loop, {"exception": RuntimeError("Event loop is closed")})
|
||
loop.default_exception_handler.assert_not_called()
|
||
|
||
|
||
def test_does_not_suppress_oserror_other_errno(self):
|
||
"""OSError with a different errno must still propagate."""
|
||
handler = _make_suppress_fn()
|
||
loop = MagicMock()
|
||
exc = OSError(errno.EACCES, "Permission denied")
|
||
handler(loop, {"exception": exc})
|
||
loop.default_exception_handler.assert_called_once()
|
||
|
||
|
||
def test_no_exception_key(self):
|
||
"""Context without 'exception' must propagate to default handler."""
|
||
handler = _make_suppress_fn()
|
||
loop = MagicMock()
|
||
handler(loop, {"message": "some log"})
|
||
loop.default_exception_handler.assert_called_once()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Outer except block – EIO handling
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestOuterExceptEIO:
|
||
"""Verify the outer ``except (KeyError, OSError)`` block logic."""
|
||
|
||
def test_eio_does_not_reraise(self):
|
||
"""OSError with errno.EIO should be silently suppressed."""
|
||
exc = OSError(errno.EIO, "Input/output error")
|
||
# Simulate the condition check from the outer except block:
|
||
assert isinstance(exc, OSError)
|
||
assert getattr(exc, "errno", None) == errno.EIO
|
||
|
||
|
||
def test_other_oserror_reraises(self):
|
||
"""Other OSError variants must not match the EIO guard."""
|
||
exc = OSError(errno.EACCES, "Permission denied")
|
||
assert not (getattr(exc, "errno", None) == errno.EIO)
|
||
assert "is not registered" not in str(exc)
|
||
assert "Bad file descriptor" not in str(exc)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Signal handler – guarded logger.debug (#13710 regression)
|
||
# ---------------------------------------------------------------------------
|
||
#
|
||
# CPython's logging module is not reentrant-safe. ``Logger.isEnabledFor``
|
||
# caches level results in ``Logger._cache``; under shutdown races the cache
|
||
# can be cleared (``Logger._clear_cache``) or mid-mutation when the signal
|
||
# fires, raising ``KeyError: <level_int>`` (e.g. ``KeyError: 10`` for DEBUG)
|
||
# from inside the handler. If that KeyError escapes, it bypasses the
|
||
# ``raise KeyboardInterrupt()`` on the next line, which in turn bypasses
|
||
# prompt_toolkit's normal interrupt unwind and surfaces as the EIO cascade
|
||
# from #13710.
|
||
#
|
||
# The fix: wrap the ``logger.debug`` call in the signal handler in a bare
|
||
# ``try/except Exception: pass`` so logging can never raise through it.
|
||
#
|
||
# These tests verify the contract: the handler must raise KeyboardInterrupt
|
||
# (and nothing else) regardless of whether logger.debug succeeds or blows up.
|
||
|
||
|
||
def _make_signal_handler(logger, agent_state):
|
||
"""Build a standalone copy of ``_signal_handler``.
|
||
|
||
The real handler is defined as a closure inside ``CLI._run_interactive``;
|
||
we reconstruct an equivalent here so the unit tests don't need a full
|
||
CLI instance. Mirrors cli.py:_signal_handler as of #13710 regression
|
||
fix — guarded logger.debug + agent interrupt + KeyboardInterrupt.
|
||
"""
|
||
def _signal_handler(signum, frame):
|
||
# Guarded: logging must never raise through a signal handler.
|
||
try:
|
||
logger.debug("Received signal %s, triggering graceful shutdown", signum)
|
||
except Exception:
|
||
pass # never let logging raise from a signal handler (#13710 regression)
|
||
try:
|
||
if agent_state.get("agent") and agent_state.get("running"):
|
||
agent_state["agent"].interrupt(f"received signal {signum}")
|
||
except Exception:
|
||
pass # never block signal handling
|
||
raise KeyboardInterrupt()
|
||
return _signal_handler
|
||
|
||
|
||
class TestSignalHandlerLoggingRace:
|
||
"""#13710 regression — logger.debug in signal handler must not escape.
|
||
|
||
If the DEBUG-level ``logging._cache`` lookup races with a concurrent
|
||
``_clear_cache`` (e.g. from another thread reconfiguring logging during
|
||
shutdown), ``logger.debug`` can raise ``KeyError: 10``. The signal
|
||
handler must swallow that and still raise KeyboardInterrupt.
|
||
"""
|
||
|
||
def test_keyboard_interrupt_raised_on_normal_path(self):
|
||
"""Sanity: handler raises KeyboardInterrupt when logging works."""
|
||
logger = MagicMock()
|
||
handler = _make_signal_handler(logger, {})
|
||
with pytest.raises(KeyboardInterrupt):
|
||
handler(15, None) # SIGTERM
|
||
logger.debug.assert_called_once()
|
||
|
||
def test_keyboard_interrupt_raised_when_logger_raises_keyerror(self):
|
||
"""logger.debug raising KeyError(10) must not escape — KeyboardInterrupt wins.
|
||
|
||
This is the exact failure signature from the #13710 regression: the
|
||
CPython 3.11 ``Logger._cache[level]`` race surfaces as KeyError on
|
||
the integer level value, and previously propagated out of the
|
||
signal handler before the ``raise KeyboardInterrupt()`` could fire.
|
||
"""
|
||
logger = MagicMock()
|
||
logger.debug.side_effect = KeyError(10) # DEBUG level int
|
||
handler = _make_signal_handler(logger, {})
|
||
# Must still raise KeyboardInterrupt, NOT KeyError.
|
||
with pytest.raises(KeyboardInterrupt):
|
||
handler(15, None)
|
||
|
||
|
||
|
||
|