mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-27 17:58:07 +00:00
ece050ac30
Root cause: delegate_task children run through three nested daemon-thread layers (async-delegation executor -> per-child timeout executor -> the interrupt worker interruptible_api_call spawns). After multi-day gateway uptime the deepest layer wedges BEFORE the socket opens — the same fingerprint as the gateway-cron hang (#62151): zero stale-detector output (the worker never reaches dispatch), all providers, foreground/restart works. The cron fix (should_use_direct_api_call) explicitly excluded delegation 'for lack of evidence' — #60203 is that evidence. - should_use_direct_api_call: extend the inline gate to delegated children, detected via the delegation ContextVar set by _run_single_child (platform='subagent' stamp as fallback). Scope unchanged otherwise: chat_completions wire only; Codex/Anthropic/ Bedrock/MoA keep their established workers. Interrupts still work — the inline path registers _active_request_abort, which interrupt() invokes cross-thread (same mechanism the #72227 stall monitor uses). - _dump_subagent_timeout_diagnostic: dump ALL thread stacks (bounded, 40), not just the conversation worker — a pre-HTTP wedge is indistinguishable from a slow provider without seeing where the nested helper threads sit.
99 lines
3.8 KiB
Python
99 lines
3.8 KiB
Python
"""Regression guard for #62151 — gateway cron must not wedge on 2nd+ API call.
|
|
|
|
Gateway-fired cron jobs hung forever on the 2nd+ non-streaming API call when
|
|
``interruptible_api_call`` spawned a daemon worker inside nested cron thread
|
|
pools. The worker logged client creation but never opened a TCP connection.
|
|
The same job succeeded via ``hermes cron tick``. Cron has no interactive
|
|
interrupt surface, so the fix routes cron through a synchronous direct call on
|
|
the conversation thread instead of the interrupt worker.
|
|
"""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
from agent.chat_completion_helpers import (
|
|
direct_api_call,
|
|
should_use_direct_api_call,
|
|
)
|
|
|
|
|
|
def _make_agent(*, platform="cron"):
|
|
agent = MagicMock()
|
|
agent.platform = platform
|
|
agent.api_mode = "chat_completions"
|
|
agent.provider = "openrouter"
|
|
agent._interrupt_requested = False
|
|
agent._touch_activity = MagicMock()
|
|
agent._create_request_openai_client = MagicMock()
|
|
agent._close_request_openai_client = MagicMock()
|
|
return agent
|
|
|
|
|
|
def test_should_use_direct_api_call_only_for_cron_openai_wire():
|
|
assert should_use_direct_api_call(_make_agent(platform="cron")) is True
|
|
assert should_use_direct_api_call(_make_agent(platform="cli")) is False
|
|
assert should_use_direct_api_call(_make_agent(platform="telegram")) is False
|
|
assert should_use_direct_api_call(_make_agent(platform=None)) is False
|
|
|
|
for api_mode in ("codex_responses", "anthropic_messages", "bedrock_converse"):
|
|
agent = _make_agent(platform="cron")
|
|
agent.api_mode = api_mode
|
|
assert should_use_direct_api_call(agent) is False
|
|
|
|
moa = _make_agent(platform="cron")
|
|
moa.provider = "moa"
|
|
assert should_use_direct_api_call(moa) is False
|
|
|
|
|
|
def test_should_use_direct_api_call_for_delegated_children():
|
|
"""#60203: delegated children share the cron nested-pool wedge and must
|
|
take the inline path — via the delegation ContextVar (how the runner
|
|
executes them) or the platform='subagent' stamp as a fallback."""
|
|
from agent.delegation_context import delegated_child_context
|
|
|
|
# Platform stamp alone (child agents are built with platform="subagent").
|
|
assert should_use_direct_api_call(_make_agent(platform="subagent")) is True
|
|
|
|
# ContextVar path: any platform, running inside _run_single_child's
|
|
# delegated_child_context().
|
|
agent = _make_agent(platform="cli")
|
|
with delegated_child_context():
|
|
assert should_use_direct_api_call(agent) is True
|
|
assert should_use_direct_api_call(agent) is False # reset outside
|
|
|
|
# Non-OpenAI-wire children keep their established transports.
|
|
for api_mode in ("codex_responses", "anthropic_messages", "bedrock_converse"):
|
|
child = _make_agent(platform="subagent")
|
|
child.api_mode = api_mode
|
|
assert should_use_direct_api_call(child) is False
|
|
|
|
# MoA children keep the worker path.
|
|
moa_child = _make_agent(platform="subagent")
|
|
moa_child.provider = "moa"
|
|
assert should_use_direct_api_call(moa_child) is False
|
|
|
|
|
|
def test_direct_api_call_runs_two_sequential_requests_on_same_thread():
|
|
"""Mirror the 2nd+ call failure mode: two back-to-back completions.create."""
|
|
agent = _make_agent()
|
|
calls = {"n": 0}
|
|
fake_client = MagicMock()
|
|
|
|
def _create(**_kwargs):
|
|
calls["n"] += 1
|
|
return fake_client
|
|
|
|
fake_client.chat.completions.create.side_effect = [
|
|
SimpleNamespace(id="first"),
|
|
SimpleNamespace(id="second"),
|
|
]
|
|
agent._create_request_openai_client.side_effect = _create
|
|
|
|
first = direct_api_call(agent, {"model": "m", "messages": []})
|
|
second = direct_api_call(agent, {"model": "m", "messages": []})
|
|
|
|
assert first.id == "first"
|
|
assert second.id == "second"
|
|
assert calls["n"] == 2
|
|
assert fake_client.chat.completions.create.call_count == 2
|
|
assert agent._close_request_openai_client.call_count == 2
|