From ece050ac300c1e31ca8187c550de6fc13911eb2f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:47:27 -0700 Subject: [PATCH] fix(delegation): route delegated-child API calls inline to avoid nested-pool wedge (#60203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: delegate_task children run through three nested daemon-thread layers (async-delegation executor -> per-child timeout executor -> the interrupt worker interruptible_api_call spawns). After multi-day gateway uptime the deepest layer wedges BEFORE the socket opens — the same fingerprint as the gateway-cron hang (#62151): zero stale-detector output (the worker never reaches dispatch), all providers, foreground/restart works. The cron fix (should_use_direct_api_call) explicitly excluded delegation 'for lack of evidence' — #60203 is that evidence. - should_use_direct_api_call: extend the inline gate to delegated children, detected via the delegation ContextVar set by _run_single_child (platform='subagent' stamp as fallback). Scope unchanged otherwise: chat_completions wire only; Codex/Anthropic/ Bedrock/MoA keep their established workers. Interrupts still work — the inline path registers _active_request_abort, which interrupt() invokes cross-thread (same mechanism the #72227 stall monitor uses). - _dump_subagent_timeout_diagnostic: dump ALL thread stacks (bounded, 40), not just the conversation worker — a pre-HTTP wedge is indistinguishable from a slow provider without seeing where the nested helper threads sit. --- agent/chat_completion_helpers.py | 58 ++++++++++++++----- tests/cron/test_cron_direct_api_call_62151.py | 28 +++++++++ tools/delegate_tool.py | 32 ++++++++++ 3 files changed, 104 insertions(+), 14 deletions(-) diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 3689b4d5be83..dcf88e7fd0d9 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -433,26 +433,56 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): def should_use_direct_api_call(agent) -> bool: - """Whether a cron OpenAI-wire request should skip the interrupt worker. + """Whether an OpenAI-wire request should skip the interrupt worker. - Issue #62151 is specific to OpenRouter's chat-completions path inside the - gateway cron thread stack. Keep native/Codex/Bedrock/MoA transports on their - established workers: their cancellation and client ownership differ, and - the report provides no evidence that those paths share the pre-HTTP wedge. + Two nested-pool contexts wedge before the socket opens when the request + is pushed onto yet another daemon worker thread: + + - Gateway cron turns (#62151): gateway asyncio loop → cron thread → + interrupt worker. Fixed by running inline. + - Delegated children (#60203): gateway loop → async-delegation executor + (module-lifetime daemon pool) → per-child timeout executor → interrupt + worker. Same fingerprint after multi-day gateway uptime — children hang + at their FIRST API call with zero stale-detector output (the worker + never reaches dispatch), all providers, restart cures it. The cron fix + originally excluded delegation "for lack of evidence"; #60203 is that + evidence. + + Running inline drops the deepest thread layer (whose only job is + interactive-interrupt responsiveness). Interrupts still work: the inline + path registers ``agent._active_request_abort``, which ``interrupt()`` + invokes cross-thread to shut the active sockets — the same mechanism the + async-delegation stall monitor (#72227) relies on. + + Keep native/Codex/Bedrock/MoA transports on their established workers: + their cancellation and client ownership differ. """ - return ( - getattr(agent, "platform", None) == "cron" - and getattr(agent, "api_mode", None) == "chat_completions" - and getattr(agent, "provider", None) != "moa" - ) + if getattr(agent, "api_mode", None) != "chat_completions": + return False + if getattr(agent, "provider", None) == "moa": + return False + if getattr(agent, "platform", None) == "cron": + return True + # Delegated child (delegate_task sync or background) — detected via the + # execution ContextVar set by _run_single_child, with the agent's own + # platform stamp as a fallback for callers that bypass the runner. + try: + from agent.delegation_context import is_delegated_child_context + + if is_delegated_child_context(): + return True + except Exception: + pass + return getattr(agent, "platform", None) == "subagent" def direct_api_call(agent, api_kwargs: dict): """Run a non-streaming LLM call inline on the conversation thread. - Used when ``should_use_direct_api_call`` is True. Skips the interrupt worker - (whose only job is interactive-interrupt responsiveness, which this context - does not have) so the nested-pool deadlock (#62151) cannot occur. Because the + Used when ``should_use_direct_api_call`` is True (cron turns and + delegated children). Skips the interrupt worker (whose only job is + interactive-interrupt responsiveness, which these contexts do not have) + so the nested-pool deadlock (#62151, #60203) cannot occur. Because the request runs in-flight normally, the per-request OpenAI client's own httpx timeout (provider ``request_timeout_seconds`` / ``HERMES_API_TIMEOUT``) bounds a genuinely hung provider — the same bound interactive calls already rely on. @@ -463,7 +493,7 @@ def direct_api_call(agent, api_kwargs: dict): request_client_lock = threading.Lock() def _abort_active_request(reason: str) -> None: - """Abort the inline request from cron's watchdog/interrupt thread.""" + """Abort the inline request from a watchdog/interrupt thread.""" with request_client_lock: request_client = request_client_holder["client"] if request_client is not None: diff --git a/tests/cron/test_cron_direct_api_call_62151.py b/tests/cron/test_cron_direct_api_call_62151.py index f6c4c19d5305..5e63339997b1 100644 --- a/tests/cron/test_cron_direct_api_call_62151.py +++ b/tests/cron/test_cron_direct_api_call_62151.py @@ -45,6 +45,34 @@ def test_should_use_direct_api_call_only_for_cron_openai_wire(): assert should_use_direct_api_call(moa) is False +def test_should_use_direct_api_call_for_delegated_children(): + """#60203: delegated children share the cron nested-pool wedge and must + take the inline path — via the delegation ContextVar (how the runner + executes them) or the platform='subagent' stamp as a fallback.""" + from agent.delegation_context import delegated_child_context + + # Platform stamp alone (child agents are built with platform="subagent"). + assert should_use_direct_api_call(_make_agent(platform="subagent")) is True + + # ContextVar path: any platform, running inside _run_single_child's + # delegated_child_context(). + agent = _make_agent(platform="cli") + with delegated_child_context(): + assert should_use_direct_api_call(agent) is True + assert should_use_direct_api_call(agent) is False # reset outside + + # Non-OpenAI-wire children keep their established transports. + for api_mode in ("codex_responses", "anthropic_messages", "bedrock_converse"): + child = _make_agent(platform="subagent") + child.api_mode = api_mode + assert should_use_direct_api_call(child) is False + + # MoA children keep the worker path. + moa_child = _make_agent(platform="subagent") + moa_child.provider = "moa" + assert should_use_direct_api_call(moa_child) is False + + def test_direct_api_call_runs_two_sequential_requests_on_same_thread(): """Mirror the 2nd+ call failure mode: two back-to-back completions.create.""" agent = _make_agent() diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index b1dcd80127ba..b3019af05032 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -1623,6 +1623,7 @@ def _dump_subagent_timeout_diagnostic( import datetime as _dt import sys as _sys import traceback as _traceback + import threading as _threading hermes_home = get_hermes_home() logs_dir = hermes_home / "logs" @@ -1729,6 +1730,37 @@ def _dump_subagent_timeout_diagnostic( _w(" ") _w("") + # All other live threads. The conversation worker's own stack often + # shows it parked waiting on a nested helper thread (interrupt worker, + # daemon-pool sibling) — without the full picture, a pre-HTTP wedge + # (#60203/#62151) is indistinguishable from a slow provider. Best + # effort and bounded: names + stacks for up to 40 threads. + _w("## All thread stacks at timeout") + try: + frames = _sys._current_frames() + by_ident = { + th.ident: th for th in _threading.enumerate() if th.ident + } + worker_ident = worker_thread.ident if worker_thread else None + dumped = 0 + for ident, frame in frames.items(): + if ident == worker_ident: + continue # already dumped above + if dumped >= 40: + _w(f" <{len(frames) - dumped - 1} more threads omitted>") + break + th = by_ident.get(ident) + name = th.name if th else f"ident={ident}" + daemon = " daemon" if (th and th.daemon) else "" + _w(f" --- {name}{daemon} ---") + for frame_line in _traceback.format_stack(frame): + for sub in frame_line.rstrip().split("\n"): + _w(f" {sub}") + dumped += 1 + except Exception as exc: + _w(f" ") + _w("") + _w("## Notes") _w(" This file is written ONLY when a subagent times out with 0 API calls.") _w(" 0-API-call timeouts mean the child never reached its first LLM request.")