diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 45a88f8decf0..ad2c3cfae229 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -17,6 +17,7 @@ from __future__ import annotations import json import logging +import math import os import re import threading @@ -191,6 +192,30 @@ def _env_float(name: str, default: float) -> float: return default +def _codex_wait_notice_recovery( + *, + stale_timeout: float, + ttfb_enabled: bool, + ttfb_timeout: float, + last_event_ts: Optional[float], + call_start: float, + idle_enabled: bool, + idle_timeout: float, +) -> str: + """Describe the earliest enabled Codex watchdog on the call timeline.""" + deadlines: list[float] = [] + if math.isfinite(stale_timeout): + deadlines.append(stale_timeout) + if last_event_ts is None: + if ttfb_enabled and math.isfinite(ttfb_timeout): + deadlines.append(ttfb_timeout) + elif idle_enabled and math.isfinite(idle_timeout): + deadlines.append(max(0.0, last_event_ts - call_start) + idle_timeout) + if not deadlines: + return "" + return f"; auto-reconnect at {int(min(deadlines))}s" + + # ── Cross-turn stale-call circuit breaker (#58962) ───────────────────── # A session wedged against an unresponsive provider hits the stale detector # on every call and loops forever (observed: 494 consecutive failures over @@ -611,16 +636,19 @@ def interruptible_api_call(agent, api_kwargs: dict): # usually a slow/overloaded provider, but the UI never said so). if _poll_count % 100 == 0: # 100 × 0.3s = 30s _elapsed = time.time() - _call_start - _deadline = _stale_timeout - if ( - _ttfb_enabled - and getattr(agent, "_codex_stream_last_event_ts", None) is None - ): - _deadline = min(_deadline, _ttfb_timeout) + _recovery = _codex_wait_notice_recovery( + stale_timeout=_stale_timeout, + ttfb_enabled=_ttfb_enabled, + ttfb_timeout=_ttfb_timeout, + last_event_ts=getattr(agent, "_codex_stream_last_event_ts", None), + call_start=_call_start, + idle_enabled=_codex_idle_enabled, + idle_timeout=_codex_idle_timeout, + ) agent._emit_wait_notice( f"⏳ waiting on {api_kwargs.get('model', 'the provider')} — " f"{int(_elapsed)}s with no response yet (provider may be slow " - f"or overloaded; auto-reconnect at {int(_deadline)}s)" + f"or overloaded{_recovery})" ) _elapsed = time.time() - _call_start diff --git a/tests/agent/test_codex_ttfb_watchdog.py b/tests/agent/test_codex_ttfb_watchdog.py index 6a647164f93d..977e56335047 100644 --- a/tests/agent/test_codex_ttfb_watchdog.py +++ b/tests/agent/test_codex_ttfb_watchdog.py @@ -316,6 +316,24 @@ def test_event_idle_kills_after_first_event_then_silence(tmp_path, monkeypatch): stop["flag"] = True +def test_wait_notice_handles_infinite_local_stale_timeout(): + """After the first SSE event, a local endpoint's infinite wall-clock + timeout must not reach ``int()``; report the finite idle watchdog instead.""" + from agent import chat_completion_helpers as h + + recovery = h._codex_wait_notice_recovery( + stale_timeout=float("inf"), + ttfb_enabled=True, + ttfb_timeout=120.0, + last_event_ts=130.0, + call_start=100.0, + idle_enabled=True, + idle_timeout=60.0, + ) + + assert recovery == "; auto-reconnect at 90s" + + def test_ttfb_disabled_via_env_zero(tmp_path, monkeypatch): """Setting HERMES_CODEX_TTFB_TIMEOUT_SECONDS=0 disables the TTFB watchdog; a no-event stall then falls through to the (here, 60s) stale timeout, so a