fix: handle infinite Codex wait deadlines

This commit is contained in:
发飙的公牛 2026-07-16 19:48:05 +08:00 committed by Teknium
parent 6dcbcd0277
commit 1a5d2a12d8
2 changed files with 53 additions and 7 deletions

View file

@ -17,6 +17,7 @@ from __future__ import annotations
import json
import logging
import math
import os
import re
import threading
@ -191,6 +192,30 @@ def _env_float(name: str, default: float) -> float:
return default
def _codex_wait_notice_recovery(
*,
stale_timeout: float,
ttfb_enabled: bool,
ttfb_timeout: float,
last_event_ts: Optional[float],
call_start: float,
idle_enabled: bool,
idle_timeout: float,
) -> str:
"""Describe the earliest enabled Codex watchdog on the call timeline."""
deadlines: list[float] = []
if math.isfinite(stale_timeout):
deadlines.append(stale_timeout)
if last_event_ts is None:
if ttfb_enabled and math.isfinite(ttfb_timeout):
deadlines.append(ttfb_timeout)
elif idle_enabled and math.isfinite(idle_timeout):
deadlines.append(max(0.0, last_event_ts - call_start) + idle_timeout)
if not deadlines:
return ""
return f"; auto-reconnect at {int(min(deadlines))}s"
# ── Cross-turn stale-call circuit breaker (#58962) ─────────────────────
# A session wedged against an unresponsive provider hits the stale detector
# on every call and loops forever (observed: 494 consecutive failures over
@ -611,16 +636,19 @@ def interruptible_api_call(agent, api_kwargs: dict):
# usually a slow/overloaded provider, but the UI never said so).
if _poll_count % 100 == 0: # 100 × 0.3s = 30s
_elapsed = time.time() - _call_start
_deadline = _stale_timeout
if (
_ttfb_enabled
and getattr(agent, "_codex_stream_last_event_ts", None) is None
):
_deadline = min(_deadline, _ttfb_timeout)
_recovery = _codex_wait_notice_recovery(
stale_timeout=_stale_timeout,
ttfb_enabled=_ttfb_enabled,
ttfb_timeout=_ttfb_timeout,
last_event_ts=getattr(agent, "_codex_stream_last_event_ts", None),
call_start=_call_start,
idle_enabled=_codex_idle_enabled,
idle_timeout=_codex_idle_timeout,
)
agent._emit_wait_notice(
f"⏳ waiting on {api_kwargs.get('model', 'the provider')}"
f"{int(_elapsed)}s with no response yet (provider may be slow "
f"or overloaded; auto-reconnect at {int(_deadline)}s)"
f"or overloaded{_recovery})"
)
_elapsed = time.time() - _call_start

View file

@ -316,6 +316,24 @@ def test_event_idle_kills_after_first_event_then_silence(tmp_path, monkeypatch):
stop["flag"] = True
def test_wait_notice_handles_infinite_local_stale_timeout():
"""After the first SSE event, a local endpoint's infinite wall-clock
timeout must not reach ``int()``; report the finite idle watchdog instead."""
from agent import chat_completion_helpers as h
recovery = h._codex_wait_notice_recovery(
stale_timeout=float("inf"),
ttfb_enabled=True,
ttfb_timeout=120.0,
last_event_ts=130.0,
call_start=100.0,
idle_enabled=True,
idle_timeout=60.0,
)
assert recovery == "; auto-reconnect at 90s"
def test_ttfb_disabled_via_env_zero(tmp_path, monkeypatch):
"""Setting HERMES_CODEX_TTFB_TIMEOUT_SECONDS=0 disables the TTFB watchdog;
a no-event stall then falls through to the (here, 60s) stale timeout, so a