mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-22 16:25:58 +00:00
fix: handle infinite Codex wait deadlines
This commit is contained in:
parent
6dcbcd0277
commit
1a5d2a12d8
2 changed files with 53 additions and 7 deletions
|
|
@ -17,6 +17,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
|
|
@ -191,6 +192,30 @@ def _env_float(name: str, default: float) -> float:
|
|||
return default
|
||||
|
||||
|
||||
def _codex_wait_notice_recovery(
|
||||
*,
|
||||
stale_timeout: float,
|
||||
ttfb_enabled: bool,
|
||||
ttfb_timeout: float,
|
||||
last_event_ts: Optional[float],
|
||||
call_start: float,
|
||||
idle_enabled: bool,
|
||||
idle_timeout: float,
|
||||
) -> str:
|
||||
"""Describe the earliest enabled Codex watchdog on the call timeline."""
|
||||
deadlines: list[float] = []
|
||||
if math.isfinite(stale_timeout):
|
||||
deadlines.append(stale_timeout)
|
||||
if last_event_ts is None:
|
||||
if ttfb_enabled and math.isfinite(ttfb_timeout):
|
||||
deadlines.append(ttfb_timeout)
|
||||
elif idle_enabled and math.isfinite(idle_timeout):
|
||||
deadlines.append(max(0.0, last_event_ts - call_start) + idle_timeout)
|
||||
if not deadlines:
|
||||
return ""
|
||||
return f"; auto-reconnect at {int(min(deadlines))}s"
|
||||
|
||||
|
||||
# ── Cross-turn stale-call circuit breaker (#58962) ─────────────────────
|
||||
# A session wedged against an unresponsive provider hits the stale detector
|
||||
# on every call and loops forever (observed: 494 consecutive failures over
|
||||
|
|
@ -611,16 +636,19 @@ def interruptible_api_call(agent, api_kwargs: dict):
|
|||
# usually a slow/overloaded provider, but the UI never said so).
|
||||
if _poll_count % 100 == 0: # 100 × 0.3s = 30s
|
||||
_elapsed = time.time() - _call_start
|
||||
_deadline = _stale_timeout
|
||||
if (
|
||||
_ttfb_enabled
|
||||
and getattr(agent, "_codex_stream_last_event_ts", None) is None
|
||||
):
|
||||
_deadline = min(_deadline, _ttfb_timeout)
|
||||
_recovery = _codex_wait_notice_recovery(
|
||||
stale_timeout=_stale_timeout,
|
||||
ttfb_enabled=_ttfb_enabled,
|
||||
ttfb_timeout=_ttfb_timeout,
|
||||
last_event_ts=getattr(agent, "_codex_stream_last_event_ts", None),
|
||||
call_start=_call_start,
|
||||
idle_enabled=_codex_idle_enabled,
|
||||
idle_timeout=_codex_idle_timeout,
|
||||
)
|
||||
agent._emit_wait_notice(
|
||||
f"⏳ waiting on {api_kwargs.get('model', 'the provider')} — "
|
||||
f"{int(_elapsed)}s with no response yet (provider may be slow "
|
||||
f"or overloaded; auto-reconnect at {int(_deadline)}s)"
|
||||
f"or overloaded{_recovery})"
|
||||
)
|
||||
|
||||
_elapsed = time.time() - _call_start
|
||||
|
|
|
|||
|
|
@ -316,6 +316,24 @@ def test_event_idle_kills_after_first_event_then_silence(tmp_path, monkeypatch):
|
|||
stop["flag"] = True
|
||||
|
||||
|
||||
def test_wait_notice_handles_infinite_local_stale_timeout():
|
||||
"""After the first SSE event, a local endpoint's infinite wall-clock
|
||||
timeout must not reach ``int()``; report the finite idle watchdog instead."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
recovery = h._codex_wait_notice_recovery(
|
||||
stale_timeout=float("inf"),
|
||||
ttfb_enabled=True,
|
||||
ttfb_timeout=120.0,
|
||||
last_event_ts=130.0,
|
||||
call_start=100.0,
|
||||
idle_enabled=True,
|
||||
idle_timeout=60.0,
|
||||
)
|
||||
|
||||
assert recovery == "; auto-reconnect at 90s"
|
||||
|
||||
|
||||
def test_ttfb_disabled_via_env_zero(tmp_path, monkeypatch):
|
||||
"""Setting HERMES_CODEX_TTFB_TIMEOUT_SECONDS=0 disables the TTFB watchdog;
|
||||
a no-event stall then falls through to the (here, 60s) stale timeout, so a
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue