mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-29 06:31:32 +00:00
b1adb95038
The ChatGPT Codex backend (chatgpt.com/backend-api/codex) has historically silently dropped certain model requests: the connection is accepted but no stream events are emitted and no error is raised. PR #31967 lowered the implicit stale-call default from 300s to 90s so fallbacks kick in faster, but users still see an opaque "No response from provider for 90s (non-streaming, ...)" message that gives no path forward. This patch adds a narrow heuristic — gpt-5.5 family on the Codex backend via codex_responses api_mode — that substitutes the generic timeout message with actionable text naming the gpt-5.4-codex workaround and pointing at #21444 for symptom history. Changes: - run_agent.py — new ``AIAgent._codex_silent_hang_hint(model=...)`` method. Returns ``None`` for any request that does not match all three guards (codex_responses api_mode, openai-codex provider or chatgpt.com Codex base URL, gpt-5.5-family model name with word-boundary regex anchoring to avoid false-positives on e.g. ``gpt-5.50``). - agent/chat_completion_helpers.py — the non-stream stale-call site consults the hint via ``getattr(...)`` so the call site stays robust if the helper is ever removed or stubbed in tests. Hint is appended to both the ``_emit_status`` warning and the ``TimeoutError`` message so the user sees it in their terminal AND it lands in any retry-loop diagnostics. - tests/run_agent/test_codex_silent_hang_hint.py — 10 regression tests covering positive cases (bare gpt-5.5, vendor-prefixed openai/gpt-5.5, gpt-5.5-codex SKU, model=None fallback to self.model) and negative cases (gpt-5.4-codex workaround, gpt-5.50 false-positive guard, non-codex api_mode, non-codex provider, empty/None model, unrelated models on Codex). Does NOT fix the backend-side issue (that's an upstream OpenAI/ChatGPT problem we cannot patch from here). Only converts an opaque timeout into text that names the workaround so users do not have to dig through logs or wait for a forum post to learn what to do. Closes #22046
121 lines
4.2 KiB
Python
121 lines
4.2 KiB
Python
"""Tests for the ``_codex_silent_hang_hint`` heuristic.
|
|
|
|
The helper substitutes an actionable hint into the stale-call timeout
|
|
warning when the request matches a known Codex silent-reject pattern
|
|
(gpt-5.5 family on the ChatGPT Codex backend). See issue #21444 for
|
|
symptom history.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
def _make_agent(tmp_path: Path, **overrides):
|
|
from run_agent import AIAgent
|
|
kwargs = dict(
|
|
model="gpt-5.5",
|
|
provider="openai-codex",
|
|
api_key="sk-dummy",
|
|
base_url="https://chatgpt.com/backend-api/codex",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
platform="cli",
|
|
)
|
|
kwargs.update(overrides)
|
|
return AIAgent(**kwargs)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_hermes_home(monkeypatch, tmp_path):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
(tmp_path / ".env").write_text("", encoding="utf-8")
|
|
|
|
|
|
# ── positive cases: hint fires ─────────────────────────────────────────────
|
|
|
|
|
|
def test_hint_fires_for_bare_gpt_5_5_on_codex(tmp_path):
|
|
agent = _make_agent(tmp_path)
|
|
agent.api_mode = "codex_responses"
|
|
hint = agent._codex_silent_hang_hint(model="gpt-5.5")
|
|
assert hint is not None
|
|
assert "gpt-5.4-codex" in hint
|
|
assert "fallback chain" in hint
|
|
|
|
|
|
def test_hint_fires_for_vendor_prefixed_gpt_5_5(tmp_path):
|
|
agent = _make_agent(tmp_path, model="openai/gpt-5.5")
|
|
agent.api_mode = "codex_responses"
|
|
hint = agent._codex_silent_hang_hint(model="openai/gpt-5.5")
|
|
assert hint is not None
|
|
|
|
|
|
def test_hint_fires_for_gpt_5_5_codex_suffix(tmp_path):
|
|
agent = _make_agent(tmp_path, model="gpt-5.5-codex")
|
|
agent.api_mode = "codex_responses"
|
|
hint = agent._codex_silent_hang_hint(model="gpt-5.5-codex")
|
|
assert hint is not None
|
|
|
|
|
|
def test_hint_fires_when_model_arg_omitted(tmp_path):
|
|
"""The helper falls back to ``self.model`` when ``model=`` not passed."""
|
|
agent = _make_agent(tmp_path)
|
|
agent.api_mode = "codex_responses"
|
|
hint = agent._codex_silent_hang_hint()
|
|
assert hint is not None
|
|
|
|
|
|
# ── negative cases: hint stays None ────────────────────────────────────────
|
|
|
|
|
|
def test_hint_skipped_for_gpt_5_4_codex(tmp_path):
|
|
"""gpt-5.4-codex is the recommended workaround — must not trigger."""
|
|
agent = _make_agent(tmp_path, model="gpt-5.4-codex")
|
|
agent.api_mode = "codex_responses"
|
|
assert agent._codex_silent_hang_hint(model="gpt-5.4-codex") is None
|
|
|
|
|
|
def test_hint_skipped_for_gpt_5_50_false_positive(tmp_path):
|
|
"""``gpt-5.50`` (hypothetical future SKU) must not regex-match gpt-5.5."""
|
|
agent = _make_agent(tmp_path, model="gpt-5.50")
|
|
agent.api_mode = "codex_responses"
|
|
assert agent._codex_silent_hang_hint(model="gpt-5.50") is None
|
|
|
|
|
|
def test_hint_skipped_for_non_codex_api_mode(tmp_path):
|
|
"""Hint only fires on the Codex Responses path."""
|
|
agent = _make_agent(tmp_path)
|
|
agent.api_mode = "chat_completions"
|
|
assert agent._codex_silent_hang_hint(model="gpt-5.5") is None
|
|
|
|
|
|
def test_hint_skipped_for_non_codex_provider(tmp_path):
|
|
"""Same model on a non-Codex provider does not trigger."""
|
|
agent = _make_agent(
|
|
tmp_path,
|
|
provider="openrouter",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
model="openai/gpt-5.5",
|
|
)
|
|
agent.api_mode = "codex_responses"
|
|
assert agent._codex_silent_hang_hint(model="openai/gpt-5.5") is None
|
|
|
|
|
|
def test_hint_skipped_for_empty_model(tmp_path):
|
|
"""Explicit empty string ``model`` short-circuits the regex."""
|
|
agent = _make_agent(tmp_path, model="gpt-5.4-codex") # self.model non-matching
|
|
agent.api_mode = "codex_responses"
|
|
# Explicit empty string: regex won't match
|
|
assert agent._codex_silent_hang_hint(model="") is None
|
|
# model=None falls back to self.model which is gpt-5.4-codex, also no match
|
|
assert agent._codex_silent_hang_hint(model=None) is None
|
|
|
|
|
|
def test_hint_skipped_for_unrelated_model_on_codex(tmp_path):
|
|
agent = _make_agent(tmp_path, model="gpt-4-turbo")
|
|
agent.api_mode = "codex_responses"
|
|
assert agent._codex_silent_hang_hint(model="gpt-4-turbo") is None
|