fix: keep plain custom GPT-5 relays on chat completions

Generic provider:custom relays were force-routed to the OpenAI Responses
API whenever the model matched gpt-5*, and a stale persisted
model.api_mode=codex_responses survived /reset and upgrades. Some
OpenAI-compatible relays do not implement Responses semantics, which
surfaced as malformed function_call.name replay errors in gateway sessions.

- runtime_provider: route custom-provider api_mode through
  _resolve_plain_custom_api_mode(), which drops a stale codex_responses
  unless the URL is direct OpenAI/xAI
- run_agent: _provider_model_requires_responses_api returns False for
  custom; direct api.openai.com / api.x.ai URLs still upgrade via
  _is_direct_openai_url() / URL detection
- regression coverage for plain relays vs direct OpenAI/xAI URLs

Co-authored-by: HiddenPuppy <HiddenPuppy@users.noreply.github.com>

hermes_cli/runtime_provider.py

@@ -115,6 +115,28 @@ def _detect_api_mode_for_url(base_url: str) -> Optional[str]:
     return None
 
 
+def _resolve_plain_custom_api_mode(model_cfg: Dict[str, Any], base_url: str) -> str:
+    """Resolve api_mode for legacy/plain ``provider: custom`` endpoints.
+
+    Custom endpoints should stay conservative by default. Only direct OpenAI/xAI
+    URLs imply Responses API automatically; named custom providers can opt in via
+    their own ``api_mode`` field. This also prevents a stale persisted
+    ``model.api_mode: codex_responses`` from forcing generic relays onto the
+    Responses path after upgrades or /reset.
+    """
+    configured_mode = _parse_api_mode(model_cfg.get("api_mode"))
+    detected_mode = _detect_api_mode_for_url(base_url)
+
+    if configured_mode == "codex_responses" and detected_mode != "codex_responses":
+        logger.info(
+            "Ignoring persisted custom api_mode=codex_responses for non-OpenAI endpoint %s",
+            base_url or "(unknown)",
+        )
+        configured_mode = None
+
+    return configured_mode or detected_mode or "chat_completions"
+
+
 def _host_derived_api_key(base_url: str) -> str:
     """Look up `<VENDOR>_API_KEY` in the env, derived from the base URL host.
 
@@ -1093,7 +1115,9 @@ def _resolve_openrouter_runtime(
 
     return {
         "provider": effective_provider,
-        "api_mode": _parse_api_mode(model_cfg.get("api_mode"))
+        "api_mode": _resolve_plain_custom_api_mode(model_cfg, base_url)
+        if effective_provider == "custom"
+        else _parse_api_mode(model_cfg.get("api_mode"))
         or _detect_api_mode_for_url(base_url)
         or "chat_completions",
         "base_url": base_url,

run_agent.py

@@ -1314,6 +1314,11 @@ class AIAgent:
         # completions endpoint; its /v1/responses endpoint returns 404.
         if normalized_provider == "nous":
             return False
+        if normalized_provider == "custom":
+            # Generic custom endpoints are conservative by default. They may
+            # relay GPT-5 models without full Responses semantics, so only
+            # direct OpenAI/xAI URL detection should auto-upgrade them.
+            return False
         if normalized_provider == "copilot":
             try:
                 from hermes_cli.models import _should_use_copilot_responses_api

tests/hermes_cli/test_runtime_provider_resolution.py

@@ -1244,8 +1244,8 @@ def test_resolve_requested_provider_precedence(monkeypatch):
 # ── api_mode config override tests ──────────────────────────────────────
 
 
-def test_model_config_api_mode(monkeypatch):
-    """model.api_mode in config.yaml should override the default chat_completions."""
+def test_model_config_codex_api_mode_is_ignored_for_plain_custom_relays(monkeypatch):
+    """Plain custom relays should not inherit stale Responses routing."""
     monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
     monkeypatch.setattr(
         rp, "_get_model_config",
@@ -1262,10 +1262,32 @@ def test_model_config_api_mode(monkeypatch):
 
     resolved = rp.resolve_runtime_provider(requested="custom")
 
-    assert resolved["api_mode"] == "codex_responses"
+    assert resolved["api_mode"] == "chat_completions"
     assert resolved["base_url"] == "http://127.0.0.1:9208/v1"
 
 
+def test_model_config_codex_api_mode_still_applies_to_direct_openai_url(monkeypatch):
+    """Direct OpenAI URLs should continue to route through Responses API."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
+    monkeypatch.setattr(
+        rp, "_get_model_config",
+        lambda: {
+            "provider": "custom",
+            "base_url": "https://api.openai.com/v1",
+            "api_mode": "codex_responses",
+        },
+    )
+    monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+    monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+
+    resolved = rp.resolve_runtime_provider(requested="custom")
+
+    assert resolved["api_mode"] == "codex_responses"
+    assert resolved["base_url"] == "https://api.openai.com/v1"
+
+
 def test_model_config_api_mode_ignored_when_provider_differs(monkeypatch):
     monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "zai")
     monkeypatch.setattr(
@@ -1308,6 +1330,42 @@ def test_invalid_api_mode_ignored(monkeypatch):
     assert resolved["api_mode"] == "chat_completions"
 
 
+def test_custom_provider_ignores_stale_codex_responses_api_mode(monkeypatch):
+    """Legacy custom endpoints should not inherit stale Responses routing."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
+    monkeypatch.setattr(rp, "_get_model_config", lambda: {
+        "provider": "custom",
+        "base_url": "https://relay.example.com/v1",
+        "api_key": "sk-relay-key",
+        "api_mode": "codex_responses",
+    })
+    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
+
+    resolved = rp.resolve_runtime_provider(requested="custom")
+
+    assert resolved["provider"] == "custom"
+    assert resolved["api_mode"] == "chat_completions"
+
+
+def test_custom_provider_keeps_configured_non_responses_api_mode(monkeypatch):
+    """Only stale codex_responses should be ignored for plain custom endpoints."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
+    monkeypatch.setattr(rp, "_get_model_config", lambda: {
+        "provider": "custom",
+        "base_url": "https://proxy.example.com/anthropic",
+        "api_key": "sk-proxy-key",
+        "api_mode": "anthropic_messages",
+    })
+    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
+
+    resolved = rp.resolve_runtime_provider(requested="custom")
+
+    assert resolved["provider"] == "custom"
+    assert resolved["api_mode"] == "anthropic_messages"
+
+
 def test_named_custom_provider_api_mode(monkeypatch):
     """custom_providers entries with api_mode should use it."""
     monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "my-server")

tests/run_agent/test_run_agent_codex_responses.py

@@ -268,6 +268,38 @@ def test_copilot_acp_stays_on_chat_completions_for_gpt_5_models(monkeypatch):
     assert agent.api_mode == "chat_completions"
 
 
+def test_custom_provider_gpt5_stays_on_chat_completions(monkeypatch):
+    _patch_agent_bootstrap(monkeypatch)
+    agent = run_agent.AIAgent(
+        model="gpt-5.4",
+        base_url="https://relay.example.com/v1",
+        provider="custom",
+        api_key="relay-token",
+        quiet_mode=True,
+        max_iterations=1,
+        skip_context_files=True,
+        skip_memory=True,
+    )
+    assert agent.provider == "custom"
+    assert agent.api_mode == "chat_completions"
+
+
+def test_custom_provider_direct_openai_url_still_uses_responses(monkeypatch):
+    _patch_agent_bootstrap(monkeypatch)
+    agent = run_agent.AIAgent(
+        model="gpt-5.4",
+        base_url="https://api.openai.com/v1",
+        provider="custom",
+        api_key="openai-token",
+        quiet_mode=True,
+        max_iterations=1,
+        skip_context_files=True,
+        skip_memory=True,
+    )
+    assert agent.provider == "custom"
+    assert agent.api_mode == "codex_responses"
+
+
 def test_copilot_gpt_5_mini_stays_on_chat_completions(monkeypatch):
     _patch_agent_bootstrap(monkeypatch)
     agent = run_agent.AIAgent(