From 0e4c879a3b7c4851fbe27a4a3eac29b3acb40b9a Mon Sep 17 00:00:00 2001 From: HiddenPuppy Date: Tue, 30 Jun 2026 15:37:46 -0700 Subject: [PATCH] fix: keep plain custom GPT-5 relays on chat completions Generic provider:custom relays were force-routed to the OpenAI Responses API whenever the model matched gpt-5*, and a stale persisted model.api_mode=codex_responses survived /reset and upgrades. Some OpenAI-compatible relays do not implement Responses semantics, which surfaced as malformed function_call.name replay errors in gateway sessions. - runtime_provider: route custom-provider api_mode through _resolve_plain_custom_api_mode(), which drops a stale codex_responses unless the URL is direct OpenAI/xAI - run_agent: _provider_model_requires_responses_api returns False for custom; direct api.openai.com / api.x.ai URLs still upgrade via _is_direct_openai_url() / URL detection - regression coverage for plain relays vs direct OpenAI/xAI URLs Co-authored-by: HiddenPuppy --- hermes_cli/runtime_provider.py | 26 +++++++- run_agent.py | 5 ++ .../test_runtime_provider_resolution.py | 64 ++++++++++++++++++- .../test_run_agent_codex_responses.py | 32 ++++++++++ 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index a30bdcc3a17..7f4692b838c 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -115,6 +115,28 @@ def _detect_api_mode_for_url(base_url: str) -> Optional[str]: return None +def _resolve_plain_custom_api_mode(model_cfg: Dict[str, Any], base_url: str) -> str: + """Resolve api_mode for legacy/plain ``provider: custom`` endpoints. + + Custom endpoints should stay conservative by default. Only direct OpenAI/xAI + URLs imply Responses API automatically; named custom providers can opt in via + their own ``api_mode`` field. This also prevents a stale persisted + ``model.api_mode: codex_responses`` from forcing generic relays onto the + Responses path after upgrades or /reset. + """ + configured_mode = _parse_api_mode(model_cfg.get("api_mode")) + detected_mode = _detect_api_mode_for_url(base_url) + + if configured_mode == "codex_responses" and detected_mode != "codex_responses": + logger.info( + "Ignoring persisted custom api_mode=codex_responses for non-OpenAI endpoint %s", + base_url or "(unknown)", + ) + configured_mode = None + + return configured_mode or detected_mode or "chat_completions" + + def _host_derived_api_key(base_url: str) -> str: """Look up `_API_KEY` in the env, derived from the base URL host. @@ -1093,7 +1115,9 @@ def _resolve_openrouter_runtime( return { "provider": effective_provider, - "api_mode": _parse_api_mode(model_cfg.get("api_mode")) + "api_mode": _resolve_plain_custom_api_mode(model_cfg, base_url) + if effective_provider == "custom" + else _parse_api_mode(model_cfg.get("api_mode")) or _detect_api_mode_for_url(base_url) or "chat_completions", "base_url": base_url, diff --git a/run_agent.py b/run_agent.py index ef8a0b69c9f..8157c01caa8 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1314,6 +1314,11 @@ class AIAgent: # completions endpoint; its /v1/responses endpoint returns 404. if normalized_provider == "nous": return False + if normalized_provider == "custom": + # Generic custom endpoints are conservative by default. They may + # relay GPT-5 models without full Responses semantics, so only + # direct OpenAI/xAI URL detection should auto-upgrade them. + return False if normalized_provider == "copilot": try: from hermes_cli.models import _should_use_copilot_responses_api diff --git a/tests/hermes_cli/test_runtime_provider_resolution.py b/tests/hermes_cli/test_runtime_provider_resolution.py index de32869c4a5..5ec30cfe71c 100644 --- a/tests/hermes_cli/test_runtime_provider_resolution.py +++ b/tests/hermes_cli/test_runtime_provider_resolution.py @@ -1244,8 +1244,8 @@ def test_resolve_requested_provider_precedence(monkeypatch): # ── api_mode config override tests ────────────────────────────────────── -def test_model_config_api_mode(monkeypatch): - """model.api_mode in config.yaml should override the default chat_completions.""" +def test_model_config_codex_api_mode_is_ignored_for_plain_custom_relays(monkeypatch): + """Plain custom relays should not inherit stale Responses routing.""" monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter") monkeypatch.setattr( rp, "_get_model_config", @@ -1262,10 +1262,32 @@ def test_model_config_api_mode(monkeypatch): resolved = rp.resolve_runtime_provider(requested="custom") - assert resolved["api_mode"] == "codex_responses" + assert resolved["api_mode"] == "chat_completions" assert resolved["base_url"] == "http://127.0.0.1:9208/v1" +def test_model_config_codex_api_mode_still_applies_to_direct_openai_url(monkeypatch): + """Direct OpenAI URLs should continue to route through Responses API.""" + monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter") + monkeypatch.setattr( + rp, "_get_model_config", + lambda: { + "provider": "custom", + "base_url": "https://api.openai.com/v1", + "api_mode": "codex_responses", + }, + ) + monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + + resolved = rp.resolve_runtime_provider(requested="custom") + + assert resolved["api_mode"] == "codex_responses" + assert resolved["base_url"] == "https://api.openai.com/v1" + + def test_model_config_api_mode_ignored_when_provider_differs(monkeypatch): monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "zai") monkeypatch.setattr( @@ -1308,6 +1330,42 @@ def test_invalid_api_mode_ignored(monkeypatch): assert resolved["api_mode"] == "chat_completions" +def test_custom_provider_ignores_stale_codex_responses_api_mode(monkeypatch): + """Legacy custom endpoints should not inherit stale Responses routing.""" + monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter") + monkeypatch.setattr(rp, "_get_model_config", lambda: { + "provider": "custom", + "base_url": "https://relay.example.com/v1", + "api_key": "sk-relay-key", + "api_mode": "codex_responses", + }) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False) + + resolved = rp.resolve_runtime_provider(requested="custom") + + assert resolved["provider"] == "custom" + assert resolved["api_mode"] == "chat_completions" + + +def test_custom_provider_keeps_configured_non_responses_api_mode(monkeypatch): + """Only stale codex_responses should be ignored for plain custom endpoints.""" + monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter") + monkeypatch.setattr(rp, "_get_model_config", lambda: { + "provider": "custom", + "base_url": "https://proxy.example.com/anthropic", + "api_key": "sk-proxy-key", + "api_mode": "anthropic_messages", + }) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False) + + resolved = rp.resolve_runtime_provider(requested="custom") + + assert resolved["provider"] == "custom" + assert resolved["api_mode"] == "anthropic_messages" + + def test_named_custom_provider_api_mode(monkeypatch): """custom_providers entries with api_mode should use it.""" monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "my-server") diff --git a/tests/run_agent/test_run_agent_codex_responses.py b/tests/run_agent/test_run_agent_codex_responses.py index 14e01d9fecd..825150b9c02 100644 --- a/tests/run_agent/test_run_agent_codex_responses.py +++ b/tests/run_agent/test_run_agent_codex_responses.py @@ -268,6 +268,38 @@ def test_copilot_acp_stays_on_chat_completions_for_gpt_5_models(monkeypatch): assert agent.api_mode == "chat_completions" +def test_custom_provider_gpt5_stays_on_chat_completions(monkeypatch): + _patch_agent_bootstrap(monkeypatch) + agent = run_agent.AIAgent( + model="gpt-5.4", + base_url="https://relay.example.com/v1", + provider="custom", + api_key="relay-token", + quiet_mode=True, + max_iterations=1, + skip_context_files=True, + skip_memory=True, + ) + assert agent.provider == "custom" + assert agent.api_mode == "chat_completions" + + +def test_custom_provider_direct_openai_url_still_uses_responses(monkeypatch): + _patch_agent_bootstrap(monkeypatch) + agent = run_agent.AIAgent( + model="gpt-5.4", + base_url="https://api.openai.com/v1", + provider="custom", + api_key="openai-token", + quiet_mode=True, + max_iterations=1, + skip_context_files=True, + skip_memory=True, + ) + assert agent.provider == "custom" + assert agent.api_mode == "codex_responses" + + def test_copilot_gpt_5_mini_stays_on_chat_completions(monkeypatch): _patch_agent_bootstrap(monkeypatch) agent = run_agent.AIAgent(