mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-26 17:38:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat(api): honor provider-aware request routing

Browse source 
Carry model, provider, and model_options through the API server's
execution surfaces (session chat, Chat Completions, Responses, /v1/runs)
without mutating global configuration. Precedence: session /model
override -> model_routes alias -> direct request selection -> global
defaults. Conflicting route/provider mixes fail closed with 400.
model_options stays request-scoped regardless of which selection wins.

Salvaged from PR #54426 by @abundantbeing.
This commit is contained in:
abundantbeing 2026-07-24 09:24:36 -07:00 • committed by Teknium
parent 077e41330d
commit d66a82000c
5 changed files with 927 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

1
contributors/emails/beingsabundant@gmail.com Normal file
View file

@ -0,0 +1 @@
abundantbeing

481
gateway/platforms/api_server.py
View file

@ -168,6 +168,157 @@ def _coerce_request_bool(value: Any, default: bool = False) -> bool:
return default
_REQUEST_OPTION_MISSING = object()
_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
_RUNTIME_AGENT_OVERRIDE_KEYS = (
"api_key",
"base_url",
"provider",
"api_mode",
"command",
"args",
"credential_pool",
"max_tokens",
)
def _clean_request_string(value: Any) -> Optional[str]:
"""Return a stripped request string, or None for absent/non-string values."""
if not isinstance(value, str):
return None
cleaned = value.strip()
return cleaned or None
def _request_reasoning_config(model_options: Any) -> Optional[Dict[str, Any]]:
"""Translate browser/API model_options into AIAgent reasoning_config.
The browser extension sends both a structured ``reasoning`` object and a
compatibility ``reasoning_effort`` scalar. Keep this parser permissive so
older clients can send either shape, but ignore unknown effort values rather
than raising on a chat request.
"""
if not isinstance(model_options, dict):
return None
reasoning = model_options.get("reasoning")
enabled: Any = None
effort: Any = model_options.get("reasoning_effort")
if isinstance(reasoning, dict):
enabled = reasoning.get("enabled")
effort = reasoning.get("effort", effort)
effort_norm = str(effort).strip().lower() if effort is not None else ""
if enabled is False or effort_norm == "none":
return {"enabled": False}
if effort_norm in _REASONING_EFFORTS and effort_norm != "none":
return {"enabled": True, "effort": effort_norm}
if enabled is True:
return {"enabled": True}
return None
def _request_service_tier(model_options: Any) -> Any:
"""Return a per-request service_tier override or _REQUEST_OPTION_MISSING."""
if not isinstance(model_options, dict):
return _REQUEST_OPTION_MISSING
if "service_tier" in model_options:
raw_tier = model_options.get("service_tier")
if raw_tier is None:
return None
if isinstance(raw_tier, str):
return raw_tier.strip() or None
return raw_tier
if "fast" in model_options:
return "priority" if _coerce_request_bool(model_options.get("fast"), default=False) else None
return _REQUEST_OPTION_MISSING
def _apply_runtime_agent_overrides(
runtime_kwargs: Dict[str, Any], overrides: Optional[Dict[str, Any]]
) -> Dict[str, Any]:
"""Merge resolved provider/runtime fields into ``runtime_kwargs`` in place."""
if not isinstance(overrides, dict):
return runtime_kwargs
for key in _RUNTIME_AGENT_OVERRIDE_KEYS:
if key not in overrides:
continue
value = overrides.get(key)
if value is None:
continue
runtime_kwargs[key] = list(value) if key == "args" and isinstance(value, (list, tuple)) else value
return runtime_kwargs
def _resolve_request_runtime_agent_kwargs(provider: str, target_model: Optional[str] = None) -> Dict[str, Any]:
"""Resolve runtime kwargs for a one-request provider override.
This mirrors gateway.run._resolve_runtime_agent_kwargs(), but accepts an
explicit provider/model so an API caller can use the same authenticated
provider catalog as the TUI without mutating config.yaml.
"""
from hermes_cli.runtime_provider import resolve_runtime_provider, format_runtime_provider_error, _get_model_config
try:
runtime = resolve_runtime_provider(requested=provider, target_model=target_model)
except Exception as exc:
raise RuntimeError(format_runtime_provider_error(exc)) from exc
model_cfg = _get_model_config()
max_tokens = None
env_max_tokens = os.environ.get("HERMES_MAX_TOKENS")
if env_max_tokens:
try:
max_tokens = int(env_max_tokens)
except (ValueError, TypeError):
max_tokens = None
elif isinstance(model_cfg, dict):
cfg_max_tokens = model_cfg.get("max_tokens")
if isinstance(cfg_max_tokens, int):
max_tokens = cfg_max_tokens
if max_tokens is None:
runtime_max_tokens = runtime.get("max_output_tokens")
if isinstance(runtime_max_tokens, int) and runtime_max_tokens > 0:
max_tokens = runtime_max_tokens
return {
"api_key": runtime.get("api_key"),
"base_url": runtime.get("base_url"),
"provider": runtime.get("provider"),
"api_mode": runtime.get("api_mode"),
"command": runtime.get("command"),
"args": list(runtime.get("args") or []),
"credential_pool": runtime.get("credential_pool"),
"max_tokens": max_tokens,
}
def _request_agent_overrides(body: Any, *, virtual_model: Optional[str] = None) -> Dict[str, Any]:
"""Extract per-request model/provider/options for _run_agent.
``/v1/models`` advertises a stable virtual model (usually ``hermes-agent``)
for OpenAI-compatible clients. Treat that alias as "use the gateway
default"; real model picker selections from the browser extension send the
raw provider model id plus a provider slug and should override this turn.
"""
if not isinstance(body, dict):
return {}
overrides: Dict[str, Any] = {}
model = _clean_request_string(body.get("model"))
if model and model != virtual_model:
overrides["requested_model"] = model
provider = _clean_request_string(body.get("provider"))
if provider:
overrides["requested_provider"] = provider
model_options = body.get("model_options")
if isinstance(model_options, dict):
overrides["model_options"] = dict(model_options)
return overrides
def _message_text_prefix(content: Any) -> str:
if isinstance(content, str):
return content[:128]
@ -1814,11 +1965,56 @@ class APIServerAdapter(BasePlatformAdapter):
runner = _gateway_runner_ref()
if runner is None:
return None
try:
rehydrate = getattr(runner, "_rehydrate_session_model_override", None)
if callable(rehydrate):
rehydrate(session_key)
except Exception:
logger.debug(
"api_server failed to rehydrate session /model override for %s",
session_key,
exc_info=True,
)
override = runner._session_model_overrides.get(session_key)
return dict(override) if isinstance(override, dict) else None
except Exception:
return None
def _request_route_conflict_error(
self,
*,
session_id: Optional[str],
gateway_session_key: Optional[str],
requested_model: Optional[str],
requested_provider: Optional[str],
route: Optional[Dict[str, Any]],
) -> Optional[str]:
"""Return a 400-worthy conflict string for ambiguous route/provider mixes."""
request_provider = _clean_request_string(requested_provider)
if not request_provider or not isinstance(route, dict):
return None
if self._session_model_override_for(gateway_session_key or session_id):
# Session /model wins over both the route and the request override, so
# there is no ambiguity to reject on this request path.
return None
route_provider = _clean_request_string(route.get("provider"))
route_api_key = _clean_request_string(route.get("api_key"))
route_base_url = _clean_request_string(route.get("base_url"))
route_alias = _clean_request_string(requested_model) or "requested model"
if route_provider and request_provider != route_provider:
return (
f"Model route '{route_alias}' is pinned to provider '{route_provider}'. "
f"Remove 'provider' or use '{route_provider}'."
)
if not route_provider and (route_api_key or route_base_url):
return (
f"Model route '{route_alias}' pins route credentials/base_url. "
"Do not combine it with an explicit 'provider'."
)
return None
def _create_agent(
self,
ephemeral_system_prompt: Optional[str] = None,
@ -1828,6 +2024,9 @@ class APIServerAdapter(BasePlatformAdapter):
tool_start_callback=None,
tool_complete_callback=None,
gateway_session_key: Optional[str] = None,
requested_model: Optional[str] = None,
requested_provider: Optional[str] = None,
model_options: Optional[Dict[str, Any]] = None,
route: Optional[Dict[str, Any]] = None,
) -> Any:
"""
@ -1877,51 +2076,105 @@ class APIServerAdapter(BasePlatformAdapter):
if runtime_model:
model = runtime_model
# Per-client model routing (model_routes config). The route was
# resolved from the request's ``model`` field by the HTTP handler.
# Precedence (highest first): session ``/model`` override → model_routes
# route → global config — an explicit user-issued ``/model`` on the
# session always beats static per-client route config.
session_override = self._session_model_override_for(
gateway_session_key or session_id
)
if route and not session_override:
if route.get("provider"):
# Resolve real credentials for the routed provider (mirrors
# the channel_overrides path in gateway/run.py) so a route
# without an explicit api_key/base_url still gets the right
# provider auth instead of the default provider's key.
request_reasoning_config = _request_reasoning_config(model_options)
if request_reasoning_config is not None:
reasoning_config = request_reasoning_config
request_service_tier = _request_service_tier(model_options)
request_model = _clean_request_string(requested_model)
request_provider = _clean_request_string(requested_provider)
route_model = _clean_request_string(route.get("model")) if isinstance(route, dict) else None
route_provider = _clean_request_string(route.get("provider")) if isinstance(route, dict) else None
route_api_key = _clean_request_string(route.get("api_key")) if isinstance(route, dict) else None
route_base_url = _clean_request_string(route.get("base_url")) if isinstance(route, dict) else None
def _resolve_provider_runtime(
provider: Optional[str],
*,
target_model: Optional[str],
required: bool,
) -> Optional[Dict[str, Any]]:
provider_name = _clean_request_string(provider)
if not provider_name:
return None
try:
return _resolve_request_runtime_agent_kwargs(
provider_name,
target_model=target_model or None,
)
except Exception:
try:
from gateway.run import _resolve_runtime_agent_kwargs_for_provider
provider_kwargs = _resolve_runtime_agent_kwargs_for_provider(
route["provider"]
)
provider_kwargs.pop("model", None)
runtime_kwargs.update(provider_kwargs)
except Exception:
# Fall back to just switching the provider name; explicit
# per-route api_key/base_url below can still complete auth.
runtime_kwargs["provider"] = route["provider"]
if route.get("model"):
model = route["model"]
# Per-route secrets are upstream provider credentials. Never log
# them (compare _check_auth: caller auth stays the global bearer
# key checked with hmac.compare_digest).
if route.get("api_key"):
runtime_kwargs["api_key"] = route["api_key"]
if route.get("base_url"):
runtime_kwargs["base_url"] = route["base_url"]
logger.debug(
"api_server model route applied: model=%s provider=%s",
model,
runtime_kwargs.get("provider"),
)
elif route and session_override:
logger.debug(
"api_server model route skipped: session /model override wins for %s",
gateway_session_key or session_id,
)
return _resolve_runtime_agent_kwargs_for_provider(provider_name)
except Exception:
pass
if required:
raise
logger.debug(
"api_server provider-runtime refresh failed for provider=%s model=%s",
provider_name,
target_model or "",
exc_info=True,
)
return None
# Final precedence mirrors the gateway contract:
# session /model override → model_routes mapping selected by the request
# model alias → direct per-request provider/model → global defaults.
# model_options stay request-scoped regardless of which selection wins.
session_key = gateway_session_key or session_id
session_override = self._session_model_override_for(session_key)
if session_override:
session_model = _clean_request_string(session_override.get("model")) or model
session_provider = _clean_request_string(session_override.get("provider"))
current_provider = _clean_request_string(runtime_kwargs.get("provider"))
provider_runtime = _resolve_provider_runtime(
session_provider or current_provider,
target_model=session_model,
required=False,
)
if provider_runtime:
_apply_runtime_agent_overrides(runtime_kwargs, provider_runtime)
_apply_runtime_agent_overrides(runtime_kwargs, session_override)
model = session_model
if route or request_model or request_provider:
logger.debug(
"api_server request selection skipped: session /model override wins for %s",
session_key or "",
)
else:
effective_model = route_model or request_model or model
current_provider = _clean_request_string(runtime_kwargs.get("provider"))
effective_provider = request_provider or route_provider or current_provider
provider_runtime = None
if effective_provider and (
bool(request_provider or route_provider) or effective_model != model
):
provider_runtime = _resolve_provider_runtime(
effective_provider,
target_model=effective_model,
required=bool(request_provider),
)
if provider_runtime:
_apply_runtime_agent_overrides(runtime_kwargs, provider_runtime)
elif effective_provider and effective_provider != current_provider:
runtime_kwargs["provider"] = effective_provider
model = effective_model
# Per-route explicit transport secrets/base URLs win within the
# route contract after provider resolution.
if route_api_key:
runtime_kwargs["api_key"] = route_api_key
if route_base_url:
runtime_kwargs["base_url"] = route_base_url
if route:
logger.debug(
"api_server request selection applied: model=%s provider=%s route_provider=%s request_provider=%s",
model,
runtime_kwargs.get("provider"),
route_provider or "",
request_provider or "",
)
user_config = _load_gateway_config()
enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server"))
@ -1931,26 +2184,30 @@ class APIServerAdapter(BasePlatformAdapter):
# same fallback behaviour as Telegram/Discord/Slack (fixes #4954).
fallback_model = GatewayRunner._load_fallback_model()
agent = AIAgent(
model=model,
agent_kwargs = {
"model": model,
**runtime_kwargs,
**_checkpoint_agent_kwargs(user_config),
max_iterations=max_iterations,
quiet_mode=True,
verbose_logging=False,
ephemeral_system_prompt=ephemeral_system_prompt or None,
enabled_toolsets=enabled_toolsets,
session_id=session_id,
platform="api_server",
stream_delta_callback=stream_delta_callback,
tool_progress_callback=tool_progress_callback,
tool_start_callback=tool_start_callback,
tool_complete_callback=tool_complete_callback,
session_db=self._ensure_session_db(),
fallback_model=fallback_model,
reasoning_config=reasoning_config,
gateway_session_key=gateway_session_key,
)
"max_iterations": max_iterations,
"quiet_mode": True,
"verbose_logging": False,
"ephemeral_system_prompt": ephemeral_system_prompt or None,
"enabled_toolsets": enabled_toolsets,
"session_id": session_id,
"platform": "api_server",
"stream_delta_callback": stream_delta_callback,
"tool_progress_callback": tool_progress_callback,
"tool_start_callback": tool_start_callback,
"tool_complete_callback": tool_complete_callback,
"session_db": self._ensure_session_db(),
"fallback_model": fallback_model,
"reasoning_config": reasoning_config,
"gateway_session_key": gateway_session_key,
}
if request_service_tier is not _REQUEST_OPTION_MISSING:
agent_kwargs["service_tier"] = request_service_tier
agent = AIAgent(**agent_kwargs)
return agent
# ------------------------------------------------------------------
@ -2565,6 +2822,17 @@ class APIServerAdapter(BasePlatformAdapter):
system_prompt = body.get("system_message") or body.get("instructions")
if system_prompt is not None and not isinstance(system_prompt, str):
return web.json_response(_openai_error("system_message must be a string", code="invalid_system_message"), status=400)
route = self._resolve_route(body.get("model"))
agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name)
selection_error = self._request_route_conflict_error(
session_id=session_id,
gateway_session_key=gateway_session_key,
requested_model=agent_overrides.get("requested_model"),
requested_provider=agent_overrides.get("requested_provider"),
route=route,
)
if selection_error:
return web.json_response(_openai_error(selection_error), status=400)
history = await self._conversation_history_for_session(session_id)
result, usage = await self._run_agent(
user_message=user_message,
@ -2572,6 +2840,8 @@ class APIServerAdapter(BasePlatformAdapter):
ephemeral_system_prompt=system_prompt,
session_id=session_id,
gateway_session_key=gateway_session_key,
route=route,
**agent_overrides,
)
effective_session_id = result.get("session_id") if isinstance(result, dict) else session_id
final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "")
@ -2607,6 +2877,17 @@ class APIServerAdapter(BasePlatformAdapter):
system_prompt = body.get("system_message") or body.get("instructions")
if system_prompt is not None and not isinstance(system_prompt, str):
return web.json_response(_openai_error("system_message must be a string", code="invalid_system_message"), status=400)
route = self._resolve_route(body.get("model"))
agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name)
selection_error = self._request_route_conflict_error(
session_id=session_id,
gateway_session_key=gateway_session_key,
requested_model=agent_overrides.get("requested_model"),
requested_provider=agent_overrides.get("requested_provider"),
route=route,
)
if selection_error:
return web.json_response(_openai_error(selection_error), status=400)
loop = asyncio.get_running_loop()
queue: "asyncio.Queue[Optional[tuple[str, Dict[str, Any]]]]" = asyncio.Queue()
@ -2658,10 +2939,12 @@ class APIServerAdapter(BasePlatformAdapter):
conversation_history=history,
ephemeral_system_prompt=system_prompt,
session_id=session_id,
stream_delta_callback=_delta,
tool_progress_callback=_tool_progress,
gateway_session_key=gateway_session_key,
)
stream_delta_callback=_delta,
tool_progress_callback=_tool_progress,
gateway_session_key=gateway_session_key,
route=route,
**agent_overrides,
)
final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "")
effective_session_id = result.get("session_id", session_id) if isinstance(result, dict) else session_id
turn_messages = self._turn_transcript_messages(history, user_message, result) if isinstance(result, dict) else []
@ -2861,6 +3144,16 @@ class APIServerAdapter(BasePlatformAdapter):
# configured model_routes alias, this request's agent is created
# with that route's model/provider instead of the global default.
route = self._resolve_route(model_name)
agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name)
selection_error = self._request_route_conflict_error(
session_id=session_id,
gateway_session_key=gateway_session_key,
requested_model=agent_overrides.get("requested_model"),
requested_provider=agent_overrides.get("requested_provider"),
route=route,
)
if selection_error:
return web.json_response(_openai_error(selection_error), status=400)
if stream:
import queue as _q
@ -2944,6 +3237,7 @@ class APIServerAdapter(BasePlatformAdapter):
tool_complete_callback=_on_tool_complete,
agent_ref=agent_ref,
gateway_session_key=gateway_session_key,
**agent_overrides,
route=route,
))
# Ensure SSE drain loops can terminate without relying on polling
@ -2964,12 +3258,16 @@ class APIServerAdapter(BasePlatformAdapter):
ephemeral_system_prompt=system_prompt,
session_id=session_id,
gateway_session_key=gateway_session_key,
**agent_overrides,
route=route,
)
idempotency_key = request.headers.get("Idempotency-Key")
if idempotency_key:
fp = _make_request_fingerprint(body, keys=["model", "messages", "tools", "tool_choice", "stream"])
fp = _make_request_fingerprint(
body,
keys=["model", "provider", "model_options", "messages", "tools", "tool_choice", "stream"],
)
try:
result, usage = await _idem_cache.get_or_set(idempotency_key, fp, _compute_completion)
except Exception as e:
@ -3979,10 +4277,18 @@ class APIServerAdapter(BasePlatformAdapter):
# groups the entire conversation under one session entry.
session_id = stored_session_id or str(uuid.uuid4())
# Per-client model routing for /v1/responses (see model_routes).
route = self._resolve_route(body.get("model"))
stream = _coerce_request_bool(body.get("stream"), default=False)
route = self._resolve_route(body.get("model"))
agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name)
selection_error = self._request_route_conflict_error(
session_id=session_id,
gateway_session_key=gateway_session_key,
requested_model=agent_overrides.get("requested_model"),
requested_provider=agent_overrides.get("requested_provider"),
route=route,
)
if selection_error:
return web.json_response(_openai_error(selection_error), status=400)
if stream:
# Streaming branch — emit OpenAI Responses SSE events as the
# agent runs so frontends can render text deltas and tool
@ -4035,6 +4341,7 @@ class APIServerAdapter(BasePlatformAdapter):
tool_complete_callback=_on_tool_complete,
agent_ref=agent_ref,
gateway_session_key=gateway_session_key,
**agent_overrides,
route=route,
))
# Ensure SSE drain loops can terminate without relying on polling
@ -4069,6 +4376,7 @@ class APIServerAdapter(BasePlatformAdapter):
ephemeral_system_prompt=instructions,
session_id=session_id,
gateway_session_key=gateway_session_key,
**agent_overrides,
route=route,
)
@ -4076,7 +4384,16 @@ class APIServerAdapter(BasePlatformAdapter):
if idempotency_key:
fp = _make_request_fingerprint(
body,
keys=["input", "instructions", "previous_response_id", "conversation", "model", "tools"],
keys=[
"input",
"instructions",
"previous_response_id",
"conversation",
"model",
"provider",
"model_options",
"tools",
],
)
try:
result, usage = await _idem_cache.get_or_set(idempotency_key, fp, _compute_response)
@ -4747,6 +5064,9 @@ class APIServerAdapter(BasePlatformAdapter):
tool_complete_callback=None,
agent_ref: Optional[list] = None,
gateway_session_key: Optional[str] = None,
requested_model: Optional[str] = None,
requested_provider: Optional[str] = None,
model_options: Optional[Dict[str, Any]] = None,
route: Optional[Dict[str, Any]] = None,
) -> tuple:
"""
@ -4788,6 +5108,9 @@ class APIServerAdapter(BasePlatformAdapter):
tool_start_callback=tool_start_callback,
tool_complete_callback=tool_complete_callback,
gateway_session_key=gateway_session_key,
requested_model=requested_model,
requested_provider=requested_provider,
model_options=model_options,
route=route,
)
if agent_ref is not None:
@ -4974,8 +5297,21 @@ class APIServerAdapter(BasePlatformAdapter):
)
conversation_history.append({"role": msg["role"], "content": str(content)})
session_id = body.get("session_id") or stored_session_id
route = self._resolve_route(body.get("model"))
agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name)
selection_error = self._request_route_conflict_error(
session_id=session_id,
gateway_session_key=gateway_session_key,
requested_model=agent_overrides.get("requested_model"),
requested_provider=agent_overrides.get("requested_provider"),
route=route,
)
if selection_error:
return web.json_response(_openai_error(selection_error), status=400)
run_id = f"run_{uuid.uuid4().hex}"
session_id = body.get("session_id") or stored_session_id or run_id
session_id = session_id or run_id
# Approval queues gate host-side tool execution and must be isolated
# per API run. Client-provided session IDs and memory session keys are
# conversation/memory scopes, not authorization namespaces: multiple
@ -5021,8 +5357,6 @@ class APIServerAdapter(BasePlatformAdapter):
model=body.get("model", self._model_name),
)
# Per-client model routing for /v1/runs (see model_routes).
route = self._resolve_route(body.get("model"))
# Background task outlives the HTTP response (and thus the middleware
# profile scope). Capture now and re-enter inside the task/executor.
request_profile = _api_request_profile.get()
@ -5049,6 +5383,9 @@ class APIServerAdapter(BasePlatformAdapter):
stream_delta_callback=_text_cb,
tool_progress_callback=event_cb,
gateway_session_key=gateway_session_key,
requested_model=agent_overrides.get("requested_model"),
requested_provider=agent_overrides.get("requested_provider"),
model_options=agent_overrides.get("model_options"),
route=route,
)
self._active_run_agents[run_id] = agent

407
tests/gateway/test_api_server.py
View file

@ -16,7 +16,9 @@ import asyncio
import json
import os
import stat
import sys
import time
import types
import uuid
from unittest.mock import AsyncMock, MagicMock, patch
@ -661,6 +663,8 @@ def _create_app(adapter: APIServerAdapter) -> web.Application:
app.router.add_get("/v1/capabilities", adapter._handle_capabilities)
app.router.add_get("/v1/skills", adapter._handle_skills)
app.router.add_get("/v1/toolsets", adapter._handle_toolsets)
app.router.add_post("/api/sessions/{session_id}/chat", adapter._handle_session_chat)
app.router.add_post("/api/sessions/{session_id}/chat/stream", adapter._handle_session_chat_stream)
app.router.add_post("/v1/chat/completions", adapter._handle_chat_completions)
app.router.add_post("/v1/responses", adapter._handle_responses)
app.router.add_get("/v1/responses/{response_id}", adapter._handle_get_response)
@ -711,11 +715,15 @@ class TestAgentExecution:
mock_agent.session_completion_tokens = 2
mock_agent.session_total_tokens = 3
with patch.object(adapter, "_create_agent", return_value=mock_agent):
model_options = {"reasoning": {"enabled": False}, "fast": False}
with patch.object(adapter, "_create_agent", return_value=mock_agent) as mock_create_agent:
result, usage = await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="session-123",
requested_model="MiniMax-M3",
requested_provider="minimax",
model_options=model_options,
)
# _run_agent annotates result with the effective agent.session_id
@ -725,12 +733,161 @@ class TestAgentExecution:
# the annotation — header will fall back to the provided session_id.
assert result["final_response"] == "ok"
assert usage == {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3}
create_kwargs = mock_create_agent.call_args.kwargs
assert create_kwargs["requested_model"] == "MiniMax-M3"
assert create_kwargs["requested_provider"] == "minimax"
assert create_kwargs["model_options"] == model_options
mock_agent.run_conversation.assert_called_once_with(
user_message="hello",
conversation_history=[],
task_id="session-123",
)
def test_create_agent_honors_request_model_provider_and_options(self, adapter, monkeypatch):
import gateway.run as gateway_run
import hermes_cli.runtime_provider as runtime_provider
import hermes_cli.tools_config as tools_config
class _CapturingAgent:
last_kwargs = None
def __init__(self, **kwargs):
type(self).last_kwargs = dict(kwargs)
fake_run_agent = types.ModuleType("run_agent")
fake_run_agent.AIAgent = _CapturingAgent
monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent)
monkeypatch.setattr(gateway_run, "_current_max_iterations", lambda: 7)
monkeypatch.setattr(gateway_run, "_resolve_gateway_model", lambda: "gpt-5.5")
monkeypatch.setattr(gateway_run, "_load_gateway_config", lambda: {})
monkeypatch.setattr(
gateway_run,
"_resolve_runtime_agent_kwargs",
lambda: {
"api_key": "codex-key",
"base_url": "https://chatgpt.com/backend-api/codex",
"provider": "openai-codex",
"api_mode": "codex_responses",
"command": None,
"args": [],
"credential_pool": None,
"max_tokens": None,
},
)
monkeypatch.setattr(gateway_run.GatewayRunner, "_load_reasoning_config", staticmethod(lambda: {"enabled": True, "effort": "medium"}))
monkeypatch.setattr(gateway_run.GatewayRunner, "_load_fallback_model", staticmethod(lambda: None))
monkeypatch.setattr(tools_config, "_get_platform_tools", lambda _cfg, _platform: {"web"})
monkeypatch.setattr(adapter, "_ensure_session_db", lambda: None)
def _fake_resolve_runtime_provider(*, requested=None, target_model=None, **_kwargs):
assert requested == "minimax"
assert target_model == "MiniMax-M3"
return {
"api_key": "minimax-key",
"base_url": "https://api.minimax.io/v1",
"provider": "minimax",
"api_mode": "anthropic_messages",
"command": None,
"args": [],
"credential_pool": None,
"max_output_tokens": 32000,
}
monkeypatch.setattr(runtime_provider, "resolve_runtime_provider", _fake_resolve_runtime_provider)
monkeypatch.setattr(runtime_provider, "_get_model_config", lambda: {})
adapter._create_agent(
session_id="session-123",
requested_model="MiniMax-M3",
requested_provider="minimax",
model_options={
"reasoning": {"enabled": True, "effort": "high"},
"reasoning_effort": "high",
"fast": True,
},
)
kwargs = _CapturingAgent.last_kwargs
assert kwargs is not None
assert kwargs["model"] == "MiniMax-M3"
assert kwargs["provider"] == "minimax"
assert kwargs["api_mode"] == "anthropic_messages"
assert kwargs["base_url"] == "https://api.minimax.io/v1"
assert kwargs["api_key"] == "minimax-key"
assert kwargs["max_tokens"] == 32000
assert kwargs["reasoning_config"] == {"enabled": True, "effort": "high"}
assert kwargs["service_tier"] == "priority"
assert kwargs["enabled_toolsets"] == ["web"]
def test_create_agent_session_override_beats_request_and_route_but_keeps_model_options(
self, monkeypatch
):
captured = {}
class FakeAgent:
def __init__(self, **kwargs):
captured.update(kwargs)
_patch_create_agent_runtime(monkeypatch, captured, FakeAgent)
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda requested=None, target_model=None, **_kwargs: {
"api_key": f"sk-{requested}",
"base_url": f"https://{requested}.example/v1",
"provider": requested,
"api_mode": "chat_completions",
"command": None,
"args": [],
"credential_pool": f"pool-{requested}",
"max_output_tokens": 64000,
},
)
monkeypatch.setattr("hermes_cli.runtime_provider._get_model_config", lambda: {})
adapter = _make_routing_adapter(
{
"alias": {
"model": "route/model",
"api_key": "sk-route",
"base_url": "https://route.example/v1",
}
}
)
monkeypatch.setattr(adapter, "_ensure_session_db", lambda: None)
monkeypatch.setattr(
adapter,
"_session_model_override_for",
lambda *_: {
"model": "session/model",
"provider": "sessionprov",
"api_key": "sk-session",
"base_url": "https://session.example/v1",
"api_mode": "responses",
"credential_pool": "pool-session",
},
)
adapter._create_agent(
session_id="session-123",
route=adapter._resolve_route("alias"),
requested_model="MiniMax-M3",
requested_provider="minimax",
model_options={
"reasoning": {"enabled": True, "effort": "high"},
"fast": True,
},
)
assert captured["model"] == "session/model"
assert captured["provider"] == "sessionprov"
assert captured["api_key"] == "sk-session"
assert captured["base_url"] == "https://session.example/v1"
assert captured["api_mode"] == "responses"
assert captured["credential_pool"] == "pool-session"
assert captured["reasoning_config"] == {"enabled": True, "effort": "high"}
assert captured["service_tier"] == "priority"
# ---------------------------------------------------------------------------
# /health endpoint
@ -1246,6 +1403,209 @@ class TestChatCompletionsEndpoint:
resp = await cli.post("/v1/chat/completions", json={"model": "test", "messages": []})
assert resp.status == 400
@pytest.mark.asyncio
async def test_chat_completions_passes_request_model_provider_options(self, adapter):
app = _create_app(adapter)
model_options = {
"reasoning": {"enabled": True, "effort": "high"},
"reasoning_effort": "high",
"service_tier": "priority",
"fast": True,
}
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
mock_run.return_value = (
{"final_response": "ok", "messages": [], "api_calls": 1},
{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
)
resp = await cli.post(
"/v1/chat/completions",
json={
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": model_options,
"messages": [{"role": "user", "content": "hi"}],
},
)
assert resp.status == 200
kwargs = mock_run.call_args.kwargs
assert kwargs["requested_model"] == "MiniMax-M3"
assert kwargs["requested_provider"] == "minimax"
assert kwargs["model_options"] == model_options
@pytest.mark.asyncio
async def test_chat_completions_stream_passes_request_model_provider_options(self, adapter):
app = _create_app(adapter)
model_options = {"reasoning": {"enabled": False}, "reasoning_effort": "none", "fast": False}
async def _mock_run_agent(**kwargs):
cb = kwargs.get("stream_delta_callback")
if cb:
cb("ok")
return (
{"final_response": "ok", "messages": [], "api_calls": 1},
{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
)
app = _create_app(adapter)
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_run_agent", side_effect=_mock_run_agent) as mock_run:
resp = await cli.post(
"/v1/chat/completions",
json={
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": model_options,
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
},
)
assert resp.status == 200
body = await resp.text()
assert "data: " in body
kwargs = mock_run.call_args.kwargs
assert kwargs["requested_model"] == "MiniMax-M3"
assert kwargs["requested_provider"] == "minimax"
assert kwargs["model_options"] == model_options
@pytest.mark.asyncio
async def test_session_chat_passes_request_model_provider_options(self, adapter):
app = _create_app(adapter)
model_options = {"reasoning": {"enabled": True, "effort": "low"}, "fast": True}
async with TestClient(TestServer(app)) as cli:
with (
patch.object(adapter, "_get_existing_session_or_404", return_value=({"id": "s1"}, None)),
patch.object(adapter, "_conversation_history_for_session", return_value=[]),
patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run,
):
mock_run.return_value = (
{"final_response": "ok", "messages": [], "api_calls": 1},
{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
)
resp = await cli.post(
"/api/sessions/s1/chat",
json={
"message": "hi",
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": model_options,
},
)
assert resp.status == 200
kwargs = mock_run.call_args.kwargs
assert kwargs["requested_model"] == "MiniMax-M3"
assert kwargs["requested_provider"] == "minimax"
assert kwargs["model_options"] == model_options
@pytest.mark.asyncio
async def test_session_chat_stream_passes_request_model_provider_options(self, adapter):
app = _create_app(adapter)
model_options = {"reasoning_effort": "medium", "service_tier": "priority"}
async with TestClient(TestServer(app)) as cli:
with (
patch.object(adapter, "_get_existing_session_or_404", return_value=({"id": "s1"}, None)),
patch.object(adapter, "_conversation_history_for_session", return_value=[]),
patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run,
):
mock_run.return_value = (
{"final_response": "ok", "messages": [], "api_calls": 1},
{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
)
resp = await cli.post(
"/api/sessions/s1/chat/stream",
json={
"message": "hi",
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": model_options,
},
)
assert resp.status == 200
body = await resp.text()
assert "event: run.completed" in body
kwargs = mock_run.call_args.kwargs
assert kwargs["requested_model"] == "MiniMax-M3"
assert kwargs["requested_provider"] == "minimax"
assert kwargs["model_options"] == model_options
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "body", "needs_session"),
[
(
"/v1/chat/completions",
{
"model": "alias",
"provider": "minimax",
"messages": [{"role": "user", "content": "hi"}],
},
False,
),
(
"/v1/responses",
{
"model": "alias",
"provider": "minimax",
"input": "hi",
},
False,
),
(
"/api/sessions/s1/chat",
{
"model": "alias",
"provider": "minimax",
"message": "hi",
},
True,
),
(
"/api/sessions/s1/chat/stream",
{
"model": "alias",
"provider": "minimax",
"message": "hi",
},
True,
),
],
)
async def test_handlers_reject_conflicting_route_and_request_provider(
self, path, body, needs_session
):
adapter = _make_routing_adapter(
{"alias": {"model": "route/model", "provider": "openrouter"}}
)
app = _create_app(adapter)
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
if needs_session:
with (
patch.object(
adapter,
"_get_existing_session_or_404",
return_value=({"id": "s1"}, None),
),
patch.object(
adapter,
"_conversation_history_for_session",
return_value=[],
),
):
resp = await cli.post(path, json=body)
data = await resp.json()
else:
resp = await cli.post(path, json=body)
data = await resp.json()
assert resp.status == 400
assert "provider" in data["error"]["message"].lower()
mock_run.assert_not_called()
@pytest.mark.asyncio
async def test_stream_true_returns_sse(self, adapter):
"""stream=true returns SSE format with the full response."""
@ -1948,6 +2308,35 @@ class TestResponsesEndpoint:
assert data["output"][0]["content"][0]["type"] == "output_text"
assert data["output"][0]["content"][0]["text"] == "Paris is the capital of France."
@pytest.mark.asyncio
async def test_response_passes_request_model_provider_options(self, adapter):
app = _create_app(adapter)
model_options = {
"reasoning": {"enabled": True, "effort": "medium"},
"service_tier": "priority",
}
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
mock_run.return_value = (
{"final_response": "ok", "messages": [], "api_calls": 1},
{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
)
resp = await cli.post(
"/v1/responses",
json={
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": model_options,
"input": "hi",
},
)
assert resp.status == 200
kwargs = mock_run.call_args.kwargs
assert kwargs["requested_model"] == "MiniMax-M3"
assert kwargs["requested_provider"] == "minimax"
assert kwargs["model_options"] == model_options
@pytest.mark.asyncio
async def test_successful_response_with_array_input(self, adapter):
"""Array input with role/content objects."""
@ -4658,15 +5047,21 @@ class TestModelRoutesAgentCreation:
monkeypatch.setattr(
adapter,
"_session_model_override_for",
lambda key: {"model": "session/override-model"},
lambda key: {
"model": "session/override-model",
"provider": "sessionprov",
"api_key": "sk-session",
"base_url": "https://session.example/v1",
"api_mode": "responses",
"credential_pool": "pool-session",
},
)
adapter._create_agent(session_id="s1", route=adapter._resolve_route("alias"))
# The route must NOT be applied — the session override path (global
# runtime here, since the gateway applies /model separately) wins.
assert captured["model"] == "global/model"
assert captured["api_key"] == "sk-global"
assert captured["model"] == "session/override-model"
assert captured["provider"] == "sessionprov"
assert captured["api_key"] == "sk-session"
def test_session_override_lookup_reads_gateway_runner(self, monkeypatch):
"""_session_model_override_for consults GatewayRunner._session_model_overrides."""

67
tests/gateway/test_api_server_runs.py
View file

@ -255,6 +255,73 @@ class TestStartRun:
)
assert resp.status == 202
@pytest.mark.asyncio
async def test_start_rejects_conflicting_route_and_request_provider(self):
adapter = APIServerAdapter(
PlatformConfig(
enabled=True,
extra={
"model_routes": {
"alias": {
"model": "route/model",
"provider": "openrouter",
}
}
},
)
)
app = _create_runs_app(adapter)
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_create_agent") as mock_create:
resp = await cli.post(
"/v1/runs",
json={
"input": "hello",
"model": "alias",
"provider": "minimax",
},
)
data = await resp.json()
assert resp.status == 400
assert "provider" in data["error"]["message"].lower()
assert adapter._run_streams == {}
assert adapter._run_statuses == {}
mock_create.assert_not_called()
@pytest.mark.asyncio
async def test_start_passes_request_model_provider_options_to_create_agent(self, adapter):
app = _create_runs_app(adapter)
model_options = {"reasoning_effort": "medium", "service_tier": "priority"}
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_create_agent") as mock_create:
mock_agent = MagicMock()
mock_agent.run_conversation.return_value = {"final_response": "done"}
mock_agent.session_prompt_tokens = 0
mock_agent.session_completion_tokens = 0
mock_agent.session_total_tokens = 0
mock_create.return_value = mock_agent
resp = await cli.post(
"/v1/runs",
json={
"input": "hello",
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": model_options,
},
)
assert resp.status == 202
for _ in range(20):
if mock_create.call_args is not None:
break
await asyncio.sleep(0.05)
kwargs = mock_create.call_args.kwargs
assert kwargs["requested_model"] == "MiniMax-M3"
assert kwargs["requested_provider"] == "minimax"
assert kwargs["model_options"] == model_options
# ---------------------------------------------------------------------------
# GET /v1/runs/{run_id} — poll run status

50
website/docs/user-guide/features/api-server.md
View file

@ -221,6 +221,52 @@ Returns a machine-readable description of the API server's stable surface for ex
Use this endpoint when integrating dashboards, browser UIs, or control planes so they can discover whether the running Hermes version supports runs, streaming, cancellation, and session continuity without depending on private Python internals.
## Per-request model selection
Authenticated clients can override Hermes' default model selection per request
by sending:
- `model` — the target model id for this turn
- `provider` — the Hermes provider slug to resolve credentials/runtime for this turn
- `model_options` — request-scoped reasoning / service-tier controls
The same request fields are accepted on:
- `POST /v1/chat/completions`
- `POST /v1/responses`
- `POST /v1/runs`
- `POST /api/sessions/{session_id}/chat`
- `POST /api/sessions/{session_id}/chat/stream`
Precedence is deterministic:
1. Session `/model` override, if that session already has one
2. A static `gateway.platforms.api_server.model_routes` mapping selected when
the request's `model` is a configured route alias
3. Direct request `model` / `provider` when no route alias matches
4. Global gateway config / environment defaults
`model_options` stays request-scoped regardless of which model/provider wins.
If a request sends a `provider` that conflicts with a configured `model_routes`
alias, Hermes rejects the request with `400` instead of silently remixing route
credentials with another provider.
Example:
```json
{
"model": "MiniMax-M3",
"provider": "minimax",
"model_options": {
"reasoning_effort": "high",
"service_tier": "priority"
},
"messages": [
{"role": "user", "content": "Summarize the repo status."}
]
}
```
### GET /health
Health check. Returns `{"status": "ok"}`. Also available at **GET /v1/health** for OpenAI-compatible clients that expect the `/v1/` prefix.
@ -521,7 +567,9 @@ In Open WebUI, add each as a separate connection. The model dropdown shows `alic
- **Response storage** — stored responses (for `previous_response_id`) are persisted in SQLite and survive gateway restarts. Max 100 stored responses (LRU eviction).
- **No file upload** — inline images are supported on both `/v1/chat/completions` and `/v1/responses`, but uploaded files (`file`, `input_file`, `file_id`) and non-image document inputs are not supported through the API.
- **Model field is cosmetic** — the `model` field in requests is accepted but the actual LLM model used is configured server-side in config.yaml.
- **Simple OpenAI clients still see an alias** — `/v1/models` advertises the
stable Hermes alias (`hermes-agent` or the active profile name). Richer
clients can send explicit `provider` / `model_options` overrides on requests.
## Proxy Mode