diff --git a/contributors/emails/beingsabundant@gmail.com b/contributors/emails/beingsabundant@gmail.com new file mode 100644 index 000000000000..a54f6effbc82 --- /dev/null +++ b/contributors/emails/beingsabundant@gmail.com @@ -0,0 +1 @@ +abundantbeing diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index e9d79bf71d56..6a41fba03c9c 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -168,6 +168,157 @@ def _coerce_request_bool(value: Any, default: bool = False) -> bool: return default +_REQUEST_OPTION_MISSING = object() +_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) +_RUNTIME_AGENT_OVERRIDE_KEYS = ( + "api_key", + "base_url", + "provider", + "api_mode", + "command", + "args", + "credential_pool", + "max_tokens", +) + + +def _clean_request_string(value: Any) -> Optional[str]: + """Return a stripped request string, or None for absent/non-string values.""" + if not isinstance(value, str): + return None + cleaned = value.strip() + return cleaned or None + + +def _request_reasoning_config(model_options: Any) -> Optional[Dict[str, Any]]: + """Translate browser/API model_options into AIAgent reasoning_config. + + The browser extension sends both a structured ``reasoning`` object and a + compatibility ``reasoning_effort`` scalar. Keep this parser permissive so + older clients can send either shape, but ignore unknown effort values rather + than raising on a chat request. + """ + if not isinstance(model_options, dict): + return None + + reasoning = model_options.get("reasoning") + enabled: Any = None + effort: Any = model_options.get("reasoning_effort") + if isinstance(reasoning, dict): + enabled = reasoning.get("enabled") + effort = reasoning.get("effort", effort) + + effort_norm = str(effort).strip().lower() if effort is not None else "" + if enabled is False or effort_norm == "none": + return {"enabled": False} + if effort_norm in _REASONING_EFFORTS and effort_norm != "none": + return {"enabled": True, "effort": effort_norm} + if enabled is True: + return {"enabled": True} + return None + + +def _request_service_tier(model_options: Any) -> Any: + """Return a per-request service_tier override or _REQUEST_OPTION_MISSING.""" + if not isinstance(model_options, dict): + return _REQUEST_OPTION_MISSING + if "service_tier" in model_options: + raw_tier = model_options.get("service_tier") + if raw_tier is None: + return None + if isinstance(raw_tier, str): + return raw_tier.strip() or None + return raw_tier + if "fast" in model_options: + return "priority" if _coerce_request_bool(model_options.get("fast"), default=False) else None + return _REQUEST_OPTION_MISSING + + +def _apply_runtime_agent_overrides( + runtime_kwargs: Dict[str, Any], overrides: Optional[Dict[str, Any]] +) -> Dict[str, Any]: + """Merge resolved provider/runtime fields into ``runtime_kwargs`` in place.""" + if not isinstance(overrides, dict): + return runtime_kwargs + for key in _RUNTIME_AGENT_OVERRIDE_KEYS: + if key not in overrides: + continue + value = overrides.get(key) + if value is None: + continue + runtime_kwargs[key] = list(value) if key == "args" and isinstance(value, (list, tuple)) else value + return runtime_kwargs + + +def _resolve_request_runtime_agent_kwargs(provider: str, target_model: Optional[str] = None) -> Dict[str, Any]: + """Resolve runtime kwargs for a one-request provider override. + + This mirrors gateway.run._resolve_runtime_agent_kwargs(), but accepts an + explicit provider/model so an API caller can use the same authenticated + provider catalog as the TUI without mutating config.yaml. + """ + from hermes_cli.runtime_provider import resolve_runtime_provider, format_runtime_provider_error, _get_model_config + + try: + runtime = resolve_runtime_provider(requested=provider, target_model=target_model) + except Exception as exc: + raise RuntimeError(format_runtime_provider_error(exc)) from exc + + model_cfg = _get_model_config() + max_tokens = None + env_max_tokens = os.environ.get("HERMES_MAX_TOKENS") + if env_max_tokens: + try: + max_tokens = int(env_max_tokens) + except (ValueError, TypeError): + max_tokens = None + elif isinstance(model_cfg, dict): + cfg_max_tokens = model_cfg.get("max_tokens") + if isinstance(cfg_max_tokens, int): + max_tokens = cfg_max_tokens + if max_tokens is None: + runtime_max_tokens = runtime.get("max_output_tokens") + if isinstance(runtime_max_tokens, int) and runtime_max_tokens > 0: + max_tokens = runtime_max_tokens + + return { + "api_key": runtime.get("api_key"), + "base_url": runtime.get("base_url"), + "provider": runtime.get("provider"), + "api_mode": runtime.get("api_mode"), + "command": runtime.get("command"), + "args": list(runtime.get("args") or []), + "credential_pool": runtime.get("credential_pool"), + "max_tokens": max_tokens, + } + + +def _request_agent_overrides(body: Any, *, virtual_model: Optional[str] = None) -> Dict[str, Any]: + """Extract per-request model/provider/options for _run_agent. + + ``/v1/models`` advertises a stable virtual model (usually ``hermes-agent``) + for OpenAI-compatible clients. Treat that alias as "use the gateway + default"; real model picker selections from the browser extension send the + raw provider model id plus a provider slug and should override this turn. + """ + if not isinstance(body, dict): + return {} + + overrides: Dict[str, Any] = {} + model = _clean_request_string(body.get("model")) + if model and model != virtual_model: + overrides["requested_model"] = model + + provider = _clean_request_string(body.get("provider")) + if provider: + overrides["requested_provider"] = provider + + model_options = body.get("model_options") + if isinstance(model_options, dict): + overrides["model_options"] = dict(model_options) + return overrides + + def _message_text_prefix(content: Any) -> str: if isinstance(content, str): return content[:128] @@ -1814,11 +1965,56 @@ class APIServerAdapter(BasePlatformAdapter): runner = _gateway_runner_ref() if runner is None: return None + try: + rehydrate = getattr(runner, "_rehydrate_session_model_override", None) + if callable(rehydrate): + rehydrate(session_key) + except Exception: + logger.debug( + "api_server failed to rehydrate session /model override for %s", + session_key, + exc_info=True, + ) override = runner._session_model_overrides.get(session_key) return dict(override) if isinstance(override, dict) else None except Exception: return None + def _request_route_conflict_error( + self, + *, + session_id: Optional[str], + gateway_session_key: Optional[str], + requested_model: Optional[str], + requested_provider: Optional[str], + route: Optional[Dict[str, Any]], + ) -> Optional[str]: + """Return a 400-worthy conflict string for ambiguous route/provider mixes.""" + request_provider = _clean_request_string(requested_provider) + if not request_provider or not isinstance(route, dict): + return None + if self._session_model_override_for(gateway_session_key or session_id): + # Session /model wins over both the route and the request override, so + # there is no ambiguity to reject on this request path. + return None + + route_provider = _clean_request_string(route.get("provider")) + route_api_key = _clean_request_string(route.get("api_key")) + route_base_url = _clean_request_string(route.get("base_url")) + route_alias = _clean_request_string(requested_model) or "requested model" + + if route_provider and request_provider != route_provider: + return ( + f"Model route '{route_alias}' is pinned to provider '{route_provider}'. " + f"Remove 'provider' or use '{route_provider}'." + ) + if not route_provider and (route_api_key or route_base_url): + return ( + f"Model route '{route_alias}' pins route credentials/base_url. " + "Do not combine it with an explicit 'provider'." + ) + return None + def _create_agent( self, ephemeral_system_prompt: Optional[str] = None, @@ -1828,6 +2024,9 @@ class APIServerAdapter(BasePlatformAdapter): tool_start_callback=None, tool_complete_callback=None, gateway_session_key: Optional[str] = None, + requested_model: Optional[str] = None, + requested_provider: Optional[str] = None, + model_options: Optional[Dict[str, Any]] = None, route: Optional[Dict[str, Any]] = None, ) -> Any: """ @@ -1877,51 +2076,105 @@ class APIServerAdapter(BasePlatformAdapter): if runtime_model: model = runtime_model - # Per-client model routing (model_routes config). The route was - # resolved from the request's ``model`` field by the HTTP handler. - # Precedence (highest first): session ``/model`` override → model_routes - # route → global config — an explicit user-issued ``/model`` on the - # session always beats static per-client route config. - session_override = self._session_model_override_for( - gateway_session_key or session_id - ) - if route and not session_override: - if route.get("provider"): - # Resolve real credentials for the routed provider (mirrors - # the channel_overrides path in gateway/run.py) so a route - # without an explicit api_key/base_url still gets the right - # provider auth instead of the default provider's key. + request_reasoning_config = _request_reasoning_config(model_options) + if request_reasoning_config is not None: + reasoning_config = request_reasoning_config + request_service_tier = _request_service_tier(model_options) + + request_model = _clean_request_string(requested_model) + request_provider = _clean_request_string(requested_provider) + route_model = _clean_request_string(route.get("model")) if isinstance(route, dict) else None + route_provider = _clean_request_string(route.get("provider")) if isinstance(route, dict) else None + route_api_key = _clean_request_string(route.get("api_key")) if isinstance(route, dict) else None + route_base_url = _clean_request_string(route.get("base_url")) if isinstance(route, dict) else None + + def _resolve_provider_runtime( + provider: Optional[str], + *, + target_model: Optional[str], + required: bool, + ) -> Optional[Dict[str, Any]]: + provider_name = _clean_request_string(provider) + if not provider_name: + return None + try: + return _resolve_request_runtime_agent_kwargs( + provider_name, + target_model=target_model or None, + ) + except Exception: try: from gateway.run import _resolve_runtime_agent_kwargs_for_provider - provider_kwargs = _resolve_runtime_agent_kwargs_for_provider( - route["provider"] - ) - provider_kwargs.pop("model", None) - runtime_kwargs.update(provider_kwargs) - except Exception: - # Fall back to just switching the provider name; explicit - # per-route api_key/base_url below can still complete auth. - runtime_kwargs["provider"] = route["provider"] - if route.get("model"): - model = route["model"] - # Per-route secrets are upstream provider credentials. Never log - # them (compare _check_auth: caller auth stays the global bearer - # key checked with hmac.compare_digest). - if route.get("api_key"): - runtime_kwargs["api_key"] = route["api_key"] - if route.get("base_url"): - runtime_kwargs["base_url"] = route["base_url"] - logger.debug( - "api_server model route applied: model=%s provider=%s", - model, - runtime_kwargs.get("provider"), - ) - elif route and session_override: - logger.debug( - "api_server model route skipped: session /model override wins for %s", - gateway_session_key or session_id, - ) + return _resolve_runtime_agent_kwargs_for_provider(provider_name) + except Exception: + pass + if required: + raise + logger.debug( + "api_server provider-runtime refresh failed for provider=%s model=%s", + provider_name, + target_model or "", + exc_info=True, + ) + return None + + # Final precedence mirrors the gateway contract: + # session /model override → model_routes mapping selected by the request + # model alias → direct per-request provider/model → global defaults. + # model_options stay request-scoped regardless of which selection wins. + session_key = gateway_session_key or session_id + session_override = self._session_model_override_for(session_key) + if session_override: + session_model = _clean_request_string(session_override.get("model")) or model + session_provider = _clean_request_string(session_override.get("provider")) + current_provider = _clean_request_string(runtime_kwargs.get("provider")) + provider_runtime = _resolve_provider_runtime( + session_provider or current_provider, + target_model=session_model, + required=False, + ) + if provider_runtime: + _apply_runtime_agent_overrides(runtime_kwargs, provider_runtime) + _apply_runtime_agent_overrides(runtime_kwargs, session_override) + model = session_model + if route or request_model or request_provider: + logger.debug( + "api_server request selection skipped: session /model override wins for %s", + session_key or "", + ) + else: + effective_model = route_model or request_model or model + current_provider = _clean_request_string(runtime_kwargs.get("provider")) + effective_provider = request_provider or route_provider or current_provider + provider_runtime = None + if effective_provider and ( + bool(request_provider or route_provider) or effective_model != model + ): + provider_runtime = _resolve_provider_runtime( + effective_provider, + target_model=effective_model, + required=bool(request_provider), + ) + if provider_runtime: + _apply_runtime_agent_overrides(runtime_kwargs, provider_runtime) + elif effective_provider and effective_provider != current_provider: + runtime_kwargs["provider"] = effective_provider + model = effective_model + # Per-route explicit transport secrets/base URLs win within the + # route contract after provider resolution. + if route_api_key: + runtime_kwargs["api_key"] = route_api_key + if route_base_url: + runtime_kwargs["base_url"] = route_base_url + if route: + logger.debug( + "api_server request selection applied: model=%s provider=%s route_provider=%s request_provider=%s", + model, + runtime_kwargs.get("provider"), + route_provider or "", + request_provider or "", + ) user_config = _load_gateway_config() enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server")) @@ -1931,26 +2184,30 @@ class APIServerAdapter(BasePlatformAdapter): # same fallback behaviour as Telegram/Discord/Slack (fixes #4954). fallback_model = GatewayRunner._load_fallback_model() - agent = AIAgent( - model=model, + agent_kwargs = { + "model": model, **runtime_kwargs, **_checkpoint_agent_kwargs(user_config), - max_iterations=max_iterations, - quiet_mode=True, - verbose_logging=False, - ephemeral_system_prompt=ephemeral_system_prompt or None, - enabled_toolsets=enabled_toolsets, - session_id=session_id, - platform="api_server", - stream_delta_callback=stream_delta_callback, - tool_progress_callback=tool_progress_callback, - tool_start_callback=tool_start_callback, - tool_complete_callback=tool_complete_callback, - session_db=self._ensure_session_db(), - fallback_model=fallback_model, - reasoning_config=reasoning_config, - gateway_session_key=gateway_session_key, - ) + "max_iterations": max_iterations, + "quiet_mode": True, + "verbose_logging": False, + "ephemeral_system_prompt": ephemeral_system_prompt or None, + "enabled_toolsets": enabled_toolsets, + "session_id": session_id, + "platform": "api_server", + "stream_delta_callback": stream_delta_callback, + "tool_progress_callback": tool_progress_callback, + "tool_start_callback": tool_start_callback, + "tool_complete_callback": tool_complete_callback, + "session_db": self._ensure_session_db(), + "fallback_model": fallback_model, + "reasoning_config": reasoning_config, + "gateway_session_key": gateway_session_key, + } + if request_service_tier is not _REQUEST_OPTION_MISSING: + agent_kwargs["service_tier"] = request_service_tier + + agent = AIAgent(**agent_kwargs) return agent # ------------------------------------------------------------------ @@ -2565,6 +2822,17 @@ class APIServerAdapter(BasePlatformAdapter): system_prompt = body.get("system_message") or body.get("instructions") if system_prompt is not None and not isinstance(system_prompt, str): return web.json_response(_openai_error("system_message must be a string", code="invalid_system_message"), status=400) + route = self._resolve_route(body.get("model")) + agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name) + selection_error = self._request_route_conflict_error( + session_id=session_id, + gateway_session_key=gateway_session_key, + requested_model=agent_overrides.get("requested_model"), + requested_provider=agent_overrides.get("requested_provider"), + route=route, + ) + if selection_error: + return web.json_response(_openai_error(selection_error), status=400) history = await self._conversation_history_for_session(session_id) result, usage = await self._run_agent( user_message=user_message, @@ -2572,6 +2840,8 @@ class APIServerAdapter(BasePlatformAdapter): ephemeral_system_prompt=system_prompt, session_id=session_id, gateway_session_key=gateway_session_key, + route=route, + **agent_overrides, ) effective_session_id = result.get("session_id") if isinstance(result, dict) else session_id final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "") @@ -2607,6 +2877,17 @@ class APIServerAdapter(BasePlatformAdapter): system_prompt = body.get("system_message") or body.get("instructions") if system_prompt is not None and not isinstance(system_prompt, str): return web.json_response(_openai_error("system_message must be a string", code="invalid_system_message"), status=400) + route = self._resolve_route(body.get("model")) + agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name) + selection_error = self._request_route_conflict_error( + session_id=session_id, + gateway_session_key=gateway_session_key, + requested_model=agent_overrides.get("requested_model"), + requested_provider=agent_overrides.get("requested_provider"), + route=route, + ) + if selection_error: + return web.json_response(_openai_error(selection_error), status=400) loop = asyncio.get_running_loop() queue: "asyncio.Queue[Optional[tuple[str, Dict[str, Any]]]]" = asyncio.Queue() @@ -2658,10 +2939,12 @@ class APIServerAdapter(BasePlatformAdapter): conversation_history=history, ephemeral_system_prompt=system_prompt, session_id=session_id, - stream_delta_callback=_delta, - tool_progress_callback=_tool_progress, - gateway_session_key=gateway_session_key, - ) + stream_delta_callback=_delta, + tool_progress_callback=_tool_progress, + gateway_session_key=gateway_session_key, + route=route, + **agent_overrides, + ) final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "") effective_session_id = result.get("session_id", session_id) if isinstance(result, dict) else session_id turn_messages = self._turn_transcript_messages(history, user_message, result) if isinstance(result, dict) else [] @@ -2861,6 +3144,16 @@ class APIServerAdapter(BasePlatformAdapter): # configured model_routes alias, this request's agent is created # with that route's model/provider instead of the global default. route = self._resolve_route(model_name) + agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name) + selection_error = self._request_route_conflict_error( + session_id=session_id, + gateway_session_key=gateway_session_key, + requested_model=agent_overrides.get("requested_model"), + requested_provider=agent_overrides.get("requested_provider"), + route=route, + ) + if selection_error: + return web.json_response(_openai_error(selection_error), status=400) if stream: import queue as _q @@ -2944,6 +3237,7 @@ class APIServerAdapter(BasePlatformAdapter): tool_complete_callback=_on_tool_complete, agent_ref=agent_ref, gateway_session_key=gateway_session_key, + **agent_overrides, route=route, )) # Ensure SSE drain loops can terminate without relying on polling @@ -2964,12 +3258,16 @@ class APIServerAdapter(BasePlatformAdapter): ephemeral_system_prompt=system_prompt, session_id=session_id, gateway_session_key=gateway_session_key, + **agent_overrides, route=route, ) idempotency_key = request.headers.get("Idempotency-Key") if idempotency_key: - fp = _make_request_fingerprint(body, keys=["model", "messages", "tools", "tool_choice", "stream"]) + fp = _make_request_fingerprint( + body, + keys=["model", "provider", "model_options", "messages", "tools", "tool_choice", "stream"], + ) try: result, usage = await _idem_cache.get_or_set(idempotency_key, fp, _compute_completion) except Exception as e: @@ -3979,10 +4277,18 @@ class APIServerAdapter(BasePlatformAdapter): # groups the entire conversation under one session entry. session_id = stored_session_id or str(uuid.uuid4()) - # Per-client model routing for /v1/responses (see model_routes). - route = self._resolve_route(body.get("model")) - stream = _coerce_request_bool(body.get("stream"), default=False) + route = self._resolve_route(body.get("model")) + agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name) + selection_error = self._request_route_conflict_error( + session_id=session_id, + gateway_session_key=gateway_session_key, + requested_model=agent_overrides.get("requested_model"), + requested_provider=agent_overrides.get("requested_provider"), + route=route, + ) + if selection_error: + return web.json_response(_openai_error(selection_error), status=400) if stream: # Streaming branch — emit OpenAI Responses SSE events as the # agent runs so frontends can render text deltas and tool @@ -4035,6 +4341,7 @@ class APIServerAdapter(BasePlatformAdapter): tool_complete_callback=_on_tool_complete, agent_ref=agent_ref, gateway_session_key=gateway_session_key, + **agent_overrides, route=route, )) # Ensure SSE drain loops can terminate without relying on polling @@ -4069,6 +4376,7 @@ class APIServerAdapter(BasePlatformAdapter): ephemeral_system_prompt=instructions, session_id=session_id, gateway_session_key=gateway_session_key, + **agent_overrides, route=route, ) @@ -4076,7 +4384,16 @@ class APIServerAdapter(BasePlatformAdapter): if idempotency_key: fp = _make_request_fingerprint( body, - keys=["input", "instructions", "previous_response_id", "conversation", "model", "tools"], + keys=[ + "input", + "instructions", + "previous_response_id", + "conversation", + "model", + "provider", + "model_options", + "tools", + ], ) try: result, usage = await _idem_cache.get_or_set(idempotency_key, fp, _compute_response) @@ -4747,6 +5064,9 @@ class APIServerAdapter(BasePlatformAdapter): tool_complete_callback=None, agent_ref: Optional[list] = None, gateway_session_key: Optional[str] = None, + requested_model: Optional[str] = None, + requested_provider: Optional[str] = None, + model_options: Optional[Dict[str, Any]] = None, route: Optional[Dict[str, Any]] = None, ) -> tuple: """ @@ -4788,6 +5108,9 @@ class APIServerAdapter(BasePlatformAdapter): tool_start_callback=tool_start_callback, tool_complete_callback=tool_complete_callback, gateway_session_key=gateway_session_key, + requested_model=requested_model, + requested_provider=requested_provider, + model_options=model_options, route=route, ) if agent_ref is not None: @@ -4974,8 +5297,21 @@ class APIServerAdapter(BasePlatformAdapter): ) conversation_history.append({"role": msg["role"], "content": str(content)}) + session_id = body.get("session_id") or stored_session_id + route = self._resolve_route(body.get("model")) + agent_overrides = _request_agent_overrides(body, virtual_model=self._model_name) + selection_error = self._request_route_conflict_error( + session_id=session_id, + gateway_session_key=gateway_session_key, + requested_model=agent_overrides.get("requested_model"), + requested_provider=agent_overrides.get("requested_provider"), + route=route, + ) + if selection_error: + return web.json_response(_openai_error(selection_error), status=400) + run_id = f"run_{uuid.uuid4().hex}" - session_id = body.get("session_id") or stored_session_id or run_id + session_id = session_id or run_id # Approval queues gate host-side tool execution and must be isolated # per API run. Client-provided session IDs and memory session keys are # conversation/memory scopes, not authorization namespaces: multiple @@ -5021,8 +5357,6 @@ class APIServerAdapter(BasePlatformAdapter): model=body.get("model", self._model_name), ) - # Per-client model routing for /v1/runs (see model_routes). - route = self._resolve_route(body.get("model")) # Background task outlives the HTTP response (and thus the middleware # profile scope). Capture now and re-enter inside the task/executor. request_profile = _api_request_profile.get() @@ -5049,6 +5383,9 @@ class APIServerAdapter(BasePlatformAdapter): stream_delta_callback=_text_cb, tool_progress_callback=event_cb, gateway_session_key=gateway_session_key, + requested_model=agent_overrides.get("requested_model"), + requested_provider=agent_overrides.get("requested_provider"), + model_options=agent_overrides.get("model_options"), route=route, ) self._active_run_agents[run_id] = agent diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 7ed7d66be6a0..a478ca8c142e 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -16,7 +16,9 @@ import asyncio import json import os import stat +import sys import time +import types import uuid from unittest.mock import AsyncMock, MagicMock, patch @@ -661,6 +663,8 @@ def _create_app(adapter: APIServerAdapter) -> web.Application: app.router.add_get("/v1/capabilities", adapter._handle_capabilities) app.router.add_get("/v1/skills", adapter._handle_skills) app.router.add_get("/v1/toolsets", adapter._handle_toolsets) + app.router.add_post("/api/sessions/{session_id}/chat", adapter._handle_session_chat) + app.router.add_post("/api/sessions/{session_id}/chat/stream", adapter._handle_session_chat_stream) app.router.add_post("/v1/chat/completions", adapter._handle_chat_completions) app.router.add_post("/v1/responses", adapter._handle_responses) app.router.add_get("/v1/responses/{response_id}", adapter._handle_get_response) @@ -711,11 +715,15 @@ class TestAgentExecution: mock_agent.session_completion_tokens = 2 mock_agent.session_total_tokens = 3 - with patch.object(adapter, "_create_agent", return_value=mock_agent): + model_options = {"reasoning": {"enabled": False}, "fast": False} + with patch.object(adapter, "_create_agent", return_value=mock_agent) as mock_create_agent: result, usage = await adapter._run_agent( user_message="hello", conversation_history=[], session_id="session-123", + requested_model="MiniMax-M3", + requested_provider="minimax", + model_options=model_options, ) # _run_agent annotates result with the effective agent.session_id @@ -725,12 +733,161 @@ class TestAgentExecution: # the annotation — header will fall back to the provided session_id. assert result["final_response"] == "ok" assert usage == {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3} + create_kwargs = mock_create_agent.call_args.kwargs + assert create_kwargs["requested_model"] == "MiniMax-M3" + assert create_kwargs["requested_provider"] == "minimax" + assert create_kwargs["model_options"] == model_options mock_agent.run_conversation.assert_called_once_with( user_message="hello", conversation_history=[], task_id="session-123", ) + def test_create_agent_honors_request_model_provider_and_options(self, adapter, monkeypatch): + import gateway.run as gateway_run + import hermes_cli.runtime_provider as runtime_provider + import hermes_cli.tools_config as tools_config + + class _CapturingAgent: + last_kwargs = None + + def __init__(self, **kwargs): + type(self).last_kwargs = dict(kwargs) + + fake_run_agent = types.ModuleType("run_agent") + fake_run_agent.AIAgent = _CapturingAgent + monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent) + + monkeypatch.setattr(gateway_run, "_current_max_iterations", lambda: 7) + monkeypatch.setattr(gateway_run, "_resolve_gateway_model", lambda: "gpt-5.5") + monkeypatch.setattr(gateway_run, "_load_gateway_config", lambda: {}) + monkeypatch.setattr( + gateway_run, + "_resolve_runtime_agent_kwargs", + lambda: { + "api_key": "codex-key", + "base_url": "https://chatgpt.com/backend-api/codex", + "provider": "openai-codex", + "api_mode": "codex_responses", + "command": None, + "args": [], + "credential_pool": None, + "max_tokens": None, + }, + ) + monkeypatch.setattr(gateway_run.GatewayRunner, "_load_reasoning_config", staticmethod(lambda: {"enabled": True, "effort": "medium"})) + monkeypatch.setattr(gateway_run.GatewayRunner, "_load_fallback_model", staticmethod(lambda: None)) + monkeypatch.setattr(tools_config, "_get_platform_tools", lambda _cfg, _platform: {"web"}) + monkeypatch.setattr(adapter, "_ensure_session_db", lambda: None) + + def _fake_resolve_runtime_provider(*, requested=None, target_model=None, **_kwargs): + assert requested == "minimax" + assert target_model == "MiniMax-M3" + return { + "api_key": "minimax-key", + "base_url": "https://api.minimax.io/v1", + "provider": "minimax", + "api_mode": "anthropic_messages", + "command": None, + "args": [], + "credential_pool": None, + "max_output_tokens": 32000, + } + + monkeypatch.setattr(runtime_provider, "resolve_runtime_provider", _fake_resolve_runtime_provider) + monkeypatch.setattr(runtime_provider, "_get_model_config", lambda: {}) + + adapter._create_agent( + session_id="session-123", + requested_model="MiniMax-M3", + requested_provider="minimax", + model_options={ + "reasoning": {"enabled": True, "effort": "high"}, + "reasoning_effort": "high", + "fast": True, + }, + ) + + kwargs = _CapturingAgent.last_kwargs + assert kwargs is not None + assert kwargs["model"] == "MiniMax-M3" + assert kwargs["provider"] == "minimax" + assert kwargs["api_mode"] == "anthropic_messages" + assert kwargs["base_url"] == "https://api.minimax.io/v1" + assert kwargs["api_key"] == "minimax-key" + assert kwargs["max_tokens"] == 32000 + assert kwargs["reasoning_config"] == {"enabled": True, "effort": "high"} + assert kwargs["service_tier"] == "priority" + assert kwargs["enabled_toolsets"] == ["web"] + + def test_create_agent_session_override_beats_request_and_route_but_keeps_model_options( + self, monkeypatch + ): + captured = {} + + class FakeAgent: + def __init__(self, **kwargs): + captured.update(kwargs) + + _patch_create_agent_runtime(monkeypatch, captured, FakeAgent) + monkeypatch.setattr( + "hermes_cli.runtime_provider.resolve_runtime_provider", + lambda requested=None, target_model=None, **_kwargs: { + "api_key": f"sk-{requested}", + "base_url": f"https://{requested}.example/v1", + "provider": requested, + "api_mode": "chat_completions", + "command": None, + "args": [], + "credential_pool": f"pool-{requested}", + "max_output_tokens": 64000, + }, + ) + monkeypatch.setattr("hermes_cli.runtime_provider._get_model_config", lambda: {}) + + adapter = _make_routing_adapter( + { + "alias": { + "model": "route/model", + "api_key": "sk-route", + "base_url": "https://route.example/v1", + } + } + ) + monkeypatch.setattr(adapter, "_ensure_session_db", lambda: None) + monkeypatch.setattr( + adapter, + "_session_model_override_for", + lambda *_: { + "model": "session/model", + "provider": "sessionprov", + "api_key": "sk-session", + "base_url": "https://session.example/v1", + "api_mode": "responses", + "credential_pool": "pool-session", + }, + ) + + adapter._create_agent( + session_id="session-123", + route=adapter._resolve_route("alias"), + requested_model="MiniMax-M3", + requested_provider="minimax", + model_options={ + "reasoning": {"enabled": True, "effort": "high"}, + "fast": True, + }, + ) + + assert captured["model"] == "session/model" + assert captured["provider"] == "sessionprov" + assert captured["api_key"] == "sk-session" + assert captured["base_url"] == "https://session.example/v1" + assert captured["api_mode"] == "responses" + assert captured["credential_pool"] == "pool-session" + assert captured["reasoning_config"] == {"enabled": True, "effort": "high"} + assert captured["service_tier"] == "priority" + # --------------------------------------------------------------------------- # /health endpoint @@ -1246,6 +1403,209 @@ class TestChatCompletionsEndpoint: resp = await cli.post("/v1/chat/completions", json={"model": "test", "messages": []}) assert resp.status == 400 + @pytest.mark.asyncio + async def test_chat_completions_passes_request_model_provider_options(self, adapter): + app = _create_app(adapter) + model_options = { + "reasoning": {"enabled": True, "effort": "high"}, + "reasoning_effort": "high", + "service_tier": "priority", + "fast": True, + } + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = ( + {"final_response": "ok", "messages": [], "api_calls": 1}, + {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + resp = await cli.post( + "/v1/chat/completions", + json={ + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": model_options, + "messages": [{"role": "user", "content": "hi"}], + }, + ) + + assert resp.status == 200 + kwargs = mock_run.call_args.kwargs + assert kwargs["requested_model"] == "MiniMax-M3" + assert kwargs["requested_provider"] == "minimax" + assert kwargs["model_options"] == model_options + + @pytest.mark.asyncio + async def test_chat_completions_stream_passes_request_model_provider_options(self, adapter): + app = _create_app(adapter) + model_options = {"reasoning": {"enabled": False}, "reasoning_effort": "none", "fast": False} + + async def _mock_run_agent(**kwargs): + cb = kwargs.get("stream_delta_callback") + if cb: + cb("ok") + return ( + {"final_response": "ok", "messages": [], "api_calls": 1}, + {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", side_effect=_mock_run_agent) as mock_run: + resp = await cli.post( + "/v1/chat/completions", + json={ + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": model_options, + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }, + ) + assert resp.status == 200 + body = await resp.text() + + assert "data: " in body + kwargs = mock_run.call_args.kwargs + assert kwargs["requested_model"] == "MiniMax-M3" + assert kwargs["requested_provider"] == "minimax" + assert kwargs["model_options"] == model_options + + @pytest.mark.asyncio + async def test_session_chat_passes_request_model_provider_options(self, adapter): + app = _create_app(adapter) + model_options = {"reasoning": {"enabled": True, "effort": "low"}, "fast": True} + async with TestClient(TestServer(app)) as cli: + with ( + patch.object(adapter, "_get_existing_session_or_404", return_value=({"id": "s1"}, None)), + patch.object(adapter, "_conversation_history_for_session", return_value=[]), + patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run, + ): + mock_run.return_value = ( + {"final_response": "ok", "messages": [], "api_calls": 1}, + {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + resp = await cli.post( + "/api/sessions/s1/chat", + json={ + "message": "hi", + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": model_options, + }, + ) + + assert resp.status == 200 + kwargs = mock_run.call_args.kwargs + assert kwargs["requested_model"] == "MiniMax-M3" + assert kwargs["requested_provider"] == "minimax" + assert kwargs["model_options"] == model_options + + @pytest.mark.asyncio + async def test_session_chat_stream_passes_request_model_provider_options(self, adapter): + app = _create_app(adapter) + model_options = {"reasoning_effort": "medium", "service_tier": "priority"} + async with TestClient(TestServer(app)) as cli: + with ( + patch.object(adapter, "_get_existing_session_or_404", return_value=({"id": "s1"}, None)), + patch.object(adapter, "_conversation_history_for_session", return_value=[]), + patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run, + ): + mock_run.return_value = ( + {"final_response": "ok", "messages": [], "api_calls": 1}, + {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + resp = await cli.post( + "/api/sessions/s1/chat/stream", + json={ + "message": "hi", + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": model_options, + }, + ) + assert resp.status == 200 + body = await resp.text() + + assert "event: run.completed" in body + kwargs = mock_run.call_args.kwargs + assert kwargs["requested_model"] == "MiniMax-M3" + assert kwargs["requested_provider"] == "minimax" + assert kwargs["model_options"] == model_options + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("path", "body", "needs_session"), + [ + ( + "/v1/chat/completions", + { + "model": "alias", + "provider": "minimax", + "messages": [{"role": "user", "content": "hi"}], + }, + False, + ), + ( + "/v1/responses", + { + "model": "alias", + "provider": "minimax", + "input": "hi", + }, + False, + ), + ( + "/api/sessions/s1/chat", + { + "model": "alias", + "provider": "minimax", + "message": "hi", + }, + True, + ), + ( + "/api/sessions/s1/chat/stream", + { + "model": "alias", + "provider": "minimax", + "message": "hi", + }, + True, + ), + ], + ) + async def test_handlers_reject_conflicting_route_and_request_provider( + self, path, body, needs_session + ): + adapter = _make_routing_adapter( + {"alias": {"model": "route/model", "provider": "openrouter"}} + ) + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + if needs_session: + with ( + patch.object( + adapter, + "_get_existing_session_or_404", + return_value=({"id": "s1"}, None), + ), + patch.object( + adapter, + "_conversation_history_for_session", + return_value=[], + ), + ): + resp = await cli.post(path, json=body) + data = await resp.json() + else: + resp = await cli.post(path, json=body) + data = await resp.json() + + assert resp.status == 400 + assert "provider" in data["error"]["message"].lower() + mock_run.assert_not_called() + @pytest.mark.asyncio async def test_stream_true_returns_sse(self, adapter): """stream=true returns SSE format with the full response.""" @@ -1948,6 +2308,35 @@ class TestResponsesEndpoint: assert data["output"][0]["content"][0]["type"] == "output_text" assert data["output"][0]["content"][0]["text"] == "Paris is the capital of France." + @pytest.mark.asyncio + async def test_response_passes_request_model_provider_options(self, adapter): + app = _create_app(adapter) + model_options = { + "reasoning": {"enabled": True, "effort": "medium"}, + "service_tier": "priority", + } + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = ( + {"final_response": "ok", "messages": [], "api_calls": 1}, + {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + resp = await cli.post( + "/v1/responses", + json={ + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": model_options, + "input": "hi", + }, + ) + + assert resp.status == 200 + kwargs = mock_run.call_args.kwargs + assert kwargs["requested_model"] == "MiniMax-M3" + assert kwargs["requested_provider"] == "minimax" + assert kwargs["model_options"] == model_options + @pytest.mark.asyncio async def test_successful_response_with_array_input(self, adapter): """Array input with role/content objects.""" @@ -4658,15 +5047,21 @@ class TestModelRoutesAgentCreation: monkeypatch.setattr( adapter, "_session_model_override_for", - lambda key: {"model": "session/override-model"}, + lambda key: { + "model": "session/override-model", + "provider": "sessionprov", + "api_key": "sk-session", + "base_url": "https://session.example/v1", + "api_mode": "responses", + "credential_pool": "pool-session", + }, ) adapter._create_agent(session_id="s1", route=adapter._resolve_route("alias")) - # The route must NOT be applied — the session override path (global - # runtime here, since the gateway applies /model separately) wins. - assert captured["model"] == "global/model" - assert captured["api_key"] == "sk-global" + assert captured["model"] == "session/override-model" + assert captured["provider"] == "sessionprov" + assert captured["api_key"] == "sk-session" def test_session_override_lookup_reads_gateway_runner(self, monkeypatch): """_session_model_override_for consults GatewayRunner._session_model_overrides.""" diff --git a/tests/gateway/test_api_server_runs.py b/tests/gateway/test_api_server_runs.py index ed0240a9ff2b..f98d38e94c23 100644 --- a/tests/gateway/test_api_server_runs.py +++ b/tests/gateway/test_api_server_runs.py @@ -255,6 +255,73 @@ class TestStartRun: ) assert resp.status == 202 + @pytest.mark.asyncio + async def test_start_rejects_conflicting_route_and_request_provider(self): + adapter = APIServerAdapter( + PlatformConfig( + enabled=True, + extra={ + "model_routes": { + "alias": { + "model": "route/model", + "provider": "openrouter", + } + } + }, + ) + ) + app = _create_runs_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_create_agent") as mock_create: + resp = await cli.post( + "/v1/runs", + json={ + "input": "hello", + "model": "alias", + "provider": "minimax", + }, + ) + data = await resp.json() + + assert resp.status == 400 + assert "provider" in data["error"]["message"].lower() + assert adapter._run_streams == {} + assert adapter._run_statuses == {} + mock_create.assert_not_called() + + @pytest.mark.asyncio + async def test_start_passes_request_model_provider_options_to_create_agent(self, adapter): + app = _create_runs_app(adapter) + model_options = {"reasoning_effort": "medium", "service_tier": "priority"} + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_create_agent") as mock_create: + mock_agent = MagicMock() + mock_agent.run_conversation.return_value = {"final_response": "done"} + mock_agent.session_prompt_tokens = 0 + mock_agent.session_completion_tokens = 0 + mock_agent.session_total_tokens = 0 + mock_create.return_value = mock_agent + + resp = await cli.post( + "/v1/runs", + json={ + "input": "hello", + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": model_options, + }, + ) + assert resp.status == 202 + for _ in range(20): + if mock_create.call_args is not None: + break + await asyncio.sleep(0.05) + + kwargs = mock_create.call_args.kwargs + assert kwargs["requested_model"] == "MiniMax-M3" + assert kwargs["requested_provider"] == "minimax" + assert kwargs["model_options"] == model_options + # --------------------------------------------------------------------------- # GET /v1/runs/{run_id} — poll run status diff --git a/website/docs/user-guide/features/api-server.md b/website/docs/user-guide/features/api-server.md index b67a896dd837..8bb22e401f0f 100644 --- a/website/docs/user-guide/features/api-server.md +++ b/website/docs/user-guide/features/api-server.md @@ -221,6 +221,52 @@ Returns a machine-readable description of the API server's stable surface for ex Use this endpoint when integrating dashboards, browser UIs, or control planes so they can discover whether the running Hermes version supports runs, streaming, cancellation, and session continuity without depending on private Python internals. +## Per-request model selection + +Authenticated clients can override Hermes' default model selection per request +by sending: + +- `model` — the target model id for this turn +- `provider` — the Hermes provider slug to resolve credentials/runtime for this turn +- `model_options` — request-scoped reasoning / service-tier controls + +The same request fields are accepted on: + +- `POST /v1/chat/completions` +- `POST /v1/responses` +- `POST /v1/runs` +- `POST /api/sessions/{session_id}/chat` +- `POST /api/sessions/{session_id}/chat/stream` + +Precedence is deterministic: + +1. Session `/model` override, if that session already has one +2. A static `gateway.platforms.api_server.model_routes` mapping selected when + the request's `model` is a configured route alias +3. Direct request `model` / `provider` when no route alias matches +4. Global gateway config / environment defaults + +`model_options` stays request-scoped regardless of which model/provider wins. +If a request sends a `provider` that conflicts with a configured `model_routes` +alias, Hermes rejects the request with `400` instead of silently remixing route +credentials with another provider. + +Example: + +```json +{ + "model": "MiniMax-M3", + "provider": "minimax", + "model_options": { + "reasoning_effort": "high", + "service_tier": "priority" + }, + "messages": [ + {"role": "user", "content": "Summarize the repo status."} + ] +} +``` + ### GET /health Health check. Returns `{"status": "ok"}`. Also available at **GET /v1/health** for OpenAI-compatible clients that expect the `/v1/` prefix. @@ -521,7 +567,9 @@ In Open WebUI, add each as a separate connection. The model dropdown shows `alic - **Response storage** — stored responses (for `previous_response_id`) are persisted in SQLite and survive gateway restarts. Max 100 stored responses (LRU eviction). - **No file upload** — inline images are supported on both `/v1/chat/completions` and `/v1/responses`, but uploaded files (`file`, `input_file`, `file_id`) and non-image document inputs are not supported through the API. -- **Model field is cosmetic** — the `model` field in requests is accepted but the actual LLM model used is configured server-side in config.yaml. +- **Simple OpenAI clients still see an alias** — `/v1/models` advertises the + stable Hermes alias (`hermes-agent` or the active profile name). Richer + clients can send explicit `provider` / `model_options` overrides on requests. ## Proxy Mode