hermes-agent/tests/gateway/test_session_api.py
abundantbeing 7cd48733db feat(api): backend-acknowledged session model lock with runtime routing
Add a persisted, backend-confirmed provider/model lock for Hermes
Browser and other session API clients. A confirmed lock is an
execution contract rather than response metadata:

- POST /api/sessions/{session_id}/model validates and persists a
  confirmed browser_model_lock (advertised in /v1/capabilities)
- session chat + chat/stream consume the persisted lock on body-only
  follow-up turns; a confirmed lock wins over an older gateway session
  /model override and the session-persisted model
- a later successful session /model switch explicitly clears and
  replaces the lock while preserving lineage markers (_branched_from)
  and invalidating cached system-prompt model/provider metadata
- ordinary one-off request overrides never replace a confirmed lock
- provider-resolution failure fails closed as a typed provider-auth
  error (controlled response, never global-credential reuse)
- confirmed locks disable the global fallback model chain
- the completed agent's actual provider/model must match the locked
  route or the turn fails with a runtime-mismatch error
- responses carry sanitized runtime metadata reporting actual vs
  requested provider/model and lock state

Rebased onto the provider-aware request routing (#70853) and
session-model parity (#70931) that landed since the original branch;
the lock now slots into that precedence chain as the top rung.

Salvaged from PR #61236 by @abundantbeing.
2026-07-24 13:39:21 -07:00

1197 lines
48 KiB
Python

"""Focused tests for API server session-control endpoints."""
from unittest.mock import AsyncMock, patch
import pytest
from aiohttp import web
from aiohttp.test_utils import TestClient, TestServer
from gateway.config import PlatformConfig
from gateway.platforms.api_server import APIServerAdapter
from hermes_state import SessionDB
@pytest.fixture
def session_db(tmp_path):
db = SessionDB(tmp_path / "state.db")
try:
yield db
finally:
close = getattr(db, "close", None)
if callable(close):
close()
@pytest.fixture
def adapter(session_db):
adapter = APIServerAdapter(PlatformConfig(enabled=True))
adapter._session_db = session_db
return adapter
@pytest.fixture
def auth_adapter(session_db):
adapter = APIServerAdapter(PlatformConfig(enabled=True, extra={"key": "sk-test"}))
adapter._session_db = session_db
return adapter
def _create_session_app(adapter: APIServerAdapter) -> web.Application:
app = web.Application()
app.router.add_get("/v1/capabilities", adapter._handle_capabilities)
app.router.add_get("/api/sessions", adapter._handle_list_sessions)
app.router.add_post("/api/sessions", adapter._handle_create_session)
app.router.add_get("/api/sessions/{session_id}", adapter._handle_get_session)
app.router.add_patch("/api/sessions/{session_id}", adapter._handle_patch_session)
app.router.add_delete("/api/sessions/{session_id}", adapter._handle_delete_session)
app.router.add_get("/api/sessions/{session_id}/messages", adapter._handle_session_messages)
app.router.add_post("/api/sessions/{session_id}/fork", adapter._handle_fork_session)
app.router.add_post("/api/sessions/{session_id}/chat", adapter._handle_session_chat)
app.router.add_post("/api/sessions/{session_id}/chat/stream", adapter._handle_session_chat_stream)
return app
@pytest.mark.asyncio
async def test_capabilities_advertises_session_control_surface(adapter):
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.get("/v1/capabilities")
assert resp.status == 200
data = await resp.json()
features = data["features"]
assert features["session_resources"] is True
assert features["session_chat"] is True
assert features["session_chat_streaming"] is True
assert features["session_fork"] is True
assert features["admin_config_rw"] is False
assert features["memory_write_api"] is False
assert features["skills_api"] is True
assert features["realtime_voice"] is False
assert data["endpoints"]["sessions"] == {"method": "GET", "path": "/api/sessions"}
assert data["endpoints"]["session_chat_stream"] == {
"method": "POST",
"path": "/api/sessions/{session_id}/chat/stream",
}
@pytest.mark.asyncio
async def test_run_agent_binds_api_session_context_for_tool_env(adapter, monkeypatch):
"""API-server request sessions should reach tools and terminal subprocess env."""
monkeypatch.setenv("HERMES_SESSION_ID", "stale-session")
observed = {}
class FakeAgent:
session_prompt_tokens = 0
session_completion_tokens = 0
session_total_tokens = 0
def __init__(self, session_id: str):
self.session_id = session_id
def run_conversation(self, user_message, conversation_history, task_id):
from gateway.session_context import get_session_env
from tools.environments.local import _make_run_env
observed["task_id"] = task_id
observed["context_session_id"] = get_session_env("HERMES_SESSION_ID")
observed["context_platform"] = get_session_env("HERMES_SESSION_PLATFORM")
observed["context_session_key"] = get_session_env("HERMES_SESSION_KEY")
observed["child_session_id"] = _make_run_env({}).get("HERMES_SESSION_ID")
return {"final_response": "ok"}
def fake_create_agent(**kwargs):
return FakeAgent(kwargs["session_id"])
monkeypatch.setattr(adapter, "_create_agent", fake_create_agent)
result, usage = await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="request-session",
gateway_session_key="request-key",
)
assert result["session_id"] == "request-session"
assert usage["input_tokens"] == 0
assert usage["output_tokens"] == 0
assert usage["total_tokens"] == 0
assert "runtime" not in usage
assert observed == {
"task_id": "request-session",
"context_session_id": "request-session",
"context_platform": "api_server",
"context_session_key": "request-key",
"child_session_id": "request-session",
}
@pytest.mark.asyncio
async def test_session_crud_and_message_history(adapter, session_db):
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
create_resp = await cli.post("/api/sessions", json={"title": "Mobile chat", "model": "test-model"})
assert create_resp.status == 201
created = await create_resp.json()
session_id = created["session"]["id"]
assert created["object"] == "hermes.session"
assert created["session"]["title"] == "Mobile chat"
session_db.append_message(session_id, "user", "hello from phone")
session_db.append_message(session_id, "assistant", "hello from hermes")
list_resp = await cli.get("/api/sessions?limit=10&offset=0")
assert list_resp.status == 200
listed = await list_resp.json()
assert listed["object"] == "list"
assert [s["id"] for s in listed["data"]] == [session_id]
assert listed["data"][0]["message_count"] == 2
get_resp = await cli.get(f"/api/sessions/{session_id}")
assert get_resp.status == 200
got = await get_resp.json()
assert got["session"]["id"] == session_id
assert got["session"]["message_count"] == 2
messages_resp = await cli.get(f"/api/sessions/{session_id}/messages")
assert messages_resp.status == 200
messages = await messages_resp.json()
assert messages["object"] == "list"
assert [m["role"] for m in messages["data"]] == ["user", "assistant"]
assert messages["data"][0]["content"] == "hello from phone"
patch_resp = await cli.patch(f"/api/sessions/{session_id}", json={"title": "Renamed"})
assert patch_resp.status == 200
patched = await patch_resp.json()
assert patched["session"]["title"] == "Renamed"
delete_resp = await cli.delete(f"/api/sessions/{session_id}")
assert delete_resp.status == 200
deleted = await delete_resp.json()
assert deleted == {"object": "hermes.session.deleted", "id": session_id, "deleted": True}
assert session_db.get_session(session_id) is None
@pytest.mark.asyncio
async def test_session_messages_follow_compression_tip(adapter, session_db):
source_id = session_db.create_session("source-session", "api_server")
session_db.append_message(source_id, "user", "before compression")
session_db.end_session(source_id, "compression")
session_db.create_session("tip-session", "api_server", parent_session_id=source_id)
session_db.replace_messages(source_id, [])
session_db.append_message("tip-session", "user", "after compression")
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
messages_resp = await cli.get(f"/api/sessions/{source_id}/messages")
assert messages_resp.status == 200
messages = await messages_resp.json()
assert messages["object"] == "list"
assert messages["session_id"] == "tip-session"
assert [m["content"] for m in messages["data"]] == ["after compression"]
@pytest.mark.asyncio
async def test_session_fork_uses_current_sessiondb_branch_primitives(adapter, session_db):
source_id = session_db.create_session("source-session", "api_server", model="test-model")
session_db.set_session_title(source_id, "Original")
session_db.append_message(source_id, "user", "first path")
session_db.append_message(source_id, "assistant", "answer")
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(f"/api/sessions/{source_id}/fork", json={"title": "Alternative"})
assert resp.status == 201
payload = await resp.json()
fork = payload["session"]
assert payload["object"] == "hermes.session"
assert fork["id"] != source_id
assert fork["parent_session_id"] == source_id
assert fork["title"] == "Alternative"
assert [m["content"] for m in session_db.get_messages(fork["id"])] == ["first path", "answer"]
assert session_db.get_session(source_id)["end_reason"] == "branched"
@pytest.mark.asyncio
async def test_session_chat_loads_history_and_preserves_session_headers(auth_adapter, session_db):
session_id = session_db.create_session("chat-session", "api_server")
session_db.set_session_title(session_id, "Chat")
session_db.append_message(session_id, "user", "earlier")
session_db.append_message(session_id, "assistant", "prior answer")
mock_run = AsyncMock(return_value=({"final_response": "fresh answer", "session_id": session_id}, {"total_tokens": 3}))
app = _create_session_app(auth_adapter)
with patch.object(auth_adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": "next", "system_message": "stay focused"},
headers={"Authorization": "Bearer sk-test", "X-Hermes-Session-Key": "client-42"},
)
assert resp.status == 200
payload = await resp.json()
assert resp.headers["X-Hermes-Session-Id"] == session_id
assert resp.headers["X-Hermes-Session-Key"] == "client-42"
assert payload["object"] == "hermes.session.chat.completion"
assert payload["session_id"] == session_id
assert payload["message"]["role"] == "assistant"
assert payload["message"]["content"] == "fresh answer"
mock_run.assert_awaited_once()
_, kwargs = mock_run.call_args
assert kwargs["session_id"] == session_id
assert kwargs["gateway_session_key"] == "client-42"
assert kwargs["ephemeral_system_prompt"] == "stay focused"
history = kwargs["conversation_history"]
assert len(history) == 2
assert isinstance(history[0].pop("timestamp"), (int, float))
assert isinstance(history[1].pop("timestamp"), (int, float))
assert history == [
{"role": "user", "content": "earlier"},
{"role": "assistant", "content": "prior answer"},
]
@pytest.mark.asyncio
async def test_session_chat_accepts_multimodal_message(auth_adapter, session_db):
session_id = session_db.create_session("image-session", "api_server")
image_payload = [
{"type": "input_text", "text": "What's in this image?"},
{"type": "input_image", "image_url": "data:image/png;base64,AAAA"},
]
expected_user_message = [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
]
mock_run = AsyncMock(return_value=({"final_response": "A cat.", "session_id": session_id}, {"total_tokens": 4}))
app = _create_session_app(auth_adapter)
with patch.object(auth_adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": image_payload},
headers={"Authorization": "Bearer sk-test"},
)
assert resp.status == 200, await resp.text()
_, kwargs = mock_run.call_args
assert kwargs["user_message"] == expected_user_message
@pytest.mark.asyncio
async def test_session_chat_stream_accepts_multimodal_message(adapter, session_db):
session_id = session_db.create_session("image-stream-session", "api_server")
image_payload = [
{"type": "input_text", "text": "What's in this image?"},
{"type": "input_image", "image_url": "data:image/png;base64,AAAA"},
]
expected_user_message = [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
]
captured_kwargs = {}
async def fake_run(**kwargs):
captured_kwargs.update(kwargs)
kwargs["stream_delta_callback"]("A cat.")
return {"final_response": "A cat.", "session_id": session_id}, {"total_tokens": 4}
app = _create_session_app(adapter)
with patch.object(adapter, "_run_agent", side_effect=fake_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat/stream",
json={"message": image_payload},
)
assert resp.status == 200, await resp.text()
assert resp.headers["Content-Type"].startswith("text/event-stream")
body = await resp.text()
assert "event: assistant.completed" in body
assert captured_kwargs["user_message"] == expected_user_message
@pytest.mark.asyncio
async def test_session_chat_stream_emits_lifecycle_events_and_keepalive_safe_shape(adapter, session_db):
session_id = session_db.create_session("stream-session", "api_server")
session_db.set_session_title(session_id, "Stream")
async def fake_run(**kwargs):
kwargs["stream_delta_callback"]("Hello")
kwargs["stream_delta_callback"](" world")
kwargs["tool_progress_callback"]("reasoning.available", tool_name="_thinking", preview="thinking")
return {"final_response": "Hello world", "session_id": session_id}, {"total_tokens": 2}
app = _create_session_app(adapter)
with patch.object(adapter, "_run_agent", side_effect=fake_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(f"/api/sessions/{session_id}/chat/stream", json={"message": "stream please"})
assert resp.status == 200
assert resp.headers["Content-Type"].startswith("text/event-stream")
body = await resp.text()
assert "event: run.started" in body
assert "event: message.started" in body
assert "event: assistant.delta" in body
assert "Hello world" in body
assert "event: tool.progress" in body
assert "event: assistant.completed" in body
assert "event: run.completed" in body
assert "event: done" in body
@pytest.mark.asyncio
async def test_session_chat_stream_run_completed_carries_turn_transcript(adapter, session_db):
"""run.completed must include the full interleaved turn transcript so a
client that lost intermediate (pre-tool-call) assistant text from the live
delta stream can reconcile without a separate /messages fetch. Refs #34703.
"""
import json as _json
session_id = session_db.create_session("transcript-session", "api_server")
async def fake_run(**kwargs):
# Stream the intermediate planning text the way a real turn would.
kwargs["stream_delta_callback"]("Let me search for that:")
kwargs["stream_delta_callback"]("Here is the summary.")
result = {
"final_response": "Here is the summary.",
"session_id": session_id,
"messages": [
{"role": "user", "content": "search then summarize"},
{
"role": "assistant",
"content": "Let me search for that:",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "web_search", "arguments": "{}"},
}
],
},
{"role": "tool", "content": "results", "tool_call_id": "call_1", "tool_name": "web_search"},
{"role": "assistant", "content": "Here is the summary."},
],
}
return result, {"total_tokens": 6}
app = _create_session_app(adapter)
with patch.object(adapter, "_run_agent", side_effect=fake_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat/stream",
json={"message": "search then summarize"},
)
assert resp.status == 200
body = await resp.text()
# Pull the run.completed event payload out of the SSE body.
run_completed_payload = None
for block in body.split("\n\n"):
if "event: run.completed" in block:
for line in block.splitlines():
if line.startswith("data: "):
run_completed_payload = _json.loads(line[len("data: "):])
break
assert run_completed_payload is not None, body
messages = run_completed_payload.get("messages")
assert isinstance(messages, list) and messages, run_completed_payload
# The colon-ended intermediate text that preceded the tool call must be present.
contents = [m.get("content") for m in messages]
assert "Let me search for that:" in contents
assert "Here is the summary." in contents
# No prior-turn user message should leak into the per-turn slice.
assert all(m.get("role") in ("assistant", "tool") for m in messages)
# The tool call is preserved alongside the intermediate text.
assert any(m.get("tool_calls") for m in messages)
@pytest.mark.asyncio
async def test_session_endpoints_require_auth_when_key_configured(auth_adapter):
app = _create_session_app(auth_adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.get("/api/sessions")
assert resp.status == 401
body = await resp.json()
assert body["error"]["code"] == "gateway_auth_failed"
ok = await cli.get("/api/sessions", headers={"Authorization": "Bearer sk-test"})
assert ok.status == 200
data = await ok.json()
assert data["object"] == "list"
assert data["data"] == []
@pytest.mark.asyncio
async def test_session_header_rejected_without_api_key(adapter, session_db):
session_id = session_db.create_session("unsafe-session", "api_server")
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": "hello"},
headers={"X-Hermes-Session-Key": "client-42"},
)
assert resp.status == 403
data = await resp.json()
assert "X-Hermes-Session-Key requires API key" in data["error"]["message"]
# ---------------------------------------------------------------------------
# Session-persisted model threading + provider-auth failure surfacing
# (salvaged from PR #57947 by @FvanW and PR #59941 by @kaishi00)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_session_chat_threads_session_model_to_run_agent(auth_adapter, session_db):
"""POST /api/sessions persists a per-session model, but the chat handler
previously fetched the session record and threw it away — the session's
chosen model silently had no effect on any chat turn."""
session_id = session_db.create_session("model-pinned-session", "api_server", model="claude-sonnet-4-6")
mock_run = AsyncMock(return_value=({"final_response": "ok", "session_id": session_id}, {"total_tokens": 1}))
app = _create_session_app(auth_adapter)
with patch.object(auth_adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": "hi"},
headers={"Authorization": "Bearer sk-test"},
)
assert resp.status == 200
mock_run.assert_awaited_once()
_, kwargs = mock_run.call_args
assert kwargs["session_model"] == "claude-sonnet-4-6"
@pytest.mark.asyncio
async def test_session_chat_stream_threads_session_model_to_run_agent(adapter, session_db):
"""Streaming twin of the session-model threading test above."""
session_id = session_db.create_session("model-pinned-stream-session", "api_server", model="gpt-5.5")
mock_run = AsyncMock(return_value=({"final_response": "ok", "session_id": session_id}, {"total_tokens": 1}))
app = _create_session_app(adapter)
with patch.object(adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat/stream",
json={"message": "hi"},
)
assert resp.status == 200
await resp.read()
mock_run.assert_awaited_once()
_, kwargs = mock_run.call_args
assert kwargs["session_model"] == "gpt-5.5"
@pytest.mark.asyncio
async def test_session_chat_resolves_stored_model_route_alias(session_db, monkeypatch):
"""A session-persisted model that matches a model_routes alias must go
through the route path (so route provider/credentials apply) and NOT be
passed as a raw session_model (idea from PR #59941 by @kaishi00)."""
adapter = APIServerAdapter(
PlatformConfig(
enabled=True,
extra={"model_routes": {"alias": {"model": "route/model", "provider": "openrouter"}}},
)
)
adapter._session_db = session_db
session_id = session_db.create_session("route-pinned-session", "api_server", model="alias")
mock_run = AsyncMock(return_value=({"final_response": "ok", "session_id": session_id}, {"total_tokens": 1}))
app = _create_session_app(adapter)
with patch.object(adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": "hi"},
)
assert resp.status == 200
_, kwargs = mock_run.call_args
assert kwargs["route"] == {"model": "route/model", "provider": "openrouter"}
assert kwargs["session_model"] is None
@pytest.mark.asyncio
async def test_run_agent_returns_controlled_response_on_provider_auth_failure(adapter, monkeypatch):
"""_resolve_runtime_agent_kwargs() (inside _create_agent()) raises
RuntimeError on provider auth/credential failure. Previously this
propagated unhandled out of _run_agent(): /v1/chat/completions caught it
as a generic 500, and /api/sessions/{id}/chat didn't catch it at all
(raw aiohttp 500, no JSON body). Must now return run.py's controlled
response shape instead of raising. Exercises the REAL boundary
(gateway.run._resolve_runtime_agent_kwargs, the sole raiser)."""
monkeypatch.setattr(
"gateway.run._resolve_runtime_agent_kwargs",
lambda: (_ for _ in ()).throw(
RuntimeError("No credentials found for provider 'nous' — run `hermes auth add nous`")
),
)
result, usage = await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="request-session",
)
assert result == {
"final_response": "⚠️ Provider authentication failed: No credentials found for provider 'nous' — run `hermes auth add nous`",
"messages": [],
"api_calls": 0,
"tools": [],
}
assert usage == {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
@pytest.mark.asyncio
async def test_run_agent_does_not_swallow_unrelated_exceptions(adapter, monkeypatch):
"""The _ProviderAuthResolutionError catch must stay narrow — a TypeError
elsewhere in _create_agent()/run_conversation() must still propagate."""
def fake_create_agent(**kwargs):
raise TypeError("unrelated bug: unexpected keyword argument")
monkeypatch.setattr(adapter, "_create_agent", fake_create_agent)
with pytest.raises(TypeError, match="unrelated bug"):
await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="request-session",
)
@pytest.mark.asyncio
async def test_run_agent_does_not_swallow_unrelated_runtime_error_from_run_conversation(adapter, monkeypatch):
"""agent.run_conversation() can legitimately raise a RuntimeError
unrelated to provider auth (e.g. run_agent.py's "Failed to recreate
closed OpenAI client"). A bare `except RuntimeError` around the whole
_create_agent()+run_conversation() span would mislabel it as
"Provider authentication failed". Only _ProviderAuthResolutionError —
raised exclusively inside _create_agent() at the
_resolve_runtime_agent_kwargs() call site — may trigger the controlled
response; this unrelated RuntimeError must propagate unhandled."""
class _FakeAgent:
def run_conversation(self, **kwargs):
raise RuntimeError("Failed to recreate closed OpenAI client")
monkeypatch.setattr(adapter, "_create_agent", lambda **kwargs: _FakeAgent())
with pytest.raises(RuntimeError, match="Failed to recreate closed OpenAI client"):
await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="request-session",
)
@pytest.mark.asyncio
async def test_session_chat_surfaces_controlled_response_on_provider_auth_failure(auth_adapter, session_db, monkeypatch):
"""End-to-end: POST /api/sessions/{id}/chat previously had zero wrapping
around _run_agent() — an unhandled RuntimeError produced a raw aiohttp
500 with no JSON body. Must now return 200 with the controlled error
message as the assistant content. Exercises the real
gateway.run._resolve_runtime_agent_kwargs() boundary, not a mocked
_create_agent()."""
session_id = session_db.create_session("auth-fail-session", "api_server")
monkeypatch.setattr(
"gateway.run._resolve_runtime_agent_kwargs",
lambda: (_ for _ in ()).throw(RuntimeError("Auth failed: token expired")),
)
app = _create_session_app(auth_adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": "hi"},
headers={"Authorization": "Bearer sk-test"},
)
assert resp.status == 200
payload = await resp.json()
assert payload["message"]["content"] == "⚠️ Provider authentication failed: Auth failed: token expired"
def _register_session_model_route(app, adapter):
app.router.add_post("/api/sessions/{session_id}/model", adapter._handle_session_model_lock)
def _patch_api_server_runtime(monkeypatch):
monkeypatch.setattr(
"gateway.run._resolve_runtime_agent_kwargs",
lambda: {
"provider": "openrouter",
"api_key": "sk-global",
"base_url": "https://openrouter.example/v1",
"api_mode": "chat_completions",
},
)
monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda: "global/model")
monkeypatch.setattr("gateway.run._load_gateway_config", lambda: {})
monkeypatch.setattr(
"gateway.run.GatewayRunner._load_reasoning_config",
staticmethod(lambda model="": {}),
)
monkeypatch.setattr(
"gateway.run.GatewayRunner._load_fallback_model",
staticmethod(lambda: None),
)
monkeypatch.setattr("gateway.run._current_max_iterations", lambda: 90)
monkeypatch.setattr("hermes_cli.tools_config._get_platform_tools", lambda *_: set())
monkeypatch.setattr(
"gateway.run._resolve_runtime_agent_kwargs_for_provider",
lambda provider: {
"provider": provider,
"api_key": f"sk-{provider}",
"base_url": f"https://{provider}.example/v1",
"api_mode": "chat_completions",
},
)
@pytest.mark.asyncio
async def test_session_chat_builds_raw_provider_model_route_when_alias_missing(adapter, session_db):
session_id = session_db.create_session("route-session", "api_server")
mock_run = AsyncMock(
return_value=(
{
"final_response": "ok",
"session_id": session_id,
"runtime": {"provider": "nous", "model": "x-ai/grok-4.5", "route_source": "raw_request"},
},
{"total_tokens": 2, "runtime": {"provider": "nous", "model": "x-ai/grok-4.5"}},
)
)
app = _create_session_app(adapter)
with patch.object(adapter, "_resolve_route", return_value=None), patch.object(adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={
"message": "hello",
"provider": "nous",
"model": "x-ai/grok-4.5",
"require_model_lock": True,
},
)
assert resp.status == 200, await resp.text()
payload = await resp.json()
kwargs = mock_run.call_args.kwargs
assert kwargs["route"] == {"provider": "nous", "model": "x-ai/grok-4.5"}
assert payload["runtime"]["provider"] == "nous"
assert payload["runtime"]["model"] == "x-ai/grok-4.5"
assert payload["runtime"]["requested"]["model"] == "x-ai/grok-4.5"
@pytest.mark.asyncio
async def test_session_chat_passes_runtime_options_to_run_agent(adapter, session_db):
session_id = session_db.create_session("options-session", "api_server")
mock_run = AsyncMock(return_value=({"final_response": "ok", "session_id": session_id}, {}))
app = _create_session_app(adapter)
with patch.object(adapter, "_resolve_route", return_value=None), patch.object(adapter, "_run_agent", mock_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={
"message": "hello",
"provider": "nous",
"model": "x-ai/grok-4.5",
"model_options": {
"reasoning": {"enabled": True, "effort": "xhigh"},
"service_tier": "priority",
"fast": True,
},
},
)
assert resp.status == 200, await resp.text()
kwargs = mock_run.call_args.kwargs
# In the merged design model_options travel raw to _create_agent, which
# parses reasoning/service-tier itself (see _request_reasoning_config /
# _request_service_tier) — there is no separate runtime_options kwarg.
assert kwargs["model_options"] == {
"reasoning": {"enabled": True, "effort": "xhigh"},
"service_tier": "priority",
"fast": True,
}
@pytest.mark.asyncio
async def test_session_chat_stream_uses_same_runtime_lock(adapter, session_db):
session_id = session_db.create_session("stream-lock-session", "api_server")
captured = {}
async def fake_run(**kwargs):
captured.update(kwargs)
kwargs["stream_delta_callback"]("hi")
return (
{
"final_response": "hi",
"session_id": session_id,
"runtime": {
"provider": "nous",
"model": "x-ai/grok-4.5",
"requested": {"provider": "nous", "model": "x-ai/grok-4.5"},
"route_source": "raw_request",
},
},
{
"total_tokens": 1,
"runtime": {
"provider": "nous",
"model": "x-ai/grok-4.5",
"requested": {"provider": "nous", "model": "x-ai/grok-4.5"},
},
},
)
app = _create_session_app(adapter)
with patch.object(adapter, "_resolve_route", return_value=None), patch.object(adapter, "_run_agent", side_effect=fake_run):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat/stream",
json={
"message": "stream",
"provider": "nous",
"model": "x-ai/grok-4.5",
"model_options": {"reasoning": {"enabled": False}},
"require_model_lock": True,
},
)
assert resp.status == 200, await resp.text()
body = await resp.text()
assert captured["route"] == {"provider": "nous", "model": "x-ai/grok-4.5"}
assert captured["model_options"] == {"reasoning": {"enabled": False}}
assert captured["confirmed_runtime_lock"] is True
assert "x-ai/grok-4.5" in body
assert "run.started" in body or "event: run.started" in body
@pytest.mark.asyncio
async def test_create_session_respects_browser_source_and_model_lock(adapter, session_db):
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
"/api/sessions",
json={
"id": "browser-lock-session",
"source": "hermes_browser",
"provider": "nous",
"model": "x-ai/grok-4.5",
"require_model_lock": True,
"title": "Browser lock",
"system_prompt": "browser prompt",
},
)
assert resp.status == 201, await resp.text()
payload = await resp.json()
assert payload["session"]["source"] == "hermes_browser"
assert payload["session"]["model"] == "x-ai/grok-4.5"
row = session_db.get_session("browser-lock-session")
assert row["source"] == "hermes_browser"
assert row["model"] == "x-ai/grok-4.5"
import json as _json
model_config = row.get("model_config")
if isinstance(model_config, str):
model_config = _json.loads(model_config)
assert model_config["browser_model_lock"]["provider"] == "nous"
assert model_config["browser_model_lock"]["model"] == "x-ai/grok-4.5"
assert model_config["browser_model_lock"]["confirmed"] is True
@pytest.mark.asyncio
async def test_session_model_lock_endpoint_persists_and_invalidates_prompt(adapter, session_db):
session_id = session_db.create_session(
"lock-endpoint-session",
"api_server",
model="gpt-5.5",
model_config={"_branched_from": "parent-session"},
system_prompt="Conversation started:\nModel: gpt-5.5\nProvider: openai-codex\n",
)
app = _create_session_app(adapter)
_register_session_model_route(app, adapter)
with patch.object(adapter, "_resolve_route", return_value=None):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/model",
json={
"provider": "nous",
"model": "x-ai/grok-4.5",
"model_options": {"reasoning": {"enabled": True, "effort": "high"}},
"require_model_lock": True,
},
)
assert resp.status == 200, await resp.text()
payload = await resp.json()
assert payload["object"] == "hermes.session.model_lock"
assert payload["runtime"]["requested"]["provider"] == "nous"
assert payload["runtime"]["model"] == "x-ai/grok-4.5"
assert payload["runtime"]["model_lock"] in {"accepted", "confirmed"}
row = session_db.get_session(session_id)
assert row["model"] == "x-ai/grok-4.5"
assert row["system_prompt"] is None
import json as _json
model_config = row.get("model_config")
if isinstance(model_config, str):
model_config = _json.loads(model_config)
assert model_config["_branched_from"] == "parent-session"
assert model_config["browser_model_lock"]["provider"] == "nous"
@pytest.mark.asyncio
async def test_session_model_lock_endpoint_then_chat_reuses_persisted_lock_and_provider_credentials(
adapter,
session_db,
monkeypatch,
):
session_id = session_db.create_session(
"endpoint-lock-chat",
"api_server",
model="gpt-5.5",
system_prompt="Conversation started:\nModel: gpt-5.5\nProvider: openai-codex\n",
)
captured = {}
class FakeAgent:
session_prompt_tokens = 0
session_completion_tokens = 0
session_total_tokens = 0
def __init__(self, **kwargs):
captured.update(kwargs)
self.session_id = kwargs["session_id"]
self.provider = kwargs.get("provider") or ""
self.model = kwargs.get("model") or ""
def run_conversation(self, user_message, conversation_history, task_id):
return {"final_response": "locked", "session_id": self.session_id}
_patch_api_server_runtime(monkeypatch)
monkeypatch.setattr("run_agent.AIAgent", FakeAgent)
monkeypatch.setattr(
adapter,
"_session_model_override_for",
lambda *_: {
"model": "session/override-model",
"provider": "openai-codex",
"api_key": "sk-session-override",
"base_url": "https://override.example/v1",
"api_mode": "codex_responses",
},
)
app = _create_session_app(adapter)
_register_session_model_route(app, adapter)
with patch.object(adapter, "_resolve_route", return_value=None):
async with TestClient(TestServer(app)) as cli:
lock_resp = await cli.post(
f"/api/sessions/{session_id}/model",
json={
"provider": "nous",
"model": "x-ai/grok-4.5",
"require_model_lock": True,
},
)
assert lock_resp.status == 200, await lock_resp.text()
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={"message": "use the stored lock"},
)
assert resp.status == 200, await resp.text()
payload = await resp.json()
assert captured["provider"] == "nous"
assert captured["model"] == "x-ai/grok-4.5"
assert captured["api_key"] == "sk-nous"
assert captured["base_url"] == "https://nous.example/v1"
assert payload["runtime"]["provider"] == "nous"
assert payload["runtime"]["model"] == "x-ai/grok-4.5"
assert payload["runtime"]["requested"] == {
"provider": "nous",
"model": "x-ai/grok-4.5",
}
assert payload["runtime"]["route_source"] == "session_model_lock"
@pytest.mark.asyncio
async def test_session_model_lock_endpoint_then_chat_stream_reuses_persisted_lock(
adapter,
session_db,
):
session_id = session_db.create_session("endpoint-lock-stream", "api_server")
captured = {}
async def fake_run(**kwargs):
captured.update(kwargs)
kwargs["stream_delta_callback"]("hi")
return (
{
"final_response": "hi",
"session_id": session_id,
"runtime": {
"provider": "nous",
"model": "x-ai/grok-4.5",
"requested": {"provider": "nous", "model": "x-ai/grok-4.5"},
"route_source": "session_model_lock",
},
},
{
"total_tokens": 1,
"runtime": {
"provider": "nous",
"model": "x-ai/grok-4.5",
"requested": {"provider": "nous", "model": "x-ai/grok-4.5"},
"route_source": "session_model_lock",
},
},
)
app = _create_session_app(adapter)
_register_session_model_route(app, adapter)
with patch.object(adapter, "_resolve_route", return_value=None), patch.object(
adapter,
"_run_agent",
side_effect=fake_run,
):
async with TestClient(TestServer(app)) as cli:
lock_resp = await cli.post(
f"/api/sessions/{session_id}/model",
json={
"provider": "nous",
"model": "x-ai/grok-4.5",
"require_model_lock": True,
},
)
assert lock_resp.status == 200, await lock_resp.text()
resp = await cli.post(
f"/api/sessions/{session_id}/chat/stream",
json={"message": "stream with stored lock"},
)
assert resp.status == 200, await resp.text()
body = await resp.text()
assert captured["route"] == {"provider": "nous", "model": "x-ai/grok-4.5"}
assert captured["requested_runtime"]["provider"] == "nous"
assert captured["requested_runtime"]["model"] == "x-ai/grok-4.5"
assert captured["route_source"] == "session_model_lock"
assert "x-ai/grok-4.5" in body
@pytest.mark.asyncio
async def test_run_agent_reports_actual_agent_runtime_not_requested_metadata(adapter, monkeypatch):
class FakeAgent:
session_prompt_tokens = 0
session_completion_tokens = 0
session_total_tokens = 0
def __init__(self):
self.session_id = "runtime-session"
self.provider = "actual-provider"
self.model = "actual-model"
self._hermes_api_runtime = {
"provider": "requested-provider",
"model": "requested-model",
"route_source": "raw_request",
}
def run_conversation(self, user_message, conversation_history, task_id):
return {"final_response": "ok", "session_id": self.session_id}
monkeypatch.setattr(adapter, "_create_agent", lambda **kwargs: FakeAgent())
result, usage = await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="runtime-session",
route={"provider": "requested-provider", "model": "requested-model"},
requested_runtime={
"provider": "requested-provider",
"model": "requested-model",
},
route_source="session_model_lock",
)
assert result["runtime"]["provider"] == "actual-provider"
assert result["runtime"]["model"] == "actual-model"
assert result["runtime"]["requested"] == {
"provider": "requested-provider",
"model": "requested-model",
}
assert usage["runtime"]["provider"] == "actual-provider"
assert usage["runtime"]["model"] == "actual-model"
@pytest.mark.asyncio
async def test_confirmed_runtime_lock_rejects_actual_runtime_mismatch(adapter, monkeypatch):
class FakeAgent:
session_prompt_tokens = 0
session_completion_tokens = 0
session_total_tokens = 0
session_id = "mismatch-session"
provider = "fallback-provider"
model = "fallback-model"
def run_conversation(self, user_message, conversation_history, task_id):
return {"final_response": "wrong runtime", "session_id": self.session_id}
monkeypatch.setattr(adapter, "_create_agent", lambda **kwargs: FakeAgent())
with pytest.raises(RuntimeError, match="confirmed model lock runtime mismatch"):
await adapter._run_agent(
user_message="hello",
conversation_history=[],
session_id="mismatch-session",
route={"provider": "nous", "model": "x-ai/grok-4.5"},
requested_runtime={"provider": "nous", "model": "x-ai/grok-4.5"},
route_source="session_model_lock",
confirmed_runtime_lock=True,
)
def test_confirmed_runtime_lock_fails_closed_on_provider_resolution_error(adapter, monkeypatch):
_patch_api_server_runtime(monkeypatch)
# Break BOTH resolution paths (primary picker-based resolver + the
# gateway fallback) — a confirmed lock must propagate the failure
# instead of constructing an agent on the previous global credentials.
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("provider unavailable")),
)
monkeypatch.setattr(
"gateway.run._resolve_runtime_agent_kwargs_for_provider",
lambda provider: (_ for _ in ()).throw(RuntimeError("provider unavailable")),
)
agent_ctor = patch("run_agent.AIAgent")
with agent_ctor as mocked_agent:
with pytest.raises(RuntimeError, match="provider unavailable"):
adapter._create_agent(
session_id="locked-session",
route={"provider": "nous", "model": "x-ai/grok-4.5"},
confirmed_runtime_lock=True,
)
mocked_agent.assert_not_called()
def test_confirmed_runtime_lock_disables_global_fallback_model(adapter, monkeypatch):
_patch_api_server_runtime(monkeypatch)
monkeypatch.setattr(
"gateway.run.GatewayRunner._load_fallback_model",
staticmethod(lambda: "openrouter/fallback-model"),
)
captured = {}
class FakeAgent:
provider = "nous"
model = "x-ai/grok-4.5"
def __init__(self, **kwargs):
captured.update(kwargs)
monkeypatch.setattr("run_agent.AIAgent", FakeAgent)
adapter._create_agent(
session_id="locked-session",
route={"provider": "nous", "model": "x-ai/grok-4.5"},
confirmed_runtime_lock=True,
)
assert captured["fallback_model"] is None
@pytest.mark.asyncio
async def test_unconfirmed_request_does_not_replace_confirmed_session_lock(adapter, session_db):
session_id = session_db.create_session("one-off-override", "api_server")
session_db.update_session_runtime_lock(
session_id,
provider="nous",
model="x-ai/grok-4.5",
route_source="raw_request",
confirmed=True,
)
mock_run = AsyncMock(
return_value=(
{
"final_response": "ok",
"session_id": session_id,
"runtime": {"provider": "openrouter", "model": "anthropic/claude-sonnet"},
},
{"total_tokens": 1},
)
)
app = _create_session_app(adapter)
with patch.object(adapter, "_resolve_route", return_value=None), patch.object(
adapter,
"_run_agent",
mock_run,
):
async with TestClient(TestServer(app)) as cli:
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={
"message": "one turn only",
"provider": "openrouter",
"model": "anthropic/claude-sonnet",
},
)
assert resp.status == 200, await resp.text()
import json as _json
row = session_db.get_session(session_id)
config = row["model_config"]
if isinstance(config, str):
config = _json.loads(config)
assert config["browser_model_lock"]["provider"] == "nous"
assert config["browser_model_lock"]["model"] == "x-ai/grok-4.5"
assert config["browser_model_lock"]["confirmed"] is True
@pytest.mark.asyncio
async def test_require_model_lock_hard_fails_when_global_default_would_be_used(adapter, session_db, monkeypatch):
session_id = session_db.create_session("lock-fail-session", "api_server")
monkeypatch.setattr(adapter, "_model_name", "gpt-5.5")
app = _create_session_app(adapter)
with patch.object(adapter, "_resolve_route", return_value=None), patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
async with TestClient(TestServer(app)) as cli:
# empty model + require_model_lock must not silently fall through
resp = await cli.post(
f"/api/sessions/{session_id}/chat",
json={
"message": "hello",
"provider": "nous",
"model": "",
"require_model_lock": True,
},
)
assert resp.status in (400, 409), await resp.text()
body = await resp.json()
assert body["error"]["code"] in {"model_lock_unavailable", "invalid_model_lock", "missing_model"}
mock_run.assert_not_called()
@pytest.mark.asyncio
async def test_capabilities_advertises_session_model_lock(adapter):
app = _create_session_app(adapter)
async with TestClient(TestServer(app)) as cli:
resp = await cli.get("/v1/capabilities")
assert resp.status == 200
data = await resp.json()
assert data["features"]["session_model_lock"] is True
assert data["endpoints"]["session_model_lock"] == {
"method": "POST",
"path": "/api/sessions/{session_id}/model",
}