mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
533 lines
16 KiB
Python
533 lines
16 KiB
Python
"""Tests for the core Relay-managed physical LLM attempt adapter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextvars
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from agent import relay_llm, relay_runtime
|
|
|
|
|
|
@pytest.fixture()
|
|
def relay_turn(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile"))
|
|
relay_runtime._reset_for_tests()
|
|
lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation(
|
|
profile_key=relay_runtime.current_profile_key(),
|
|
session_id="session-1",
|
|
platform="cli",
|
|
)
|
|
turn = relay_runtime.SESSION_COORDINATOR.begin_turn(
|
|
lease,
|
|
turn_id="turn-1",
|
|
task_id="task-1",
|
|
)
|
|
try:
|
|
yield lease.host.relay, turn
|
|
finally:
|
|
relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success")
|
|
relay_runtime.SESSION_COORDINATOR.release_conversation(lease)
|
|
relay_runtime._reset_for_tests()
|
|
|
|
|
|
def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn):
|
|
relay, turn = relay_turn
|
|
captured_requests = []
|
|
|
|
def rewrite_request(name, request, annotated):
|
|
del name
|
|
content = {**request.content, "temperature": 0.25}
|
|
return relay.LLMRequestInterceptOutcome(
|
|
relay.LLMRequest(request.headers, content),
|
|
annotated,
|
|
)
|
|
|
|
def rewrite_stream(request, next_call):
|
|
async def generate():
|
|
upstream = await next_call(request)
|
|
async for chunk in upstream:
|
|
updated = dict(chunk)
|
|
choices = [dict(choice) for choice in updated.get("choices", [])]
|
|
if choices:
|
|
delta = dict(choices[0].get("delta") or {})
|
|
if delta.get("content"):
|
|
delta["content"] = delta["content"].upper()
|
|
choices[0]["delta"] = delta
|
|
updated["choices"] = choices
|
|
yield updated
|
|
|
|
return generate()
|
|
|
|
def raw_stream(request):
|
|
captured_requests.append(request)
|
|
return iter([
|
|
SimpleNamespace(
|
|
model="test-model",
|
|
choices=[
|
|
SimpleNamespace(
|
|
delta=SimpleNamespace(content="hello", tool_calls=None),
|
|
finish_reason=None,
|
|
)
|
|
],
|
|
usage=None,
|
|
),
|
|
SimpleNamespace(
|
|
model="test-model",
|
|
choices=[
|
|
SimpleNamespace(
|
|
delta=SimpleNamespace(content=None, tool_calls=None),
|
|
finish_reason="stop",
|
|
)
|
|
],
|
|
usage=None,
|
|
),
|
|
])
|
|
|
|
relay.intercepts.register_llm_request(
|
|
"hermes-test-request",
|
|
1,
|
|
False,
|
|
rewrite_request,
|
|
)
|
|
relay.intercepts.register_llm_stream_execution(
|
|
"hermes-test-stream",
|
|
1,
|
|
rewrite_stream,
|
|
)
|
|
try:
|
|
stream = relay_llm.stream(
|
|
{"model": "test-model", "messages": []},
|
|
raw_stream,
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
finalizer=lambda: {
|
|
"model": "test-model",
|
|
"choices": [
|
|
{
|
|
"message": {"role": "assistant", "content": "HELLO"},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
},
|
|
metadata={
|
|
"api_mode": "custom",
|
|
"api_request_id": "request-1",
|
|
"call_role": "primary",
|
|
},
|
|
)
|
|
chunks = list(stream)
|
|
finally:
|
|
relay.intercepts.deregister_llm_stream_execution("hermes-test-stream")
|
|
relay.intercepts.deregister_llm_request("hermes-test-request")
|
|
|
|
assert captured_requests[0]["temperature"] == 0.25
|
|
assert chunks[0].choices[0].delta.content == "HELLO"
|
|
assert stream.output_modified is True
|
|
assert turn.logical_llm_calls == {}
|
|
|
|
|
|
def test_stream_preserves_provider_error_and_keeps_logical_scope_for_retry(relay_turn):
|
|
_relay, turn = relay_turn
|
|
|
|
class ProviderError(Exception):
|
|
pass
|
|
|
|
provider_error = ProviderError("provider failed")
|
|
|
|
def failing_stream(_request):
|
|
def generate():
|
|
raise provider_error
|
|
yield # pragma: no cover
|
|
|
|
return generate()
|
|
|
|
stream = relay_llm.stream(
|
|
{"model": "test-model", "messages": []},
|
|
failing_stream,
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
finalizer=dict,
|
|
metadata={
|
|
"api_mode": "chat_completions",
|
|
"api_request_id": "request-2",
|
|
},
|
|
)
|
|
|
|
with pytest.raises(ProviderError) as caught:
|
|
list(stream)
|
|
|
|
assert caught.value is provider_error
|
|
assert "request-2" in turn.logical_llm_calls
|
|
|
|
|
|
def test_non_stream_preserves_raw_provider_response_identity(relay_turn):
|
|
_relay, _turn = relay_turn
|
|
raw_response = SimpleNamespace(model="test-model", content="raw")
|
|
|
|
result = relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: raw_response,
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-raw"},
|
|
)
|
|
|
|
assert result is raw_response
|
|
|
|
|
|
def test_non_stream_defers_logical_success_and_reuses_scope_for_retry(relay_turn):
|
|
_relay, turn = relay_turn
|
|
metadata = {"api_mode": "custom", "api_request_id": "request-retry"}
|
|
|
|
first = relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: {"content": "invalid"},
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata=metadata,
|
|
defer_logical_completion=True,
|
|
)
|
|
first_handle = turn.logical_llm_calls["request-retry"]
|
|
|
|
second = relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: {"content": "valid"},
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata=metadata,
|
|
defer_logical_completion=True,
|
|
)
|
|
|
|
assert first == {"content": "invalid"}
|
|
assert second == {"content": "valid"}
|
|
assert turn.logical_llm_calls == {"request-retry": first_handle}
|
|
|
|
relay_llm.complete_logical_call("request-retry", outcome="success")
|
|
|
|
assert turn.logical_llm_calls == {}
|
|
|
|
|
|
def test_non_stream_result_survives_logical_scope_close_failure(
|
|
relay_turn, monkeypatch
|
|
):
|
|
relay, turn = relay_turn
|
|
original_pop = relay.scope.pop
|
|
pop_calls = 0
|
|
|
|
def fail_first_pop(*args, **kwargs):
|
|
nonlocal pop_calls
|
|
pop_calls += 1
|
|
if pop_calls == 1:
|
|
raise RuntimeError("simulated logical scope close failure")
|
|
return original_pop(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(relay.scope, "pop", fail_first_pop)
|
|
raw_response = SimpleNamespace(model="test-model", content="raw")
|
|
|
|
result = relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: raw_response,
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-close"},
|
|
)
|
|
|
|
assert result is raw_response
|
|
assert "request-close" in turn.logical_llm_calls
|
|
relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success")
|
|
assert turn.logical_llm_calls == {}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_non_stream_preserves_raw_provider_response_identity(relay_turn):
|
|
_relay, _turn = relay_turn
|
|
raw_response = SimpleNamespace(model="test-model", content="raw")
|
|
|
|
async def provider(_request):
|
|
return raw_response
|
|
|
|
result = await relay_llm.execute_current_async(
|
|
{"model": "test-model", "messages": []},
|
|
provider,
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-async"},
|
|
)
|
|
|
|
assert result is raw_response
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_non_stream_defers_logical_success_for_validation(relay_turn):
|
|
_relay, turn = relay_turn
|
|
|
|
async def provider(_request):
|
|
return {"content": "pending-validation"}
|
|
|
|
await relay_llm.execute_current_async(
|
|
{"model": "test-model", "messages": []},
|
|
provider,
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-async-defer"},
|
|
defer_logical_completion=True,
|
|
)
|
|
|
|
assert "request-async-defer" in turn.logical_llm_calls
|
|
|
|
relay_llm.complete_logical_call("request-async-defer", outcome="success")
|
|
|
|
assert turn.logical_llm_calls == {}
|
|
|
|
|
|
def test_stream_defers_logical_success_for_response_validation(relay_turn):
|
|
_relay, turn = relay_turn
|
|
|
|
stream = relay_llm.stream(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: iter([{"delta": "pending-validation"}]),
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
finalizer=lambda: {"content": "pending-validation"},
|
|
metadata={"api_mode": "custom", "api_request_id": "request-stream-defer"},
|
|
defer_logical_completion=True,
|
|
)
|
|
|
|
assert list(stream) == [{"delta": "pending-validation"}]
|
|
assert stream.output_modified is False
|
|
assert "request-stream-defer" in turn.logical_llm_calls
|
|
|
|
relay_llm.complete_logical_call("request-stream-defer", outcome="success")
|
|
|
|
assert turn.logical_llm_calls == {}
|
|
|
|
|
|
def test_current_attempt_bypasses_relay_without_an_active_turn(monkeypatch):
|
|
monkeypatch.setattr(relay_runtime, "current_turn", lambda: None)
|
|
request = {"model": "test-model", "messages": []}
|
|
|
|
result = relay_llm.execute_current(
|
|
request,
|
|
lambda value: value,
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
)
|
|
|
|
assert result is request
|
|
|
|
|
|
def test_current_attempt_bypasses_a_closed_turn_from_a_copied_context(
|
|
relay_turn,
|
|
monkeypatch,
|
|
):
|
|
_relay, turn = relay_turn
|
|
stale_context = contextvars.copy_context()
|
|
relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success")
|
|
request = {"model": "test-model", "messages": []}
|
|
|
|
def fail_execute(*_args, **_kwargs):
|
|
raise AssertionError("a closed turn must not manage later provider work")
|
|
|
|
monkeypatch.setattr(relay_llm, "execute", fail_execute)
|
|
|
|
result = stale_context.run(
|
|
relay_llm.execute_current,
|
|
request,
|
|
lambda value: value,
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
)
|
|
|
|
assert result is request
|
|
|
|
|
|
def test_non_stream_returns_post_execution_interceptor_result(relay_turn, monkeypatch):
|
|
relay, _turn = relay_turn
|
|
|
|
async def post_execute(_name, request, callback, **_kwargs):
|
|
response = callback(request)
|
|
return {**response, "post_interceptor": True}
|
|
|
|
monkeypatch.setattr(relay.llm, "execute", post_execute)
|
|
|
|
result = relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: {"content": "raw"},
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-post"},
|
|
)
|
|
|
|
assert result == {"content": "raw", "post_interceptor": True}
|
|
|
|
|
|
def test_non_stream_preserves_provider_error_from_relay_wrapper_suffix(
|
|
relay_turn, monkeypatch
|
|
):
|
|
relay, turn = relay_turn
|
|
|
|
class ProviderError(Exception):
|
|
pass
|
|
|
|
provider_error = ProviderError("provider failed")
|
|
|
|
async def wrapping_execute(_name, request, callback, **_kwargs):
|
|
try:
|
|
return callback(request)
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"internal error: {type(exc).__name__}: {exc} (retried 3x)"
|
|
) from None
|
|
|
|
monkeypatch.setattr(relay.llm, "execute", wrapping_execute)
|
|
|
|
with pytest.raises(ProviderError) as caught:
|
|
relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: (_ for _ in ()).throw(provider_error),
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-error"},
|
|
)
|
|
|
|
assert caught.value is provider_error
|
|
assert "request-error" in turn.logical_llm_calls
|
|
|
|
|
|
def test_non_stream_does_not_mask_relay_error_after_callback_failure(
|
|
relay_turn, monkeypatch
|
|
):
|
|
relay, _turn = relay_turn
|
|
provider_error = RuntimeError("provider failed")
|
|
relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked")
|
|
|
|
async def translating_execute(_name, request, callback, **_kwargs):
|
|
try:
|
|
callback(request)
|
|
except Exception:
|
|
raise relay_error
|
|
|
|
monkeypatch.setattr(relay.llm, "execute", translating_execute)
|
|
|
|
with pytest.raises(RuntimeError) as caught:
|
|
relay_llm.execute(
|
|
{"model": "test-model", "messages": []},
|
|
lambda _request: (_ for _ in ()).throw(provider_error),
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={"api_mode": "custom", "api_request_id": "request-policy"},
|
|
)
|
|
|
|
assert caught.value is relay_error
|
|
|
|
|
|
def test_chat_codec_preserves_provider_message_extensions_after_rewrite(relay_turn):
|
|
relay, _turn = relay_turn
|
|
captured_requests = []
|
|
|
|
def rewrite_request(name, request, annotated):
|
|
del name
|
|
annotated.params = {**(annotated.params or {}), "temperature": 0.25}
|
|
return relay.LLMRequestInterceptOutcome(request, annotated)
|
|
|
|
def provider(request):
|
|
captured_requests.append(request)
|
|
return {
|
|
"id": "chatcmpl-test",
|
|
"object": "chat.completion",
|
|
"created": 0,
|
|
"model": "test-model",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": "ok"},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
}
|
|
|
|
relay.intercepts.register_llm_request(
|
|
"hermes-provider-extension-request",
|
|
1,
|
|
False,
|
|
rewrite_request,
|
|
)
|
|
try:
|
|
relay_llm.execute(
|
|
{
|
|
"model": "test-model",
|
|
"messages": [
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"reasoning_content": "provider scratchpad",
|
|
}
|
|
],
|
|
},
|
|
provider,
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={
|
|
"api_mode": "chat_completions",
|
|
"api_request_id": "request-3",
|
|
},
|
|
)
|
|
finally:
|
|
relay.intercepts.deregister_llm_request(
|
|
"hermes-provider-extension-request"
|
|
)
|
|
|
|
assert captured_requests[0]["temperature"] == 0.25
|
|
assert captured_requests[0]["messages"][0]["reasoning_content"] == (
|
|
"provider scratchpad"
|
|
)
|
|
|
|
|
|
def test_request_rewrite_preserves_unmodified_provider_objects(relay_turn):
|
|
relay, _turn = relay_turn
|
|
timeout = object()
|
|
captured_requests = []
|
|
|
|
def rewrite_request(name, request, annotated):
|
|
del name
|
|
annotated.params = {**(annotated.params or {}), "temperature": 0.25}
|
|
return relay.LLMRequestInterceptOutcome(request, annotated)
|
|
|
|
relay.intercepts.register_llm_request(
|
|
"hermes-provider-object-request",
|
|
1,
|
|
False,
|
|
rewrite_request,
|
|
)
|
|
try:
|
|
relay_llm.execute(
|
|
{"model": "test-model", "messages": [], "timeout": timeout},
|
|
lambda request: captured_requests.append(request) or {"content": "ok"},
|
|
session_id="session-1",
|
|
name="test-provider",
|
|
model_name="test-model",
|
|
metadata={
|
|
"api_mode": "chat_completions",
|
|
"api_request_id": "request-provider-object",
|
|
},
|
|
)
|
|
finally:
|
|
relay.intercepts.deregister_llm_request(
|
|
"hermes-provider-object-request"
|
|
)
|
|
|
|
assert captured_requests[0]["timeout"] is timeout
|
|
assert captured_requests[0]["temperature"] == 0.25
|