"""Tests for the core Relay-managed physical LLM attempt adapter.""" from __future__ import annotations import asyncio import contextvars import json import threading from types import SimpleNamespace import pytest pytest.importorskip("nemo_relay") from agent import relay_llm, relay_runtime @pytest.fixture() def relay_turn(tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) relay_runtime._reset_for_tests() lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( profile_key=relay_runtime.current_profile_key(), session_id="session-1", platform="cli", ) turn = relay_runtime.SESSION_COORDINATOR.begin_turn( lease, turn_id="turn-1", task_id="task-1", ) lease.host.retain_managed_execution("test.relay_llm") try: yield lease.host.relay, turn finally: lease.host.release_managed_execution("test.relay_llm") relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") relay_runtime.SESSION_COORDINATOR.release_conversation(lease) relay_runtime._reset_for_tests() def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn): relay, turn = relay_turn captured_requests = [] def rewrite_request(name, request, annotated): del name content = {**request.content, "temperature": 0.25} return relay.LLMRequestInterceptOutcome( relay.LLMRequest(request.headers, content), annotated, ) def rewrite_stream(request, next_call): async def generate(): upstream = await next_call(request) async for chunk in upstream: updated = dict(chunk) choices = [dict(choice) for choice in updated.get("choices", [])] if choices: delta = dict(choices[0].get("delta") or {}) if delta.get("content"): delta["content"] = delta["content"].upper() choices[0]["delta"] = delta updated["choices"] = choices yield updated return generate() def raw_stream(request): captured_requests.append(request) return iter([ SimpleNamespace( model="test-model", choices=[ SimpleNamespace( delta=SimpleNamespace(content="hello", tool_calls=None), finish_reason=None, ) ], usage=None, ), SimpleNamespace( model="test-model", choices=[ SimpleNamespace( delta=SimpleNamespace(content=None, tool_calls=None), finish_reason="stop", ) ], usage=None, ), ]) relay.intercepts.register_llm_request( "hermes-test-request", 1, False, rewrite_request, ) relay.intercepts.register_llm_stream_execution( "hermes-test-stream", 1, rewrite_stream, ) try: stream = relay_llm.stream( { "model": "test-model", "messages": [], "extra_headers": {"authorization": "Bearer provider-token"}, }, raw_stream, session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: { "model": "test-model", "choices": [ { "message": {"role": "assistant", "content": "HELLO"}, "finish_reason": "stop", } ], }, metadata={ "api_mode": "custom", "api_request_id": "request-1", "call_role": "primary", }, ) chunks = list(stream) finally: relay.intercepts.deregister_llm_stream_execution("hermes-test-stream") relay.intercepts.deregister_llm_request("hermes-test-request") assert captured_requests[0]["temperature"] == 0.25 assert captured_requests[0]["extra_headers"] == { "authorization": "Bearer provider-token" } assert chunks[0].choices[0].delta.content == "HELLO" assert stream.output_modified is True assert turn.logical_llm_calls == {} def test_deferred_stream_preserves_provider_error_and_logical_scope_for_retry( relay_turn, ): _relay, turn = relay_turn class ProviderError(Exception): pass provider_error = ProviderError("provider failed") def failing_stream(_request): def generate(): raise provider_error yield # pragma: no cover return generate() stream = relay_llm.stream( {"model": "test-model", "messages": []}, failing_stream, session_id="session-1", name="test-provider", model_name="test-model", finalizer=dict, metadata={ "api_mode": "chat_completions", "api_request_id": "request-2", }, defer_logical_completion=True, ) with pytest.raises(ProviderError) as caught: list(stream) assert caught.value is provider_error assert "request-2" in turn.logical_llm_calls def test_stream_provider_error_is_not_replaced_by_finalizer_error(relay_turn): _relay, turn = relay_turn class ProviderError(Exception): pass provider_error = ProviderError("provider failed before first chunk") finalizer_called = False def failing_stream(_request): def generate(): raise provider_error yield # pragma: no cover return generate() def failing_finalizer(): nonlocal finalizer_called finalizer_called = True raise RuntimeError("missing terminal response") stream = relay_llm.stream( {"model": "test-model", "input": "hi"}, failing_stream, session_id="session-1", name="test-provider", model_name="test-model", finalizer=failing_finalizer, metadata={ "api_mode": "codex_responses", "api_request_id": "request-provider-before-finalizer", }, defer_logical_completion=True, ) with pytest.raises(ProviderError) as caught: list(stream) assert caught.value is provider_error assert finalizer_called is False assert "request-provider-before-finalizer" in turn.logical_llm_calls def test_non_deferred_partial_stream_close_cancels_logical_call( relay_turn, monkeypatch, ): relay, turn = relay_turn original_pop = relay.scope.pop terminal_outputs = [] def record_pop(handle, *args, **kwargs): terminal_outputs.append(kwargs.get("output")) return original_pop(handle, *args, **kwargs) monkeypatch.setattr(relay.scope, "pop", record_pop) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter([{"delta": "partial"}, {"delta": "unused"}]), session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "partial"}, metadata={ "api_mode": "custom", "api_request_id": "request-partial-close", }, ) assert next(stream) == {"delta": "partial"} assert "request-partial-close" in turn.logical_llm_calls stream.close() assert "request-partial-close" not in turn.logical_llm_calls assert {"outcome": "cancelled"} in terminal_outputs def test_direct_stream_close_reaches_original_provider_resource(monkeypatch): class ProviderStream: def __init__(self): self.closed = False def __iter__(self): return iter([{"delta": "partial"}, {"delta": "unused"}]) def close(self): self.closed = True provider_stream = ProviderStream() monkeypatch.setattr( relay_runtime, "resolve_execution_context", lambda _session_id: (None, None, None), ) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: provider_stream, session_id="session-1", name="test-provider", model_name="test-model", finalizer=dict, ) assert next(stream) == {"delta": "partial"} stream.close() assert provider_stream.closed is True def test_anthropic_stream_accumulator_merges_terminal_usage(): accumulator = relay_llm.AnthropicStreamAccumulator() accumulator.observe({ "type": "message_start", "message": { "id": "message-1", "type": "message", "role": "assistant", "model": "claude-test", "usage": { "input_tokens": 100, "output_tokens": 1, "cache_creation_input_tokens": 20, "cache_read_input_tokens": 30, }, }, }) accumulator.observe({ "type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 12}, }) response = accumulator.finalize() assert response["usage"] == { "input_tokens": 100, "output_tokens": 12, "cache_creation_input_tokens": 20, "cache_read_input_tokens": 30, } def test_anthropic_stream_accumulator_merges_plain_provider_object(): accumulator = relay_llm.AnthropicStreamAccumulator() accumulator.observe({ "type": "message_start", "message": { "id": "message-1", "type": "message", "role": "assistant", "model": "claude-test", "usage": {"input_tokens": 10}, }, }) accumulator.observe({ "type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": "hello"}, }) response = accumulator.response( SimpleNamespace( id="message-1", type="message", role="assistant", model="claude-test", content=[], stop_reason=None, usage={"input_tokens": 10}, ) ) assert response.id == "message-1" assert response.content[0].text == "hello" assert response.usage.input_tokens == 10 def test_jsonable_does_not_probe_dynamic_attributes(): class DynamicProviderObject: def __getattr__(self, name): raise AssertionError(f"unexpected dynamic attribute lookup: {name}") def __str__(self): return "opaque-provider-object" assert relay_llm._jsonable(DynamicProviderObject()) == "opaque-provider-object" def test_non_stream_preserves_raw_provider_response_identity(relay_turn): _relay, _turn = relay_turn raw_response = SimpleNamespace(model="test-model", content="raw") result = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: raw_response, session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-raw"}, ) assert result is raw_response def test_non_stream_provider_callback_preserves_caller_context(relay_turn): del relay_turn caller_value = contextvars.ContextVar("llm_caller_value", default="default") caller_value.set("caller") result = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: {"caller_value": caller_value.get()}, session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-context"}, ) assert result == {"caller_value": "caller"} @pytest.mark.asyncio async def test_async_provider_callback_preserves_caller_context(relay_turn): del relay_turn caller_value = contextvars.ContextVar( "async_llm_caller_value", default="default", ) caller_value.set("caller") async def provider(_request): await asyncio.sleep(0) return {"caller_value": caller_value.get()} result = await relay_llm.execute_async( {"model": "test-model", "messages": []}, provider, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "custom", "api_request_id": "request-async-context", }, ) assert result == {"caller_value": "caller"} def test_stream_provider_callbacks_preserve_caller_context(relay_turn): del relay_turn caller_value = contextvars.ContextVar( "stream_llm_caller_value", default="default", ) caller_value.set("caller") observed = [] def stream_factory(_request): observed.append(("factory", caller_value.get())) def generate(): observed.append(("next", caller_value.get())) yield {"delta": "hello"} return generate() def on_chunk(_chunk): observed.append(("chunk", caller_value.get())) def finalizer(): observed.append(("finalizer", caller_value.get())) return {"content": "hello"} stream = relay_llm.stream( {"model": "test-model", "messages": []}, stream_factory, session_id="session-1", name="test-provider", model_name="test-model", finalizer=finalizer, on_chunk=on_chunk, metadata={ "api_mode": "custom", "api_request_id": "request-stream-context", }, ) assert list(stream) == [{"delta": "hello"}] assert observed == [ ("factory", "caller"), ("next", "caller"), ("chunk", "caller"), ("finalizer", "caller"), ] def test_anthropic_stream_callbacks_do_not_reenter_captured_context( relay_turn, monkeypatch, ): del relay_turn caller_value = contextvars.ContextVar( "anthropic_stream_caller_value", default="default", ) caller_value.set("caller") callback_context = contextvars.copy_context() real_copy_context = contextvars.copy_context copy_count = 0 def capture_callback_context(): nonlocal copy_count copy_count += 1 if copy_count == 1: return callback_context return real_copy_context() monkeypatch.setattr( relay_llm.contextvars, "copy_context", capture_callback_context, ) observed = [] accumulator = relay_llm.AnthropicStreamAccumulator() def observe_chunk(chunk): observed.append(caller_value.get()) accumulator.observe(chunk) chunks = [ { "type": "message_start", "message": { "id": "message-1", "type": "message", "role": "assistant", "model": "claude-test", "usage": {"input_tokens": 1, "output_tokens": 0}, }, }, { "type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 1}, }, ] stream = relay_llm.stream( { "model": "claude-test", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}], }, lambda _request: iter(chunks), session_id="session-1", name="anthropic", model_name="claude-test", finalizer=accumulator.finalize, on_chunk=observe_chunk, metadata={ "api_mode": "anthropic_messages", "api_request_id": "request-anthropic-context-reentry", }, ) entered = threading.Event() release = threading.Event() def hold_callback_context() -> None: def wait() -> None: entered.set() assert release.wait(timeout=5) callback_context.run(wait) holder = threading.Thread(target=hold_callback_context) holder.start() assert entered.wait(timeout=1) try: assert list(stream) == chunks finally: release.set() holder.join(timeout=1) assert holder.is_alive() is False assert observed == ["caller", "caller"] def test_explicit_stream_close_surfaces_provider_close_failure(relay_turn): del relay_turn class FailingCloseStream: def __init__(self): self._chunks = iter([{"delta": "partial"}]) self.close_calls = 0 def __iter__(self): return self def __next__(self): return next(self._chunks) def close(self): self.close_calls += 1 raise RuntimeError("provider close failed") raw_stream = FailingCloseStream() stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: raw_stream, session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "partial"}, metadata={ "api_mode": "custom", "api_request_id": "request-close-failure", }, ) assert next(stream) == {"delta": "partial"} with pytest.raises(RuntimeError, match="provider close failed"): stream.close() assert raw_stream.close_calls == 1 stream.close() def test_non_stream_does_not_forward_relay_session_headers(relay_turn): _relay, _turn = relay_turn captured_requests = [] relay_llm.execute( { "model": "test-model", "messages": [], "extra_headers": {"x-provider-header": "provider-value"}, }, lambda request: captured_requests.append(request) or {"content": "ok"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-headers"}, ) assert captured_requests[0]["extra_headers"] == { "x-provider-header": "provider-value" } def test_non_stream_defers_logical_success_and_reuses_scope_for_retry(relay_turn): _relay, turn = relay_turn metadata = {"api_mode": "custom", "api_request_id": "request-retry"} first = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: {"content": "invalid"}, session_id="session-1", name="test-provider", model_name="test-model", metadata=metadata, defer_logical_completion=True, ) first_handle = turn.logical_llm_calls["request-retry"] second = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: {"content": "valid"}, session_id="session-1", name="test-provider", model_name="test-model", metadata=metadata, defer_logical_completion=True, ) assert first == {"content": "invalid"} assert second == {"content": "valid"} assert turn.logical_llm_calls == {"request-retry": first_handle} relay_llm.complete_logical_call("request-retry", outcome="success") assert turn.logical_llm_calls == {} def test_non_stream_result_survives_logical_scope_close_failure( relay_turn, monkeypatch ): relay, turn = relay_turn original_pop = relay.scope.pop pop_calls = 0 def fail_first_pop(*args, **kwargs): nonlocal pop_calls pop_calls += 1 if pop_calls == 1: raise RuntimeError("simulated logical scope close failure") return original_pop(*args, **kwargs) monkeypatch.setattr(relay.scope, "pop", fail_first_pop) raw_response = SimpleNamespace(model="test-model", content="raw") result = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: raw_response, session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-close"}, ) assert result is raw_response assert "request-close" in turn.logical_llm_calls relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") assert turn.logical_llm_calls == {} def test_non_stream_returns_provider_response_after_relay_post_processing_failure( relay_turn, monkeypatch, caplog ): relay, turn = relay_turn raw_response = SimpleNamespace(model="test-model", content="raw") async def fail_after_callback(_name, request, callback, **_kwargs): callback(request) raise RuntimeError("simulated Relay post-processing failure") monkeypatch.setattr(relay.llm, "execute", fail_after_callback) with caplog.at_level("WARNING", logger="agent.relay_llm"): result = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: raw_response, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "custom", "api_request_id": "request-post-failure", }, ) assert result is raw_response assert turn.logical_llm_calls == {} assert "returning the provider response" in caplog.text def test_non_stream_does_not_swallow_interrupt_after_provider_success( relay_turn, monkeypatch ): relay, turn = relay_turn async def interrupt_after_callback(_name, request, callback, **_kwargs): callback(request) raise KeyboardInterrupt monkeypatch.setattr(relay.llm, "execute", interrupt_after_callback) with pytest.raises(KeyboardInterrupt): relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: {"content": "already returned"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "custom", "api_request_id": "request-post-interrupt", }, ) assert "request-post-interrupt" in turn.logical_llm_calls relay_llm.complete_logical_call("request-post-interrupt", outcome="cancelled") @pytest.mark.asyncio async def test_async_non_stream_preserves_raw_provider_response_identity(relay_turn): _relay, _turn = relay_turn raw_response = SimpleNamespace(model="test-model", content="raw") async def provider(_request): return raw_response result = await relay_llm.execute_current_async( {"model": "test-model", "messages": []}, provider, name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-async"}, ) assert result is raw_response @pytest.mark.asyncio async def test_async_non_stream_returns_provider_response_after_relay_failure( relay_turn, monkeypatch, caplog ): relay, turn = relay_turn raw_response = SimpleNamespace(model="test-model", content="raw") async def provider(_request): return raw_response async def fail_after_callback(_name, request, callback, **_kwargs): await callback(request) raise RuntimeError("simulated Relay post-processing failure") monkeypatch.setattr(relay.llm, "execute", fail_after_callback) with caplog.at_level("WARNING", logger="agent.relay_llm"): result = await relay_llm.execute_current_async( {"model": "test-model", "messages": []}, provider, name="test-provider", model_name="test-model", metadata={ "api_mode": "custom", "api_request_id": "request-async-post-failure", }, ) assert result is raw_response assert turn.logical_llm_calls == {} assert "returning the provider response" in caplog.text @pytest.mark.asyncio async def test_async_non_stream_does_not_swallow_cancellation_after_provider_success( relay_turn, monkeypatch ): relay, turn = relay_turn async def provider(_request): return {"content": "already returned"} async def cancel_after_callback(_name, request, callback, **_kwargs): await callback(request) raise asyncio.CancelledError monkeypatch.setattr(relay.llm, "execute", cancel_after_callback) with pytest.raises(asyncio.CancelledError): await relay_llm.execute_current_async( {"model": "test-model", "messages": []}, provider, name="test-provider", model_name="test-model", metadata={ "api_mode": "custom", "api_request_id": "request-async-post-cancel", }, ) assert "request-async-post-cancel" in turn.logical_llm_calls relay_llm.complete_logical_call( "request-async-post-cancel", outcome="cancelled", ) @pytest.mark.asyncio async def test_async_non_stream_defers_logical_success_for_validation(relay_turn): _relay, turn = relay_turn async def provider(_request): return {"content": "pending-validation"} await relay_llm.execute_current_async( {"model": "test-model", "messages": []}, provider, name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-async-defer"}, defer_logical_completion=True, ) assert "request-async-defer" in turn.logical_llm_calls relay_llm.complete_logical_call("request-async-defer", outcome="success") assert turn.logical_llm_calls == {} def test_stream_finishes_after_relay_post_processing_failure( relay_turn, monkeypatch, caplog ): relay, turn = relay_turn async def fail_after_stream( _name, request, callback, observe_chunk, finalizer, **_kwargs, ): async def generate(): upstream = callback(request) async for chunk in upstream: observe_chunk(chunk) yield chunk finalizer() raise RuntimeError("simulated Relay post-processing failure") return generate() monkeypatch.setattr(relay.llm, "stream_execute", fail_after_stream) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter([{"delta": "complete"}]), session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "complete"}, metadata={ "api_mode": "custom", "api_request_id": "request-stream-post-failure", }, ) with caplog.at_level("WARNING", logger="agent.relay_llm"): chunks = list(stream) assert chunks == [{"delta": "complete"}] assert turn.logical_llm_calls == {} assert "preserving the provider result" in caplog.text def test_stream_flushes_buffered_provider_chunks_after_relay_failure( relay_turn, monkeypatch ): relay, turn = relay_turn raw_chunks = [{"delta": "first"}, {"delta": "second"}] async def fail_with_buffered_chunk( _name, request, callback, observe_chunk, finalizer, **_kwargs, ): async def generate(): upstream = callback(request) first = await anext(upstream) observe_chunk(first) yield first second = await anext(upstream) observe_chunk(second) with pytest.raises(StopAsyncIteration): await anext(upstream) finalizer() raise RuntimeError("simulated buffered Relay failure") return generate() monkeypatch.setattr(relay.llm, "stream_execute", fail_with_buffered_chunk) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter(raw_chunks), session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "complete"}, metadata={ "api_mode": "custom", "api_request_id": "request-buffered-failure", }, ) assert list(stream) == raw_chunks assert turn.logical_llm_calls == {} def test_stream_constructor_flushes_provider_chunks_after_relay_failure( relay_turn, monkeypatch ): relay, turn = relay_turn raw_chunks = [{"delta": "first"}, {"delta": "second"}] async def fail_during_stream_setup( _name, request, callback, observe_chunk, finalizer, **_kwargs, ): upstream = callback(request) async for chunk in upstream: observe_chunk(chunk) finalizer() raise RuntimeError("simulated Relay setup failure") monkeypatch.setattr(relay.llm, "stream_execute", fail_during_stream_setup) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter(raw_chunks), session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "complete"}, metadata={ "api_mode": "custom", "api_request_id": "request-setup-failure", }, ) assert list(stream) == raw_chunks assert turn.logical_llm_calls == {} def test_stream_does_not_swallow_interrupt_after_provider_success( relay_turn, monkeypatch ): relay, turn = relay_turn async def interrupt_after_stream( _name, request, callback, observe_chunk, finalizer, **_kwargs, ): async def generate(): upstream = callback(request) async for chunk in upstream: observe_chunk(chunk) yield chunk finalizer() raise KeyboardInterrupt return generate() monkeypatch.setattr(relay.llm, "stream_execute", interrupt_after_stream) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter([{"delta": "complete"}]), session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "complete"}, metadata={ "api_mode": "custom", "api_request_id": "request-stream-post-interrupt", }, ) assert next(stream) == {"delta": "complete"} with pytest.raises(KeyboardInterrupt): next(stream) assert turn.logical_llm_calls == {} def test_stream_does_not_swallow_hermes_finalizer_failure(relay_turn, monkeypatch): relay, _turn = relay_turn finalizer_error = RuntimeError("Hermes finalizer failed") def fail_finalizer(): raise finalizer_error async def execute_stream( _name, request, callback, _observe_chunk, finalizer, **_kwargs, ): async def generate(): upstream = callback(request) async for chunk in upstream: yield chunk finalizer() return generate() monkeypatch.setattr(relay.llm, "stream_execute", execute_stream) stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter([{"delta": "complete"}]), session_id="session-1", name="test-provider", model_name="test-model", finalizer=fail_finalizer, metadata={ "api_mode": "custom", "api_request_id": "request-finalizer-failure", }, ) with pytest.raises(Exception) as caught: list(stream) assert caught.value is finalizer_error def test_stream_defers_logical_success_for_response_validation(relay_turn): _relay, turn = relay_turn stream = relay_llm.stream( {"model": "test-model", "messages": []}, lambda _request: iter([{"delta": "pending-validation"}]), session_id="session-1", name="test-provider", model_name="test-model", finalizer=lambda: {"content": "pending-validation"}, metadata={"api_mode": "custom", "api_request_id": "request-stream-defer"}, defer_logical_completion=True, ) assert list(stream) == [{"delta": "pending-validation"}] assert stream.output_modified is False assert "request-stream-defer" in turn.logical_llm_calls relay_llm.complete_logical_call("request-stream-defer", outcome="success") assert turn.logical_llm_calls == {} def test_current_attempt_bypasses_relay_without_an_active_turn(monkeypatch): monkeypatch.setattr(relay_runtime, "current_turn", lambda: None) request = {"model": "test-model", "messages": []} result = relay_llm.execute_current( request, lambda value: value, name="test-provider", model_name="test-model", ) assert result is request def test_non_stream_bypasses_relay_without_an_active_consumer(relay_turn, monkeypatch): relay, turn = relay_turn turn.lease.host.release_managed_execution("test.relay_llm") request = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]} monkeypatch.setattr( relay.llm, "execute", lambda *_args, **_kwargs: (_ for _ in ()).throw( AssertionError("inactive Relay must not manage the provider call") ), ) result = relay_llm.execute( request, lambda value: value, session_id="session-1", name="test-provider", model_name="test-model", ) assert result is request def test_stream_bypasses_relay_without_an_active_consumer(relay_turn, monkeypatch): relay, turn = relay_turn turn.lease.host.release_managed_execution("test.relay_llm") request = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]} observed = [] monkeypatch.setattr( relay.llm, "stream_execute", lambda *_args, **_kwargs: (_ for _ in ()).throw( AssertionError("inactive Relay must not manage the provider stream") ), ) stream = relay_llm.stream( request, lambda value: (observed.append(value), iter([{"delta": "ok"}]))[1], session_id="session-1", name="test-provider", model_name="test-model", finalizer=dict, ) assert list(stream) == [{"delta": "ok"}] assert observed == [request] def test_bypassed_stream_still_honors_chunk_acceptance(relay_turn): _relay, turn = relay_turn turn.lease.host.release_managed_execution("test.relay_llm") provider_closed = [] def provider_stream(_request): try: yield {"delta": "accepted"} yield {"delta": "rejected"} yield {"delta": "unreachable"} finally: provider_closed.append(True) stream = relay_llm.stream( {"model": "test-model", "messages": []}, provider_stream, session_id="session-1", name="test-provider", model_name="test-model", finalizer=dict, accept_chunk=lambda chunk: chunk["delta"] != "rejected", ) assert list(stream) == [{"delta": "accepted"}] assert provider_closed == [True] def test_anthropic_codec_preserves_tool_history_and_cached_system_blocks(relay_turn): _relay, _turn = relay_turn request = { "model": "claude-sonnet-4-5", "max_tokens": 512, "system": [ { "type": "text", "text": "You are Hermes.", "cache_control": {"type": "ephemeral"}, } ], "messages": [ {"role": "user", "content": [{"type": "text", "text": "Run pwd"}]}, { "role": "assistant", "content": [ { "type": "tool_use", "id": "toolu_01", "name": "terminal", "input": {"command": "pwd"}, } ], }, { "role": "user", "content": [ { "type": "tool_result", "tool_use_id": "toolu_01", "content": [{"type": "text", "text": "/tmp/worktree"}], } ], }, ], } original_wire = json.dumps(request, ensure_ascii=False, separators=(",", ":")) observed_body_wire = "" def provider(final_request): nonlocal observed_body_wire provider_body = { key: value for key, value in final_request.items() if key != "extra_headers" } observed_body_wire = json.dumps( provider_body, ensure_ascii=False, separators=(",", ":"), ) return { "id": "msg_01", "type": "message", "role": "assistant", "model": "claude-sonnet-4-5", "content": [{"type": "text", "text": "Done"}], "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 10, "output_tokens": 1}, } relay_llm.execute( request, provider, session_id="session-1", name="anthropic", model_name="claude-sonnet-4-5", metadata={ "api_mode": "anthropic_messages", "api_request_id": "request-anthropic", }, ) assert observed_body_wire == original_wire def test_current_attempt_bypasses_a_closed_turn_from_a_copied_context( relay_turn, monkeypatch, ): _relay, turn = relay_turn stale_context = contextvars.copy_context() relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") request = {"model": "test-model", "messages": []} def fail_execute(*_args, **_kwargs): raise AssertionError("a closed turn must not manage later provider work") monkeypatch.setattr(relay_llm, "execute", fail_execute) result = stale_context.run( relay_llm.execute_current, request, lambda value: value, name="test-provider", model_name="test-model", ) assert result is request def test_non_stream_returns_post_execution_interceptor_result(relay_turn, monkeypatch): relay, _turn = relay_turn async def post_execute(_name, request, callback, **_kwargs): response = callback(request) return {**response, "post_interceptor": True} monkeypatch.setattr(relay.llm, "execute", post_execute) result = relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: {"content": "raw"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-post"}, ) assert result.content == "raw" assert result.post_interceptor is True @pytest.mark.asyncio async def test_async_non_stream_returns_namespaced_interceptor_result( relay_turn, monkeypatch, ): relay, _turn = relay_turn async def post_execute(_name, request, callback, **_kwargs): response = await callback(request) return { **response, "post_interceptor": True, "usage": {"input_tokens": 10}, } monkeypatch.setattr(relay.llm, "execute", post_execute) async def provider(_request): return {"content": "raw"} result = await relay_llm.execute_async( {"model": "test-model", "messages": []}, provider, session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-async-post"}, ) assert result.content == "raw" assert result.post_interceptor is True assert result.usage.input_tokens == 10 def test_non_stream_preserves_provider_error_from_relay_wrapper_suffix( relay_turn, monkeypatch ): relay, turn = relay_turn class ProviderError(Exception): pass provider_error = ProviderError("provider failed") async def wrapping_execute(_name, request, callback, **_kwargs): try: return callback(request) except Exception as exc: raise RuntimeError( f"internal error: {type(exc).__name__}: {exc} (retried 3x)" ) from None monkeypatch.setattr(relay.llm, "execute", wrapping_execute) with pytest.raises(ProviderError) as caught: relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: (_ for _ in ()).throw(provider_error), session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-error"}, ) assert caught.value is provider_error assert "request-error" in turn.logical_llm_calls def test_non_stream_does_not_mask_relay_error_after_callback_failure( relay_turn, monkeypatch ): relay, _turn = relay_turn provider_error = RuntimeError("provider failed") relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") async def translating_execute(_name, request, callback, **_kwargs): try: callback(request) except Exception: raise relay_error monkeypatch.setattr(relay.llm, "execute", translating_execute) with pytest.raises(RuntimeError) as caught: relay_llm.execute( {"model": "test-model", "messages": []}, lambda _request: (_ for _ in ()).throw(provider_error), session_id="session-1", name="test-provider", model_name="test-model", metadata={"api_mode": "custom", "api_request_id": "request-policy"}, ) assert caught.value is relay_error def test_chat_codec_preserves_provider_message_extensions_after_rewrite(relay_turn): relay, _turn = relay_turn captured_requests = [] def rewrite_request(name, request, annotated): del name annotated.params = {**(annotated.params or {}), "temperature": 0.25} return relay.LLMRequestInterceptOutcome(request, annotated) def provider(request): captured_requests.append(request) return { "id": "chatcmpl-test", "object": "chat.completion", "created": 0, "model": "test-model", "choices": [ { "index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", } ], } relay.intercepts.register_llm_request( "hermes-provider-extension-request", 1, False, rewrite_request, ) try: relay_llm.execute( { "model": "test-model", "messages": [ { "role": "assistant", "content": "", "reasoning_content": "provider scratchpad", } ], }, provider, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "chat_completions", "api_request_id": "request-3", }, ) finally: relay.intercepts.deregister_llm_request( "hermes-provider-extension-request" ) assert captured_requests[0]["temperature"] == 0.25 assert captured_requests[0]["messages"][0]["reasoning_content"] == ( "provider scratchpad" ) def test_request_rewrite_preserves_unmodified_provider_objects(relay_turn): relay, _turn = relay_turn timeout = object() captured_requests = [] def rewrite_request(name, request, annotated): del name annotated.params = {**(annotated.params or {}), "temperature": 0.25} return relay.LLMRequestInterceptOutcome(request, annotated) relay.intercepts.register_llm_request( "hermes-provider-object-request", 1, False, rewrite_request, ) try: relay_llm.execute( {"model": "test-model", "messages": [], "timeout": timeout}, lambda request: captured_requests.append(request) or {"content": "ok"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "chat_completions", "api_request_id": "request-provider-object", }, ) finally: relay.intercepts.deregister_llm_request( "hermes-provider-object-request" ) assert captured_requests[0]["timeout"] is timeout assert captured_requests[0]["temperature"] == 0.25 def test_request_rewrite_preserves_fields_dropped_by_codec(relay_turn, monkeypatch): relay, _turn = relay_turn captured_requests = [] vendor_body = { "routing": {"provider": "nim", "region": "us-west-2"}, "trace_vendor_request": False, } async def lossy_execute(_name, request, callback, **_kwargs): content = { key: value for key, value in request.content.items() if key != "extra_body" } content["temperature"] = 0.25 return callback(relay.LLMRequest(request.headers, content)) monkeypatch.setattr(relay.llm, "execute", lossy_execute) monkeypatch.setattr( relay_llm, "_codec_round_trip_request_body", lambda *_args, relay_request_body, **_kwargs: { key: value for key, value in relay_request_body.items() if key != "extra_body" }, ) relay_llm.execute( { "model": "test-model", "messages": [], "temperature": 0.0, "extra_body": vendor_body, }, lambda request: captured_requests.append(request) or {"content": "ok"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "chat_completions", "api_request_id": "request-lossy-codec", }, ) assert captured_requests == [ { "model": "test-model", "messages": [], "temperature": 0.25, "extra_body": vendor_body, } ] def test_request_rewrite_is_ignored_when_codec_baseline_fails( relay_turn, monkeypatch ): relay, _turn = relay_turn captured_requests = [] original = { "model": "test-model", "messages": [], "temperature": 0.0, "extra_body": {"routing": {"provider": "nim"}}, } async def lossy_execute(_name, request, callback, **_kwargs): rewritten = { key: value for key, value in request.content.items() if key != "extra_body" } rewritten["temperature"] = 0.25 return callback(relay.LLMRequest(request.headers, rewritten)) monkeypatch.setattr(relay.llm, "execute", lossy_execute) monkeypatch.setattr( relay_llm, "_codec_round_trip_request_body", lambda *_args, **_kwargs: None, ) relay_llm.execute( original, lambda request: captured_requests.append(request) or {"content": "ok"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "chat_completions", "api_request_id": "request-codec-failure", }, ) assert captured_requests == [original] def test_codec_baseline_failure_is_explicit(relay_turn, monkeypatch, caplog): relay, _turn = relay_turn request_body = {"model": "test-model", "messages": []} request = relay.LLMRequest({}, request_body) class FailingCodec: def decode(self, _request): raise RuntimeError("simulated codec failure") monkeypatch.setattr(relay_llm, "_codec", lambda *_args, **_kwargs: FailingCodec()) with caplog.at_level("WARNING", logger="agent.relay_llm"): baseline = relay_llm._codec_round_trip_request_body( relay, request, relay_request_body=request_body, metadata={"api_mode": "chat_completions"}, ) assert baseline is None assert "ignoring request rewrites" in caplog.text def test_request_rewrite_can_remove_codec_represented_field(relay_turn, monkeypatch): relay, _turn = relay_turn captured_requests = [] async def remove_temperature(_name, request, callback, **_kwargs): content = dict(request.content) content.pop("temperature") return callback(relay.LLMRequest(request.headers, content)) monkeypatch.setattr(relay.llm, "execute", remove_temperature) relay_llm.execute( { "model": "test-model", "messages": [], "temperature": 0.25, "extra_body": {"routing": {"provider": "nim"}}, }, lambda request: captured_requests.append(request) or {"content": "ok"}, session_id="session-1", name="test-provider", model_name="test-model", metadata={ "api_mode": "chat_completions", "api_request_id": "request-remove-field", }, ) assert captured_requests == [ { "model": "test-model", "messages": [], "extra_body": {"routing": {"provider": "nim"}}, } ]