From 4dedaa4237371778f024a98468527608cc237ef8 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Sun, 19 Jul 2026 08:59:06 -0400 Subject: [PATCH] refactor(runtime): consolidate Relay lifecycle ownership Signed-off-by: Alex Fournier --- agent/auxiliary_client.py | 324 ++++- agent/chat_completion_helpers.py | 50 +- agent/conversation_loop.py | 9 +- agent/relay_llm.py | 163 ++- agent/relay_runtime.py | 95 +- agent/relay_tools.py | 19 +- agent/turn_context.py | 2 +- agent/turn_finalizer.py | 6 +- cli.py | 31 +- gateway/run.py | 12 +- gateway/slash_commands.py | 7 +- hermes_cli/kanban_db.py | 2 +- hermes_cli/lifecycle.py | 63 + hermes_cli/observability/__init__.py | 28 +- .../observability/relay_shared_metrics.py | 30 +- hermes_cli/plugins.py | 33 +- model_tools.py | 4 +- plugins/observability/nemo_relay/README.md | 64 +- plugins/observability/nemo_relay/__init__.py | 601 ++-------- plugins/observability/nemo_relay/plugin.yaml | 5 - run_agent.py | 12 +- tests/agent/test_auxiliary_relay.py | 186 +++ tests/agent/test_relay_llm.py | 171 ++- tests/agent/test_relay_tools.py | 55 + tests/cli/test_session_boundary_hooks.py | 19 +- tests/hermes_cli/test_lifecycle.py | 60 + .../test_relay_shared_metrics_runtime.py | 199 +++- tests/plugins/test_nemo_relay_plugin.py | 1056 ++--------------- tests/run_agent/test_run_agent.py | 61 +- tools/approval.py | 2 +- tools/delegate_tool.py | 4 +- tools/terminal_tool.py | 2 +- tui_gateway/server.py | 20 +- 33 files changed, 1648 insertions(+), 1747 deletions(-) create mode 100644 hermes_cli/lifecycle.py create mode 100644 tests/agent/test_auxiliary_relay.py create mode 100644 tests/hermes_cli/test_lifecycle.py diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index da49a695180a..fe6536d34c21 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -42,6 +42,7 @@ Payment / credit exhaustion fallback: import contextlib import contextvars +import functools import hashlib import inspect import json @@ -50,6 +51,7 @@ import os import re import threading import time +import uuid from pathlib import Path # noqa: F401 — used by test mocks from types import SimpleNamespace from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING @@ -2335,6 +2337,155 @@ _RUNTIME_MAIN_AUTH_MODE: str = "" _RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( contextvars.ContextVar("auxiliary_runtime_main", default=None) ) + +_RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( + contextvars.ContextVar("auxiliary_relay_call", default=None) +) + + +def _relay_auxiliary_call(callback): + """Give every physical retry in one auxiliary call a shared Relay identity.""" + + @functools.wraps(callback) + def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return callback(*args, **kwargs) + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _relay_auxiliary_call_async(callback): + """Async counterpart to :func:`_relay_auxiliary_call`.""" + + @functools.wraps(callback) + async def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return await callback(*args, **kwargs) + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _set_relay_auxiliary_route( + provider: str | None, + model: str | None, + api_mode: str | None, +) -> None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + context["provider"] = str(provider or "auxiliary") + context["model"] = str(model or "unknown") + context["api_mode"] = str(api_mode or "chat_completions") + + +def _relay_auxiliary_metadata( + *, + provider: str | None = None, + api_mode: str | None = None, +) -> tuple[str, str, dict[str, Any]] | None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return None + attempt_count = int(context.get("attempt_count") or 0) + context["attempt_count"] = attempt_count + 1 + provider_name = str(provider or context.get("provider") or "auxiliary") + model_name = str(context.get("model") or "unknown") + return provider_name, model_name, { + "api_mode": str(api_mode or context.get("api_mode") or "chat_completions"), + "api_request_id": str(context["request_id"]), + "call_role": f"auxiliary:{context['task']}", + "retry_count": attempt_count, + "auxiliary_task": str(context["task"]), + } + + +def _relay_sync_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, +) -> Any: + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return client.chat.completions.create(**kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.execute_current( + kwargs, + lambda request: client.chat.completions.create(**request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + ) + + +async def _relay_async_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, +) -> Any: + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return await client.chat.completions.create(**kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return await relay_llm.execute_current_async( + kwargs, + lambda request: client.chat.completions.create(**request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + ) + + +def _relay_sync_stream( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, +) -> Any: + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return client.chat.completions.create(**kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.stream_current( + kwargs, + lambda request: client.chat.completions.create(**request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + finalizer=dict, + metadata=metadata, + ) _RUNTIME_MAIN_COMPAT_SNAPSHOT: Tuple[Any, ...] = ("", "", "", "", "", "") _RUNTIME_MAIN_COMPAT_LOCK = threading.Lock() @@ -3627,7 +3778,13 @@ def _retry_same_provider_sync( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task, + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -3686,7 +3843,13 @@ async def _retry_same_provider_async( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task, + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -3873,7 +4036,7 @@ def _call_fallback_candidate_sync( base_url=fb_base) try: return _validate_llm_response( - fb_client.chat.completions.create(**fb_kwargs), task) + _relay_sync_completion(fb_client, fb_kwargs, provider=fb_label), task) except Exception as fb_err: if not _is_auth_error(fb_err): raise @@ -3890,7 +4053,13 @@ def _call_fallback_candidate_sync( base_url=str(getattr(retry_client, "base_url", "") or fb_base)) try: return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=fb_provider, + ), + task, + ) except Exception as retry_err: if not _is_auth_error(retry_err): raise @@ -3939,7 +4108,13 @@ async def _call_fallback_candidate_async( base_url=fb_base) try: return _validate_llm_response( - await fb_client.chat.completions.create(**fb_kwargs), task) + await _relay_async_completion( + fb_client, + fb_kwargs, + provider=fb_label, + ), + task, + ) except Exception as fb_err: if not _is_auth_error(fb_err): raise @@ -3957,7 +4132,13 @@ async def _call_fallback_candidate_async( base_url=str(getattr(retry_client, "base_url", "") or fb_base)) try: return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=fb_provider, + ), + task, + ) except Exception as retry_err: if not _is_auth_error(retry_err): raise @@ -6906,6 +7087,7 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any: return value +@_relay_auxiliary_call def call_llm( task: str = None, *, @@ -7042,6 +7224,11 @@ def call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Log what we're about to do — makes auxiliary operations visible _base_info = str(getattr(client, "base_url", resolved_base_url) or "") @@ -7077,7 +7264,12 @@ def call_llm( kwargs["stream"] = True if stream_options: kwargs["stream_options"] = stream_options - return client.chat.completions.create(**kwargs) + return _relay_sync_stream( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ) # Handle unsupported temperature, max_tokens vs max_completion_tokens retry, # then payment fallback. @@ -7100,7 +7292,12 @@ def call_llm( # for the transient retry every auxiliary task shares. (PR #16587) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task, + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task, provider=resolved_provider, base_url=_base_info) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -7133,7 +7330,12 @@ def call_llm( time.sleep(_backoff) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_transient: if not _is_transient_transport_error(retry_transient): raise @@ -7150,7 +7352,12 @@ def call_llm( ) try: return _validate_llm_response( - client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) # If retry still fails, fall through to the max_tokens / @@ -7188,7 +7395,12 @@ def call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. @@ -7218,7 +7430,12 @@ def call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -7251,7 +7468,12 @@ def call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -7279,7 +7501,12 @@ def call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry ─────────────────────────────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -7328,7 +7555,12 @@ def call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise @@ -7573,6 +7805,7 @@ def extract_content_or_reasoning(response) -> str: return "" +@_relay_auxiliary_call_async async def async_call_llm( task: str = None, *, @@ -7664,6 +7897,11 @@ async def async_call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Pass the client's actual base_url (not just resolved_base_url) so # endpoint-specific temperature overrides can distinguish @@ -7686,7 +7924,12 @@ async def async_call_llm( # for the rationale. (PR #16587) try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task, + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task, provider=resolved_provider, base_url=_client_base) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -7707,7 +7950,12 @@ async def async_call_llm( task or "call", transient_err, ) return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) @@ -7718,7 +7966,12 @@ async def async_call_llm( ) try: return _validate_llm_response( - await client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) if not ( @@ -7752,7 +8005,12 @@ async def async_call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. @@ -7781,7 +8039,12 @@ async def async_call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -7813,7 +8076,12 @@ async def async_call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -7840,7 +8108,12 @@ async def async_call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry (mirrors sync call_llm) ─────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -7884,7 +8157,12 @@ async def async_call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 2ec624179a14..da1ae4c70ea3 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1908,6 +1908,26 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: """Request a summary when max iterations are reached. Returns the final response text.""" print(f"⚠️ Reached maximum iterations ({agent.max_iterations}). Requesting summary...") + summary_api_request_id = f"iteration-summary:{uuid.uuid4()}" + + def _managed_summary_call(request, callback, *, retry_count: int): + from agent import relay_llm + + return relay_llm.execute_current( + request, + callback, + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + metadata={ + "api_mode": str( + getattr(agent, "api_mode", "") or "chat_completions" + ), + "api_request_id": summary_api_request_id, + "call_role": "iteration_summary", + "retry_count": retry_count, + }, + ) + summary_request = ( "You've reached the maximum number of tool-calling iterations allowed. " "Please provide a final response summarizing what you've found and accomplished so far, " @@ -2087,11 +2107,22 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, is_oauth=agent._is_anthropic_oauth, preserve_dots=agent._anthropic_preserve_dots()) - summary_response = agent._anthropic_messages_create(_ant_kw) + summary_response = _managed_summary_call( + _ant_kw, + agent._anthropic_messages_create, + retry_count=0, + ) _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_summary_result.content or "").strip() else: - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=0, + ) _summary_result = agent._get_transport().normalize_response(summary_response) final_response = (_summary_result.content or "").strip() @@ -2117,7 +2148,11 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: is_oauth=agent._is_anthropic_oauth, max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, preserve_dots=agent._anthropic_preserve_dots()) - retry_response = agent._anthropic_messages_create(_ant_kw2) + retry_response = _managed_summary_call( + _ant_kw2, + agent._anthropic_messages_create, + retry_count=1, + ) _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_retry_result.content or "").strip() else: @@ -2134,7 +2169,14 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if summary_extra_body: summary_kwargs["extra_body"] = summary_extra_body - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary_retry").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary_retry" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=1, + ) _retry_result = agent._get_transport().normalize_response(summary_response) final_response = (_retry_result.content or "").strip() diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 039c3ba73a42..c88e63f72b22 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -385,7 +385,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # session is created (not on continuation). Plugins can use this # to initialise session-scoped state (e.g. warm a memory cache). try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_start", session_id=agent.session_id, @@ -1320,7 +1320,7 @@ def run_conversation( _llm_middleware_trace = [] try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -4486,7 +4486,7 @@ def run_conversation( assistant_message.content = str(raw) try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -5582,7 +5582,8 @@ def run_conversation( _attempt = getattr(agent, "_pre_verify_nudges", 0) try: from agent.verify_hooks import max_verify_nudges - from hermes_cli.plugins import get_pre_verify_continue_message, has_hook + from hermes_cli.lifecycle import has_hook + from hermes_cli.plugins import get_pre_verify_continue_message if _edited and has_hook("pre_verify") and _attempt < max_verify_nudges(): # Posture is fixed for the session — resolve once + cache. diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 98d0f83baea1..02bde00d4d10 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -71,8 +71,9 @@ def execute( ) ) except BaseException as exc: - if callback_error is not None and _relay_wrapped_callback_error( - exc, callback_error + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) ): raise callback_error raise @@ -84,6 +85,139 @@ def execute( return managed +async def execute_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, +) -> Any: + """Run one asynchronous physical provider attempt through Relay.""" + runtime, session, parent = _execution_context(session_id) + if runtime is None or session is None: + return await callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + + async def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw = await callback(final_request) + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = await runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + raise + + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return managed + + +def execute_current( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, +) -> Any: + """Run a provider attempt under the inherited Hermes turn when present.""" + turn = relay_runtime.current_turn() + if turn is None: + return callback(request) + return execute( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + ) + + +async def execute_current_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, +) -> Any: + """Run an async provider attempt under the inherited turn when present.""" + turn = relay_runtime.current_turn() + if turn is None: + return await callback(request) + return await execute_async( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + ) + + +def stream_current( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + finalizer: Callable[[], Any], + metadata: dict[str, Any] | None = None, +) -> Any: + """Run a provider stream under the inherited Hermes turn when present.""" + turn = relay_runtime.current_turn() + if turn is None: + return stream_factory(request) + return stream( + request, + stream_factory, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + metadata=metadata, + ) + + def stream( request: dict[str, Any], stream_factory: Callable[[dict[str, Any]], Any], @@ -262,8 +396,9 @@ class ManagedLlmStream(Iterator[Any]): except BaseException as exc: callback_error = self._callback_error self.close() - if callback_error is not None and _relay_wrapped_callback_error( - exc, callback_error + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) ): raise callback_error raise @@ -500,6 +635,12 @@ def _provider_request( for key, value in original.items(): if key not in relay_request_body and value is None: final.setdefault(key, value) + elif ( + key in relay_request_body + and key in final + and _json_equal(final[key], relay_request_body[key]) + ): + final[key] = value _restore_provider_message_extensions(original, final) headers = getattr(request, "headers", None) if isinstance(headers, dict) and headers: @@ -657,20 +798,6 @@ def _json_equal(left: Any, right: Any) -> bool: return False -def _relay_wrapped_callback_error( - exc: BaseException, - callback_error: BaseException, -) -> bool: - if exc is callback_error: - return True - expected_suffix = f"{callback_error.__class__.__name__}: {callback_error}" - return ( - isinstance(exc, RuntimeError) - and str(exc).startswith("internal error: ") - and str(exc).endswith(expected_suffix) - ) - - def _run_awaitable(value: Any) -> Any: if not inspect.isawaitable(value): return value diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py index c7e78403403f..d9a3d11e7380 100644 --- a/agent/relay_runtime.py +++ b/agent/relay_runtime.py @@ -24,10 +24,6 @@ RUNTIME_SCHEMA_KEY = "hermes.relay.schema_version" RUNTIME_SCHEMA_VERSION = "hermes.relay.runtime.v1" RUNTIME_INSTANCE_KEY = "hermes.relay.runtime_instance" -SESSION_START_HOOKS = frozenset({"on_session_start"}) -SESSION_CLOSE_HOOKS = frozenset({"on_session_finalize", "on_session_reset"}) -HANDLED_HOOKS = SESSION_START_HOOKS | SESSION_CLOSE_HOOKS - @dataclass class RelaySession: @@ -453,6 +449,42 @@ class RelaySessionCoordinator: def __init__(self, registry: RelayHostRegistry = HOST_REGISTRY) -> None: self.registry = registry + self._initializer_lock = threading.RLock() + self._session_initializers: dict[ + str, + Callable[[RelayRuntime, dict[str, Any]], None], + ] = {} + + def register_session_initializer( + self, + name: str, + callback: Callable[[RelayRuntime, dict[str, Any]], None], + ) -> None: + """Register idempotent profile/session preparation before scope creation.""" + with self._initializer_lock: + self._session_initializers[name] = callback + + def unregister_session_initializer(self, name: str) -> None: + """Remove a previously registered session initializer.""" + with self._initializer_lock: + self._session_initializers.pop(name, None) + + def _prepare_session( + self, + host: RelayRuntime, + context: dict[str, Any], + ) -> None: + with self._initializer_lock: + initializers = list(self._session_initializers.items()) + for name, callback in initializers: + try: + callback(host, context) + except Exception: + logger.warning( + "Hermes Relay session initializer failed: %s", + name, + exc_info=True, + ) def acquire_conversation( self, @@ -461,6 +493,7 @@ class RelaySessionCoordinator: session_id: str, platform: str, parent_session_id: str = "", + model: str = "", ) -> ConversationLease: host = self.registry.for_profile(profile_key) if host is None: @@ -468,6 +501,14 @@ class RelaySessionCoordinator: session = None if isinstance(host, RelayRuntime): try: + session_context = { + "profile_key": profile_key, + "session_id": session_id, + "platform": platform, + "parent_session_id": parent_session_id, + "model": model, + } + self._prepare_session(host, session_context) metadata = {"hermes.execution_surface": platform or "unknown"} if parent_session_id and parent_session_id != session_id: session = host.register_subagent( @@ -609,30 +650,6 @@ def current_turn() -> RelayTurnContext | None: return _CURRENT_TURN.get() -def handles_hook(hook_name: str) -> bool: - """Return whether the core Relay host consumes this lifecycle hook.""" - return hook_name in HANDLED_HOOKS - - -def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: - """Apply session lifecycle events to the core Relay host.""" - if not handles_hook(hook_name): - return - # Session hooks do not activate Relay by themselves. A direct core - # producer or an enabled built-in consumer creates the host lazily, after - # which these hooks keep its session lifetime correct. - runtime = get_runtime(create=False) - if runtime is None: - return - try: - if hook_name in SESSION_START_HOOKS: - runtime.ensure_session(kwargs) - elif hook_name in SESSION_CLOSE_HOOKS: - runtime.close_session(kwargs) - except Exception: - logger.warning("Hermes Relay lifecycle failed: %s", hook_name, exc_info=True) - - def emit_mark( name: str, *, @@ -729,6 +746,28 @@ def get_session_handle(session_id: str) -> Any: return None if runtime is None else runtime.get_session_handle(session_id) +def _is_relay_wrapped_callback_error( + relay_error: BaseException, + callback_error: BaseException, +) -> bool: + """Match Relay's native callback wrapper without masking policy errors.""" + if relay_error is callback_error: + return True + if not isinstance(relay_error, RuntimeError): + return False + callback_type = callback_error.__class__ + type_names = { + callback_type.__name__, + callback_type.__qualname__, + f"{callback_type.__module__}.{callback_type.__qualname__}", + } + message = str(relay_error) + return any( + message.startswith(f"internal error: {type_name}: {callback_error}") + for type_name in type_names + ) + + def get_runtime( *, create: bool = True, diff --git a/agent/relay_tools.py b/agent/relay_tools.py index 16867cd464e1..4ce4eb3c95e5 100644 --- a/agent/relay_tools.py +++ b/agent/relay_tools.py @@ -53,8 +53,9 @@ def execute( ) ) except BaseException as exc: - if callback_error is not None and _relay_wrapped_callback_error( - exc, callback_error + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) ): raise callback_error raise @@ -107,20 +108,6 @@ def _json_equal(left: Any, right: Any) -> bool: return left == right -def _relay_wrapped_callback_error( - relay_error: BaseException, callback_error: BaseException -) -> bool: - message = str(relay_error) - callback_type = type(callback_error) - type_names = { - callback_type.__name__, - f"{callback_type.__module__}.{callback_type.__qualname__}", - } - return "internal error" in message.lower() and any( - type_name in message for type_name in type_names - ) - - def _run_awaitable(value: Any) -> Any: if not inspect.isawaitable(value): return value diff --git a/agent/turn_context.py b/agent/turn_context.py index 419c95dab1aa..9538dcc4955b 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -697,7 +697,7 @@ def build_turn_context( # Plugin hook: pre_llm_call (context injected into user message, not system prompt). plugin_user_context = "" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _pre_results = _invoke_hook( "pre_llm_call", session_id=agent.session_id, diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index d4293bebd523..de9f404707b1 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -411,7 +411,7 @@ def finalize_turn( # First hook to return a string wins; None/empty return leaves text unchanged. if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _transform_results = _invoke_hook( "transform_llm_output", response_text=final_response, @@ -433,7 +433,7 @@ def finalize_turn( # to an external memory system). if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "post_llm_call", session_id=agent.session_id, @@ -564,7 +564,7 @@ def finalize_turn( # Fired at the very end of every run_conversation call. # Plugins can use this for cleanup, flushing buffers, etc. try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=agent.session_id, diff --git a/cli.py b/cli.py index f2c9773aef74..16f34b30cc92 100644 --- a/cli.py +++ b/cli.py @@ -1210,9 +1210,8 @@ def _notify_session_finalize( reason: str = "shutdown", ) -> None: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=session_id, platform=platform, reason=reason, @@ -1240,7 +1239,7 @@ def _emit_interrupted_session_end(cli, *, reason: str = "keyboard_interrupt") -> pass try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=session_id, @@ -6965,13 +6964,21 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): lifecycle point (shutdown, /new, /reset). """ try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - event_type, - session_id=self.agent.session_id if self.agent else None, - platform=getattr(self, "platform", None) or "cli", - reason="new_session" if event_type == "on_session_reset" else "session_boundary", - ) + from hermes_cli.lifecycle import finalize_session, invoke_hook + + context = { + "session_id": self.agent.session_id if self.agent else None, + "platform": getattr(self, "platform", None) or "cli", + "reason": ( + "new_session" + if event_type == "on_session_reset" + else "session_boundary" + ), + } + if event_type == "on_session_finalize": + finalize_session(**context) + else: + invoke_hook(event_type, **context) except Exception: pass @@ -15257,7 +15264,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # the exit occurred, meaning run_conversation's hook didn't fire. if self.agent and getattr(self, '_agent_running', False): try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=self.agent.session_id, diff --git a/gateway/run.py b/gateway/run.py index b071e4854f39..06e20444a553 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -6362,9 +6362,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _e: logger.debug("Shutdown transcript flush failed: %s", _e) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=getattr(agent, "session_id", None), platform="gateway", reason="shutdown", @@ -8347,11 +8346,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew for key, entry in _expired_entries: try: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import finalize_session _parts = key.split(":") _platform = _parts[2] if len(_parts) > 2 else "" - _invoke_hook( - "on_session_finalize", + finalize_session( session_id=entry.session_id, platform=_platform, reason="session_expired", @@ -9945,7 +9943,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # (e.g. customer handover ingest) without triggering the pairing flow. if not is_internal: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _hook_results = _invoke_hook( "pre_gateway_dispatch", event=event, diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 03c3017c8bae..f4c89afd0cf2 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -210,9 +210,8 @@ class GatewaySlashCommandsMixin: # Fire plugin on_session_finalize hook (session boundary) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=_old_sid, platform=source.platform.value if source.platform else "", reason="new_session", @@ -289,7 +288,7 @@ class GatewaySlashCommandsMixin: # Fire plugin on_session_reset hook (new session guaranteed to exist) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _new_sid = new_entry.session_id if new_entry else None _invoke_hook( "on_session_reset", diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index de332f36ee44..55ec683f5188 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -152,7 +152,7 @@ def _fire_kanban_lifecycle_hook(event: str, task_id: str, **fields: Any) -> None it through. """ try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook from hermes_cli.profiles import get_active_profile_name try: profile_name = get_active_profile_name() diff --git a/hermes_cli/lifecycle.py b/hermes_cli/lifecycle.py new file mode 100644 index 000000000000..350e3880aff9 --- /dev/null +++ b/hermes_cli/lifecycle.py @@ -0,0 +1,63 @@ +"""Hermes lifecycle dispatch for first-party observers and plugins.""" + +from __future__ import annotations + +import logging +from typing import Any, List + +logger = logging.getLogger(__name__) + + +def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]: + """Notify first-party observers, then invoke compatibility plugin hooks.""" + try: + from hermes_cli.observability import observe_lifecycle + + observe_lifecycle(hook_name, **kwargs) + except Exception: + logger.warning("Built-in observability hook failed", exc_info=True) + + from hermes_cli import plugins + + return plugins.invoke_hook(hook_name, **kwargs) + + +def has_hook(hook_name: str) -> bool: + """Return whether a first-party observer or plugin consumes a hook.""" + try: + from hermes_cli.observability import handles_hook + + if handles_hook(hook_name): + return True + except Exception: + logger.warning("Unable to inspect built-in observability hooks", exc_info=True) + + from hermes_cli import plugins + + return plugins.has_hook(hook_name) + + +def finalize_session(**kwargs: Any) -> List[Any]: + """Notify observers and hard-close one core-owned Relay conversation.""" + try: + from hermes_cli.observability import observe_lifecycle + + observe_lifecycle("on_session_finalize", **kwargs) + except Exception: + logger.warning("Built-in observability hook failed", exc_info=True) + + session_id = str(kwargs.get("session_id") or "") + if session_id: + try: + from agent import relay_runtime + + relay_runtime.SESSION_COORDINATOR.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=session_id, + ) + except Exception: + logger.warning("Core Relay session finalization failed", exc_info=True) + + from hermes_cli import plugins + + return plugins.invoke_hook("on_session_finalize", **kwargs) diff --git a/hermes_cli/observability/__init__.py b/hermes_cli/observability/__init__.py index a95e2fe43f73..1c4e70300c6f 100644 --- a/hermes_cli/observability/__init__.py +++ b/hermes_cli/observability/__init__.py @@ -8,44 +8,18 @@ from typing import Any logger = logging.getLogger(__name__) -def prepare_lifecycle(hook_name: str, **kwargs: Any) -> None: - """Prepare subscribers that must observe a session's start event.""" - from agent import relay_runtime - - from . import relay_shared_metrics - - if hook_name in relay_runtime.SESSION_START_HOOKS: - try: - relay_shared_metrics.prepare_session_start() - except Exception: - logger.warning("Built-in observability preparation failed", exc_info=True) - - def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: """Dispatch a Hermes lifecycle event to built-in observability features.""" - from agent import relay_runtime - from . import relay_shared_metrics - # Session-start plugin callbacks register optional per-session subscribers - # before this completion step opens the shared core scope. On teardown, - # metrics finish child LLM scopes before the neutral host closes the owner. - if hook_name not in relay_runtime.SESSION_CLOSE_HOOKS: - _safe_observe(relay_runtime.observe_lifecycle, hook_name, kwargs) _safe_observe(relay_shared_metrics.observe_lifecycle, hook_name, kwargs) - if hook_name in relay_runtime.SESSION_CLOSE_HOOKS: - _safe_observe(relay_runtime.observe_lifecycle, hook_name, kwargs) def handles_hook(hook_name: str) -> bool: """Return whether any built-in observability feature handles a hook.""" - from agent import relay_runtime - from . import relay_shared_metrics - return relay_runtime.handles_hook(hook_name) or relay_shared_metrics.handles_hook( - hook_name - ) + return relay_shared_metrics.handles_hook(hook_name) def _safe_observe(callback: Any, hook_name: str, kwargs: dict[str, Any]) -> None: diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index 79130f0bd7bb..978fe0807598 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -650,6 +650,17 @@ def prepare_session_start() -> None: _get_runtime(retry_failed=True) +def _prepare_core_session( + host: relay_runtime.RelayRuntime, + context: dict[str, Any], +) -> None: + """Prepare the profile subscriber before the coordinator opens a scope.""" + del context + if host.profile_key == relay_runtime.current_profile_key(): + if enabled(): + _get_runtime(retry_failed=True, host=host) + + def start_task_run( *, session_id: str, @@ -726,18 +737,25 @@ def finish_task_run( ) -def _get_runtime(*, retry_failed: bool = False) -> _Runtime | None: +def _get_runtime( + *, + retry_failed: bool = False, + host: relay_runtime.RelayRuntime | None = None, +) -> _Runtime | None: profile_key = relay_runtime.current_profile_key() with _RUNTIME_LOCK: runtime = _RUNTIMES.get(profile_key) if isinstance(runtime, _Runtime): - return runtime + if host is None or runtime.host is host: + return runtime + runtime.deactivate() + _RUNTIMES.pop(profile_key, None) if runtime is _RUNTIME_FAILED and not retry_failed: return None if runtime is _RUNTIME_FAILED: _RUNTIMES.pop(profile_key, None) try: - runtime = _Runtime() + runtime = _Runtime(host=host) except Exception: logger.warning("Hermes shared metrics initialization failed", exc_info=True) _RUNTIMES[profile_key] = _RUNTIME_FAILED @@ -746,6 +764,12 @@ def _get_runtime(*, retry_failed: bool = False) -> _Runtime | None: return runtime +relay_runtime.SESSION_COORDINATOR.register_session_initializer( + SUBSCRIBER_NAME, + _prepare_core_session, +) + + def _reset_for_tests() -> None: """Reset all profile-scoped shared-metrics state for isolated tests.""" with _RUNTIME_LOCK: diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index 9b1d24880812..a41f6c092be8 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -2047,32 +2047,10 @@ def discover_plugins(force: bool = False) -> None: def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]: - """Invoke a lifecycle hook on built-in observers and loaded plugins. + """Invoke a lifecycle hook on loaded plugins. Returns a list of non-``None`` return values from plugin callbacks. """ - if hook_name == "on_session_start": - try: - from hermes_cli.observability import prepare_lifecycle - - prepare_lifecycle(hook_name, **kwargs) - except Exception: - logger.warning("Built-in observability preparation failed", exc_info=True) - results = get_plugin_manager().invoke_hook(hook_name, **kwargs) - try: - from hermes_cli.observability import observe_lifecycle - - observe_lifecycle(hook_name, **kwargs) - except Exception: - logger.warning("Built-in observability hook failed", exc_info=True) - return results - - try: - from hermes_cli.observability import observe_lifecycle - - observe_lifecycle(hook_name, **kwargs) - except Exception: - logger.warning("Built-in observability hook failed", exc_info=True) return get_plugin_manager().invoke_hook(hook_name, **kwargs) @@ -2094,14 +2072,7 @@ def has_middleware(kind: str) -> bool: def has_hook(hook_name: str) -> bool: - """Return True when a built-in observer or plugin handles a hook.""" - try: - from hermes_cli.observability import handles_hook - - if handles_hook(hook_name): - return True - except Exception: - logger.warning("Unable to inspect built-in shared-metrics hooks", exc_info=True) + """Return True when a loaded plugin handles a hook.""" return get_plugin_manager().has_hook(hook_name) diff --git a/model_tools.py b/model_tools.py index 51535f672be1..eba7e3b5215d 100644 --- a/model_tools.py +++ b/model_tools.py @@ -997,7 +997,7 @@ def _emit_post_tool_call_hook( listener will actually consume it). """ try: - from hermes_cli.plugins import has_hook, invoke_hook + from hermes_cli.lifecycle import has_hook, invoke_hook if not has_hook("post_tool_call"): return if status is None: @@ -1324,7 +1324,7 @@ def handle_function_call( # Gated on has_hook so the no-listener path skips both the result # field derivation and the payload dispatch. try: - from hermes_cli.plugins import has_hook, invoke_hook + from hermes_cli.lifecycle import has_hook, invoke_hook if has_hook("transform_tool_result"): status, error_type, error_message = _tool_result_observer_fields(result) hook_results = invoke_hook( diff --git a/plugins/observability/nemo_relay/README.md b/plugins/observability/nemo_relay/README.md index 52d31bb78903..3f222e0383c5 100644 --- a/plugins/observability/nemo_relay/README.md +++ b/plugins/observability/nemo_relay/README.md @@ -1,18 +1,19 @@ # NeMo Relay Observability -Optional Hermes observability plugin that maps Hermes observer hooks to -NeMo Relay scopes, LLM spans, tool spans, marks, ATOF, and ATIF. +Optional Hermes observability plugin that configures exporters and maps +Hermes-specific observer hooks to NeMo Relay marks and ATIF state. Hermes core +owns Relay session, turn, LLM, and tool execution scopes. NeMo Relay is NVIDIA's runtime layer for agent execution boundaries. It does not replace Hermes Agent's planner, tools, memory, model provider routing, or -CLI UX. Instead, this plugin lets Hermes emit NeMo Relay lifecycle events for -the work Hermes already owns: sessions, turns, provider/API calls, tool calls, -approval prompts, and delegated subagents. +CLI UX. Hermes core emits NeMo Relay lifecycle events for provider and tool +execution, while this plugin enables rich exporters and observer marks for +sessions, turns, approval prompts, and delegated subagents. With this plugin enabled, Hermes Agent can: -- Preserve Hermes execution as NeMo Relay scopes, LLM spans, tool spans, and - mark events. +- Export the Relay scopes and LLM/tool lifecycles emitted by Hermes core. +- Add Hermes session, turn, approval, and subagent mark events. - Export raw lifecycle events as Agent Trajectory Observability Format (ATOF) JSONL for debugging and offline inspection. - Export Agent Trajectory Interchange Format (ATIF) trajectories for replay, @@ -167,8 +168,9 @@ Relay owns exporter lifecycle through that config. The direct double-export trajectories on teardown. If `plugins.toml` initialization fails, Hermes keeps the direct env-var fallbacks active for that run. -To enable NeMo Relay managed execution intercepts for provider and tool calls, -include an adaptive component in the same `plugins.toml`: +Hermes core routes provider and tool execution through NeMo Relay managed APIs +regardless of whether this plugin is enabled. To install adaptive interceptors +on those boundaries, include an adaptive component in the same `plugins.toml`: ```toml [[components]] @@ -179,13 +181,10 @@ enabled = true mode = "observe_only" ``` -When the adaptive component is enabled and the installed NeMo Relay runtime -exposes `llm.execute(...)` / `tools.execute(...)`, Hermes routes LLM and tool -execution through those middleware boundaries. The observer hooks still emit -session, turn, approval, and subagent marks; the plugin skips its manual -`llm.call` and `tools.call` spans for executions that are already managed by -NeMo Relay. `tool_parallelism.mode = "observe_only"` keeps tool scheduling -observational while still wrapping the real execution boundary. +The observer hooks emit session, turn, approval, and subagent marks. They do not +create a second LLM or tool lifecycle. `tool_parallelism.mode = "observe_only"` +keeps tool scheduling observational while still intercepting the core-managed +execution boundary. ### Dynamic Plugins @@ -437,8 +436,8 @@ Sanitized ATIF excerpt: The plugin keeps NeMo Relay's native event model: - Hermes sessions map to `agent` scopes. -- Hermes API request hooks map to `llm` scope start/end events. -- Hermes tool hooks map to `tool` scope start/end events. +- Hermes core managed provider calls map to `llm` scope start/end events. +- Hermes core managed tool calls map to `tool` scope start/end events. - Turn, approval, subagent, and diagnostic fallback events map to `mark` events. @@ -448,11 +447,11 @@ subagent IDs, role/status fields when present, and derived stream lossless for later ATIF conversion that can compact subagents into separate trajectories. -## Adaptive Middleware Example +## Adaptive Execution Example -The `observability/nemo_relay` plugin uses Hermes execution middleware to hand -LLM and tool calls to NeMo Relay managed execution when an adaptive component is -enabled. +Hermes core always hands LLM and tool calls to NeMo Relay managed execution. +The `observability/nemo_relay` plugin can install adaptive components on those +boundaries. Minimal `plugins.toml`: @@ -473,26 +472,21 @@ Enable it for Hermes: export HERMES_NEMO_RELAY_PLUGINS_TOML=/tmp/hermes-middleware-test/plugins.toml ``` -When the adaptive component is enabled and the installed NeMo Relay runtime -exposes `llm.execute(...)` and `tools.execute(...)`, Hermes routes execution -through these boundaries: +Execution follows these boundaries with or without an adaptive component: ```text Hermes provider call - -> llm_execution middleware - -> nemo_relay.llm.execute(...) - -> Hermes provider adapter next_call(...) + -> nemo_relay.llm.execute(...) + -> Hermes provider adapter callback(...) Hermes tool call - -> tool_execution middleware - -> nemo_relay.tools.execute(...) - -> Hermes tool dispatcher next_call(...) + -> nemo_relay.tools.execute(...) + -> Hermes authorization and dispatch callback(...) ``` -The plugin still emits observer marks for sessions, turns, approvals, and -subagents. When adaptive managed execution is active, it skips manual -`llm.call` and `tools.call` observer spans to avoid duplicate LLM/tool events -for the same execution. +The plugin emits observer marks for sessions, turns, approvals, and subagents. +It does not register provider or tool lifecycle hooks, so each managed call +produces one Relay lifecycle. ### Local Adaptive E2E diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index a4f3c0b458ee..c173aa1a35a6 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -22,11 +22,7 @@ logger = logging.getLogger(__name__) _INIT_FAILED = object() _LOCK = threading.RLock() _RUNTIMES: dict[str, "_Runtime | object"] = {} -_RELAY_LLM_SURFACE_BY_API_MODE = { - "anthropic_messages": "anthropic.messages", - "chat_completions": "openai.chat_completions", - "codex_responses": "openai.responses", -} +_SESSION_INITIALIZER_NAME = "hermes.nemo_relay.rich_observability" @dataclass @@ -38,8 +34,6 @@ class _SessionState: atif_subscriber_name: str = "" is_embedded_subagent: bool = False parent_session_id: str = "" - llm_spans: dict[str, Any] = field(default_factory=dict) - tool_spans: dict[str, Any] = field(default_factory=dict) @dataclass @@ -53,8 +47,6 @@ class _Settings: plugins_toml_path: str = "" plugins_config: dict[str, Any] | None = None dynamic_plugins: list[dict[str, Any]] = field(default_factory=list) - adaptive_enabled: bool = False - adaptive_mode: str = "observe_only" atof_enabled: bool = False atof_output_directory: str = "" atof_filename: str = "hermes-atof.jsonl" @@ -78,6 +70,7 @@ class _Runtime: self.nemo_relay = nemo_relay self.settings = settings self.host = host + self._sessions_lock = threading.RLock() self.sessions: dict[str, _SessionState] = {} self.subagent_contexts: dict[str, _SubagentContext] = {} self.atof_exporter: Any = None @@ -241,34 +234,47 @@ class _Runtime: logger.debug("NeMo Relay ATOF deregister failed", exc_info=True) self.atof_exporter = None - def ensure_session(self, kwargs: dict[str, Any]) -> _SessionState: - self._maybe_reinitialize_plugins_toml() + def prepare_session(self, kwargs: dict[str, Any]) -> _SessionState: + """Register per-session subscribers without opening the core scope.""" session_id = _session_id(kwargs) - state = self.sessions.get(session_id) - if state is not None: + with self._sessions_lock: + self._maybe_reinitialize_plugins_toml() + state = self.sessions.get(session_id) + if state is not None: + return state + + state = _SessionState(session_id=session_id) + if self.settings.atif_enabled and not self._plugins_toml_owns_exporter("atif"): + state.atif_exporter = self.nemo_relay.AtifExporter( + session_id, + self.settings.atif_agent_name, + self.settings.atif_agent_version, + model_name=str(kwargs.get("model") or self.settings.atif_model_name), + extra={ + "source": "hermes-agent", + "plugin": "observability/nemo_relay", + }, + ) + state.atif_subscriber_name = ( + f"hermes.nemo_relay.atif.{self.host.runtime_id}.{session_id}" + ) + state.atif_exporter.register(state.atif_subscriber_name) + self.sessions[session_id] = state return state - state = _SessionState(session_id=session_id) - if self.settings.atif_enabled and not self._plugins_toml_owns_exporter("atif"): - state.atif_exporter = self.nemo_relay.AtifExporter( - session_id, - self.settings.atif_agent_name, - self.settings.atif_agent_version, - model_name=str(kwargs.get("model") or self.settings.atif_model_name), - extra={"source": "hermes-agent", "plugin": "observability/nemo_relay"}, - ) - state.atif_subscriber_name = ( - f"hermes.nemo_relay.atif.{self.host.runtime_id}.{session_id}" - ) - state.atif_exporter.register(state.atif_subscriber_name) + def ensure_session(self, kwargs: dict[str, Any]) -> _SessionState: + state = self.prepare_session(kwargs) + if state.relay_session is not None: + return state rich_metadata = _metadata(kwargs) - subagent_context = self.subagent_contexts.get(session_id) + with self._sessions_lock: + subagent_context = self.subagent_contexts.get(state.session_id) if subagent_context is not None: rich_metadata = {**rich_metadata, **subagent_context.metadata} relay_session = self.host.ensure_session( kwargs, - data={"session_id": session_id}, + data={"session_id": state.session_id}, metadata=rich_metadata, ) if relay_session is None: @@ -278,7 +284,6 @@ class _Runtime: if subagent_context is not None: state.is_embedded_subagent = True state.parent_session_id = subagent_context.parent_session_id - self.sessions[session_id] = state return state def run_in_session( @@ -297,22 +302,6 @@ class _Runtime: **kwargs, ) - async def run_in_session_async( - self, - state: _SessionState, - callback: Callable[..., Any], - *args: Any, - **kwargs: Any, - ) -> Any: - if state.relay_session is None: - raise RuntimeError("Hermes core Relay session is unavailable") - return await self.host.run_in_session_async( - state.relay_session, - callback, - *args, - **kwargs, - ) - def export_atif(self, state: _SessionState) -> None: if not self.settings.atif_enabled or state.atif_exporter is None: return @@ -332,8 +321,9 @@ class _Runtime: close_host: bool = True, ) -> None: session_id = _session_id(kwargs) - self.subagent_contexts.pop(session_id, None) - state = self.sessions.pop(session_id, None) + with self._sessions_lock: + self.subagent_contexts.pop(session_id, None) + state = self.sessions.pop(session_id, None) if state is None: return failures: list[str] = [] @@ -351,21 +341,22 @@ class _Runtime: state.atif_exporter.deregister(state.atif_subscriber_name) except Exception as exc: failures.append(f"ATIF deregister failed: {exc}") - if ( - self._plugin_config_initialized - and self._plugin_activation is None - and not self.sessions - ): - try: - self._clear_plugins_toml() - except Exception as exc: - failures.append(f"plugin configuration clear failed: {exc}") - elif ( - self.settings.plugins_config - and self._plugin_activation is None - and not self.sessions - ): - self._plugin_config_needs_reinit = True + with self._sessions_lock: + if ( + self._plugin_config_initialized + and self._plugin_activation is None + and not self.sessions + ): + try: + self._clear_plugins_toml() + except Exception as exc: + failures.append(f"plugin configuration clear failed: {exc}") + elif ( + self.settings.plugins_config + and self._plugin_activation is None + and not self.sessions + ): + self._plugin_config_needs_reinit = True if failures: logger.warning( "NeMo Relay session %s teardown completed with errors: %s", @@ -376,7 +367,9 @@ class _Runtime: def shutdown(self) -> None: """Close active sessions and the process-lifetime plugin activation.""" failures: list[str] = [] - for session_id in list(self.sessions): + with self._sessions_lock: + session_ids = list(self.sessions) + for session_id in session_ids: try: self.close_session({"session_id": session_id, "reason": "runtime_shutdown"}) except Exception as exc: @@ -412,10 +405,11 @@ class _Runtime: metadata = _metadata(kwargs) child_session_id = _child_session_id(kwargs) if child_session_id: - self.subagent_contexts[child_session_id] = _SubagentContext( - parent_session_id=parent_state.session_id, - metadata=_subagent_child_metadata(kwargs, metadata), - ) + with self._sessions_lock: + self.subagent_contexts[child_session_id] = _SubagentContext( + parent_session_id=parent_state.session_id, + metadata=_subagent_child_metadata(kwargs, metadata), + ) self.run_in_session( parent_state, self.nemo_relay.scope.event, @@ -432,151 +426,15 @@ class _Runtime: {"session_id": child_session_id}, close_host=False, ) - self.subagent_contexts.pop(child_session_id, None) + with self._sessions_lock: + self.subagent_contexts.pop(child_session_id, None) self.mark("hermes.subagent.stop", kwargs) - def managed_llm_enabled(self) -> bool: - return ( - (self.settings.adaptive_enabled or self._plugin_activation is not None) - and callable(getattr(getattr(self.nemo_relay, "llm", None), "execute", None)) - and callable(getattr(self.nemo_relay, "LLMRequest", None)) - ) - - def managed_tool_enabled(self) -> bool: - return ( - (self.settings.adaptive_enabled or self._plugin_activation is not None) - and callable(getattr(getattr(self.nemo_relay, "tools", None), "execute", None)) - ) - - def _run_managed_with_downstream_preservation( - self, - next_call: Callable[[Any], Any], - normalize_payload: Callable[[Any], Any], - shape_response: Callable[[Any], Any], - make_managed_execute: Callable[[Callable[[Any], Any]], Any], - *, - preserve_raw_response: bool, - ) -> Any: - # NeMo Relay's native managed execution may wrap a failing callback as an - # internal runtime error, hiding the real downstream provider/tool - # exception. Capture the original here and re-raise it after managed - # execution so Hermes retry classification still sees it. The LLM and tool - # paths share this scaffolding; they differ only in payload normalization, - # response shaping, and the Relay call itself. - raw_response: dict[str, Any] = {"set": False, "value": None, "normalized": None} - callback_error: Exception | None = None - downstream_error: BaseException | None = None - - def _impl(next_payload: Any) -> Any: - nonlocal callback_error, downstream_error - try: - raw = next_call(normalize_payload(next_payload)) - except Exception as exc: - callback_error = exc - downstream_error = _original_downstream_error(exc) - raise - raw_response["set"] = True - raw_response["value"] = raw - raw_response["normalized"] = shape_response(raw) - return raw_response["normalized"] - - try: - managed_result = _resolve_awaitable(make_managed_execute(_impl)) - except Exception as exc: - if downstream_error is not None and _is_relay_wrapped_callback_error(exc, callback_error): - raise downstream_error - raise - if ( - preserve_raw_response - and raw_response["set"] - and _json_semantically_equal(managed_result, raw_response["normalized"]) - ): - return raw_response["value"] - return managed_result - - def execute_llm(self, kwargs: dict[str, Any]) -> Any: - state = self.ensure_session(kwargs) - request_body = _jsonable(kwargs.get("request") or {}) - request = self.nemo_relay.LLMRequest({}, request_body) - next_call = kwargs.get("next_call") - if not callable(next_call): - return request_body - - def _normalize(next_request: Any) -> Any: - next_body = getattr(next_request, "content", next_request) - return next_body if isinstance(next_body, dict) else request_body - - def _make_managed(impl: Callable[[Any], Any]) -> Any: - async def _managed_execute() -> Any: - return await self.run_in_session_async( - state, - self.nemo_relay.llm.execute, - _relay_llm_surface(kwargs), - request, - impl, - handle=state.handle, - data=_jsonable( - { - "turn_id": kwargs.get("turn_id"), - "api_request_id": kwargs.get("api_request_id"), - "api_call_count": kwargs.get("api_call_count"), - "mode": self.settings.adaptive_mode, - } - ), - metadata=_metadata(kwargs), - model_name=str(kwargs.get("model") or ""), - ) - - return _managed_execute() - - return self._run_managed_with_downstream_preservation( - next_call, _normalize, _llm_response_payload, _make_managed, preserve_raw_response=True - ) - - def execute_tool(self, kwargs: dict[str, Any]) -> Any: - state = self.ensure_session(kwargs) - tool_name = str(kwargs.get("tool_name") or "tool") - args = _jsonable(kwargs.get("args") or {}) - next_call = kwargs.get("next_call") - if not callable(next_call): - return args - - def _normalize(next_args: Any) -> Any: - normalized = next_args if isinstance(next_args, dict) else args - if not _json_semantically_equal(normalized, args): - raise RuntimeError( - "NeMo Relay changed tool arguments after Hermes authorization" - ) - return args - - def _make_managed(impl: Callable[[Any], Any]) -> Any: - async def _managed_execute() -> Any: - return await self.run_in_session_async( - state, - self.nemo_relay.tools.execute, - tool_name, - args, - impl, - handle=state.handle, - data=_jsonable( - { - "turn_id": kwargs.get("turn_id"), - "api_request_id": kwargs.get("api_request_id"), - "tool_call_id": kwargs.get("tool_call_id"), - "mode": self.settings.adaptive_mode, - } - ), - metadata=_metadata(kwargs), - ) - - return _managed_execute() - - return self._run_managed_with_downstream_preservation( - next_call, _normalize, _jsonable, _make_managed, preserve_raw_response=False - ) - - def register(ctx) -> None: + relay_runtime.SESSION_COORDINATOR.register_session_initializer( + _SESSION_INITIALIZER_NAME, + _prepare_core_session, + ) # Activate dynamic plugins before Hermes installs the managed execution # boundaries that invoke their interceptors. if _load_settings().dynamic_plugins: @@ -587,11 +445,6 @@ def register(ctx) -> None: ctx.register_hook("on_session_reset", on_session_reset) ctx.register_hook("pre_llm_call", on_pre_llm_call) ctx.register_hook("post_llm_call", on_post_llm_call) - ctx.register_hook("pre_api_request", on_pre_api_request) - ctx.register_hook("post_api_request", on_post_api_request) - ctx.register_hook("api_request_error", on_api_request_error) - ctx.register_hook("pre_tool_call", on_pre_tool_call) - ctx.register_hook("post_tool_call", on_post_tool_call) ctx.register_hook("pre_approval_request", on_pre_approval_request) ctx.register_hook("post_approval_response", on_post_approval_response) ctx.register_hook("subagent_start", on_subagent_start) @@ -613,13 +466,13 @@ def on_session_end(**kwargs: Any) -> None: def on_session_finalize(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: - _safe(lambda: runtime.close_session(kwargs)) + _safe(lambda: runtime.close_session(kwargs, close_host=False)) def on_session_reset(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: - _safe(lambda: runtime.close_session(kwargs)) + _safe(lambda: runtime.close_session(kwargs, close_host=False)) def on_pre_llm_call(**kwargs: Any) -> None: @@ -634,132 +487,6 @@ def on_post_llm_call(**kwargs: Any) -> None: _safe(lambda: runtime.mark("hermes.turn.end", kwargs)) -def on_pre_api_request(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - request_payload = kwargs.get("request") - request_body = request_payload.get("body") if isinstance(request_payload, dict) else {} - request = runtime.nemo_relay.LLMRequest({}, _jsonable(request_body)) - span = runtime.run_in_session( - state, - runtime.nemo_relay.llm.call, - str(kwargs.get("provider") or "llm"), - request, - handle=state.handle, - data=_jsonable({"turn_id": kwargs.get("turn_id"), "api_request_id": kwargs.get("api_request_id")}), - metadata=_metadata(kwargs), - model_name=str(kwargs.get("model") or ""), - ) - state.llm_spans[_api_key(kwargs)] = span - - _safe(_record) - - -def on_post_api_request(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.llm_spans.pop(_api_key(kwargs), None) - if span is None: - runtime.mark("hermes.api.response.unmatched", kwargs) - return - runtime.run_in_session( - state, - runtime.nemo_relay.llm.call_end, - span, - _jsonable(kwargs.get("response") or {}), - data=_jsonable({"usage": kwargs.get("usage"), "finish_reason": kwargs.get("finish_reason")}), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - -def on_api_request_error(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.llm_spans.pop(_api_key(kwargs), None) - if span is None: - runtime.mark("hermes.api.error", kwargs) - return - runtime.run_in_session( - state, - runtime.nemo_relay.llm.call_end, - span, - {"error": _jsonable(kwargs.get("error") or {})}, - data=_jsonable(kwargs), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - -def on_pre_tool_call(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_tool_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = runtime.run_in_session( - state, - runtime.nemo_relay.tools.call, - str(kwargs.get("tool_name") or "tool"), - _jsonable(kwargs.get("args") or {}), - handle=state.handle, - data=_jsonable({"turn_id": kwargs.get("turn_id"), "api_request_id": kwargs.get("api_request_id")}), - metadata=_metadata(kwargs), - tool_call_id=str(kwargs.get("tool_call_id") or ""), - ) - state.tool_spans[_tool_key(kwargs)] = span - - _safe(_record) - - -def on_post_tool_call(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_tool_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.tool_spans.pop(_tool_key(kwargs), None) - if span is None: - runtime.mark("hermes.tool.response.unmatched", kwargs) - return - runtime.run_in_session( - state, - runtime.nemo_relay.tools.call_end, - span, - _jsonable(kwargs.get("result")), - data=_jsonable({"status": kwargs.get("status"), "duration_ms": kwargs.get("duration_ms")}), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - def on_pre_approval_request(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: @@ -784,44 +511,42 @@ def on_subagent_stop(**kwargs: Any) -> None: _safe(lambda: runtime.mark_subagent_stop(kwargs)) -def on_llm_execution_middleware(**kwargs: Any) -> Any: - runtime = _get_runtime() - next_call = kwargs.get("next_call") - request = kwargs.get("request") or {} - if runtime is not None and runtime.managed_llm_enabled(): - return runtime.execute_llm(kwargs) - if callable(next_call): - return next_call(request) - return request +def _prepare_core_session( + host: relay_runtime.RelayRuntime, + context: dict[str, Any], +) -> None: + """Register rich subscribers before core creates the conversation scope.""" + runtime = _get_runtime( + profile_key=str(context.get("profile_key") or host.profile_key), + host=host, + ) + if runtime is not None: + runtime.prepare_session(context) -def on_tool_execution_middleware(**kwargs: Any) -> Any: - runtime = _get_runtime() - next_call = kwargs.get("next_call") - args = kwargs.get("args") or {} - if runtime is not None and runtime.managed_tool_enabled(): - return runtime.execute_tool(kwargs) - if callable(next_call): - return next_call(args) - return args - - -def _get_runtime() -> Optional[_Runtime]: - profile_key = relay_runtime.current_profile_key() +def _get_runtime( + *, + profile_key: str | None = None, + host: relay_runtime.RelayRuntime | None = None, +) -> Optional[_Runtime]: + profile_key = profile_key or relay_runtime.current_profile_key() with _LOCK: runtime = _RUNTIMES.get(profile_key) if runtime is _INIT_FAILED: return None if isinstance(runtime, _Runtime): - return runtime + if host is None or runtime.host is host: + return runtime + runtime.shutdown() + _RUNTIMES.pop(profile_key, None) try: - host = relay_runtime.get_runtime() - if host is None: + resolved_host = host or relay_runtime.get_runtime(profile_key=profile_key) + if resolved_host is None: raise RuntimeError("Hermes core Relay runtime is unavailable") runtime = _Runtime( - nemo_relay=host.relay, + nemo_relay=resolved_host.relay, settings=_load_settings(), - host=host, + host=resolved_host, ) except Exception as exc: logger.debug("NeMo Relay plugin disabled: init failed: %s", exc, exc_info=True) @@ -834,13 +559,10 @@ def _get_runtime() -> Optional[_Runtime]: def _load_settings() -> _Settings: plugins_toml_path = _env("HERMES_NEMO_RELAY_PLUGINS_TOML") plugins_config = _load_plugins_config(plugins_toml_path) - adaptive_config = _enabled_component_config(plugins_config, "adaptive") return _Settings( plugins_toml_path=plugins_toml_path, plugins_config=plugins_config, dynamic_plugins=_dynamic_plugin_specs(plugins_config, plugins_toml_path), - adaptive_enabled=adaptive_config is not None, - adaptive_mode=_adaptive_mode(adaptive_config), atof_enabled=_env_bool("HERMES_NEMO_RELAY_ATOF_ENABLED"), atof_output_directory=_env("HERMES_NEMO_RELAY_ATOF_OUTPUT_DIRECTORY"), atof_filename=_env("HERMES_NEMO_RELAY_ATOF_FILENAME") or "hermes-atof.jsonl", @@ -1014,20 +736,6 @@ def _enabled_component_config( return None -def _adaptive_mode(config: dict[str, Any] | None) -> str: - if not isinstance(config, dict): - return "observe_only" - tool_parallelism = config.get("tool_parallelism") - if isinstance(tool_parallelism, dict): - mode = tool_parallelism.get("mode") - if isinstance(mode, str) and mode.strip(): - return mode.strip() - mode = config.get("mode") - if isinstance(mode, str) and mode.strip(): - return mode.strip() - return "observe_only" - - def _observability_exporter_enabled( plugins_config: dict[str, Any] | None, exporter_name: str, @@ -1089,25 +797,6 @@ def _subagent_child_metadata( return metadata -def _api_key(kwargs: dict[str, Any]) -> str: - return str(kwargs.get("api_request_id") or f"{_session_id(kwargs)}:{kwargs.get('api_call_count') or 'api'}") - - -def _tool_key(kwargs: dict[str, Any]) -> str: - return str( - kwargs.get("tool_call_id") - or f"{_session_id(kwargs)}:{kwargs.get('turn_id') or ''}:{kwargs.get('tool_name') or 'tool'}" - ) - - -def _relay_llm_surface(kwargs: dict[str, Any]) -> str: - api_mode = str(kwargs.get("api_mode") or "").strip().lower() - return _RELAY_LLM_SURFACE_BY_API_MODE.get( - api_mode, - str(kwargs.get("provider") or "llm"), - ) - - def _metadata(kwargs: dict[str, Any]) -> dict[str, Any]: keys = ( "telemetry_schema_version", @@ -1167,105 +856,6 @@ def _jsonable(value: Any) -> Any: return str(value) -def _json_semantically_equal(left: Any, right: Any) -> bool: - """Compare JSON-compatible values without conflating booleans and numbers.""" - try: - left_json = json.dumps( - _jsonable(left), ensure_ascii=False, sort_keys=True, separators=(",", ":") - ) - right_json = json.dumps( - _jsonable(right), ensure_ascii=False, sort_keys=True, separators=(",", ":") - ) - return left_json == right_json - except (TypeError, ValueError): - return False - - -def _value(obj: Any, key: str, default: Any = None) -> Any: - if isinstance(obj, dict): - return obj.get(key, default) - return getattr(obj, key, default) - - -def _original_downstream_error(exc: Exception) -> BaseException: - # Hermes wraps downstream execution failures in a local/private exception - # class, so detect the wrapper by shape instead of importing it here. - original = getattr(exc, "original", None) - if exc.__class__.__name__ == "_DownstreamExecutionError" and isinstance(original, BaseException): - return original - return exc - - -def _is_relay_wrapped_callback_error(exc: Exception, callback_error: Exception | None) -> bool: - # NeMo Relay re-wraps a failing callback as ``RuntimeError("internal error: - # : ")``. Match by prefix rather than exact equality so a - # trailing traceback/suffix in a future Relay version doesn't silently defeat - # the unwrap; the class-name + message prefix still discriminates the real - # downstream failure from unrelated Relay-internal errors. If Relay drops the - # leading ``internal error:`` shape entirely, this returns False and Hermes - # falls back to surfacing Relay's error (the pre-fix behavior) rather than - # masking it. - if callback_error is None or not isinstance(exc, RuntimeError): - return False - expected = f"internal error: {callback_error.__class__.__name__}: {callback_error}" - return str(exc).startswith(expected) - - -def _llm_response_payload(response: Any) -> Any: - """Return the LLM response shape NeMo Relay's ATIF conversion expects.""" - payload = _jsonable(response) - if isinstance(payload, dict) and "assistant_message" in payload: - return payload - - choices = _value(response, "choices") - if choices is None and isinstance(payload, dict): - choices = payload.get("choices") - first_choice = choices[0] if isinstance(choices, list) and choices else None - message = _value(first_choice, "message") - finish_reason = _value(first_choice, "finish_reason") - - assistant_message: dict[str, Any] = {"role": "assistant", "content": ""} - if message is not None: - assistant_message["role"] = _value(message, "role", "assistant") or "assistant" - content = _value(message, "content") - if content is not None: - assistant_message["content"] = _jsonable(content) - tool_calls = _tool_calls_payload(_value(message, "tool_calls")) - if tool_calls: - assistant_message["tool_calls"] = tool_calls - reasoning = _value(message, "reasoning_content") - if reasoning is not None: - assistant_message["reasoning_content"] = _jsonable(reasoning) - elif isinstance(payload, dict): - assistant_message["content"] = payload.get("content") or payload.get("output_text") or "" - - return { - "model": _value(response, "model", payload.get("model") if isinstance(payload, dict) else None), - "assistant_message": assistant_message, - "finish_reason": finish_reason, - "usage": _jsonable(_value(response, "usage", payload.get("usage") if isinstance(payload, dict) else None)), - } - - -def _tool_calls_payload(tool_calls: Any) -> list[dict[str, Any]]: - if not isinstance(tool_calls, list): - return [] - normalized: list[dict[str, Any]] = [] - for call in tool_calls: - function = _value(call, "function") - normalized.append( - { - "id": _value(call, "id"), - "type": _value(call, "type", "function") or "function", - "function": { - "name": _value(function, "name"), - "arguments": _value(function, "arguments"), - }, - } - ) - return normalized - - def _safe(fn) -> None: try: fn() @@ -1303,6 +893,9 @@ def _resolve_awaitable(value: Any) -> Any: def reset_for_tests() -> None: + relay_runtime.SESSION_COORDINATOR.unregister_session_initializer( + _SESSION_INITIALIZER_NAME + ) with _LOCK: runtimes = list(_RUNTIMES.values()) _RUNTIMES.clear() diff --git a/plugins/observability/nemo_relay/plugin.yaml b/plugins/observability/nemo_relay/plugin.yaml index b1b00f25d819..046d5d0d8510 100644 --- a/plugins/observability/nemo_relay/plugin.yaml +++ b/plugins/observability/nemo_relay/plugin.yaml @@ -9,11 +9,6 @@ hooks: - on_session_reset - pre_llm_call - post_llm_call - - pre_api_request - - post_api_request - - api_request_error - - pre_tool_call - - post_tool_call - pre_approval_request - post_approval_response - subagent_start diff --git a/run_agent.py b/run_agent.py index 2bd2a9458de3..82e6065c9fdd 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2600,17 +2600,16 @@ class AIAgent: retryable: Optional[bool] = None, reason: Optional[str] = None, ) -> None: - # Lazy module import (not from-import) so tests that - # ``monkeypatch.setattr("hermes_cli.plugins.has_hook", ...)`` still - # take effect on this call site. After first call the import is a + # Lazy module import (not from-import) so tests can replace lifecycle + # dispatch at this call site. After first call the import is a # ``sys.modules`` dict lookup, so retries don't repay any real cost. try: - from hermes_cli import plugins as _plugins + from hermes_cli import lifecycle as _lifecycle - if not _plugins.has_hook("api_request_error"): + if not _lifecycle.has_hook("api_request_error"): return ended_at = time.time() - _plugins.invoke_hook( + _lifecycle.invoke_hook( "api_request_error", task_id=task_id, turn_id=turn_id, @@ -6374,6 +6373,7 @@ class AIAgent: session_id=task_context["session_id"], platform=task_context["platform"], parent_session_id=relay_parent_session_id, + model=str(getattr(self, "model", None) or ""), ) relay_turn = relay_runtime.SESSION_COORDINATOR.begin_turn( relay_lease, diff --git a/tests/agent/test_auxiliary_relay.py b/tests/agent/test_auxiliary_relay.py new file mode 100644 index 000000000000..8fc7ce168d3e --- /dev/null +++ b/tests/agent/test_auxiliary_relay.py @@ -0,0 +1,186 @@ +from types import SimpleNamespace + +import pytest + +from agent import auxiliary_client, relay_llm, relay_runtime + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + try: + yield lease.host.relay, turn + finally: + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_auxiliary_retries_share_logical_relay_identity(monkeypatch): + attempts = [] + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **kwargs: {"request": kwargs}, + ) + ) + ) + + def execute_current(request, callback, **kwargs): + attempts.append(kwargs) + return callback(request) + + monkeypatch.setattr(relay_llm, "execute_current", execute_current) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + first = auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ) + second = auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ) + return first, second + + first, second = run("compression") + + assert first["request"]["model"] == "test-model" + assert second["request"]["model"] == "test-model" + assert attempts[0]["metadata"]["api_request_id"] == ( + attempts[1]["metadata"]["api_request_id"] + ) + assert [attempt["metadata"]["retry_count"] for attempt in attempts] == [0, 1] + assert attempts[0]["metadata"]["call_role"] == "auxiliary:compression" + + +@pytest.mark.asyncio +async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch): + captured = {} + + async def create(**kwargs): + return {"request": kwargs} + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + async def execute_current_async(request, callback, **kwargs): + captured.update(kwargs) + return await callback(request) + + monkeypatch.setattr( + relay_llm, + "execute_current_async", + execute_current_async, + ) + + @auxiliary_client._relay_auxiliary_call_async + async def run(task): + auxiliary_client._set_relay_auxiliary_route( + "anthropic", + "claude-test", + "chat_completions", + ) + return await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ) + + result = await run("title_generation") + + assert result["request"]["model"] == "claude-test" + assert captured["name"] == "anthropic" + assert captured["metadata"]["call_role"] == "auxiliary:title_generation" + + +def test_auxiliary_stream_uses_streaming_relay_primitive(monkeypatch): + captured = {} + raw_stream = iter([{"delta": "one"}, {"delta": "two"}]) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace(create=lambda **_kwargs: raw_stream) + ) + ) + + def stream_current(request, stream_factory, **kwargs): + captured.update(kwargs) + return stream_factory(request) + + monkeypatch.setattr(relay_llm, "stream_current", stream_current) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "moa-model", + "chat_completions", + ) + return auxiliary_client._relay_sync_stream( + client, + {"model": "moa-model", "messages": [], "stream": True}, + ) + + assert list(run("moa")) == [{"delta": "one"}, {"delta": "two"}] + assert captured["metadata"]["call_role"] == "auxiliary:moa" + + +def test_auxiliary_attempt_uses_real_relay_request_intercepts(relay_turn): + relay, turn = relay_turn + captured_requests = [] + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **kwargs: captured_requests.append(kwargs) + or {"content": "ok"}, + ) + ) + ) + + def rewrite_request(_name, request, annotated): + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + relay.intercepts.register_llm_request( + "hermes-auxiliary-request", + 1, + False, + rewrite_request, + ) + try: + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ) + + result = run("compression") + finally: + relay.intercepts.deregister_llm_request("hermes-auxiliary-request") + + assert result == {"content": "ok"} + assert captured_requests[0]["temperature"] == 0.25 + assert turn.logical_llm_calls == {} diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py index 358f608c2c75..84d5b8fdb49c 100644 --- a/tests/agent/test_relay_llm.py +++ b/tests/agent/test_relay_llm.py @@ -127,7 +127,7 @@ def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn): assert turn.logical_llm_calls == {} -def test_stream_preserves_provider_error_and_turn_closes_logical_scope(relay_turn): +def test_stream_preserves_provider_error_and_keeps_logical_scope_for_retry(relay_turn): _relay, turn = relay_turn class ProviderError(Exception): @@ -162,6 +162,138 @@ def test_stream_preserves_provider_error_and_turn_closes_logical_scope(relay_tur assert "request-2" in turn.logical_llm_calls +def test_non_stream_preserves_raw_provider_response_identity(relay_turn): + _relay, _turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-raw"}, + ) + + assert result is raw_response + + +@pytest.mark.asyncio +async def test_async_non_stream_preserves_raw_provider_response_identity(relay_turn): + _relay, _turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def provider(_request): + return raw_response + + result = await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async"}, + ) + + assert result is raw_response + + +def test_current_attempt_bypasses_relay_without_an_active_turn(monkeypatch): + monkeypatch.setattr(relay_runtime, "current_turn", lambda: None) + request = {"model": "test-model", "messages": []} + + result = relay_llm.execute_current( + request, + lambda value: value, + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_non_stream_returns_post_execution_interceptor_result(relay_turn, monkeypatch): + relay, _turn = relay_turn + + async def post_execute(_name, request, callback, **_kwargs): + response = callback(request) + return {**response, "post_interceptor": True} + + monkeypatch.setattr(relay.llm, "execute", post_execute) + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "raw"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-post"}, + ) + + assert result == {"content": "raw", "post_interceptor": True} + + +def test_non_stream_preserves_provider_error_from_relay_wrapper_suffix( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + class ProviderError(Exception): + pass + + provider_error = ProviderError("provider failed") + + async def wrapping_execute(_name, request, callback, **_kwargs): + try: + return callback(request) + except Exception as exc: + raise RuntimeError( + f"internal error: {type(exc).__name__}: {exc} (retried 3x)" + ) from None + + monkeypatch.setattr(relay.llm, "execute", wrapping_execute) + + with pytest.raises(ProviderError) as caught: + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: (_ for _ in ()).throw(provider_error), + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-error"}, + ) + + assert caught.value is provider_error + assert "request-error" in turn.logical_llm_calls + + +def test_non_stream_does_not_mask_relay_error_after_callback_failure( + relay_turn, monkeypatch +): + relay, _turn = relay_turn + provider_error = RuntimeError("provider failed") + relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") + + async def translating_execute(_name, request, callback, **_kwargs): + try: + callback(request) + except Exception: + raise relay_error + + monkeypatch.setattr(relay.llm, "execute", translating_execute) + + with pytest.raises(RuntimeError) as caught: + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: (_ for _ in ()).throw(provider_error), + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-policy"}, + ) + + assert caught.value is relay_error + + def test_chat_codec_preserves_provider_message_extensions_after_rewrite(relay_turn): relay, _turn = relay_turn captured_requests = [] @@ -223,3 +355,40 @@ def test_chat_codec_preserves_provider_message_extensions_after_rewrite(relay_tu assert captured_requests[0]["messages"][0]["reasoning_content"] == ( "provider scratchpad" ) + + +def test_request_rewrite_preserves_unmodified_provider_objects(relay_turn): + relay, _turn = relay_turn + timeout = object() + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + relay.intercepts.register_llm_request( + "hermes-provider-object-request", + 1, + False, + rewrite_request, + ) + try: + relay_llm.execute( + {"model": "test-model", "messages": [], "timeout": timeout}, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-provider-object", + }, + ) + finally: + relay.intercepts.deregister_llm_request( + "hermes-provider-object-request" + ) + + assert captured_requests[0]["timeout"] is timeout + assert captured_requests[0]["temperature"] == 0.25 diff --git a/tests/agent/test_relay_tools.py b/tests/agent/test_relay_tools.py index e1d0fffb869c..f352d0b8497a 100644 --- a/tests/agent/test_relay_tools.py +++ b/tests/agent/test_relay_tools.py @@ -83,3 +83,58 @@ def test_provider_error_identity_is_preserved(relay_turn): ) assert caught.value is tool_error + + +def test_tool_error_is_preserved_from_relay_wrapper_suffix(relay_turn, monkeypatch): + relay = relay_turn + + class ToolError(Exception): + pass + + tool_error = ToolError("dispatch failed") + + async def wrapping_execute(_name, args, callback, **_kwargs): + try: + return callback(args) + except Exception as exc: + raise RuntimeError( + f"internal error: {type(exc).__name__}: {exc} (worker trace)" + ) from None + + monkeypatch.setattr(relay.tools, "execute", wrapping_execute) + + with pytest.raises(ToolError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + lambda _args: (_ for _ in ()).throw(tool_error), + session_id="session-1", + ) + + assert caught.value is tool_error + + +def test_tool_adapter_does_not_mask_relay_error_after_callback_failure( + relay_turn, monkeypatch +): + relay = relay_turn + tool_error = RuntimeError("dispatch failed") + relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") + + async def translating_execute(_name, args, callback, **_kwargs): + try: + callback(args) + except Exception: + raise relay_error + + monkeypatch.setattr(relay.tools, "execute", translating_execute) + + with pytest.raises(RuntimeError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + lambda _args: (_ for _ in ()).throw(tool_error), + session_id="session-1", + ) + + assert caught.value is relay_error diff --git a/tests/cli/test_session_boundary_hooks.py b/tests/cli/test_session_boundary_hooks.py index 52c64c01c2d8..ba4f13d2c100 100644 --- a/tests/cli/test_session_boundary_hooks.py +++ b/tests/cli/test_session_boundary_hooks.py @@ -10,8 +10,9 @@ def test_session_hooks_in_valid_hooks(): assert "on_session_reset" in VALID_HOOKS -@patch("hermes_cli.plugins.invoke_hook") -def test_session_finalize_on_reset(mock_invoke_hook): +@patch("hermes_cli.lifecycle.invoke_hook") +@patch("hermes_cli.lifecycle.finalize_session") +def test_session_finalize_on_reset(mock_finalize_session, mock_invoke_hook): """Verify on_session_finalize fires when /new or /reset is used.""" cli = HermesCLI() cli.agent = MagicMock() @@ -22,10 +23,10 @@ def test_session_finalize_on_reset(mock_invoke_hook): # Check if on_session_finalize was called for the old session assert any( - c.args == ("on_session_finalize",) + not c.args and c.kwargs["session_id"] == "test-session-id" and c.kwargs["platform"] == "cli" - for c in mock_invoke_hook.call_args_list + for c in mock_finalize_session.call_args_list ) # Check if on_session_reset was called for the new session assert any( @@ -36,8 +37,8 @@ def test_session_finalize_on_reset(mock_invoke_hook): ) -@patch("hermes_cli.plugins.invoke_hook") -def test_session_finalize_on_cleanup(mock_invoke_hook): +@patch("hermes_cli.lifecycle.finalize_session") +def test_session_finalize_on_cleanup(mock_finalize_session): """Verify on_session_finalize fires during CLI exit cleanup.""" import cli as cli_mod @@ -49,15 +50,15 @@ def test_session_finalize_on_cleanup(mock_invoke_hook): cli_mod._run_cleanup() assert any( - c.args == ("on_session_finalize",) + not c.args and c.kwargs["session_id"] == "cleanup-session-id" and c.kwargs["platform"] == "cli" and c.kwargs["reason"] == "shutdown" - for c in mock_invoke_hook.call_args_list + for c in mock_finalize_session.call_args_list ) -@patch("hermes_cli.plugins.invoke_hook") +@patch("hermes_cli.lifecycle.invoke_hook") def test_interrupted_session_end_helper_emits_observer_shape(mock_invoke_hook): """Verify quiet single-query interruption emits a correlated session end.""" import cli as cli_mod diff --git a/tests/hermes_cli/test_lifecycle.py b/tests/hermes_cli/test_lifecycle.py new file mode 100644 index 000000000000..999d6057b3bf --- /dev/null +++ b/tests/hermes_cli/test_lifecycle.py @@ -0,0 +1,60 @@ +from types import SimpleNamespace + +from agent import relay_runtime +from hermes_cli import lifecycle, observability, plugins + + +def test_invoke_hook_notifies_builtin_observers_before_plugins(monkeypatch): + calls = [] + manager = SimpleNamespace( + invoke_hook=lambda name, **kwargs: calls.append(("plugin", name, kwargs)) or ["ok"] + ) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda name, **kwargs: calls.append(("builtin", name, kwargs)), + ) + monkeypatch.setattr(plugins, "invoke_hook", manager.invoke_hook) + + result = lifecycle.invoke_hook("on_session_start", session_id="session-1") + + assert result == ["ok"] + assert [call[0] for call in calls] == ["builtin", "plugin"] + + +def test_finalize_session_closes_core_before_plugin_export(monkeypatch): + calls = [] + manager = SimpleNamespace( + invoke_hook=lambda name, **kwargs: calls.append(("plugin", name, kwargs)) or [] + ) + coordinator = SimpleNamespace( + finalize_conversation=lambda **kwargs: calls.append(("core", kwargs)) + ) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda name, **kwargs: calls.append(("builtin", name, kwargs)), + ) + monkeypatch.setattr(plugins, "invoke_hook", manager.invoke_hook) + monkeypatch.setattr(relay_runtime, "SESSION_COORDINATOR", coordinator) + monkeypatch.setattr(relay_runtime, "current_profile_key", lambda: "profile-1") + + lifecycle.finalize_session(session_id="session-1", platform="cli") + + assert [call[0] for call in calls] == ["builtin", "core", "plugin"] + assert calls[1][1] == { + "profile_key": "profile-1", + "session_id": "session-1", + } + + +def test_plugin_only_dispatch_does_not_reenter_builtin_observers(monkeypatch): + manager = SimpleNamespace(invoke_hook=lambda name, **kwargs: [name, kwargs]) + monkeypatch.setattr(plugins, "get_plugin_manager", lambda: manager) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unexpected")), + ) + + assert plugins.invoke_hook("custom", value=1) == ["custom", {"value": 1}] diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py index a261b51ce3ba..a1953508bd54 100644 --- a/tests/hermes_cli/test_relay_shared_metrics_runtime.py +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -12,7 +12,7 @@ from typing import Any import pytest -from hermes_cli import plugins +from hermes_cli import lifecycle, plugins from hermes_cli.observability import relay_runtime, relay_shared_metrics from hermes_cli.plugins import PluginManager @@ -168,15 +168,15 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa "base_url": "http://127.0.0.1:11434/v1", } - assert plugins.has_hook("pre_api_request") - plugins.invoke_hook("on_session_start", **base) - plugins.invoke_hook("pre_llm_call", **base) - plugins.invoke_hook( + assert lifecycle.has_hook("pre_api_request") + lifecycle.invoke_hook("on_session_start", **base) + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook( "pre_api_request", **base, request={"body": {"messages": ["sensitive-prompt"]}}, ) - plugins.invoke_hook( + lifecycle.invoke_hook( "post_tool_call", **base, tool_call_id="sensitive-tool-call", @@ -185,13 +185,13 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa result={"output": "sensitive-tool-result"}, status="ok", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "api_request_error", **base, retryable=True, error={"message": "sensitive-error"}, ) - plugins.invoke_hook( + lifecycle.invoke_hook( "pre_api_request", **{ **base, @@ -201,7 +201,7 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa }, request={"body": {"messages": ["sensitive-prompt"]}}, ) - plugins.invoke_hook( + lifecycle.invoke_hook( "post_api_request", **{ **base, @@ -211,7 +211,7 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa }, response={"content": "sensitive-response"}, ) - plugins.invoke_hook( + lifecycle.invoke_hook( "on_session_end", **base, completed=True, @@ -219,7 +219,7 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa interrupted=False, turn_exit_reason="text_response(stop)", ) - plugins.invoke_hook("on_session_finalize", session_id=base["session_id"]) + lifecycle.finalize_session(session_id=base["session_id"]) starts = [event for event in direct_runtime.events if event[0] == "llm.call"] ends = [event for event in direct_runtime.events if event[0] == "llm.call_end"] @@ -310,8 +310,8 @@ def test_direct_runtime_is_disabled_by_default(tmp_path, monkeypatch): monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) assert not plugins.has_hook("pre_api_request") - plugins.invoke_hook("on_session_start", session_id="s1", platform="cli") - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") + lifecycle.finalize_session(session_id="s1") assert fake.events == [] assert not (tmp_path / "hermes-home" / "telemetry").exists() @@ -345,7 +345,7 @@ def test_core_runtime_is_fail_open_without_a_published_binding(monkeypatch, capl def test_core_mark_uses_the_shared_session_handle_without_a_plugin(direct_runtime): - plugins.invoke_hook("on_session_start", session_id="s1", platform="cli") + lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") handle = relay_runtime.get_session_handle("s1") assert handle is not None @@ -473,6 +473,117 @@ def test_session_coordinator_separates_turn_release_from_hard_finalize( assert runtime.get_session("coordinated-session") is None +def test_session_coordinator_prepares_subscribers_before_opening_scope( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + initializer_name = "test.pre_scope_subscriber" + + def prepare(host, context): + assert host.profile_key == relay_runtime.current_profile_key() + direct_runtime.events.append(("session.prepare", dict(context))) + + coordinator.register_session_initializer(initializer_name, prepare) + try: + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="prepared-session", + platform="cli", + model="demo-model", + ) + finally: + coordinator.unregister_session_initializer(initializer_name) + + prepared = next( + index + for index, event in enumerate(direct_runtime.events) + if event[0] == "session.prepare" + ) + opened = next( + index + for index, event in enumerate(direct_runtime.events) + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ) + assert prepared < opened + assert direct_runtime.events[prepared][1] == { + "profile_key": relay_runtime.current_profile_key(), + "session_id": "prepared-session", + "platform": "cli", + "parent_session_id": "", + "model": "demo-model", + } + coordinator.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=lease.session_id, + ) + + +def test_session_initializer_failure_does_not_block_conversation( + direct_runtime, + caplog, +): + coordinator = relay_runtime.SESSION_COORDINATOR + initializer_name = "test.failing_subscriber" + + def fail(_host, _context): + raise RuntimeError("subscriber setup failed") + + coordinator.register_session_initializer(initializer_name, fail) + try: + with caplog.at_level("WARNING"): + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="initializer-failure", + platform="cli", + ) + finally: + coordinator.unregister_session_initializer(initializer_name) + + assert lease.session is not None + assert "Hermes Relay session initializer failed" in caplog.text + coordinator.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=lease.session_id, + ) + + +def test_profile_host_recreation_rebinds_shared_metrics_subscriber( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + first_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="before-restart", + platform="cli", + ) + first_metrics = relay_shared_metrics._get_runtime() + assert first_metrics is not None + first_host = first_lease.host + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=first_lease.session_id, + ) + coordinator.shutdown_profile(profile_key) + + second_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="after-restart", + platform="cli", + ) + second_metrics = relay_shared_metrics._get_runtime() + + assert second_metrics is not None + assert second_lease.host is not first_host + assert second_metrics is not first_metrics + assert second_metrics.host is second_lease.host + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=second_lease.session_id, + ) + + def test_core_mark_lazily_starts_relay_without_metrics_or_a_plugin( tmp_path, monkeypatch, @@ -490,7 +601,7 @@ def test_core_mark_lazily_starts_relay_without_metrics_or_a_plugin( session_id="s1", data={"provenance": "agent_created"}, ) - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.finalize_session(session_id="s1") assert [event[0] for event in fake.events] == [ "scope.push", @@ -923,7 +1034,7 @@ def test_subagent_stop_hook_does_not_own_child_session_lifetime(direct_runtime): ) assert child is not None - plugins.invoke_hook( + lifecycle.invoke_hook( "subagent_stop", parent_session_id="parent", child_session_id="child", @@ -1121,9 +1232,9 @@ def test_terminal_model_error_is_counted_as_failed(direct_runtime): "model": "claude-sonnet", } - plugins.invoke_hook("pre_api_request", **base) - plugins.invoke_hook("api_request_error", **base, retryable=False) - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=False) + lifecycle.finalize_session(session_id="s1") [end] = [event for event in direct_runtime.events if event[0] == "llm.call_end"] assert end[2]["outcome"] == "failed" @@ -1139,15 +1250,15 @@ def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runt "model": "nvidia/nemotron-3-super-120b-a12b", } - plugins.invoke_hook("pre_llm_call", **base) - plugins.invoke_hook("pre_api_request", **base) - plugins.invoke_hook("api_request_error", **base, retryable=True) - plugins.invoke_hook("pre_api_request", **base) - plugins.invoke_hook("api_request_error", **base, retryable=True) - plugins.invoke_hook("pre_api_request", **base) - plugins.invoke_hook("api_request_error", **base, retryable=False) + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=True) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=True) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=False) for tool_call_id in ("tool-1", "tool-1", "tool-2"): - plugins.invoke_hook( + lifecycle.invoke_hook( "post_tool_call", **base, tool_call_id=tool_call_id, @@ -1155,7 +1266,7 @@ def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runt result={"output": "private"}, status="ok", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "on_session_end", **base, completed=False, @@ -1163,7 +1274,7 @@ def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runt interrupted=False, turn_exit_reason="all_retries_exhausted_no_response", ) - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.finalize_session(session_id="s1") model_starts = [event for event in direct_runtime.events if event[0] == "llm.call"] model_ends = [ @@ -1249,7 +1360,7 @@ def test_outer_agent_boundary_closes_early_returns_and_exceptions( with pytest.raises(TimeoutError, match="private timeout detail"): AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.finalize_session(session_id="s1") task_ends = [ event[2]["output"] @@ -1315,14 +1426,14 @@ def test_outer_agent_boundary_preserves_a_returned_timeout_reason( def test_session_finalize_closes_a_pending_task_as_system_aborted(direct_runtime): - plugins.invoke_hook( + lifecycle.invoke_hook( "pre_llm_call", session_id="s1", task_id="t1", platform="cli", ) - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.finalize_session(session_id="s1") [task_end] = [ event @@ -1344,13 +1455,13 @@ def test_session_finalize_closes_a_pending_task_as_system_aborted(direct_runtime def test_sequential_tasks_in_one_session_aggregate_once_each(direct_runtime, tmp_path): for task_id in ("t1", "t2"): - plugins.invoke_hook( + lifecycle.invoke_hook( "pre_llm_call", session_id="s1", task_id=task_id, platform="cli", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "on_session_end", session_id="s1", task_id=task_id, @@ -1360,7 +1471,7 @@ def test_sequential_tasks_in_one_session_aggregate_once_each(direct_runtime, tmp interrupted=False, turn_exit_reason="text_response(stop)", ) - plugins.invoke_hook("on_session_finalize", session_id="s1") + lifecycle.finalize_session(session_id="s1") outbox = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" / "outbox" [package_path] = list(outbox.glob("*.json")) @@ -1371,13 +1482,13 @@ def test_sequential_tasks_in_one_session_aggregate_once_each(direct_runtime, tmp def test_task_ownership_survives_session_id_rotation(direct_runtime): - plugins.invoke_hook( + lifecycle.invoke_hook( "pre_llm_call", session_id="before-compression", task_id="t1", platform="cli", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "pre_api_request", session_id="after-compression", task_id="t1", @@ -1386,7 +1497,7 @@ def test_task_ownership_survives_session_id_rotation(direct_runtime): provider="nvidia", model="nvidia/nemotron-3-super-120b-a12b", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "post_api_request", session_id="after-compression", task_id="t1", @@ -1395,7 +1506,7 @@ def test_task_ownership_survives_session_id_rotation(direct_runtime): provider="nvidia", model="nvidia/nemotron-3-super-120b-a12b", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "on_session_end", session_id="after-compression", task_id="t1", @@ -1405,7 +1516,7 @@ def test_task_ownership_survives_session_id_rotation(direct_runtime): interrupted=False, turn_exit_reason="text_response(stop)", ) - plugins.invoke_hook("on_session_finalize", session_id="before-compression") + lifecycle.finalize_session(session_id="before-compression") task_starts = [ event @@ -1443,8 +1554,8 @@ def test_gateway_and_delegated_entrypoints_flow_through_relay(direct_runtime): }, ] for task in tasks: - plugins.invoke_hook("pre_llm_call", **task) - plugins.invoke_hook( + lifecycle.invoke_hook("pre_llm_call", **task) + lifecycle.invoke_hook( "on_session_end", **task, completed=True, @@ -1477,7 +1588,7 @@ def test_persistence_failure_does_not_escape_the_hook( raise OSError("store unavailable") monkeypatch.setattr(runtime.subscriber.store, "record_counter", fail_record) - plugins.invoke_hook( + lifecycle.invoke_hook( "pre_api_request", session_id="s1", task_id="t1", @@ -1485,7 +1596,7 @@ def test_persistence_failure_does_not_escape_the_hook( provider="openai", model="gpt-5", ) - plugins.invoke_hook( + lifecycle.invoke_hook( "post_api_request", session_id="s1", task_id="t1", diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 67dea13b6684..fce510b7c2bf 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import builtins import contextvars import gc import importlib @@ -16,7 +15,7 @@ from types import SimpleNamespace import pytest import yaml -from hermes_cli import plugins as plugin_api +from hermes_cli import lifecycle, plugins as plugin_api from hermes_cli.observability import relay_runtime, relay_shared_metrics from hermes_cli.plugins import PluginManager @@ -113,7 +112,13 @@ class _FakeNemoRelay: def _llm_execute(self, name, request, func, **kwargs): self.events.append(("llm.execute.start", name, request.content, kwargs)) + handle = self._llm_call(name, request, **kwargs) result = func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) + self._llm_call_end( + handle, + result, + **{key: value for key, value in kwargs.items() if key != "handle"}, + ) self.events.append(("llm.execute.end", name, result, kwargs)) return result @@ -127,7 +132,13 @@ class _FakeNemoRelay: def _tool_execute(self, name, args, func, **kwargs): self.events.append(("tool.execute.start", name, args, kwargs)) + handle = self._tool_call(name, args, **kwargs) result = func(args) + self._tool_call_end( + handle, + result, + **{key: value for key, value in kwargs.items() if key != "handle"}, + ) self.events.append(("tool.execute.end", name, result, kwargs)) return result @@ -233,33 +244,6 @@ def _fresh_plugin(monkeypatch, fake): return plugin -def _wrapped_downstream_error(original): - class _DownstreamExecutionError(Exception): - def __init__(self, original): - super().__init__(str(original)) - self.original = original - - return _DownstreamExecutionError(original) - - -def _enable_adaptive_plugin(tmp_path, monkeypatch) -> None: - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - def _enable_dynamic_plugin(tmp_path, monkeypatch) -> Path: plugins_toml = tmp_path / "plugins.toml" plugins_toml.write_text( @@ -290,11 +274,6 @@ def test_manifest_fields(): "on_session_reset", "pre_llm_call", "post_llm_call", - "pre_api_request", - "post_api_request", - "api_request_error", - "pre_tool_call", - "post_tool_call", "pre_approval_request", "post_approval_response", "subagent_start", @@ -323,7 +302,12 @@ def test_nemo_relay_plugin_uses_nemo_relay_runtime(monkeypatch): assert any(event[0] == "scope.push" for event in fake_relay.events) -def test_nemo_relay_plugin_emits_llm_tool_and_exports_atif(tmp_path, monkeypatch): +def test_nemo_relay_plugin_exports_core_managed_llm_and_tool_events( + tmp_path, + monkeypatch, +): + from agent import relay_llm, relay_tools + fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) monkeypatch.setenv("HERMES_NEMO_RELAY_ATOF_ENABLED", "1") @@ -338,30 +322,47 @@ def test_nemo_relay_plugin_emits_llm_tool_and_exports_atif(tmp_path, monkeypatch "telemetry_schema_version": "hermes.observer.v1", } plugin.on_session_start(**base, model="demo-model", platform="cli") - plugin.on_pre_api_request( - **base, - api_request_id="api-1", - provider="openai", - model="demo-model", - request={"method": "POST", "body": {"messages": [{"role": "user", "content": "hi"}]}}, + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + platform="cli", ) - plugin.on_post_api_request( - **base, - api_request_id="api-1", - response={"assistant_message": {"role": "assistant", "content": "hello"}}, + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="t1", ) - plugin.on_pre_tool_call(**base, tool_name="read_file", tool_call_id="tool-1", args={"path": "x"}) - plugin.on_post_tool_call(**base, tool_name="read_file", tool_call_id="tool-1", result='{"ok": true}', status="ok") + relay_llm.execute( + {"messages": [{"role": "user", "content": "hi"}]}, + lambda request: { + "assistant_message": {"role": "assistant", "content": "hello"}, + "request": request, + }, + session_id="s1", + name="openai", + model_name="demo-model", + metadata={"api_request_id": "api-1", "api_mode": "custom"}, + ) + relay_tools.execute( + "read_file", + {"path": "x"}, + lambda _args: {"ok": True}, + session_id="s1", + metadata={"tool_call_id": "tool-1"}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) plugin.on_session_end(**base, completed=True, interrupted=False) plugin.on_session_finalize(**base, reason="shutdown") event_names = [event[0] for event in fake.events] assert "atof.register" in event_names assert "atif.register" in event_names - assert "llm.call" in event_names - assert "llm.call_end" in event_names - assert "tool.call" in event_names - assert "tool.call_end" in event_names + assert event_names.count("llm.call") == 1 + assert event_names.count("llm.call_end") == 1 + assert event_names.count("tool.call") == 1 + assert event_names.count("tool.call_end") == 1 assert "scope.pop" in event_names assert (tmp_path / "atif" / "hermes-atif-s1.json").exists() @@ -370,6 +371,8 @@ def test_shared_metrics_and_rich_plugin_share_one_core_session( tmp_path, monkeypatch, ): + from agent import relay_llm + fake = _FakeNemoRelay() hermes_home = tmp_path / "hermes-home" monkeypatch.setenv("HERMES_HOME", str(hermes_home)) @@ -383,10 +386,12 @@ def test_shared_metrics_and_rich_plugin_share_one_core_session( ) plugin = _fresh_plugin(monkeypatch, fake) manager = PluginManager() - manager._hooks["on_session_start"] = [plugin.on_session_start] - manager._hooks["pre_api_request"] = [plugin.on_pre_api_request] - manager._hooks["post_api_request"] = [plugin.on_post_api_request] - manager._hooks["on_session_finalize"] = [plugin.on_session_finalize] + + class _Context: + def register_hook(self, name, callback): + manager._hooks.setdefault(name, []).append(callback) + + plugin.register(_Context()) monkeypatch.setattr(plugin_api, "_plugin_manager", manager) event = { @@ -397,18 +402,42 @@ def test_shared_metrics_and_rich_plugin_share_one_core_session( "model": "claude-sonnet", "platform": "cli", } - plugin_api.invoke_hook("on_session_start", **event) - plugin_api.invoke_hook( + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + platform="cli", + model=event["model"], + ) + lifecycle.invoke_hook("on_session_start", **event) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="t1", + ) + lifecycle.invoke_hook( "pre_api_request", **event, request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, ) - plugin_api.invoke_hook( + relay_llm.execute( + {"messages": [{"role": "user", "content": "hi"}]}, + lambda _request: { + "assistant_message": {"role": "assistant", "content": "hello"} + }, + session_id="s1", + name="anthropic", + model_name="claude-sonnet", + metadata={"api_request_id": "api-1", "api_mode": "custom"}, + ) + lifecycle.invoke_hook( "post_api_request", **event, response={"assistant_message": {"role": "assistant", "content": "hello"}}, ) - plugin_api.invoke_hook("on_session_finalize", session_id="s1") + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + lifecycle.finalize_session(session_id="s1") session_pushes = [ item @@ -442,38 +471,6 @@ def test_shared_metrics_and_rich_plugin_share_one_core_session( assert (tmp_path / "atif" / "hermes-atif-s1.json").exists() -def test_nemo_relay_plugin_closes_api_span_on_error(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - base = { - "session_id": "s1", - "task_id": "t1", - "turn_id": "turn-1", - "telemetry_schema_version": "hermes.observer.v1", - } - - plugin.on_pre_api_request( - **base, - api_request_id="api-err", - provider="openai", - model="demo-model", - request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, - ) - plugin.on_api_request_error( - **base, - api_request_id="api-err", - error={"type": "RateLimitError", "message": "rate limited"}, - retryable=True, - reason="rate_limit", - ) - - call_end = next(event for event in fake.events if event[0] == "llm.call_end") - assert call_end[1] == ("llm", "openai") - assert call_end[2] == {"error": {"type": "RateLimitError", "message": "rate limited"}} - assert call_end[3]["data"]["reason"] == "rate_limit" - assert not plugin._get_runtime().sessions["s1"].llm_spans - - def test_nemo_relay_plugin_emits_approval_marks(monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -486,24 +483,6 @@ def test_nemo_relay_plugin_emits_approval_marks(monkeypatch): assert "hermes.approval.response" in mark_names -def test_nemo_relay_plugin_emits_unmatched_fallback_marks(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - plugin.on_post_api_request(session_id="s1", api_request_id="missing-api", response={"ok": True}) - plugin.on_api_request_error( - session_id="s1", - api_request_id="missing-api", - error={"type": "TimeoutError", "message": "timed out"}, - ) - plugin.on_post_tool_call(session_id="s1", tool_call_id="missing-tool", result={"ok": True}) - - mark_names = [event[1] for event in fake.events if event[0] == "scope.event"] - assert "hermes.api.response.unmatched" in mark_names - assert "hermes.api.error" in mark_names - assert "hermes.tool.response.unmatched" in mark_names - - def test_nemo_relay_plugin_metadata_promotes_trajectory_and_subagent_ids(monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -716,6 +695,8 @@ enabled = true def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypatch): + from agent import relay_llm, relay_tools + fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -726,26 +707,42 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa runtime = plugin._get_runtime() assert runtime is not None assert runtime._plugin_activation is not None - llm_result = plugin.on_llm_execution_middleware( + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), session_id="s1", - provider="openai", - model="fixture", - request={"messages": []}, - next_call=lambda request: {"request": request}, + platform="cli", + ) + turn = coordinator.begin_turn(lease, turn_id="turn-1", task_id="task-1") + llm_result = relay_llm.execute( + {"messages": []}, + lambda request: {"request": request}, + session_id="s1", + name="openai", + model_name="fixture", + metadata={"api_mode": "custom", "api_request_id": "api-1"}, ) tool_args = relay_runtime.apply_tool_request_intercepts( session_id="s1", tool_name="fixture-tool", args={"value": 1}, ) - tool_result = plugin.on_tool_execution_middleware( + tool_result, final_args = relay_tools.execute( + "fixture-tool", + tool_args, + lambda args: {"args": args}, session_id="s1", - tool_name="fixture-tool", - args=tool_args, - next_call=lambda args: {"args": args}, + metadata={"tool_call_id": "tool-1"}, ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) assert llm_result["request"]["intercepted"] is True assert tool_result["args"]["intercepted"] is True + assert final_args["intercepted"] is True + relay_runtime.SESSION_COORDINATOR.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + ) plugin.on_session_finalize(session_id="s1", reason="shutdown") assert runtime._plugin_activation is not None assert not any(event[0] == "plugin.activation.close" for event in fake.events) @@ -840,128 +837,6 @@ environment_ref = "../environments/worker-fixture" runtime.shutdown() -@pytest.mark.parametrize( - ("provider", "api_mode", "expected_surface", "should_rewrite"), - [ - ("custom", "chat_completions", "openai.chat_completions", True), - ("openai-codex", "codex_responses", "openai.responses", True), - ("anthropic", "anthropic_messages", "anthropic.messages", True), - ("custom", "anthropic_messages", "anthropic.messages", True), - ("bedrock", "bedrock_converse", "bedrock", False), - ], -) -def test_nemo_relay_managed_llm_uses_wire_protocol_for_interceptor_dispatch( - tmp_path, - monkeypatch, - provider, - api_mode, - expected_surface, - should_rewrite, -): - fake = _FakeNemoRelay() - supported_surfaces = { - "anthropic.messages", - "openai.chat_completions", - "openai.responses", - } - - def execute(name, request, func, **kwargs): - fake.events.append(("llm.execute.start", name, request.content, kwargs)) - content = dict(request.content) - if name in supported_surfaces: - content["rewritten_for"] = name - result = func(_FakeLLMRequest(request.headers, content)) - fake.events.append(("llm.execute.end", name, result, kwargs)) - return result - - fake.llm.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_llm_execution_middleware( - session_id="s1", - provider=provider, - api_mode=api_mode, - model="fixture", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: request, - ) - - execute_start = next( - event for event in fake.events if event[0] == "llm.execute.start" - ) - assert execute_start[1] == expected_surface - assert execute_start[3]["metadata"]["provider"] == provider - assert execute_start[3]["metadata"]["api_mode"] == api_mode - if should_rewrite: - assert result["rewritten_for"] == expected_surface - else: - assert "rewritten_for" not in result - - -def test_nemo_relay_managed_llm_returns_post_next_interceptor_result(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - raw_response = SimpleNamespace( - model="fixture", - choices=[ - SimpleNamespace( - message=SimpleNamespace(role="assistant", content="raw", tool_calls=[]), - finish_reason="stop", - ) - ], - usage=None, - ) - - def execute(name, request, func, **kwargs): - del name, kwargs - normalized = func(_FakeLLMRequest(request.headers, request.content)) - return {**normalized, "post_next_interceptor": True} - - fake.llm.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_llm_execution_middleware( - session_id="s1", - provider="openai", - api_mode="chat_completions", - model="fixture", - request={"messages": []}, - next_call=lambda request: raw_response, - ) - - assert result["post_next_interceptor"] is True - assert result["assistant_message"]["content"] == "raw" - assert result is not raw_response - - -def test_nemo_relay_managed_tool_returns_post_interceptor_result(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def execute(name, args, func, **kwargs): - fake.events.append(("tool.execute.start", name, args, kwargs)) - raw = func(args) - result = {"compressed": True, "raw": raw} - fake.events.append(("tool.execute.end", name, result, kwargs)) - return result - - fake.tools.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="fixture-tool", - args={"value": 1}, - next_call=lambda args: {"tool_output": args}, - ) - - assert result == { - "compressed": True, - "raw": {"tool_output": {"value": 1}}, - } - - def test_relay_tool_request_rewrite_precedes_hermes_authorization_boundary( tmp_path, monkeypatch, @@ -984,50 +859,18 @@ def test_relay_tool_request_rewrite_precedes_hermes_authorization_boundary( assert result.trace[0] == {"source": "nemo_relay"} -def test_managed_tool_refuses_post_authorization_argument_rewrite( - tmp_path, - monkeypatch, -): - fake = _FakeNemoRelay() - - def execute(name, args, func, **kwargs): - del name, kwargs - return func({**args, "after_approval": True}) - - fake.tools.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - dispatched = False - - def next_call(args): - nonlocal dispatched - dispatched = True - return args - - with pytest.raises( - RuntimeError, - match="changed tool arguments after Hermes authorization", - ): - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="fixture-tool", - args={"value": 1}, - next_call=next_call, - ) - - assert not dispatched - - -def test_nemo_relay_plugin_activates_without_registering_managed_middleware( +def test_nemo_relay_plugin_activates_without_duplicate_execution_hooks( tmp_path, monkeypatch ): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) + registered_hooks = [] class _Context: def register_hook(self, name, callback): - del name, callback + del callback + registered_hooks.append(name) def register_middleware(self, name, callback): del callback @@ -1038,6 +881,13 @@ def test_nemo_relay_plugin_activates_without_registering_managed_middleware( event_names = [event[0] for event in fake.events] assert "plugin.activate_dynamic" in event_names assert "hermes.register_middleware" not in event_names + assert not { + "pre_api_request", + "post_api_request", + "api_request_error", + "pre_tool_call", + "post_tool_call", + }.intersection(registered_hooks) runtime = plugin._get_runtime() assert runtime is not None runtime.shutdown() @@ -1422,657 +1272,3 @@ output_directory = "{(tmp_path / "managed-atof").as_posix()}" assert event_names.count("plugin.initialize.attempt") == 2 assert event_names.count("atof.register") == 1 assert event_names.count("atof.deregister") == 1 - - -def test_nemo_relay_adaptive_llm_execution_middleware_preserves_raw_response(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - seen_request = {} - raw_choice = SimpleNamespace( - message=SimpleNamespace( - role="assistant", - content=None, - tool_calls=[ - SimpleNamespace( - id="tool-1", - type="function", - function=SimpleNamespace(name="terminal", arguments='{"command":"pwd"}'), - ) - ], - reasoning_content="need a tool", - ), - finish_reason="tool_calls", - ) - raw_response = SimpleNamespace( - id="resp-1", - model="demo-model", - choices=[raw_choice], - usage=SimpleNamespace(prompt_tokens=3, completion_tokens=5, total_tokens=8), - ) - - def next_call(request): - seen_request.update(request) - return raw_response - - response = plugin.on_llm_execution_middleware( - session_id="s1", - task_id="t1", - turn_id="turn-1", - api_request_id="api-1", - provider="anthropic", - model="demo-model", - api_call_count=1, - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert response is raw_response - assert response.model == "demo-model" - assert response.choices == [raw_choice] - assert seen_request["intercepted"] is True - execute_start = next(event for event in fake.events if event[0] == "llm.execute.start") - assert execute_start[3]["data"]["mode"] == "observe_only" - execute_end = next(event for event in fake.events if event[0] == "llm.execute.end") - assert execute_end[2] == { - "model": "demo-model", - "assistant_message": { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "tool-1", - "type": "function", - "function": {"name": "terminal", "arguments": '{"command":"pwd"}'}, - } - ], - "reasoning_content": "need a tool", - }, - "finish_reason": "tool_calls", - "usage": {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}, - } - - -def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def native_like_execute(name, request, func, **kwargs): - fake.events.append(("llm.execute.start", name, request.content, kwargs)) - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None - - fake.llm.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ProviderAuthError(Exception): - status_code = 403 - - provider_error = ProviderAuthError("provider auth failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(ProviderAuthError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error_with_relay_suffix( - tmp_path, monkeypatch -): - # Guards the startswith (vs exact ==) match in _is_relay_wrapped_callback_error: - # Relay re-wraps the callback failure with its canonical prefix but APPENDS a - # trailing suffix. Exact equality would miss this and surface Relay's wrapper; - # prefix matching must still recover the original downstream error. - fake = _FakeNemoRelay() - - def native_like_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc} (retried 3x)") from None - - fake.llm.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ProviderAuthError(Exception): - status_code = 403 - - provider_error = ProviderAuthError("provider auth failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(ProviderAuthError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_llm_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - relay_error = RuntimeError("internal error: relay setup failed") - - def internal_error_execute(name, request, func, **kwargs): - raise relay_error - - fake.llm.execute = internal_error_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - with pytest.raises(RuntimeError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_llm_execution_keeps_wrapped_relay_error_after_downstream_failure( - tmp_path, monkeypatch -): - fake = _FakeNemoRelay() - relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream") - - def translated_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception: - raise relay_error - - fake.llm.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - def next_call(request): - raise _wrapped_downstream_error(RuntimeError("provider failed")) - - with pytest.raises(RuntimeError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_llm_execution_keeps_relay_translated_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - class RelayPolicyError(Exception): - pass - - relay_error = RelayPolicyError("relay policy blocked") - - def translated_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception: - raise relay_error - - fake.llm.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - provider_error = RuntimeError("provider failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(RelayPolicyError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_downstream_unwrap_matches_real_middleware_wrapper_shape(monkeypatch): - # Regression guard against core/plugin drift. The synthetic tests above model - # the downstream-error wrapper with a local class, so they keep passing even - # if core middleware renames its private ``_DownstreamExecutionError`` or drops - # ``.original`` -- the exact shape the plugin matches by name at - # ``_original_downstream_error``. Capture the wrapper the REAL - # ``hermes_cli.middleware._run_execution_chain`` hands to a middleware - # callback's ``next_call`` and assert the plugin's detector unwraps it to the - # original exception. If core middleware changes the wrapper shape, this fails - # here instead of silently defeating the unwrap in production. - from hermes_cli import middleware - - from plugins.observability.nemo_relay import _original_downstream_error - - class ProviderError(Exception): - status_code = 403 - - provider_error = ProviderError("provider auth failed") - captured: dict[str, Exception] = {} - - def terminal_call(payload): - raise provider_error - - def capturing_callback(**kwargs): - next_call = kwargs["next_call"] - try: - return next_call(kwargs.get("request")) - except Exception as exc: - captured["wrapper"] = exc - # Surface the original so the chain unwinds without re-wrapping noise. - raise _original_downstream_error(exc) from None - - with pytest.raises(ProviderError) as caught: - middleware._run_execution_chain( - "llm", - [capturing_callback], - terminal_call, - request={"messages": []}, - ) - - wrapper = captured["wrapper"] - # The wrapper the plugin sees must match what _original_downstream_error keys on. - assert wrapper.__class__.__name__ == "_DownstreamExecutionError" - assert isinstance(getattr(wrapper, "original", None), BaseException) - assert _original_downstream_error(wrapper) is provider_error - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def _adaptive_llm_execute_mode(tmp_path, monkeypatch, plugins_toml_text: str) -> str: - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text(plugins_toml_text, encoding="utf-8") - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - - execute_start = next(event for event in fake.events if event[0] == "llm.execute.start") - return execute_start[3]["data"]["mode"] - - -def test_nemo_relay_adaptive_llm_execution_middleware_defaults_to_observe_only_when_mode_is_unset( - tmp_path, monkeypatch -): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -version = 1 -""", - ) - assert mode == "observe_only" - - -def test_nemo_relay_adaptive_llm_execution_middleware_accepts_legacy_top_level_mode(tmp_path, monkeypatch): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -mode = "route" -""", - ) - assert mode == "route" - - -def test_nemo_relay_adaptive_llm_execution_middleware_prefers_tool_parallelism_mode(tmp_path, monkeypatch): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -mode = "route" - -[components.config.tool_parallelism] -mode = "schedule" -""", - ) - assert mode == "schedule" - - -def test_nemo_relay_llm_execution_middleware_calls_through_without_adaptive(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - response = plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": []}, - next_call=lambda request: {"raw": request}, - ) - - assert response == {"raw": {"messages": []}} - assert not any(event[0] == "llm.execute.start" for event in fake.events) - - -def test_nemo_relay_adaptive_tool_execution_middleware_preserves_raw_response(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - seen_args = {} - - def next_call(args): - seen_args.update(args) - return {"raw": True, "args": args} - - approved_args = relay_runtime.apply_tool_request_intercepts( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - ) - response = plugin.on_tool_execution_middleware( - session_id="s1", - task_id="t1", - turn_id="turn-1", - api_request_id="api-1", - tool_name="terminal", - tool_call_id="tool-1", - args=approved_args, - next_call=next_call, - ) - - assert response == {"raw": True, "args": {"command": "pwd", "intercepted": True}} - assert seen_args["intercepted"] is True - execute_start = next(event for event in fake.events if event[0] == "tool.execute.start") - assert execute_start[3]["data"]["mode"] == "observe_only" - assert execute_start[3]["data"]["tool_call_id"] == "tool-1" - - -def test_nemo_relay_adaptive_tool_execution_preserves_downstream_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def native_like_execute(name, args, func, **kwargs): - fake.events.append(("tool.execute.start", name, args, kwargs)) - try: - return func(args) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None - - fake.tools.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ToolAuthError(Exception): - status_code = 403 - - tool_error = ToolAuthError("tool auth failed") - - def next_call(args): - raise _wrapped_downstream_error(tool_error) - - with pytest.raises(ToolAuthError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is tool_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_tool_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - relay_error = RuntimeError("internal error: relay setup failed") - - def internal_error_execute(name, args, func, **kwargs): - raise relay_error - - fake.tools.execute = internal_error_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - with pytest.raises(RuntimeError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_tool_execution_keeps_wrapped_relay_error_after_downstream_failure( - tmp_path, monkeypatch -): - fake = _FakeNemoRelay() - relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream") - - def translated_execute(name, args, func, **kwargs): - try: - return func(args) - except Exception: - raise relay_error - - fake.tools.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - def next_call(args): - raise _wrapped_downstream_error(RuntimeError("tool failed")) - - with pytest.raises(RuntimeError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_tool_execution_keeps_relay_translated_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - class RelayPolicyError(Exception): - pass - - relay_error = RelayPolicyError("relay policy blocked") - - def translated_execute(name, args, func, **kwargs): - try: - return func(args) - except Exception: - raise relay_error - - fake.tools.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - tool_error = RuntimeError("tool failed") - - def next_call(args): - raise _wrapped_downstream_error(tool_error) - - with pytest.raises(RelayPolicyError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_tool_execution_middleware_calls_through_without_adaptive(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - response = plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - assert response == {"raw": {"command": "pwd"}} - assert not any(event[0] == "tool.execute.start" for event in fake.events) - - -def test_nemo_relay_adaptive_execution_skips_duplicate_observer_spans(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - base = { - "session_id": "s1", - "task_id": "t1", - "turn_id": "turn-1", - "api_request_id": "api-1", - } - plugin.on_pre_api_request( - **base, - provider="anthropic", - model="demo-model", - request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, - ) - plugin.on_post_api_request(**base, response={"ok": True}) - plugin.on_pre_tool_call(**base, tool_name="terminal", tool_call_id="tool-1", args={"command": "pwd"}) - plugin.on_post_tool_call(**base, tool_name="terminal", tool_call_id="tool-1", result={"ok": True}) - - plugin.on_llm_execution_middleware( - **base, - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - plugin.on_tool_execution_middleware( - **base, - tool_name="terminal", - tool_call_id="tool-1", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - event_names = [event[0] for event in fake.events] - assert "llm.call" not in event_names - assert "llm.call_end" not in event_names - assert "tool.call" not in event_names - assert "tool.call_end" not in event_names - assert "llm.execute.start" in event_names - assert "tool.execute.start" in event_names - - -def test_nemo_relay_plugin_noops_without_dependency(monkeypatch): - monkeypatch.delitem(sys.modules, "nemo_relay", raising=False) - sys.modules.pop("plugins.observability.nemo_relay", None) - plugin = importlib.import_module("plugins.observability.nemo_relay") - plugin.reset_for_tests() - - real_import = builtins.__import__ - - def blocked_import(name, *args, **kwargs): - if name == "nemo_relay": - raise ModuleNotFoundError(f"No module named {name!r}") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", blocked_import) - - plugin.on_pre_api_request(session_id="s1", api_request_id="api-1") - plugin.on_post_api_request(session_id="s1", api_request_id="api-1") diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 9c3077a6b4fa..c1bc9fb4d7e8 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2439,8 +2439,8 @@ class TestExecuteToolCalls: hook_calls.append((hook_name, kwargs)) return [] - monkeypatch.setattr("hermes_cli.plugins.invoke_hook", _capture_hook) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook", _capture_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with ( patch("run_agent.handle_function_call", side_effect=KeyboardInterrupt), @@ -3185,10 +3185,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: result = agent._invoke_tool("todo", {"todos": []}, "task-1", tool_call_id="todo-1") @@ -3268,10 +3268,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: "Blocked by policy", ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("run_agent.handle_function_call", side_effect=AssertionError("should not run")): agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3295,10 +3295,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3342,10 +3342,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3380,10 +3380,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}'): agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") @@ -3853,6 +3853,31 @@ class TestHandleMaxIterations: assert len(result) > 0 assert "summary" in result.lower() + def test_summary_retries_share_relay_identity(self, agent): + agent.client.chat.completions.create.side_effect = [ + _mock_response(content=""), + _mock_response(content="Summary"), + ] + agent._cached_system_prompt = "You are helpful." + relay_calls = [] + + def execute_current(request, callback, **kwargs): + relay_calls.append(kwargs) + return callback(request) + + with patch("agent.relay_llm.execute_current", side_effect=execute_current): + result = agent._handle_max_iterations( + [{"role": "user", "content": "do stuff"}], + 60, + ) + + assert result == "Summary" + assert [call["metadata"]["retry_count"] for call in relay_calls] == [0, 1] + assert relay_calls[0]["metadata"]["api_request_id"] == ( + relay_calls[1]["metadata"]["api_request_id"] + ) + assert relay_calls[0]["metadata"]["call_role"] == "iteration_summary" + def test_api_failure_returns_error(self, agent): agent.client.chat.completions.create.side_effect = Exception("API down") agent._cached_system_prompt = "You are helpful." @@ -4323,10 +4348,10 @@ class TestRunConversation: with ( patch("run_agent.handle_function_call", return_value="search result"), patch( - "hermes_cli.plugins.has_hook", + "hermes_cli.lifecycle.has_hook", side_effect=lambda name: name in {"pre_api_request", "post_api_request"}, ), - patch("hermes_cli.plugins.invoke_hook", side_effect=_record_hook), + patch("hermes_cli.lifecycle.invoke_hook", side_effect=_record_hook), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -4365,8 +4390,8 @@ class TestRunConversation: hook_called = True return [] - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: False) - monkeypatch.setattr("hermes_cli.plugins.invoke_hook", _invoke_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: False) + monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook", _invoke_hook) monkeypatch.setattr(agent, "_api_request_payload_for_hook", _payload_for_hook) agent._invoke_api_request_error_hook( @@ -4405,12 +4430,12 @@ class TestRunConversation: payload_counts["response"] += 1 return {} - monkeypatch.setattr("hermes_cli.plugins.has_hook", _has_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", _has_hook) monkeypatch.setattr(agent, "_api_request_payload_for_hook", _request_payload) monkeypatch.setattr(agent, "_api_response_payload_for_hook", _response_payload) with ( - patch("hermes_cli.plugins.invoke_hook", return_value=[]), + patch("hermes_cli.lifecycle.invoke_hook", return_value=[]), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), diff --git a/tools/approval.py b/tools/approval.py index ea3bb8269069..95c41896b926 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -104,7 +104,7 @@ def _fire_approval_hook(hook_name: str, **kwargs) -> None: pre_approval_request, post_approval_response. """ try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook except Exception: # Plugin system not available in this execution context # (e.g. bare tool-only imports, minimal test environments). diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 9a8d842a705c..49ffb4980c3c 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -1453,7 +1453,7 @@ def _build_child_agent( logger.debug("spawn_requested relay failed: %s", exc) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "subagent_start", parent_session_id=getattr(parent_agent, "session_id", None), @@ -2798,7 +2798,7 @@ def delegate_task( # child was closed. _parent_session_id = getattr(parent_agent, "session_id", None) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook except Exception: _invoke_hook = None # Aggregate child spend here so the parent's footer/UI reflect the true diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 57422299d0de..bf8f2b05e307 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -2799,7 +2799,7 @@ def terminal_tool( # still subject to the final output limit below. # The hook is fail-open, and the first valid string return wins. try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook hook_results = invoke_hook( "transform_terminal_output", command=command, diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 5f04efaabd90..4598fcad5bde 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -429,13 +429,19 @@ def _notify_session_boundary( ) -> None: """Fire session lifecycle hooks with CLI parity.""" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import finalize_session, invoke_hook - _invoke_hook( - event_type, - session_id=session_id, - platform=_resolve_agent_platform(platform), - ) + if event_type == "on_session_finalize": + finalize_session( + session_id=session_id, + platform=_resolve_agent_platform(platform), + ) + else: + invoke_hook( + event_type, + session_id=session_id, + platform=_resolve_agent_platform(platform), + ) except Exception: pass @@ -614,7 +620,7 @@ def _finalize_session(session: dict | None, end_reason: str = "tui_close") -> No # the user Ctrl‑C's mid‑turn. if agent is not None: try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook invoke_hook( "on_session_end",