diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index b5706cadfec3..97f78b9143cb 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -2475,7 +2475,8 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False, skip_tool_request_middleware: bool = False, - tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None) -> str: + tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None, + skip_tool_execution_middleware: bool = False) -> str: """Invoke a single tool and return the result string. No display logic. Handles both agent-level tools (todo, memory, etc.) and registry-dispatched @@ -2654,8 +2655,7 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i return _finish_agent_tool(agent._dispatch_delegate_task(next_args), next_args) else: def _execute(next_args: dict) -> Any: - return _ra().handle_function_call( - function_name, next_args, effective_task_id, + dispatch_kwargs = dict( tool_call_id=tool_call_id, session_id=agent.session_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", @@ -2667,6 +2667,17 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i disabled_toolsets=getattr(agent, "disabled_toolsets", None), tool_request_middleware_trace=list(_tool_middleware_trace), ) + if skip_tool_execution_middleware: + dispatch_kwargs["skip_tool_execution_middleware"] = True + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + **dispatch_kwargs, + ) + + if skip_tool_execution_middleware: + return _execute(function_args) from hermes_cli.middleware import run_tool_execution_middleware diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index fa5bc6b80d7e..da85f03659e8 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -42,6 +42,7 @@ Payment / credit exhaustion fallback: import contextlib import contextvars +import functools import hashlib import inspect import json @@ -50,9 +51,10 @@ import os import re import threading import time +import uuid from pathlib import Path # noqa: F401 — used by test mocks from types import SimpleNamespace -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse, parse_qs, urlunparse # NOTE: `from openai import OpenAI` is deliberately NOT at module top — the @@ -2492,6 +2494,167 @@ _RUNTIME_MAIN_AUTH_MODE: str = "" _RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( contextvars.ContextVar("auxiliary_runtime_main", default=None) ) + +_RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( + contextvars.ContextVar("auxiliary_relay_call", default=None) +) + + +def _relay_auxiliary_call(callback): + """Give every physical retry in one auxiliary call a shared Relay identity.""" + + @functools.wraps(callback) + def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return callback(*args, **kwargs) + except BaseException: + _fail_relay_auxiliary_call() + raise + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _relay_auxiliary_call_async(callback): + """Async counterpart to :func:`_relay_auxiliary_call`.""" + + @functools.wraps(callback) + async def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return await callback(*args, **kwargs) + except BaseException: + _fail_relay_auxiliary_call() + raise + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _set_relay_auxiliary_route( + provider: str | None, + model: str | None, + api_mode: str | None, +) -> None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + context["provider"] = str(provider or "auxiliary") + context["model"] = str(model or "unknown") + context["api_mode"] = str(api_mode or "chat_completions") + + +def _relay_auxiliary_metadata( + *, + provider: str | None = None, + api_mode: str | None = None, +) -> tuple[str, str, dict[str, Any]] | None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return None + attempt_count = int(context.get("attempt_count") or 0) + context["attempt_count"] = attempt_count + 1 + provider_name = str(provider or context.get("provider") or "auxiliary") + model_name = str(context.get("model") or "unknown") + return provider_name, model_name, { + "api_mode": str(api_mode or context.get("api_mode") or "chat_completions"), + "api_request_id": str(context["request_id"]), + "call_role": f"auxiliary:{context['task']}", + "retry_count": attempt_count, + "auxiliary_task": str(context["task"]), + } + + +def _relay_sync_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return callback(kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.execute_current( + kwargs, + callback, + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + defer_logical_completion=True, + ) + + +async def _relay_async_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return await callback(kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return await relay_llm.execute_current_async( + kwargs, + callback, + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + defer_logical_completion=True, + ) + + +def _relay_sync_stream( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, +) -> Any: + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return client.chat.completions.create(**kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.stream_current( + kwargs, + lambda request: client.chat.completions.create(**request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + finalizer=dict, + metadata=metadata, + ) _RUNTIME_MAIN_COMPAT_SNAPSHOT: Tuple[Any, ...] = ("", "", "", "", "", "") _RUNTIME_MAIN_COMPAT_LOCK = threading.Lock() @@ -3801,7 +3964,13 @@ def _retry_same_provider_sync( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task, + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -3866,7 +4035,13 @@ async def _retry_same_provider_async( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task, + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -4071,7 +4246,7 @@ def _call_fallback_candidate_sync( base_url=fb_base, task=task) try: return _validate_llm_response( - fb_client.chat.completions.create(**fb_kwargs), task) + _relay_sync_completion(fb_client, fb_kwargs, provider=fb_label), task) except Exception as fb_err: if not _is_auth_error(fb_err): raise @@ -4088,7 +4263,13 @@ def _call_fallback_candidate_sync( base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=fb_provider, + ), + task, + ) except Exception as retry_err: if not _is_auth_error(retry_err): raise @@ -4137,7 +4318,13 @@ async def _call_fallback_candidate_async( base_url=fb_base, task=task) try: return _validate_llm_response( - await fb_client.chat.completions.create(**fb_kwargs), task) + await _relay_async_completion( + fb_client, + fb_kwargs, + provider=fb_label, + ), + task, + ) except Exception as fb_err: if not _is_auth_error(fb_err): raise @@ -4155,7 +4342,13 @@ async def _call_fallback_candidate_async( base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=fb_provider, + ), + task, + ) except Exception as retry_err: if not _is_auth_error(retry_err): raise @@ -7270,6 +7463,7 @@ def _validate_llm_response( except (AttributeError, TypeError, IndexError) as exc: recovered = _recover_aux_response_message(response) if recovered is not None: + _complete_relay_auxiliary_call() return recovered response_type = type(response).__name__ response_preview = str(response)[:120] @@ -7279,9 +7473,34 @@ def _validate_llm_response( f"Expected object with .choices[0].message — check provider " f"adapter or custom endpoint compatibility." ) from exc + _complete_relay_auxiliary_call() return response +def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None: + """Close one auxiliary logical call after acceptance or terminal failure.""" + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + from agent import relay_llm + + relay_llm.complete_logical_call( + str(context.get("request_id") or ""), + outcome=outcome, + ) + + +def _fail_relay_auxiliary_call() -> None: + """Close a terminally failed call without replacing its original error.""" + try: + _complete_relay_auxiliary_call(outcome="failed") + except Exception: + logger.warning( + "Relay auxiliary failure finalization failed", + exc_info=True, + ) + + def _recover_aux_response_message(response: Any) -> Optional[Any]: """Synthesize chat-completions shape from Responses-style text fields. @@ -7680,6 +7899,7 @@ async def _acreate_with_stream( ) +@_relay_auxiliary_call def call_llm( task: str = None, *, @@ -7820,6 +8040,11 @@ def call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Log what we're about to do — makes auxiliary operations visible _base_info = str(getattr(client, "base_url", resolved_base_url) or "") @@ -7857,7 +8082,12 @@ def call_llm( kwargs["stream"] = True if stream_options: kwargs["stream_options"] = stream_options - return client.chat.completions.create(**kwargs) + return _relay_sync_stream( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ) # Handle unsupported temperature, max_tokens vs max_completion_tokens retry, # then payment fallback. @@ -7880,10 +8110,18 @@ def call_llm( # for the transient retry every auxiliary task shares. (PR #16587) try: return _validate_llm_response( - _create_with_progress( - client, kwargs, task, - force_stream=_provider_requires_stream( - resolved_provider, _base_info or resolved_base_url, + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, _base_info or resolved_base_url, + ), ), ), task, @@ -7919,10 +8157,19 @@ def call_llm( time.sleep(_backoff) try: return _validate_llm_response( - _create_with_progress( - client, kwargs, task, - force_stream=_provider_requires_stream( - resolved_provider, _base_info or resolved_base_url, + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, + _base_info or resolved_base_url, + ), ), ), task) @@ -7942,7 +8189,12 @@ def call_llm( ) try: return _validate_llm_response( - client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) # If retry still fails, fall through to the max_tokens / @@ -7980,7 +8232,12 @@ def call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. @@ -8010,7 +8267,12 @@ def call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -8043,7 +8305,12 @@ def call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -8071,7 +8338,12 @@ def call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry ─────────────────────────────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -8121,7 +8393,12 @@ def call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise @@ -8379,6 +8656,7 @@ def extract_content_or_reasoning(response) -> str: return "" +@_relay_auxiliary_call_async async def async_call_llm( task: str = None, *, @@ -8470,6 +8748,11 @@ async def async_call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Pass the client's actual base_url (not just resolved_base_url) so # endpoint-specific temperature overrides can distinguish @@ -8508,7 +8791,14 @@ async def async_call_llm( try: return _validate_llm_response( - await _acreate(kwargs), task, + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task, provider=resolved_provider, base_url=_client_base) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -8529,7 +8819,14 @@ async def async_call_llm( task or "call", transient_err, ) return _validate_llm_response( - await _acreate(kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task) except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) @@ -8540,7 +8837,12 @@ async def async_call_llm( ) try: return _validate_llm_response( - await client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) if not ( @@ -8574,7 +8876,12 @@ async def async_call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. @@ -8603,7 +8910,12 @@ async def async_call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -8635,7 +8947,12 @@ async def async_call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -8662,7 +8979,12 @@ async def async_call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry (mirrors sync call_llm) ─────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -8706,7 +9028,12 @@ async def async_call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 5fecd0e26cfe..7ce136652125 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -15,6 +15,7 @@ sites unchanged. Symbols that tests patch on ``run_agent`` (e.g. from __future__ import annotations +import contextvars import json import logging import math @@ -57,6 +58,12 @@ _OPENROUTER_PROVIDER_SORT_VALUES = {"throughput", "latency", "price"} _FALLBACK_EXHAUSTED_COOLDOWN_S = 5.0 +def _context_thread_target(callback): + """Bind a no-argument thread target to the caller's ContextVars.""" + context = contextvars.copy_context() + return lambda: context.run(callback) + + def _ra(): """Lazy ``run_agent`` reference. @@ -810,7 +817,7 @@ def interruptible_api_call(agent, api_kwargs: dict): _call_start = time.time() agent._touch_activity("waiting for non-streaming API response") - t = threading.Thread(target=_call, daemon=True) + t = threading.Thread(target=_context_thread_target(_call), daemon=True) t.start() _poll_count = 0 while t.is_alive(): @@ -1989,6 +1996,28 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: """Request a summary when max iterations are reached. Returns the final response text.""" print(f"⚠️ Reached maximum iterations ({agent.max_iterations}). Requesting summary...") + summary_api_request_id = f"iteration-summary:{uuid.uuid4()}" + summary_call_outcome = "failed" + + def _managed_summary_call(request, callback, *, retry_count: int): + from agent import relay_llm + + return relay_llm.execute_current( + request, + callback, + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + metadata={ + "api_mode": str( + getattr(agent, "api_mode", "") or "chat_completions" + ), + "api_request_id": summary_api_request_id, + "call_role": "iteration_summary", + "retry_count": retry_count, + }, + defer_logical_completion=True, + ) + summary_request = ( "You've reached the maximum number of tool-calling iterations allowed. " "Please provide a final response summarizing what you've found and accomplished so far, " @@ -2174,17 +2203,33 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if agent.api_mode == "anthropic_messages": _tsum = agent._get_transport() - _ant_kw = _tsum.build_kwargs(model=agent.model, messages=api_messages, tools=None, - max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - is_oauth=agent._is_anthropic_oauth, - preserve_dots=agent._anthropic_preserve_dots(), - base_url=getattr(agent, "_anthropic_base_url", None)) + _ant_kw = _tsum.build_kwargs( + model=agent.model, + messages=api_messages, + tools=None, + max_tokens=agent.max_tokens, + reasoning_config=agent.reasoning_config, + is_oauth=agent._is_anthropic_oauth, + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None), + ) _ant_kw = _merge_nous_portal_messages_extra_body(agent, _ant_kw) - summary_response = agent._anthropic_messages_create(_ant_kw) + summary_response = _managed_summary_call( + _ant_kw, + agent._anthropic_messages_create, + retry_count=0, + ) _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_summary_result.content or "").strip() else: - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=0, + ) _summary_result = agent._get_transport().normalize_response(summary_response) final_response = (_summary_result.content or "").strip() @@ -2192,6 +2237,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if "" in final_response: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: + summary_call_outcome = "success" messages.append({"role": "assistant", "content": final_response}) else: final_response = "I reached the iteration limit and couldn't generate a summary." @@ -2206,13 +2252,22 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: final_response = (_cnr_retry.content or "").strip() elif agent.api_mode == "anthropic_messages": _tretry = agent._get_transport() - _ant_kw2 = _tretry.build_kwargs(model=agent.model, messages=api_messages, tools=None, - is_oauth=agent._is_anthropic_oauth, - max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - preserve_dots=agent._anthropic_preserve_dots(), - base_url=getattr(agent, "_anthropic_base_url", None)) + _ant_kw2 = _tretry.build_kwargs( + model=agent.model, + messages=api_messages, + tools=None, + is_oauth=agent._is_anthropic_oauth, + max_tokens=agent.max_tokens, + reasoning_config=agent.reasoning_config, + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None), + ) _ant_kw2 = _merge_nous_portal_messages_extra_body(agent, _ant_kw2) - retry_response = agent._anthropic_messages_create(_ant_kw2) + retry_response = _managed_summary_call( + _ant_kw2, + agent._anthropic_messages_create, + retry_count=1, + ) _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_retry_result.content or "").strip() else: @@ -2229,7 +2284,14 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if summary_extra_body: summary_kwargs["extra_body"] = summary_extra_body - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary_retry").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary_retry" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=1, + ) _retry_result = agent._get_transport().normalize_response(summary_response) final_response = (_retry_result.content or "").strip() @@ -2237,6 +2299,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if "" in final_response: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: + summary_call_outcome = "success" messages.append({"role": "assistant", "content": final_response}) else: final_response = "I reached the iteration limit and couldn't generate a summary." @@ -2246,6 +2309,13 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: except Exception as e: logger.warning(f"Failed to get summary response: {e}") final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" + finally: + from agent import relay_llm + + relay_llm.complete_logical_call( + summary_api_request_id, + outcome=summary_call_outcome, + ) return final_response @@ -2404,7 +2474,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= pass def _bedrock_call(): + stream = None try: + from agent import relay_llm from agent.bedrock_adapter import ( _get_bedrock_runtime_client, invalidate_runtime_client, @@ -2413,44 +2485,40 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= normalize_converse_response, stream_converse_with_callbacks, ) - region = api_kwargs.pop("__bedrock_region__", "us-east-1") - api_kwargs.pop("__bedrock_converse__", None) - client = _get_bedrock_runtime_client(region) - try: - raw_response = client.converse_stream(**api_kwargs) - except Exception as _bedrock_exc: - # IAM policies scoped to bedrock:InvokeModel only (no - # InvokeModelWithResponseStream) reject converse_stream() - # with AccessDeniedException. That denial is permanent for - # the session — fall back to the non-streaming converse() - # inline (it maps to bedrock:InvokeModel) and disable - # streaming for subsequent calls so we don't re-fail every - # turn. - if is_streaming_access_denied_error(_bedrock_exc): - agent._disable_streaming = True - agent._safe_print( - "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — " - "falling back to non-streaming InvokeModel.\n" - " Grant that action to restore streaming output.\n" - ) - logger.info( - "bedrock: converse_stream denied by IAM (%s) — " - "using non-streaming converse() for this session.", - type(_bedrock_exc).__name__, - ) - result["response"] = normalize_converse_response( - client.converse(**api_kwargs) - ) - return - # Evict the cached client on stale-connection failures - # so the outer retry loop builds a fresh client/pool. - if is_stale_connection_error(_bedrock_exc): - invalidate_runtime_client(region) - raise + intercepted_events = [] + writer_token = {"value": None} - # Claim the delta sink for this bedrock stream (#65991) so a - # superseded attempt's callbacks are fenced by the sink guard. - claim_stream_writer(agent) + def _open_bedrock_stream(next_api_kwargs: dict[str, Any]): + final_kwargs = dict(next_api_kwargs) + region = final_kwargs.pop("__bedrock_region__", "us-east-1") + final_kwargs.pop("__bedrock_converse__", None) + client = _get_bedrock_runtime_client(region) + try: + raw_response = client.converse_stream(**final_kwargs) + except Exception as _bedrock_exc: + # InvokeModel-only policies cannot open a stream. Keep + # the fallback inside the same managed Relay attempt so + # the real provider request and terminal response still + # share one lifecycle boundary. + if is_streaming_access_denied_error(_bedrock_exc): + agent._disable_streaming = True + agent._safe_print( + "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — " + "falling back to non-streaming InvokeModel.\n" + " Grant that action to restore streaming output.\n" + ) + logger.info( + "bedrock: converse_stream denied by IAM (%s) — " + "using non-streaming converse() for this session.", + type(_bedrock_exc).__name__, + ) + return normalize_converse_response( + client.converse(**final_kwargs) + ) + if is_stale_connection_error(_bedrock_exc): + invalidate_runtime_client(region) + raise + return raw_response.get("stream", []) def _on_text(text): _fire_first() @@ -2465,18 +2533,65 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= _fire_first() agent._fire_reasoning_delta(text) - result["response"] = stream_converse_with_callbacks( - raw_response, + def _finalize_bedrock_stream(): + return stream_converse_with_callbacks( + {"stream": list(intercepted_events)} + ) + + def _bedrock_stream_created(_stream: Any) -> None: + writer_token["value"] = claim_stream_writer(agent) + + def _accept_bedrock_event(_event: Any) -> bool: + token = writer_token["value"] + return token is None or stream_writer_is_current(agent, token) + + stream = relay_llm.stream( + dict(api_kwargs), + _open_bedrock_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "bedrock"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=_finalize_bedrock_stream, + on_stream_created=_bedrock_stream_created, + on_chunk=intercepted_events.append, + chunk_adapter=lambda chunk: chunk, + accept_chunk=_accept_bedrock_event, + completed_response_predicate=lambda response: bool( + getattr(response, "choices", None) + ), + metadata={ + "api_mode": "custom", + "api_request_id": getattr( + agent, "_current_api_request_id", None + ), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + streamed_response = stream_converse_with_callbacks( + {"stream": stream}, on_text_delta=_on_text if agent._has_stream_consumers() else None, on_tool_start=_on_tool, on_reasoning_delta=_on_reasoning if agent.reasoning_callback or agent.stream_delta_callback else None, on_interrupt_check=lambda: agent._interrupt_requested, on_event=lambda: _bedrock_last_event.__setitem__("t", time.time()), ) + result["response"] = stream.final_response or streamed_response except Exception as e: result["error"] = e + finally: + if stream is not None: + stream.close() - t = threading.Thread(target=_bedrock_call, daemon=True) + t = threading.Thread( + target=_context_thread_target(_bedrock_call), daemon=True + ) t.start() while t.is_alive(): t.join(timeout=0.3) @@ -2661,6 +2776,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= first_delta_fired = {"done": False} deltas_were_sent = {"yes": False} # Track if any deltas were fired (for fallback) + provider_tool_in_flight = {"yes": False} # Wall-clock timestamp of the last real streaming chunk. The outer # poll loop uses this to detect stale connections that keep receiving # SSE keep-alive pings but no actual data. @@ -2680,11 +2796,29 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= "discarded_chunks": 0, "discarded_bytes": 0, } + managed_stream_holder = {"stream": None} + + def _set_managed_stream(stream: Any) -> Any: + managed_stream_holder["stream"] = stream + return stream + + def _close_managed_stream() -> None: + stream = managed_stream_holder.pop("stream", None) + if stream is None: + return + close = getattr(stream, "close", None) + if callable(close): + try: + close() + except Exception: + logger.debug("Managed provider stream cleanup failed", exc_info=True) def _start_stream_attempt() -> int: with stream_attempt_lock: stream_attempt_state["current"] += 1 - return int(stream_attempt_state["current"]) + attempt_id = int(stream_attempt_state["current"]) + provider_tool_in_flight["yes"] = False + return attempt_id def _cancel_current_stream_attempt(reason: str) -> None: with stream_attempt_lock: @@ -2794,107 +2928,6 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # Cap connect/pool at 60s even when provider timeout is higher. # connect/pool cover TCP handshake, not model inference. _conn_cap = min(_base_timeout, 60.0) if _provider_timeout_cfg is not None else 30.0 - stream_kwargs = { - **api_kwargs, - "stream": True, - "timeout": _httpx.Timeout( - connect=_conn_cap, - read=_stream_read_timeout, - write=_base_timeout, - pool=_conn_cap, - ), - } - # OpenAI's `stream_options={"include_usage": True}` drives usage - # accounting on OpenAI-compatible endpoints (incl. the Gemini OpenAI - # compat shim and aggregators like OpenRouter). Google's *native* - # Gemini REST endpoint rejects the keyword outright - # (`Completions.create() got an unexpected keyword argument - # 'stream_options'`), so omit it only for that endpoint. - if not is_native_gemini_base_url(agent.base_url): - stream_kwargs["stream_options"] = {"include_usage": True} - request_client = _set_request_client( - agent._create_request_openai_client( - reason="chat_completion_stream_request", - api_kwargs=stream_kwargs, - ) - ) - # Reset stale-stream timer so the detector measures from this - # attempt's start, not a previous attempt's last chunk. - last_chunk_time["t"] = time.time() - agent._touch_activity("waiting for provider response (streaming)") - # Initialize per-attempt stream diagnostics so the retry block can - # reach for them after the stream dies. Lives on - # ``request_client_holder["diag"]`` for closure access. - _diag = agent._stream_diag_init() - request_client_holder["diag"] = _diag - stream = request_client.chat.completions.create(**stream_kwargs) - if agent.provider == "moa": - # The MoA facade is a shared singleton — abort/close of the - # registered client is a no-op, so register the stream handle - # itself for interrupt teardown (#57354). - stream = _set_request_stream_handle(stream) - # Claim the delta sink for THIS attempt (#65991). If a prior attempt's - # stream is somehow still alive (a stale-stream reconnect whose socket - # abort raced), this claim supersedes it so its late chunks are fenced - # out of the turn instead of interleaving with ours. - _writer_token = claim_stream_writer(agent) - - # Some OpenAI-compatible adapters (for example copilot-acp, and the MoA - # openai-codex aggregator) accept stream=True but still return a - # completed response object rather than an iterator of chunks. Treat - # that as "streaming unsupported" for the rest of this session instead - # of crashing on ``for chunk in stream`` with ``'types.SimpleNamespace' - # object is not iterable`` (#11732, #55933). - # - # Discriminate on the mere PRESENCE of a ``choices`` attribute, not on - # it being a non-empty list: an adapter may hand back a completed - # response whose ``choices`` is ``None`` or empty (an error / - # content-filter / terminal frame), and every such shape is still a - # whole response — not a token stream — that would crash iteration just - # the same. A genuine provider stream (SDK ``Stream`` object, - # generator) exposes no ``choices`` attribute, so it is left untouched. - if hasattr(stream, "choices"): - logger.info( - "Streaming request returned a final response object instead of " - "an iterator; switching %s/%s to non-streaming for this session.", - agent.provider or "unknown", - agent.model or "unknown", - ) - agent._disable_streaming = True - # An empty/None ``choices`` carries no message to surface; return the - # completed object as-is so the outer loop's normal invalid-response - # validation (conversation_loop.py) handles it via the retry path, - # never ``for chunk in stream``. - choices = stream.choices - first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None - message = getattr(first_choice, "message", None) - if message is not None: - reasoning_text = ( - getattr(message, "reasoning_content", None) - or getattr(message, "reasoning", None) - ) - if isinstance(reasoning_text, str) and reasoning_text: - _fire_first_delta() - agent._fire_reasoning_delta(reasoning_text) - content = getattr(message, "content", None) - if isinstance(content, str) and content: - _fire_first_delta() - agent._fire_stream_delta(content) - return stream - - # Capture rate limit headers from the initial HTTP response. - # The OpenAI SDK Stream object exposes the underlying httpx - # response via .response before any chunks are consumed. - agent._capture_rate_limits(getattr(stream, "response", None)) - agent._capture_credits(getattr(stream, "response", None)) - # Snapshot diagnostic headers (cf-ray, x-openrouter-provider, etc.) - # so they survive even when the stream dies before any chunk - # arrives. Best-effort; never raises. - agent._stream_diag_capture_response(_diag, getattr(stream, "response", None)) - - # Log OpenRouter response cache status when present. - agent._check_openrouter_cache_status(getattr(stream, "response", None)) - content_parts: list = [] tool_calls_acc: dict = {} tool_gen_notified: set = set() @@ -2909,19 +2942,123 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= role = "assistant" reasoning_parts: list = [] usage_obj = None - for chunk in stream: - # Stop the moment a newer attempt has claimed the delta sink - # (#65991): this attempt has been superseded, so it must neither - # fire deltas (incl. the tool-suppressed raw-callback path below) - # nor keep consuming a stream that would interleave into the turn. - if not stream_writer_is_current(agent, _writer_token): + _diag = agent._stream_diag_init() + request_client_holder["diag"] = _diag + _writer_token = {"value": None} + + def _open_stream(next_api_kwargs: dict[str, Any]): + stream_kwargs = { + **next_api_kwargs, + "stream": True, + "timeout": _httpx.Timeout( + connect=_conn_cap, + read=_stream_read_timeout, + write=_base_timeout, + pool=_conn_cap, + ), + } + # Native Gemini rejects OpenAI's usage-streaming extension. + if not is_native_gemini_base_url(agent.base_url): + stream_kwargs["stream_options"] = {"include_usage": True} + request_client = _set_request_client( + agent._create_request_openai_client( + reason="chat_completion_stream_request", + api_kwargs=stream_kwargs, + ) + ) + last_chunk_time["t"] = time.time() + agent._touch_activity("waiting for provider response (streaming)") + return request_client.chat.completions.create(**stream_kwargs) + + def _stream_created(raw_stream: Any) -> None: + response = getattr(raw_stream, "response", None) + agent._capture_rate_limits(response) + agent._capture_credits(response) + agent._stream_diag_capture_response(_diag, response) + agent._check_openrouter_cache_status(response) + _writer_token["value"] = claim_stream_writer(agent) + + def _accept_stream_chunk(_chunk: Any) -> bool: + # A stale-attempt fence can win while Relay is handing an + # already-received tool-call chunk back to Hermes. Preserve only + # the fact that a tool call was in flight so retry policy does not + # misclassify the attempt as a partial text response. The chunk + # itself is still rejected below and never reaches callbacks. + try: + choices = getattr(_chunk, "choices", None) + delta = getattr(choices[0], "delta", None) if choices else None + if getattr(delta, "tool_calls", None): + provider_tool_in_flight["yes"] = True + except Exception: + pass + if not _stream_attempt_is_active(stream_attempt_id): + return False + token = _writer_token["value"] + if token is not None and not stream_writer_is_current(agent, token): logger.warning( "Streaming attempt superseded by a newer stream; stopping " "consumption to preserve the single-writer invariant " "(model=%s).", api_kwargs.get("model", "unknown"), ) - break + return False + # Record provider activity before Relay processes the chunk. This + # prevents the stale watchdog from cancelling a live stream while + # an interceptor or codec is still handling an already-received + # event. + last_chunk_time["t"] = time.time() + return True + + def _relay_final_response() -> dict[str, Any]: + tool_calls = [tool_calls_acc[index] for index in sorted(tool_calls_acc)] + return { + "model": model_name, + "choices": [ + { + "message": { + "role": role, + "content": "".join(content_parts) or None, + "reasoning_content": "".join(reasoning_parts) or None, + "tool_calls": tool_calls or None, + }, + "finish_reason": finish_reason or "stop", + } + ], + "usage": usage_obj, + } + + from agent import relay_llm + + stream = _set_managed_stream( + relay_llm.stream( + api_kwargs, + _open_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=_relay_final_response, + on_stream_created=_stream_created, + accept_chunk=_accept_stream_chunk, + completed_response_predicate=lambda value: hasattr(value, "choices"), + metadata={ + "api_mode": "chat_completions", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + ) + if agent.provider == "moa": + # Hermes interrupts the managed stream; Relay retains sole + # ownership of closing the underlying provider stream. + _set_request_stream_handle(stream) + for chunk in stream: last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") @@ -3075,11 +3212,46 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if hasattr(chunk, "usage") and chunk.usage: usage_obj = chunk.usage + _close_managed_stream() + if _stream_attempt_was_cancelled(stream_attempt_id): raise _httpx.RemoteProtocolError( f"stream attempt {stream_attempt_id} was superseded" ) + # Some OpenAI-compatible adapters accept ``stream=True`` but return a + # completed response. Relay records that attempt while Hermes preserves + # its existing switch-to-non-streaming behavior for later calls. + if stream.final_response is not None: + final_response = stream.final_response + logger.info( + "Streaming request returned a final response object instead of " + "an iterator; switching %s/%s to non-streaming for this session.", + agent.provider or "unknown", + agent.model or "unknown", + ) + agent._disable_streaming = True + choices = final_response.choices + first_choice = ( + choices[0] + if isinstance(choices, (list, tuple)) and choices + else None + ) + message = getattr(first_choice, "message", None) + if message is not None: + reasoning_text = ( + getattr(message, "reasoning_content", None) + or getattr(message, "reasoning", None) + ) + if isinstance(reasoning_text, str) and reasoning_text: + _fire_first_delta() + agent._fire_reasoning_delta(reasoning_text) + content = getattr(message, "content", None) + if isinstance(content, str) and content: + _fire_first_delta() + agent._fire_stream_delta(content) + return final_response + # Build mock response matching non-streaming shape full_content = "".join(content_parts) or None mock_tool_calls = None @@ -3241,72 +3413,95 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # fabricated "successful" empty turn. saw_stream_event = False - # Reset stale-stream timer for this attempt last_chunk_time["t"] = time.time() - # Per-attempt diagnostic dict for the retry block to consume. _diag = agent._stream_diag_init() request_client_holder["diag"] = _diag - # Defensive: strip Responses-only kwargs (instructions, input, ...) - # that can leak in under an api_mode-flip race. The Anthropic SDK - # raises a non-retryable TypeError on them, killing the turn. See - # #31673 / sanitize_anthropic_kwargs(). + _writer_token = {"value": None} + _stream_context = {"manager": None, "stream": None} + base_final_message = None + + from agent import relay_llm from agent.anthropic_adapter import sanitize_anthropic_kwargs - sanitize_anthropic_kwargs( - api_kwargs, log_prefix=getattr(agent, "log_prefix", "") - ) - # Use the Anthropic SDK's streaming context manager - with request_client.messages.stream(**api_kwargs) as stream: + + accumulator = relay_llm.AnthropicStreamAccumulator() + + def _open_anthropic_stream(next_api_kwargs: dict[str, Any]): + final_kwargs = dict(next_api_kwargs) + sanitize_anthropic_kwargs( + final_kwargs, + log_prefix=getattr(agent, "log_prefix", ""), + ) + manager = request_client.messages.stream(**final_kwargs) + _stream_context["manager"] = manager + return manager.__enter__() + + def _anthropic_stream_created(raw_stream: Any) -> None: + _stream_context["stream"] = raw_stream # The Anthropic SDK exposes the raw httpx response on - # ``stream.response``. Snapshot diagnostic headers - # immediately so they survive a stream that dies before the - # first event. + # ``stream.response``. Snapshot diagnostics immediately so they + # survive a stream that dies before the first event. try: agent._stream_diag_capture_response( - _diag, getattr(stream, "response", None) + _diag, + getattr(raw_stream, "response", None), ) except Exception: pass - # Claim the delta sink for THIS attempt (#65991) — parity with the - # chat_completions path so a superseded anthropic stream is fenced. - _writer_token = claim_stream_writer(agent) + _writer_token["value"] = claim_stream_writer(agent) + + def _accept_anthropic_event(_event: Any) -> bool: + token = _writer_token["value"] + if token is None or stream_writer_is_current(agent, token): + return True + logger.warning( + "Anthropic streaming attempt superseded by a newer stream; " + "stopping consumption to preserve the single-writer " + "invariant (model=%s).", + api_kwargs.get("model", "unknown"), + ) + return False + + stream = _set_managed_stream( + relay_llm.stream( + api_kwargs, + _open_anthropic_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "anthropic"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=accumulator.finalize, + on_stream_created=_anthropic_stream_created, + on_chunk=accumulator.observe, + accept_chunk=_accept_anthropic_event, + metadata={ + "api_mode": "anthropic_messages", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + ) + try: for event in stream: - # Bail the instant a newer attempt supersedes this one so a - # stale stream can't interleave tokens into the turn. - if not stream_writer_is_current(agent, _writer_token): - logger.warning( - "Anthropic streaming attempt superseded by a newer " - "stream; stopping consumption to preserve the " - "single-writer invariant (model=%s).", - api_kwargs.get("model", "unknown"), - ) - break saw_stream_event = True - # Update stale-stream timer on every event so the - # outer poll loop knows data is flowing. Without - # this, the detector kills healthy long-running - # Opus streams after 180 s even when events are - # actively arriving (the chat_completions path - # already does this at the top of its chunk loop). last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") - - # Update per-attempt diagnostic counters (best-effort). try: _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 if _diag.get("first_chunk_at") is None: _diag["first_chunk_at"] = last_chunk_time["t"] - try: - _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) - except Exception: - pass + _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) except Exception: pass - if agent._interrupt_requested: break event_type = getattr(event, "type", None) - if event_type == "content_block_start": block = getattr(event, "content_block", None) if block and getattr(block, "type", None) == "tool_use": @@ -3315,7 +3510,6 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if tool_name: _fire_first_delta() agent._fire_tool_gen_started(tool_name) - elif event_type == "content_block_delta": delta = getattr(event, "delta", None) if delta: @@ -3331,48 +3525,49 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if thinking_text: _fire_first_delta() agent._fire_reasoning_delta(thinking_text) - - # Return the native Anthropic Message for downstream processing. - # If the stream was interrupted (the event loop broke out above on - # agent._interrupt_requested), do NOT call get_final_message() — on - # a partially-consumed stream the SDK may hang draining remaining - # events or return a Message with incomplete tool_use blocks (partial - # JSON in `input`). The outer poll loop raises InterruptedError, so - # this return value is discarded anyway. - if agent._interrupt_requested: - return None - # Zero-event guard (parity with the chat_completions zero-chunk - # guard above). Real SDK: an eventless stream has no - # message_start, so get_final_message() raises AssertionError - # (final-message snapshot is None) — normalize that to - # EmptyStreamError so it gets the transient retry budget - # instead of surfacing raw. + if not agent._interrupt_requested: + raw_stream = _stream_context["stream"] + if raw_stream is not None: + try: + base_final_message = raw_stream.get_final_message() + except AssertionError: + if not saw_stream_event: + raise EmptyStreamError( + "Provider returned an empty stream with no events " + "(possible upstream error or malformed event stream)." + ) from None + raise + finally: try: - _final_message = stream.get_final_message() - except AssertionError: - if not saw_stream_event: - raise EmptyStreamError( - "Provider returned an empty stream with no events " - "(possible upstream error or malformed event stream)." - ) from None - raise - # Shim variants of the same failure: an OpenAI-compat adapter - # may fabricate a contentless Message with no stop_reason, or - # return None where the SDK assert would have fired (e.g. - # ``python -O``). A real completed response always carries a - # stop_reason, so this cannot fire on legitimate turns. - if not saw_stream_event and ( - _final_message is None - or ( - not getattr(_final_message, "content", None) - and getattr(_final_message, "stop_reason", None) is None - ) - ): - raise EmptyStreamError( - "Provider returned an empty stream with no stop_reason " - "(possible upstream error or malformed event stream)." - ) - return _final_message + _close_managed_stream() + finally: + manager = _stream_context["manager"] + if manager is not None: + manager.__exit__(None, None, None) + + if agent._interrupt_requested: + return None + if ( + base_final_message is not None + and not getattr(base_final_message, "content", None) + and getattr(base_final_message, "stop_reason", None) is None + ): + raise EmptyStreamError( + "Provider returned an empty stream with no stop_reason " + "(possible upstream error or malformed event stream)." + ) + if base_final_message is not None and not stream.output_modified: + return base_final_message + final_message = accumulator.response(base_final_message) + if ( + not getattr(final_message, "content", None) + and getattr(final_message, "stop_reason", None) is None + ): + raise EmptyStreamError( + "Provider returned an empty stream with no stop_reason " + "(possible upstream error or malformed event stream)." + ) + return final_message def _call(): import httpx as _httpx @@ -3407,6 +3602,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= result["response"] = _call_chat_completions(stream_attempt_id) return # success except Exception as e: + _close_managed_stream() # If the main poll loop force-closed this request because # of an interrupt, the resulting transport error is the # expected consequence of our own close — NOT a transient @@ -3446,7 +3642,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if deltas_were_sent["yes"]: _partial_tool_in_flight = bool( result.get("partial_tool_names") - ) + ) or provider_tool_in_flight["yes"] _is_sse_conn_err_preview = False if not _is_timeout and not _is_conn_err: from openai import APIError as _APIError @@ -3695,6 +3891,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= result["error"] = e return finally: + _close_managed_stream() _close_request_client_once("stream_request_complete") # Provider-configured stale timeout takes priority over env default. @@ -3757,7 +3954,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if _reasoning_floor is not None: _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor) - t = threading.Thread(target=_call, daemon=True) + t = threading.Thread(target=_context_thread_target(_call), daemon=True) t.start() _last_heartbeat = time.time() _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index da3bc4f95695..0c5bb7b70503 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -1236,6 +1236,8 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta """ import httpx as _httpx + from agent import relay_llm + active_client = client or agent._ensure_primary_openai_client(reason="codex_stream_direct") max_stream_retries = 1 # Accumulate streamed text so callers / compat shims can read it. @@ -1260,48 +1262,88 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta if agent._interrupt_requested: raise InterruptedError("Agent interrupted before Codex stream retry") - stream_kwargs = dict(api_kwargs) - stream_kwargs["stream"] = True + intercepted_events = [] + writer_token = {"value": None} + + def _open_codex_stream(next_api_kwargs: dict[str, Any]): + stream_kwargs = dict(next_api_kwargs) + stream_kwargs["stream"] = True + return active_client.responses.create(**stream_kwargs) + + def _codex_stream_created(_raw_stream: Any) -> None: + # Claim the delta sink for THIS physical attempt. A newer attempt + # supersedes this token and fences late deltas out of the turn. + writer_token["value"] = claim_stream_writer(agent) + + def _accept_codex_chunk(_chunk: Any) -> bool: + token = writer_token["value"] + if token is None or stream_writer_is_current(agent, token): + return True + logger.warning( + "Codex streaming attempt superseded by a newer stream; " + "stopping consumption to preserve the single-writer " + "invariant (model=%s).", + api_kwargs.get("model", "unknown"), + ) + return False + + def _finalize_codex_stream() -> Any: + return _consume_codex_event_stream( + list(intercepted_events), + model=api_kwargs.get("model"), + ) try: - event_stream = active_client.responses.create(**stream_kwargs) - except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: + event_stream = relay_llm.stream( + dict(api_kwargs), + _open_codex_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "codex"), + model_name=str(api_kwargs.get("model") or ""), + finalizer=_finalize_codex_stream, + on_stream_created=_codex_stream_created, + on_chunk=intercepted_events.append, + chunk_adapter=lambda chunk: chunk, + accept_chunk=_accept_codex_chunk, + completed_response_predicate=lambda response: bool( + hasattr(response, "output") and not hasattr(response, "__iter__") + ), + metadata={ + "api_mode": "codex_responses", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + "retry_count": attempt, + }, + defer_logical_completion=True, + ) + except ( + _httpx.RemoteProtocolError, + _httpx.ReadTimeout, + _httpx.ConnectError, + ConnectionError, + ) as exc: if attempt < max_stream_retries: logger.debug( - "Codex Responses stream connect failed (attempt %s/%s); retrying. %s error=%s", - attempt + 1, max_stream_retries + 1, - agent._client_log_context(), exc, + "Codex Responses stream connect failed (attempt %s/%s); " + "retrying. %s error=%s", + attempt + 1, + max_stream_retries + 1, + agent._client_log_context(), + exc, ) continue raise - # Claim the delta sink for THIS attempt (#65991) — parity with the - # chat_completions/anthropic/bedrock paths. If a prior attempt's - # stream is somehow still alive, this claim supersedes it so its - # late deltas are fenced out of the turn; conversely, a newer - # attempt supersedes us and the interrupt_check below stops our - # consumption immediately. - _writer_token = claim_stream_writer(agent) - - def _interrupt_or_superseded(_tok=_writer_token) -> bool: - if agent._interrupt_requested: - return True - if not stream_writer_is_current(agent, _tok): - logger.warning( - "Codex streaming attempt superseded by a newer stream; " - "stopping consumption to preserve the single-writer " - "invariant (model=%s).", - api_kwargs.get("model", "unknown"), - ) - return True - return False + def _interrupt_or_superseded() -> bool: + return bool(agent._interrupt_requested) try: - # Compatibility: some mocks/providers return a concrete response - # instead of an iterable. Pass it straight through. - if hasattr(event_stream, "output") and not hasattr(event_stream, "__iter__"): - return event_stream - try: final = _consume_codex_event_stream( event_stream, @@ -1320,6 +1362,12 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta on_event=_on_event, interrupt_check=_interrupt_or_superseded, ) + # The terminal SSE frame is contractually last. Request the + # end-of-stream marker so Relay can run its response finalizer + # and close the physical attempt scope before Hermes returns. + if not agent._interrupt_requested: + for _ignored in event_stream: + pass except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: if attempt < max_stream_retries: logger.debug( @@ -1330,6 +1378,10 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta ) continue raise + except RuntimeError: + if event_stream.final_response is not None: + return event_stream.final_response + raise if final.status in {"incomplete", "failed"}: logger.warning( diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 9a1d91a1a9e2..d03a2cfa3d23 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -494,7 +494,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # session is created (not on continuation). Plugins can use this # to initialise session-scoped state (e.g. warm a memory cache). try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_start", session_id=agent.session_id, @@ -2063,7 +2063,7 @@ def run_conversation( _llm_middleware_trace = [] try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -2104,6 +2104,7 @@ def run_conversation( base_url=agent.base_url, api_mode=agent.api_mode, api_call_count=api_call_count, + retry_count=retry_count, request_messages=list(request_messages) if isinstance(request_messages, list) else [], @@ -2193,7 +2194,28 @@ def run_conversation( return agent._interruptible_streaming_api_call( next_api_kwargs, on_first_delta=_stop_spinner ) - return agent._interruptible_api_call(next_api_kwargs) + from agent import relay_llm + + return relay_llm.execute( + next_api_kwargs, + agent._interruptible_api_call, + session_id=str(agent.session_id or ""), + name=str(agent.provider or "provider"), + model_name=str(agent.model or ""), + metadata={ + "api_mode": agent.api_mode, + "api_request_id": api_request_id, + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + "retry_count": retry_count, + }, + defer_logical_completion=True, + ) from hermes_cli.middleware import run_llm_execution_middleware @@ -3233,6 +3255,12 @@ def run_conversation( clear_nous_rate_limit() except Exception: pass + from agent import relay_llm + + relay_llm.complete_logical_call( + api_request_id, + outcome="success", + ) agent._touch_activity(f"API call #{api_call_count} completed") break # Success, exit retry loop @@ -5360,7 +5388,7 @@ def run_conversation( assistant_message.content = str(raw) try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -6693,7 +6721,8 @@ def run_conversation( _attempt = getattr(agent, "_pre_verify_nudges", 0) try: from agent.verify_hooks import max_verify_nudges - from hermes_cli.plugins import get_pre_verify_continue_message, has_hook + from hermes_cli.lifecycle import has_hook + from hermes_cli.plugins import get_pre_verify_continue_message if _edited and has_hook("pre_verify") and _attempt < max_verify_nudges(): # Posture is fixed for the session — resolve once + cache. diff --git a/agent/relay_llm.py b/agent/relay_llm.py new file mode 100644 index 000000000000..85286c47dae7 --- /dev/null +++ b/agent/relay_llm.py @@ -0,0 +1,1108 @@ +"""Core NeMo Relay adapters for physical Hermes provider attempts.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import logging +from collections.abc import Callable, Iterator +from types import SimpleNamespace +from typing import Any + +from agent import relay_runtime + +logger = logging.getLogger(__name__) + + +_PROVIDER_MESSAGE_EXTENSION_KEYS = frozenset( + {"reasoning_content", "reasoning_details"} +) +_RELAY_INTERNAL_PROVIDER_HEADERS = frozenset( + {"x-dynamo-parent-session-id", "x-dynamo-session-id"} +) + + +def execute( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run one non-streaming physical provider attempt through Relay.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + raw = callback_context.copy().run(callback, final_request) + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = _run_awaitable( + runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if _recover_successful_callback( + raw_response, + relay_error=exc, + callback_error=callback_error, + logical=logical, + defer_logical_completion=defer_logical_completion, + ): + return raw_response["value"] + raise + + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return _namespace(managed) + + +async def execute_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run one asynchronous physical provider attempt through Relay.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return await callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + async def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + async def call_provider() -> Any: + return await callback(final_request) + + task = callback_context.copy().run( + asyncio.create_task, + call_provider(), + ) + raw = await task + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = await runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if _recover_successful_callback( + raw_response, + relay_error=exc, + callback_error=callback_error, + logical=logical, + defer_logical_completion=defer_logical_completion, + ): + return raw_response["value"] + raise + + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return _namespace(managed) + + +def execute_current( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run a provider attempt under the inherited Hermes turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return callback(request) + return execute( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +async def execute_current_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run an async provider attempt under the inherited turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return await callback(request) + return await execute_async( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +def stream_current( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + finalizer: Callable[[], Any], + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run a provider stream under the inherited Hermes turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return stream_factory(request) + return stream( + request, + stream_factory, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +def stream( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + finalizer: Callable[[], Any], + on_stream_created: Callable[[Any], None] | None = None, + on_chunk: Callable[[Any], None] | None = None, + chunk_adapter: Callable[[Any], Any] | None = None, + accept_chunk: Callable[[Any], bool] | None = None, + completed_response_predicate: Callable[[Any], bool] | None = None, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> "ManagedLlmStream": + """Return a synchronous view of one Relay-managed provider stream.""" + return ManagedLlmStream( + request, + stream_factory, + session_id=session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + on_stream_created=on_stream_created, + on_chunk=on_chunk, + chunk_adapter=chunk_adapter, + accept_chunk=accept_chunk, + completed_response_predicate=completed_response_predicate, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +class ManagedLlmStream(Iterator[Any]): + """Drive Relay's async stream from Hermes's provider worker thread.""" + + def __init__( + self, + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + finalizer: Callable[[], Any], + on_stream_created: Callable[[Any], None] | None, + on_chunk: Callable[[Any], None] | None, + chunk_adapter: Callable[[Any], Any] | None, + accept_chunk: Callable[[Any], bool] | None, + completed_response_predicate: Callable[[Any], bool] | None, + metadata: dict[str, Any] | None, + defer_logical_completion: bool, + ) -> None: + self.final_response: Any = None + self._loop: asyncio.AbstractEventLoop | None = None + self._stream: Any = None + self._raw_stream_resource: Any = None + self._closed = False + self._callback_error: BaseException | None = None + self._logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None = None + self._defer_logical_completion = defer_logical_completion + self._on_chunk = on_chunk + self._chunk_adapter = chunk_adapter or _namespace + self._accept_chunk = accept_chunk + self._relay_observes_chunks = False + self._provider_completed = False + self._raw_chunks: list[tuple[Any, Any]] = [] + self.output_modified = False + callback_context = contextvars.copy_context() + + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if ( + runtime is None + or session is None + or not runtime.managed_execution_enabled() + ): + raw_stream = stream_factory(request) + if completed_response_predicate is not None and completed_response_predicate( + raw_stream + ): + self.final_response = raw_stream + self._stream = iter(()) + else: + self._raw_stream_resource = raw_stream + if on_stream_created is not None: + on_stream_created(raw_stream) + self._stream = iter(raw_stream) + return + + self._logical = _logical_parent(runtime, session, parent, metadata) + if self._logical is not None: + parent = self._logical[1] + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + + async def provider_stream(next_request: Any): + raw_stream = None + try: + raw_stream = callback_context.run( + stream_factory, + _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + ) + if ( + completed_response_predicate is not None + and callback_context.run( + completed_response_predicate, + raw_stream, + ) + ): + self.final_response = raw_stream + self._provider_completed = True + return + if on_stream_created is not None: + callback_context.run(on_stream_created, raw_stream) + raw_iterator = callback_context.run(iter, raw_stream) + while True: + try: + chunk = callback_context.run(next, raw_iterator) + except StopIteration: + break + if self._accept_chunk is not None and not callback_context.run( + self._accept_chunk, + chunk, + ): + break + encoded_chunk = _jsonable(chunk) + self._raw_chunks.append((encoded_chunk, chunk)) + yield encoded_chunk + self._provider_completed = True + except BaseException as exc: + self._callback_error = exc + raise + finally: + close = getattr(raw_stream, "close", None) + if callable(close): + callback_context.run(close) + + def observe_chunk(chunk: Any) -> None: + if self._on_chunk is not None: + callback_context.run(self._on_chunk, _jsonable(chunk)) + + def relay_finalizer() -> Any: + try: + if self.final_response is not None: + return _jsonable(self.final_response) + return _jsonable(callback_context.run(finalizer)) + except BaseException as exc: + self._callback_error = exc + raise + + loop = asyncio.new_event_loop() + self._loop = loop + self._relay_observes_chunks = True + try: + self._stream = loop.run_until_complete( + runtime.run_in_session_async( + session, + runtime.relay.llm.stream_execute, + name, + relay_request, + provider_stream, + observe_chunk, + relay_finalizer, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + ) + except BaseException as exc: + if ( + isinstance(exc, Exception) + and self._provider_completed + and self._callback_error is None + ): + logger.warning( + "NeMo Relay stream post-processing failed after provider success; " + "preserving the provider result", + exc_info=True, + ) + self._preserve_pending_provider_chunks() + return + if not self._defer_logical_completion: + _complete_logical( + self._logical, + outcome="cancelled" if _is_cancellation(exc) else "failed", + ) + self._logical = None + loop.close() + self._loop = None + raise + + def __iter__(self) -> "ManagedLlmStream": + return self + + def __next__(self) -> Any: + if self._closed: + raise StopIteration + if self._loop is None: + try: + chunk = next(self._stream) + except StopIteration: + self.close() + raise + if self._accept_chunk is not None and not self._accept_chunk(chunk): + self.close() + raise StopIteration + return chunk + + async def next_chunk() -> Any: + return await anext(self._stream) + + try: + chunk = self._loop.run_until_complete(next_chunk()) + except StopAsyncIteration: + if self._raw_chunks: + self.output_modified = True + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome="success") + self._logical = None + self.close() + raise StopIteration from None + except BaseException as exc: + callback_error = self._callback_error + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + self._close(logical_outcome="failed") + raise callback_error + if ( + isinstance(exc, Exception) + and self._provider_completed + and callback_error is None + ): + logger.warning( + "NeMo Relay stream post-processing failed after provider success; " + "preserving the provider result", + exc_info=True, + ) + self._preserve_pending_provider_chunks() + return next(self) + self._close( + logical_outcome="cancelled" if _is_cancellation(exc) else "failed" + ) + raise + if not self._relay_observes_chunks and self._on_chunk is not None: + self._on_chunk(chunk) + for index, (encoded, raw) in enumerate(self._raw_chunks): + if _json_equal(chunk, encoded): + if index > 0: + self.output_modified = True + del self._raw_chunks[: index + 1] + return raw + self.output_modified = True + return self._chunk_adapter(chunk) + + def close(self) -> None: + """Close an explicitly abandoned stream and cancel its logical call.""" + self._close(logical_outcome="cancelled") + + def _preserve_pending_provider_chunks(self) -> None: + """Switch a failed Relay stream to its undelivered provider chunks.""" + pending = [raw for _encoded, raw in self._raw_chunks] + self._raw_chunks.clear() + loop = self._loop + relay_stream = self._stream + self._loop = None + self._stream = iter(pending) + self._raw_stream_resource = None + self._accept_chunk = None + if loop is not None: + close = getattr(relay_stream, "aclose", None) + if callable(close): + + async def close_stream() -> None: + await close() + + try: + loop.run_until_complete(close_stream()) + except Exception: + logger.debug( + "Relay stream cleanup failed during provider fallback", + exc_info=True, + ) + loop.close() + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome="success") + self._logical = None + + def _close(self, *, logical_outcome: str) -> None: + if self._closed: + return + self._closed = True + loop = self._loop + self._loop = None + if loop is None: + resources = (self._stream, self._raw_stream_resource) + self._stream = None + self._raw_stream_resource = None + closed_ids: set[int] = set() + for resource in resources: + if resource is None or id(resource) in closed_ids: + continue + closed_ids.add(id(resource)) + close = getattr(resource, "close", None) + if callable(close): + try: + close() + except Exception: + logger.debug( + "Provider stream cleanup failed", + exc_info=True, + ) + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome=logical_outcome) + self._logical = None + return + close = getattr(self._stream, "aclose", None) + if callable(close): + + async def close_stream() -> None: + await close() + + try: + loop.run_until_complete(close_stream()) + except Exception: + pass + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome=logical_outcome) + self._logical = None + loop.close() + + def __del__(self) -> None: + self.close() + + +class AnthropicStreamAccumulator: + """Rebuild an Anthropic Message from post-intercept SSE events.""" + + def __init__(self) -> None: + self._message: dict[str, Any] = {} + self._blocks: dict[int, dict[str, Any]] = {} + + def observe(self, event: Any) -> None: + payload = _jsonable(event) + if not isinstance(payload, dict): + return + event_type = payload.get("type") + if event_type == "message_start": + message = payload.get("message") + if isinstance(message, dict): + for key in ("id", "type", "role", "model", "usage"): + if key in message: + self._message[key] = message[key] + return + if event_type == "content_block_start": + index = payload.get("index") + block = payload.get("content_block") + if isinstance(index, int) and isinstance(block, dict): + self._blocks[index] = dict(block) + return + if event_type == "content_block_delta": + index = payload.get("index") + delta = payload.get("delta") + if not isinstance(index, int) or not isinstance(delta, dict): + return + block = self._blocks.setdefault(index, {}) + delta_type = delta.get("type") + if delta_type == "text_delta": + block["text"] = str(block.get("text") or "") + str( + delta.get("text") or "" + ) + elif delta_type == "thinking_delta": + block["thinking"] = str(block.get("thinking") or "") + str( + delta.get("thinking") or "" + ) + elif delta_type == "signature_delta": + block["signature"] = str(block.get("signature") or "") + str( + delta.get("signature") or "" + ) + elif delta_type == "input_json_delta": + partial = str(block.pop("_partial_json", "")) + str( + delta.get("partial_json") or "" + ) + block["_partial_json"] = partial + elif delta_type == "citations_delta" and "citation" in delta: + block.setdefault("citations", []).append(delta["citation"]) + return + if event_type == "message_delta": + delta = payload.get("delta") + if isinstance(delta, dict): + for key in ("stop_reason", "stop_sequence"): + if key in delta: + self._message[key] = delta[key] + if "usage" in payload: + usage = payload["usage"] + current_usage = self._message.get("usage") + if isinstance(current_usage, dict) and isinstance(usage, dict): + self._message["usage"] = {**current_usage, **usage} + else: + self._message["usage"] = usage + + def finalize(self) -> dict[str, Any]: + blocks = [] + for index in sorted(self._blocks): + block = dict(self._blocks[index]) + partial = block.pop("_partial_json", None) + if partial is not None: + try: + block["input"] = json.loads(partial) + except (TypeError, ValueError): + block["input"] = partial + blocks.append(block) + return {**self._message, "content": blocks} + + def response(self, base: Any = None) -> Any: + """Return the attribute-shaped response consumed by Hermes.""" + assembled = self.finalize() + base_payload = _jsonable(base) + if not isinstance(base_payload, dict): + base_payload = {} + content = assembled.pop("content", []) + merged = {**base_payload, **assembled} + if content or "content" not in merged: + merged["content"] = content + return _namespace(merged) + + +def _logical_parent( + runtime: relay_runtime.RelayRuntime, + session: Any, + parent: Any, + metadata: dict[str, Any] | None, +) -> tuple[relay_runtime.RelayTurnContext, Any, str] | None: + turn = relay_runtime.active_turn(session.session_id) + request_id = str((metadata or {}).get("api_request_id") or "") + if turn is None or not request_id or turn.lease.host is not runtime: + return None + with turn.finalize_lock: + if turn.closed: + return None + with turn.logical_llm_lock: + handle = turn.logical_llm_calls.get(request_id) + if handle is None: + handle = runtime.run_in_session( + session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=parent, + input={}, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id, + "hermes.call_role": str( + (metadata or {}).get("call_role") or "primary" + ), + }, + ) + turn.logical_llm_calls[request_id] = handle + return turn, handle, request_id + + +def _complete_logical( + logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, + *, + outcome: str, +) -> None: + if logical is None: + return + turn, handle, request_id = logical + lease = turn.lease + if not isinstance(lease.host, relay_runtime.RelayRuntime): + return + with turn.finalize_lock: + with turn.logical_llm_lock: + if turn.logical_llm_calls.get(request_id) is not handle: + return + if lease.session is None: + return + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + handle, + output={"outcome": outcome}, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + # The provider result is authoritative. Retain the handle so turn + # finalization can retry cleanup without changing that result. + logger.warning( + "Hermes Relay logical LLM finalization failed", + exc_info=True, + ) + return + with turn.logical_llm_lock: + if turn.logical_llm_calls.get(request_id) is handle: + turn.logical_llm_calls.pop(request_id, None) + + +def _recover_successful_callback( + raw_response: dict[str, Any], + *, + relay_error: BaseException, + callback_error: BaseException | None, + logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, + defer_logical_completion: bool, +) -> bool: + if ( + not isinstance(relay_error, Exception) + or callback_error is not None + or "value" not in raw_response + ): + return False + logger.warning( + "NeMo Relay LLM post-processing failed after provider success; " + "returning the provider response", + exc_info=True, + ) + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + return True + + +def _is_cancellation(error: BaseException) -> bool: + return isinstance( + error, + (asyncio.CancelledError, InterruptedError, KeyboardInterrupt), + ) + + +def complete_logical_call(api_request_id: str, *, outcome: str) -> None: + """Complete the active turn's logical LLM call after caller validation.""" + turn = relay_runtime.active_turn() + if turn is None or not api_request_id: + return + with turn.logical_llm_lock: + handle = turn.logical_llm_calls.get(api_request_id) + if handle is not None: + _complete_logical((turn, handle, api_request_id), outcome=outcome) + + +def _provider_request( + original: dict[str, Any], + request: Any, + *, + relay_request_body: dict[str, Any], + codec_baseline_body: dict[str, Any] | None, + metadata: dict[str, Any] | None, +) -> dict[str, Any]: + content = getattr(request, "content", request) + if not isinstance(content, dict): + content = relay_request_body + if codec_baseline_body is None or _json_equal(content, relay_request_body): + final = dict(original) + else: + baseline = codec_baseline_body + intercepted = _provider_request_body(content, metadata) + final = dict(original) + # Typed codecs may not represent provider-specific fields. Overlay only + # values that changed from the codec-facing baseline so unrelated + # intercepts cannot delete or normalize unknown provider arguments. + for key in baseline.keys() | intercepted.keys(): + if key not in intercepted: + final.pop(key, None) + elif key not in baseline or not _json_equal( + intercepted[key], + baseline[key], + ): + final[key] = intercepted[key] + _restore_provider_message_extensions( + original, + final, + baseline=baseline, + intercepted=intercepted, + ) + headers = getattr(request, "headers", None) + if isinstance(headers, dict): + headers = { + key: value + for key, value in headers.items() + if str(key).lower() not in _RELAY_INTERNAL_PROVIDER_HEADERS + } + if headers: + final["extra_headers"] = { + **dict(final.get("extra_headers") or {}), + **headers, + } + return final + + +def _relay_request_body( + request: dict[str, Any], metadata: dict[str, Any] | None +) -> dict[str, Any]: + body = _jsonable(request) + if not isinstance(body, dict): + return {} + # The Responses SDK accepts ``tools=None`` as "no tools", while Relay's + # typed Responses codec correctly expects either an array or an absent + # field. Normalize only the codec-facing copy; the original provider + # request is restored when no interceptor changes it. + if str((metadata or {}).get("api_mode") or "") == "codex_responses": + body = dict(body) + if body.get("tools") is None: + body.pop("tools", None) + elif isinstance(body.get("tools"), list): + body["tools"] = [ + { + "type": "function", + "function": { + key: value + for key, value in tool.items() + if key != "type" + }, + } + if isinstance(tool, dict) + and tool.get("type") == "function" + and "function" not in tool + else tool + for tool in body["tools"] + ] + elif str((metadata or {}).get("api_mode") or "") == "chat_completions": + tools = body.get("tools") + if isinstance(tools, list): + body = dict(body) + body["tools"] = [ + {"type": "function", **tool} + if isinstance(tool, dict) + and "function" in tool + and "type" not in tool + else tool + for tool in tools + ] + return body + + +def _restore_provider_message_extensions( + original: dict[str, Any], + final: dict[str, Any], + *, + baseline: dict[str, Any], + intercepted: dict[str, Any], +) -> None: + """Restore provider wire fields that Relay's typed codec cannot represent.""" + original_messages = original.get("messages") + final_messages = final.get("messages") + baseline_messages = baseline.get("messages") + intercepted_messages = intercepted.get("messages") + if not all( + isinstance(messages, list) + for messages in ( + original_messages, + final_messages, + baseline_messages, + intercepted_messages, + ) + ): + return + if not ( + len(original_messages) + == len(final_messages) + == len(baseline_messages) + == len(intercepted_messages) + ): + return + for original_message, final_message, baseline_message, intercepted_message in zip( + original_messages, + final_messages, + baseline_messages, + intercepted_messages, + strict=True, + ): + if not all( + isinstance(message, dict) + for message in ( + original_message, + final_message, + baseline_message, + intercepted_message, + ) + ): + continue + for key in _PROVIDER_MESSAGE_EXTENSION_KEYS: + if ( + key in original_message + and key not in baseline_message + and key not in intercepted_message + and key not in final_message + ): + final_message[key] = original_message[key] + + +def _codec_round_trip_request_body( + relay: Any, + relay_request: Any, + *, + relay_request_body: dict[str, Any], + metadata: dict[str, Any] | None, +) -> dict[str, Any] | None: + """Return the codec-only request shape used to identify real rewrites.""" + codec = _codec(relay, metadata) + if codec is None: + return _provider_request_body(relay_request_body, metadata) + try: + annotated = codec.decode(relay_request) + encoded = codec.encode(annotated, relay_request) + content = getattr(encoded, "content", encoded) + if isinstance(content, dict): + return _provider_request_body(content, metadata) + except Exception: + logger.warning( + "NeMo Relay request codec baseline failed; ignoring request rewrites", + exc_info=True, + ) + return None + logger.warning( + "NeMo Relay request codec returned an unsupported baseline; " + "ignoring request rewrites" + ) + return None + + +def _provider_request_body( + content: dict[str, Any], metadata: dict[str, Any] | None +) -> dict[str, Any]: + body = dict(content) + if str((metadata or {}).get("api_mode") or "") != "codex_responses": + return body + tools = body.get("tools") + if not isinstance(tools, list): + return body + body["tools"] = [ + { + "type": "function", + **dict(tool["function"]), + } + if isinstance(tool, dict) + and tool.get("type") == "function" + and isinstance(tool.get("function"), dict) + else tool + for tool in tools + ] + return body + + +def _codec(relay: Any, metadata: dict[str, Any] | None) -> Any: + api_mode = str((metadata or {}).get("api_mode") or "") + codecs = getattr(relay, "codecs", None) + if codecs is None: + return None + if api_mode == "chat_completions": + codec = getattr(codecs, "OpenAIChatCodec", None) + elif api_mode == "anthropic_messages": + codec = getattr(codecs, "AnthropicMessagesCodec", None) + elif api_mode == "codex_responses": + codec = getattr(codecs, "OpenAIResponsesCodec", None) + else: + codec = None + return codec() if callable(codec) else None + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + model_dump = getattr(type(value), "model_dump", None) + if callable(model_dump): + try: + return _jsonable(value.model_dump(mode="json")) + except Exception: + pass + try: + attributes = { + str(key): item + for key, item in vars(value).items() + if not str(key).startswith("_") + } + except (TypeError, AttributeError): + return str(value) + return _jsonable(attributes) if attributes else str(value) + + +def _namespace(value: Any) -> Any: + if isinstance(value, dict): + return SimpleNamespace(**{ + str(key): _namespace(item) for key, item in value.items() + }) + if isinstance(value, list): + return [_namespace(item) for item in value] + return value + + +def _json_equal(left: Any, right: Any) -> bool: + try: + return json.dumps( + _jsonable(left), sort_keys=True, separators=(",", ":") + ) == json.dumps(_jsonable(right), sort_keys=True, separators=(",", ":")) + except (TypeError, ValueError): + return False + + +def _run_awaitable(value: Any) -> Any: + if not inspect.isawaitable(value): + return value + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(value) + raise RuntimeError( + "Synchronous Relay LLM execution cannot run on an event-loop thread" + ) diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py new file mode 100644 index 000000000000..4e3654cf7780 --- /dev/null +++ b/agent/relay_runtime.py @@ -0,0 +1,991 @@ +"""Profile-scoped NeMo Relay runtimes owned by the Hermes agent core.""" + +from __future__ import annotations + +import atexit +import asyncio +import contextvars +import importlib +import inspect +import logging +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable + +from hermes_constants import get_hermes_home + +logger = logging.getLogger(__name__) + +SESSION_SCOPE = "hermes.session" +TURN_SCOPE = "hermes.turn" +LOGICAL_LLM_SCOPE = "hermes.logical_llm_call" +RUNTIME_SCHEMA_KEY = "hermes.relay.schema_version" +RUNTIME_SCHEMA_VERSION = "hermes.relay.runtime.v1" +RUNTIME_INSTANCE_KEY = "hermes.relay.runtime_instance" +_PROFILE_KEY_CACHE: dict[str, str] = {} + + +@dataclass +class RelaySession: + """One isolated Relay scope stack owned by a Hermes session.""" + + session_id: str + parent_session_id: str = "" + lock: threading.RLock = field(default_factory=threading.RLock, repr=False) + closing: bool = False + handle: Any = None + context: contextvars.Context | None = None + + +class RelayRuntime: + """Own Relay session scopes independently of any exporter or plugin.""" + + def __init__(self, relay: Any = None, *, profile_key: str | None = None) -> None: + self.relay = relay or _load_nemo_relay() + self.profile_key = profile_key or current_profile_key() + self.runtime_id = uuid.uuid4().hex + self._sessions_lock = threading.RLock() + self._sessions: dict[str, RelaySession] = {} + self._subagent_parents: dict[str, str] = {} + self._subagent_parent_handles: dict[str, Any] = {} + self._execution_consumers_lock = threading.RLock() + self._execution_consumers: set[str] = set() + self._shutdown_registered = True + atexit.register(self.shutdown) + + def retain_managed_execution(self, consumer: str) -> None: + """Keep managed LLM and tool execution active for one consumer.""" + if not consumer: + raise ValueError("Relay managed-execution consumer must not be empty") + with self._execution_consumers_lock: + self._execution_consumers.add(consumer) + + def release_managed_execution(self, consumer: str) -> None: + """Release a consumer's managed-execution requirement.""" + with self._execution_consumers_lock: + self._execution_consumers.discard(consumer) + + def managed_execution_enabled(self) -> bool: + """Return whether a Hermes-managed consumer needs the Relay pipeline.""" + with self._execution_consumers_lock: + return bool(self._execution_consumers) + + def ensure_session( + self, + event: dict[str, Any], + *, + data: Any = None, + metadata: dict[str, Any] | None = None, + ) -> RelaySession | None: + """Return the existing session scope or create it once.""" + session_id = _session_id(event) + if not session_id: + return None + with self._sessions_lock: + session = self._sessions.get(session_id) + if session is None: + parent_session_id = self._subagent_parents.get(session_id, "") + session = RelaySession( + session_id=session_id, + parent_session_id=parent_session_id, + ) + self._sessions[session_id] = session + with session.lock: + if session.closing: + return None + if session.handle is None: + parent_handle = None + scope_metadata = { + **(metadata or {}), + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + } + if session.parent_session_id: + with self._sessions_lock: + parent_handle = self._subagent_parent_handles.get(session_id) + if parent_handle is None: + parent = self.ensure_session({ + "session_id": session.parent_session_id + }) + if parent is not None: + parent_handle = parent.handle + scope_metadata["nemo_relay_scope_role"] = "subagent" + context = contextvars.Context() + try: + session.handle = context.run( + self.relay.scope.push, + SESSION_SCOPE, + self.relay.ScopeType.Agent, + handle=parent_handle, + data=data, + input={}, + metadata=scope_metadata, + ) + except Exception: + session.context = None + raise + session.context = context + return session + + def register_subagent( + self, + event: dict[str, Any], + *, + metadata: dict[str, Any] | None = None, + ) -> RelaySession | None: + """Open a child Agent scope under its spawning turn when available.""" + parent_session_id = str(event.get("parent_session_id") or "") + child_session_id = str(event.get("child_session_id") or "") + if ( + not parent_session_id + or not child_session_id + or parent_session_id == child_session_id + ): + return None + parent = self.ensure_session({"session_id": parent_session_id}) + parent_handle = None if parent is None else parent.handle + turn = active_turn(parent_session_id) + if ( + turn is not None + and not turn.closed + and turn.handle is not None + and turn.lease.host is self + and turn.lease.session is not None + and turn.lease.session.session_id == parent_session_id + ): + parent_handle = turn.handle + with self._sessions_lock: + self._subagent_parents[child_session_id] = parent_session_id + if parent_handle is not None: + self._subagent_parent_handles[child_session_id] = parent_handle + return self.ensure_session( + {"session_id": child_session_id}, + metadata=metadata, + ) + + def unregister_subagent(self, event: dict[str, Any]) -> None: + """Close a delegated session and forget its parent relationship.""" + child_session_id = str(event.get("child_session_id") or "") + if not child_session_id: + return + self.close_session({"session_id": child_session_id}) + with self._sessions_lock: + self._subagent_parents.pop(child_session_id, None) + self._subagent_parent_handles.pop(child_session_id, None) + + def get_session(self, session_id: str) -> RelaySession | None: + """Return an active Hermes Relay session without creating one.""" + with self._sessions_lock: + session = self._sessions.get(str(session_id or "")) + if session is None: + return None + with session.lock: + return None if session.closing else session + + def get_session_handle(self, session_id: str) -> Any: + """Return the Relay parent handle for a Hermes session, if active.""" + session = self.get_session(session_id) + return None if session is None else session.handle + + def run_in_session( + self, + session: RelaySession, + callback: Callable[..., Any], + *args: Any, + allow_closing: bool = False, + **kwargs: Any, + ) -> Any: + """Run a Relay operation against a session's isolated scope stack.""" + with session.lock: + if session.closing and not allow_closing: + raise RuntimeError("Hermes Relay session is closing") + if session.context is None or session.handle is None: + raise RuntimeError("Hermes Relay session context is unavailable") + relay_context = session.context.copy() + + context = contextvars.copy_context() + for variable, value in relay_context.items(): + context.run(variable.set, value) + + def invoke() -> Any: + self.relay.get_scope_stack() + return callback(*args, **kwargs) + + # A copy permits a helper called by an existing Relay callback to + # re-enter the same logical session without re-entering Context. + return context.run(invoke) + + async def run_in_session_async( + self, + session: RelaySession, + callback: Callable[..., Any], + *args: Any, + allow_closing: bool = False, + **kwargs: Any, + ) -> Any: + """Create and await an operation inside the session's saved context.""" + with session.lock: + if session.closing and not allow_closing: + raise RuntimeError("Hermes Relay session is closing") + if session.context is None or session.handle is None: + raise RuntimeError("Hermes Relay session context is unavailable") + relay_context = session.context.copy() + + context = contextvars.copy_context() + for variable, value in relay_context.items(): + context.run(variable.set, value) + + async def invoke() -> Any: + self.relay.get_scope_stack() + result = callback(*args, **kwargs) + if inspect.isawaitable(result): + return await result + return result + + task = context.run(asyncio.create_task, invoke()) + return await task + + def emit_mark( + self, + name: str, + event: dict[str, Any], + *, + data: Any = None, + metadata: Any = None, + ) -> bool: + """Emit a mark parented to the Hermes session identified by ``event``.""" + session = self.ensure_session(event) + if session is None: + return False + self.run_in_session( + session, + self.relay.scope.event, + name, + handle=session.handle, + data=data, + metadata=metadata, + ) + return True + + def apply_tool_request_intercepts( + self, + *, + session_id: str, + tool_name: str, + args: dict[str, Any], + ) -> dict[str, Any]: + """Apply Relay request rewriting before Hermes authorizes a tool call.""" + if not self.managed_execution_enabled(): + return args + request_intercepts = getattr( + getattr(self.relay, "tools", None), + "request_intercepts", + None, + ) + if not callable(request_intercepts): + return args + session = self.ensure_session({"session_id": session_id}) + if session is None: + return args + result = self.run_in_session( + session, + request_intercepts, + tool_name, + args, + ) + return result if isinstance(result, dict) else args + + def close_session(self, event: dict[str, Any]) -> None: + """Close one session scope and remove it from the core registry.""" + session_id = _session_id(event) + with self._sessions_lock: + session = self._sessions.get(session_id) + if session is None: + with self._sessions_lock: + self._subagent_parents.pop(session_id, None) + self._subagent_parent_handles.pop(session_id, None) + return + failures: list[str] = [] + with session.lock: + if session.closing: + return + session.closing = True + if session.handle is not None: + try: + self.run_in_session( + session, + self.relay.scope.pop, + session.handle, + output={}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + }, + allow_closing=True, + ) + except Exception as exc: + failures.append(f"session scope close failed: {exc}") + try: + self.relay.subscribers.flush() + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + with self._sessions_lock: + if self._sessions.get(session_id) is session: + self._sessions.pop(session_id, None) + self._subagent_parents.pop(session_id, None) + self._subagent_parent_handles.pop(session_id, None) + if failures: + logger.warning( + "Hermes Relay session %s closed with errors: %s", + session_id, + "; ".join(failures), + ) + + def shutdown(self) -> None: + """Close all core-owned Relay session scopes.""" + with self._sessions_lock: + session_ids = list(self._sessions) + for session_id in session_ids: + self._safe(self.close_session, {"session_id": session_id}) + if self._shutdown_registered: + try: + atexit.unregister(self.shutdown) + except Exception: + pass + self._shutdown_registered = False + + @staticmethod + def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return callback(*args, **kwargs) + except Exception: + logger.warning("Hermes Relay runtime operation failed", exc_info=True) + return None + + +@dataclass(frozen=True) +class NoopRelayRuntime: + """Explicit reduced-capability host for platforms without Relay wheels.""" + + profile_key: str + reason: str + + @property + def available(self) -> bool: + return False + + def apply_tool_request_intercepts( + self, + *, + session_id: str, + tool_name: str, + args: dict[str, Any], + ) -> dict[str, Any]: + del session_id, tool_name + return args + + @staticmethod + def retain_managed_execution(consumer: str) -> None: + del consumer + + @staticmethod + def release_managed_execution(consumer: str) -> None: + del consumer + + @staticmethod + def managed_execution_enabled() -> bool: + return False + + def shutdown(self) -> None: + """No resources are allocated on unsupported platforms.""" + + +RelayHost = RelayRuntime | NoopRelayRuntime + + +class RelayHostRegistry: + """Own exactly one Relay host for each canonical Hermes profile.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._hosts: dict[str, RelayHost] = {} + + def for_profile( + self, + profile_key: str | None = None, + *, + create: bool = True, + ) -> RelayHost | None: + key = profile_key or current_profile_key() + host = self._hosts.get(key) + if host is not None or not create: + return host + with self._lock: + host = self._hosts.get(key) + if host is not None or not create: + return host + try: + host = RelayRuntime(profile_key=key) + except Exception as exc: + logger.warning( + "Hermes Relay runtime initialization failed", exc_info=True + ) + host = NoopRelayRuntime(profile_key=key, reason=str(exc)) + self._hosts[key] = host + return host + + def shutdown_profile(self, profile_key: str) -> None: + with self._lock: + host = self._hosts.pop(profile_key, None) + if host is not None: + host.shutdown() + + def shutdown_all(self) -> None: + with self._lock: + hosts = list(self._hosts.values()) + self._hosts.clear() + for host in hosts: + host.shutdown() + + +HOST_REGISTRY = RelayHostRegistry() + + +@dataclass +class ConversationLease: + """A resumable reference to one profile-scoped conversation scope.""" + + profile_key: str + session_id: str + platform: str + host: RelayHost + session: RelaySession | None + parent_session_id: str = "" + released: bool = False + + +@dataclass +class RelayTurnContext: + """Runtime-only context for one Hermes turn or top-level task.""" + + lease: ConversationLease + turn_id: str + task_id: str + handle: Any = None + logical_llm_calls: dict[str, Any] = field(default_factory=dict, repr=False) + logical_llm_lock: threading.RLock = field( + default_factory=threading.RLock, + repr=False, + ) + finalize_lock: threading.RLock = field( + default_factory=threading.RLock, + repr=False, + ) + _token: contextvars.Token[RelayTurnContext | None] | None = field( + default=None, + repr=False, + ) + _active_registered: bool = field(default=False, repr=False) + closed: bool = False + + +_CURRENT_TURN: contextvars.ContextVar[RelayTurnContext | None] = contextvars.ContextVar( + "hermes_relay_turn", default=None +) + + +class RelaySessionCoordinator: + """Own semantic conversation and turn lifetimes for Hermes core.""" + + def __init__(self, registry: RelayHostRegistry = HOST_REGISTRY) -> None: + self.registry = registry + self._initializer_lock = threading.RLock() + self._session_initializers: dict[ + str, + Callable[[RelayRuntime, dict[str, Any]], None], + ] = {} + self._active_turns_lock = threading.RLock() + self._active_turns: dict[tuple[str, str], set[int]] = {} + + def register_session_initializer( + self, + name: str, + callback: Callable[[RelayRuntime, dict[str, Any]], None], + ) -> None: + """Register idempotent profile/session preparation before scope creation.""" + with self._initializer_lock: + self._session_initializers[name] = callback + + def unregister_session_initializer(self, name: str) -> None: + """Remove a previously registered session initializer.""" + with self._initializer_lock: + self._session_initializers.pop(name, None) + + def _prepare_session( + self, + host: RelayRuntime, + context: dict[str, Any], + ) -> None: + with self._initializer_lock: + initializers = list(self._session_initializers.items()) + for name, callback in initializers: + try: + callback(host, context) + except Exception: + logger.warning( + "Hermes Relay session initializer failed: %s", + name, + exc_info=True, + ) + + def acquire_conversation( + self, + *, + profile_key: str, + session_id: str, + platform: str, + parent_session_id: str = "", + model: str = "", + ) -> ConversationLease: + host = self.registry.for_profile(profile_key) + if host is None: + host = NoopRelayRuntime(profile_key, "Relay host creation was disabled") + session = None + if isinstance(host, RelayRuntime): + try: + session_context = { + "profile_key": profile_key, + "session_id": session_id, + "platform": platform, + "parent_session_id": parent_session_id, + "model": model, + } + self._prepare_session(host, session_context) + metadata = {"hermes.execution_surface": platform or "unknown"} + if parent_session_id and parent_session_id != session_id: + session = host.register_subagent( + { + "parent_session_id": parent_session_id, + "child_session_id": session_id, + }, + metadata=metadata, + ) + else: + session = host.ensure_session( + {"session_id": session_id}, + metadata=metadata, + ) + except Exception: + logger.warning( + "Hermes Relay conversation initialization failed", + exc_info=True, + ) + return ConversationLease( + profile_key=profile_key, + session_id=session_id, + platform=platform, + host=host, + session=session, + parent_session_id=parent_session_id, + ) + + def begin_turn( + self, + lease: ConversationLease, + *, + turn_id: str, + task_id: str, + ) -> RelayTurnContext: + if lease.released: + raise RuntimeError("Hermes Relay conversation lease is released") + turn = RelayTurnContext(lease=lease, turn_id=turn_id, task_id=task_id) + if isinstance(lease.host, RelayRuntime) and lease.session is not None: + try: + turn.handle = lease.host.run_in_session( + lease.session, + lease.host.relay.scope.push, + TURN_SCOPE, + lease.host.relay.ScopeType.Function, + handle=lease.session.handle, + input={}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + "hermes.execution_surface": lease.platform or "unknown", + }, + ) + except Exception: + logger.warning("Hermes Relay turn initialization failed", exc_info=True) + turn._token = _CURRENT_TURN.set(turn) + key = (lease.profile_key, lease.session_id) + with self._active_turns_lock: + self._active_turns.setdefault(key, set()).add(id(turn)) + turn._active_registered = True + return turn + + def end_turn( + self, + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + with turn.finalize_lock: + if turn.closed: + self._reset_turn_context(turn) + return + turn.closed = True + lease = turn.lease + try: + if isinstance(lease.host, RelayRuntime) and lease.session is not None: + self._finish_logical_calls(turn, outcome=outcome) + if turn.handle is not None: + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + turn.handle, + output={"outcome": outcome}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + logger.warning( + "Hermes Relay turn finalization failed", exc_info=True + ) + finally: + try: + # Delegated agents own one turn. Close their conversation + # while the active-turn guard is still held so a parent + # timeout fallback cannot race this terminal boundary. + if ( + lease.parent_session_id + and isinstance(lease.host, RelayRuntime) + ): + lease.host.unregister_subagent({ + "child_session_id": lease.session_id + }) + except Exception: + logger.warning( + "Hermes Relay child conversation finalization failed", + exc_info=True, + ) + finally: + self._unregister_active_turn(turn) + self._reset_turn_context(turn) + + def has_active_turn(self, *, profile_key: str, session_id: str) -> bool: + """Return whether a turn is still running for one profile/session.""" + key = (profile_key, session_id) + with self._active_turns_lock: + return bool(self._active_turns.get(key)) + + def _unregister_active_turn(self, turn: RelayTurnContext) -> None: + if not turn._active_registered: + return + key = (turn.lease.profile_key, turn.lease.session_id) + with self._active_turns_lock: + active = self._active_turns.get(key) + if active is not None: + active.discard(id(turn)) + if not active: + self._active_turns.pop(key, None) + turn._active_registered = False + + def _reset_active_turns_for_tests(self) -> None: + with self._active_turns_lock: + self._active_turns.clear() + + def finish_logical_calls( + self, + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + """Close logical LLM children before sibling task aggregation scopes.""" + with turn.finalize_lock: + if turn.closed: + return + self._finish_logical_calls(turn, outcome=outcome) + + @staticmethod + def _finish_logical_calls( + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + lease = turn.lease + if not isinstance(lease.host, RelayRuntime) or lease.session is None: + return + with turn.logical_llm_lock: + logical_calls = list(turn.logical_llm_calls.items()) + turn.logical_llm_calls.clear() + for request_id, logical_handle in logical_calls: + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + logical_handle, + output={"outcome": outcome}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + with turn.logical_llm_lock: + turn.logical_llm_calls.setdefault(request_id, logical_handle) + logger.warning( + "Hermes Relay logical LLM finalization failed", + exc_info=True, + ) + + @staticmethod + def _reset_turn_context(turn: RelayTurnContext) -> None: + """Reset the originating ContextVar token when called in that context.""" + if turn._token is None: + return + try: + _CURRENT_TURN.reset(turn._token) + except ValueError: + # A copied async/thread context may own terminal cleanup. Keep the + # token so the originating context can clear its stale reference. + return + turn._token = None + + @staticmethod + def release_conversation(lease: ConversationLease) -> None: + """Release a caller lease without closing a resumable conversation.""" + lease.released = True + + def finalize_conversation( + self, + *, + profile_key: str, + session_id: str, + ) -> None: + host = self.registry.for_profile(profile_key, create=False) + if isinstance(host, RelayRuntime): + host.close_session({"session_id": session_id}) + + def shutdown_profile(self, profile_key: str) -> None: + self.registry.shutdown_profile(profile_key) + + +SESSION_COORDINATOR = RelaySessionCoordinator() + + +def current_turn() -> RelayTurnContext | None: + """Return the turn context inherited by current async and thread work.""" + return _CURRENT_TURN.get() + + +def active_turn(session_id: str | None = None) -> RelayTurnContext | None: + """Return a live turn only when it belongs to the active profile/session.""" + turn = current_turn() + if turn is None or turn.closed or turn.lease.released: + return None + if turn.lease.profile_key != current_profile_key(): + return None + if session_id is not None and turn.lease.session_id != session_id: + return None + if isinstance(turn.lease.host, RelayRuntime): + if turn.lease.session is None: + return None + if turn.lease.host.get_session(turn.lease.session_id) is not turn.lease.session: + return None + return turn + + +def resolve_execution_context( + session_id: str, +) -> tuple[RelayRuntime | None, RelaySession | None, Any]: + """Resolve one active turn/session parent for managed Relay execution.""" + turn = active_turn(session_id) + if ( + turn is not None + and isinstance(turn.lease.host, RelayRuntime) + and turn.lease.session is not None + ): + session = turn.lease.session + return turn.lease.host, session, turn.handle or session.handle + # Managed-execution consumers create and retain the profile host before + # reaching an out-of-turn adapter. Do not initialize Relay for the default + # no-consumer path. + runtime = get_runtime(create=False) + if runtime is None: + return None, None, None + if not runtime.managed_execution_enabled(): + return None, None, None + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + return runtime, session, None if session is None else session.handle + + +def emit_mark( + name: str, + *, + session_id: str, + data: Any = None, + metadata: Any = None, +) -> bool: + """Emit a fail-open Relay mark under a Hermes session.""" + runtime = get_runtime(create=False) + if runtime is None: + return False + try: + return runtime.emit_mark( + name, + {"session_id": session_id}, + data=data, + metadata=metadata, + ) + except Exception: + logger.warning("Hermes Relay mark failed: %s", name, exc_info=True) + return False + + +def apply_tool_request_intercepts( + *, + session_id: str, + tool_name: str, + args: dict[str, Any], +) -> dict[str, Any]: + """Return Relay-rewritten arguments at Hermes's authorization boundary.""" + if not session_id: + return args + runtime = get_runtime(create=False) + if runtime is None: + return args + return runtime.apply_tool_request_intercepts( + session_id=session_id, + tool_name=tool_name, + args=args, + ) + + +def ensure_session(*, session_id: str, **context: Any) -> RelaySession | None: + """Create or return the shared Relay session used by Hermes core.""" + runtime = get_runtime() + if runtime is None: + return None + try: + return runtime.ensure_session({"session_id": session_id, **context}) + except Exception: + logger.warning("Hermes Relay session initialization failed", exc_info=True) + return None + + +def run_in_session( + session_id: str, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> Any: + """Run a scope, LLM, or tool API against a shared Hermes session.""" + runtime = get_runtime() + if runtime is None: + raise RuntimeError("Hermes Relay runtime is unavailable") + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay session is unavailable") + return runtime.run_in_session(session, callback, *args, **kwargs) + + +async def run_in_session_async( + session_id: str, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> Any: + """Await a Relay operation inside a shared Hermes session context.""" + runtime = get_runtime() + if runtime is None: + raise RuntimeError("Hermes Relay runtime is unavailable") + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay session is unavailable") + return await runtime.run_in_session_async(session, callback, *args, **kwargs) + + +def get_session_handle(session_id: str) -> Any: + """Return the shared Relay handle for direct core instrumentation.""" + runtime = get_runtime(create=False) + return None if runtime is None else runtime.get_session_handle(session_id) + + +def _is_relay_wrapped_callback_error( + relay_error: BaseException, + callback_error: BaseException, +) -> bool: + """Match Relay's native callback wrapper without masking policy errors.""" + if relay_error is callback_error: + return True + if not isinstance(relay_error, RuntimeError): + return False + callback_type = callback_error.__class__ + type_names = { + callback_type.__name__, + callback_type.__qualname__, + f"{callback_type.__module__}.{callback_type.__qualname__}", + } + message = str(relay_error) + return any( + message.startswith(f"internal error: {type_name}: {callback_error}") + for type_name in type_names + ) + + +def get_runtime( + *, + create: bool = True, + profile_key: str | None = None, +) -> RelayRuntime | None: + """Return the Relay host for the active Hermes profile.""" + host = HOST_REGISTRY.for_profile(profile_key, create=create) + return host if isinstance(host, RelayRuntime) else None + + +def get_host( + *, + create: bool = True, + profile_key: str | None = None, +) -> RelayHost | None: + """Return the explicit real or reduced-capability host for a profile.""" + return HOST_REGISTRY.for_profile(profile_key, create=create) + + +def current_profile_key() -> str: + """Return the canonical profile identity used for runtime isolation.""" + home = get_hermes_home().expanduser() + if not home.is_absolute(): + return str(home.resolve()) + raw = str(home) + cached = _PROFILE_KEY_CACHE.get(raw) + if cached is not None: + return cached + resolved = str(home.resolve()) + return _PROFILE_KEY_CACHE.setdefault(raw, resolved) + + +def _load_nemo_relay() -> Any: + """Load the binding only when a producer or consumer needs Relay.""" + return importlib.import_module("nemo_relay") + + +def _session_id(event: dict[str, Any]) -> str: + return str(event.get("session_id") or "") + + +def _reset_for_tests() -> None: + """Reset all profile-scoped Relay hosts for isolated tests.""" + SESSION_COORDINATOR._reset_active_turns_for_tests() + HOST_REGISTRY.shutdown_all() + _PROFILE_KEY_CACHE.clear() diff --git a/agent/relay_tools.py b/agent/relay_tools.py new file mode 100644 index 000000000000..5023df1bcf92 --- /dev/null +++ b/agent/relay_tools.py @@ -0,0 +1,123 @@ +"""Core NeMo Relay adapter for Hermes tool execution.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import logging +from collections.abc import Callable +from typing import Any + +from agent import relay_runtime + +logger = logging.getLogger(__name__) + + +def execute( + tool_name: str, + args: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + metadata: dict[str, Any] | None = None, +) -> tuple[Any, dict[str, Any]]: + """Run one tool call through Relay and return its final arguments.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return callback(args), args + + observed_args = args + raw_result: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + def invoke(next_args: Any) -> Any: + nonlocal callback_error, observed_args + observed_args = next_args if isinstance(next_args, dict) else args + try: + result = callback_context.copy().run(callback, observed_args) + except BaseException as exc: + callback_error = exc + raise + raw_result["value"] = result + raw_result["json"] = _jsonable(result) + return raw_result["json"] + + try: + managed = _run_awaitable( + runtime.run_in_session_async( + session, + runtime.relay.tools.execute, + tool_name, + _jsonable(args), + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + ) + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if ( + isinstance(exc, Exception) + and callback_error is None + and "value" in raw_result + ): + logger.warning( + "NeMo Relay tool post-processing failed after dispatch success; " + "returning the Hermes tool result", + exc_info=True, + ) + return raw_result["value"], observed_args + raise + + if "value" in raw_result and _json_equal(managed, raw_result["json"]): + return raw_result["value"], observed_args + if isinstance(managed, str): + return managed, observed_args + return json.dumps(_jsonable(managed), ensure_ascii=False), observed_args + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return _jsonable(model_dump(mode="json")) + except Exception: + pass + try: + return _jsonable(vars(value)) + except (TypeError, AttributeError): + return str(value) + + +def _json_equal(left: Any, right: Any) -> bool: + try: + return json.dumps( + _jsonable(left), sort_keys=True, separators=(",", ":") + ) == json.dumps(_jsonable(right), sort_keys=True, separators=(",", ":")) + except (TypeError, ValueError): + return left == right + + +def _run_awaitable(value: Any) -> Any: + if not inspect.isawaitable(value): + return value + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(value) + raise RuntimeError( + "Synchronous Hermes Relay tool execution cannot run on an active event-loop thread" + ) diff --git a/agent/tool_executor.py b/agent/tool_executor.py index d32fe99c0c55..1dfcf1ebf6ef 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -20,6 +20,7 @@ import os import random import threading import time +from dataclasses import dataclass from typing import Any, Optional from agent.display import ( @@ -292,31 +293,63 @@ def _tool_search_scoped_names(agent) -> frozenset: return names -def _apply_tool_request_middleware_for_agent( - agent, - *, - function_name: str, - function_args: dict, - effective_task_id: str, - tool_call_id: str, -) -> tuple[dict, list[dict[str, Any]]]: - try: - from hermes_cli.middleware import apply_tool_request_middleware +@dataclass +class _ManagedToolResult: + result: Any + args: dict[str, Any] + middleware_trace: list[dict[str, Any]] + blocked: bool - result = apply_tool_request_middleware( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=tool_call_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - ) - payload = result.payload if isinstance(result.payload, dict) else function_args - return payload, list(result.trace) - except Exception as exc: - logger.debug("tool_request middleware error: %s", exc) - return function_args, [] + +class _ConcurrentToolAuthorizationGate: + """Serialize policy prompts and exclude their queue from batch deadlines.""" + + def __init__(self) -> None: + self._serialization_lock = threading.Lock() + self._state_lock = threading.Lock() + self._pending = 0 + self._window_started: float | None = None + self._excluded_seconds = 0.0 + + def run(self, callback): + now = time.monotonic() + with self._state_lock: + if self._pending == 0: + self._window_started = now + self._pending += 1 + try: + with self._serialization_lock: + return callback() + finally: + now = time.monotonic() + with self._state_lock: + self._pending -= 1 + if self._pending == 0: + if self._window_started is not None: + self._excluded_seconds += max( + 0.0, now - self._window_started + ) + self._window_started = None + + def excluded_seconds(self) -> float: + """Return completed plus currently active authorization wait time.""" + now = time.monotonic() + with self._state_lock: + excluded = self._excluded_seconds + if self._window_started is not None: + excluded += max(0.0, now - self._window_started) + return excluded + + +def _managed_values( + outcome: _ManagedToolResult, +) -> tuple[Any, dict[str, Any], list[dict[str, Any]], bool]: + return ( + outcome.result, + outcome.args, + outcome.middleware_trace, + outcome.blocked, + ) def _run_agent_tool_execution_middleware( @@ -327,28 +360,271 @@ def _run_agent_tool_execution_middleware( effective_task_id: str, tool_call_id: str, execute, -) -> tuple[Any, dict]: - observed_args = function_args + scope_block: str | None = None, + display_index: int | None = None, + middleware_trace: list[dict[str, Any]] | None = None, + begin_execution=None, + authorization_gate: _ConcurrentToolAuthorizationGate | None = None, +) -> _ManagedToolResult: + """Run Relay rewrites before Hermes policy and dispatch exactly once.""" + from agent import relay_tools + from hermes_cli.middleware import ( + apply_tool_request_middleware, + run_tool_execution_middleware, + ) - def _execute(next_args: dict) -> Any: - nonlocal observed_args - observed_args = next_args if isinstance(next_args, dict) else function_args - return execute(observed_args) + trace = middleware_trace if middleware_trace is not None else [] + state = { + "args": function_args, + "middleware_trace": trace, + "blocked": False, + "dispatched": False, + } + dispatch_lock = threading.Lock() - from hermes_cli.middleware import run_tool_execution_middleware + def _authorized_dispatch(final_args: dict[str, Any]) -> Any: + with dispatch_lock: + if state["dispatched"]: + raise RuntimeError( + "Hermes tool execution callback invoked more than once" + ) + state["dispatched"] = True + state["blocked"] = False + state["args"] = final_args - result = run_tool_execution_middleware( + def _begin() -> None: + _begin_tool_execution( + agent, + function_name=function_name, + function_args=final_args, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + display_index=display_index, + ) + + def _advance_start_order(callback=None) -> None: + if begin_execution is None: + if callback is not None: + callback() + return + begin_execution(callback) + + block_message = scope_block + block_error_type = "tool_scope_block" + if block_message is None: + block_error_type = "plugin_block" + + def _resolve_pre_tool_block(): + try: + from hermes_cli.plugins import resolve_pre_tool_block + + return resolve_pre_tool_block( + function_name, + final_args, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + middleware_trace=list(state["middleware_trace"]), + ) + except Exception: + return None + + block_message = ( + _resolve_pre_tool_block() + if authorization_gate is None + else authorization_gate.run(_resolve_pre_tool_block) + ) + + guardrail_decision = None + if block_message is None: + guardrail_decision = agent._tool_guardrails.before_call( + function_name, final_args + ) + if guardrail_decision.allows_execution: + guardrail_decision = None + + if block_message is not None or guardrail_decision is not None: + _advance_start_order() + state["blocked"] = True + if block_message is not None: + result = json.dumps({"error": block_message}, ensure_ascii=False) + error_type = block_error_type + error_message = block_message + else: + result = agent._guardrail_block_result(guardrail_decision) + error_type = "guardrail_block" + error_message = ( + getattr(guardrail_decision, "message", None) + or "Tool blocked by guardrail policy" + ) + _emit_terminal_post_tool_call( + agent, + function_name=function_name, + function_args=final_args, + result=result, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + status="blocked", + error_type=error_type, + error_message=error_message, + middleware_trace=list(state["middleware_trace"]), + ) + return result + + if function_name == "memory": + agent._turns_since_memory = 0 + elif function_name == "skill_manage": + agent._iters_since_skill = 0 + + _advance_start_order(_begin) + return execute(final_args) + + def _hermes_pipeline(relay_args: dict[str, Any]) -> Any: + request_result = apply_tool_request_middleware( + function_name, + relay_args, + skip_relay=True, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") or "", + ) + request_args = ( + request_result.payload + if isinstance(request_result.payload, dict) + else relay_args + ) + trace.clear() + trace.extend(request_result.trace) + return run_tool_execution_middleware( + function_name, + request_args, + lambda next_args: _authorized_dispatch( + next_args if isinstance(next_args, dict) else request_args + ), + original_args=function_args, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") or "", + ) + + result, _relay_args = relay_tools.execute( function_name, function_args, - _execute, - original_args=function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=tool_call_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", + _hermes_pipeline, + session_id=str(getattr(agent, "session_id", "") or ""), + metadata={ + "task_id": effective_task_id or "", + "turn_id": getattr(agent, "_current_turn_id", "") or "", + "api_request_id": getattr(agent, "_current_api_request_id", "") or "", + "tool_call_id": tool_call_id or "", + }, ) - return result, observed_args + return _ManagedToolResult( + result=result, + args=state["args"], + middleware_trace=state["middleware_trace"], + blocked=bool(state["blocked"]), + ) + + +def _begin_tool_execution( + agent, + *, + function_name: str, + function_args: dict[str, Any], + effective_task_id: str, + tool_call_id: str, + display_index: int | None, +) -> None: + """Run user-visible and checkpoint preflight on final tool arguments.""" + if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": + display_args = ( + _redact_tool_args_for_display(function_name, function_args) or function_args + ) + args_str = json.dumps(display_args, ensure_ascii=False) + prefix = f"Tool {display_index}" if display_index is not None else "Tool" + if agent.verbose_logging: + print(f" 📞 {prefix}: {function_name}({list(display_args.keys())})") + print( + agent._wrap_verbose( + "Args: ", json.dumps(display_args, indent=2, ensure_ascii=False) + ) + ) + else: + args_preview = ( + args_str[: agent.log_prefix_chars] + "..." + if len(args_str) > agent.log_prefix_chars + else args_str + ) + print( + f" 📞 {prefix}: {function_name}({list(function_args.keys())}) - " + f"{args_preview}" + ) + + agent._current_tool = function_name + agent._touch_activity(f"executing tool: {function_name}") + try: + from tools.environments.base import set_activity_callback + + set_activity_callback(agent._touch_activity) + except Exception: + pass + + if agent.tool_progress_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + preview = _build_tool_preview(function_name, display_args) + agent.tool_progress_callback( + "tool.started", function_name, preview, display_args + ) + except Exception as callback_error: + logging.debug("Tool progress callback error: %s", callback_error) + + if agent.tool_start_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + agent.tool_start_callback( + tool_call_id, function_name, display_args + ) + except Exception as callback_error: + logging.debug("Tool start callback error: %s", callback_error) + + if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: + try: + _ensure_file_checkpoint( + agent, + function_name, + function_args, + effective_task_id, + ) + except Exception: + pass + + if function_name == "terminal" and agent._checkpoint_mgr.enabled: + try: + command = function_args.get("command", "") + if _is_destructive_command(command): + cwd = function_args.get("workdir") or os.getenv( + "TERMINAL_CWD", os.getcwd() + ) + agent._checkpoint_mgr.ensure_checkpoint( + cwd, f"before terminal: {command[:60]}" + ) + except Exception: + pass def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: @@ -386,7 +662,9 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe return # ── Parse args + pre-execution bookkeeping ─────────────────────── - parsed_calls = [] # list of (tool_call, function_name, function_args, middleware_trace, block_result, blocked_by_guardrail) + # (tool call, resolved name, parsed args, middleware trace, parse error, + # tool-search scope block) + parsed_calls = [] for tool_call in tool_calls: function_name = tool_call.function.name @@ -402,17 +680,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe function_args, [], malformed_args_result, - False, + None, ) ) continue - # Reset nudge counters only for a structurally valid invocation. - if function_name == "memory": - agent._turns_since_memory = 0 - elif function_name == "skill_manage": - agent._iters_since_skill = 0 - # ── Tool Search unwrap ──────────────────────────────────────── # When the model invokes the tool_call bridge, peel it open so # every downstream check (checkpointing, guardrails, plugin @@ -446,167 +718,58 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe function_name = _underlying function_args = _underlying_args else: - _ts_scope_block = json.dumps({ - "error": ( - f"'{_underlying}' is not available in this session. " - "Use tool_search to find tools you can call." - ), - }, ensure_ascii=False) + _ts_scope_block = ( + f"'{_underlying}' is not available in this session. " + "Use tool_search to find tools you can call." + ) except Exception: pass - function_args, middleware_trace = _apply_tool_request_middleware_for_agent( - agent, - function_name=function_name, - function_args=function_args, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", + parsed_calls.append( + (tool_call, function_name, function_args, [], None, _ts_scope_block) ) - # ── Block evaluation (BEFORE checkpoint preflight) ─────────── - # We must know whether the tool will execute before touching - # checkpoint state (dedup slot, real snapshots). - block_result = None - blocked_by_guardrail = False - if _ts_scope_block is not None: - # Out-of-scope tool_call: reject before hooks/guardrails/dispatch. - block_result = _ts_scope_block - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="tool_scope_block", - error_message=_ts_scope_block, - middleware_trace=list(middleware_trace), - ) - else: - try: - from hermes_cli.plugins import resolve_pre_tool_block - block_message = resolve_pre_tool_block( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=getattr(tool_call, "id", "") or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - middleware_trace=list(middleware_trace), - ) - except Exception: - block_message = None - - if block_message is not None: - block_result = json.dumps({"error": block_message}, ensure_ascii=False) - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="plugin_block", - error_message=block_message, - middleware_trace=list(middleware_trace), - ) - else: - guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - block_result = agent._guardrail_block_result(guardrail_decision) - blocked_by_guardrail = True - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="guardrail_block", - error_message=getattr(guardrail_decision, "message", None) or "Tool blocked by guardrail policy", - middleware_trace=list(middleware_trace), - ) - - # ── Checkpoint preflight (only for tools that will execute) ── - if block_result is None: - # Checkpoint for file-mutating tools - if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: - try: - _ensure_file_checkpoint( - agent, - function_name, - function_args, - effective_task_id, - ) - except Exception: - pass - - # Checkpoint before destructive terminal commands - if function_name == "terminal" and agent._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - agent._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass - - parsed_calls.append((tool_call, function_name, function_args, middleware_trace, block_result, blocked_by_guardrail)) - # ── Logging / callbacks ────────────────────────────────────────── tool_names_str = ", ".join(name for _, name, _, _, _, _ in parsed_calls) if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}") - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls, 1): - display_args = _redact_tool_args_for_display(name, args) or args - args_str = json.dumps(display_args, ensure_ascii=False) - if agent.verbose_logging: - print(f" 📞 Tool {i}: {name}({list(display_args.keys())})") - print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {name}({list(args.keys())}) - {args_preview}") - - for tc, name, args, middleware_trace, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if agent.tool_progress_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - preview = _build_tool_preview(name, display_args) - agent.tool_progress_callback("tool.started", name, preview, display_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - for tc, name, args, middleware_trace, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if agent.tool_start_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - agent.tool_start_callback(tc.id, name, display_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") # ── Concurrent execution ───────────────────────────────────────── # Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag, middleware_trace) results = [None] * num_tools - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + for i, (tc, name, args, middleware_trace, block_result, _scope_block) in enumerate(parsed_calls): if block_result is not None: results[i] = (name, args, block_result, 0.0, True, True, middleware_trace) + start_condition = threading.Condition() + next_start_order = 0 + authorization_gate = _ConcurrentToolAuthorizationGate() + + def _begin_in_order(order: int, callback=None) -> None: + nonlocal next_start_order + with start_condition: + start_condition.wait_for(lambda: order == next_start_order) + try: + if callback is not None: + callback() + finally: + next_start_order += 1 + start_condition.notify_all() + # Touch activity before launching workers so the gateway knows # we're executing tools (not stuck). agent._current_tool = tool_names_str agent._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}") - def _run_tool(index, tool_call, function_name, function_args, middleware_trace): + def _run_tool( + index, + tool_call, + function_name, + function_args, + middleware_trace, + scope_block, + start_order, + ): """Worker function executed in a thread.""" # Register this worker tid so the agent can fan out an interrupt # to it — see AIAgent.interrupt(). Must happen first thing, and @@ -636,18 +799,50 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe # ContextVars are propagated by propagate_context_to_thread() at the # submit site below (GHSA-qg5c-hvr5-hjgr, #13617). start = time.time() + blocked = False + start_advanced = False + + def _advance_start(callback=None) -> None: + nonlocal start_advanced + if start_advanced: + return + try: + _begin_in_order(start_order, callback) + finally: + start_advanced = True + try: try: - result = agent._invoke_tool( - function_name, - function_args, - effective_task_id, - tool_call.id, - messages=messages, - pre_tool_block_checked=True, - skip_tool_request_middleware=True, - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict[str, Any]) -> Any: + return agent._invoke_tool( + function_name, + next_args, + effective_task_id, + tool_call.id, + messages=messages, + pre_tool_block_checked=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + ) + + managed = _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=scope_block, + display_index=index + 1, + middleware_trace=middleware_trace, + begin_execution=_advance_start, + authorization_gate=authorization_gate, ) + result = managed.result + function_args = managed.args + middleware_trace = managed.middleware_trace + blocked = managed.blocked except KeyboardInterrupt: try: agent.interrupt("keyboard interrupt") @@ -664,7 +859,15 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe ) duration = time.time() - start logger.info("tool %s cancelled (%.2fs)", function_name, duration) - results[index] = (function_name, function_args, result, duration, True, False, middleware_trace) + results[index] = ( + function_name, + function_args, + result, + duration, + True, + False, + middleware_trace, + ) return except Exception as tool_error: result = f"Error executing tool '{function_name}': {tool_error}" @@ -675,8 +878,17 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200]) else: logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result)) - results[index] = (function_name, function_args, result, duration, is_error, False, middleware_trace) + results[index] = ( + function_name, + function_args, + result, + duration, + is_error, + blocked, + middleware_trace, + ) finally: + _advance_start() # Tear down worker-tid tracking. Clear any interrupt bit we may # have set so the next task scheduled onto this recycled tid # starts with a clean slate. This MUST be in a finally block @@ -699,9 +911,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe try: runnable_calls = [ - (i, tc, name, args) - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls) - if block_result is None + (i, tc, name, args, scope_block) + for i, (tc, name, args, _trace, parse_error, scope_block) in enumerate( + parsed_calls + ) + if parse_error is None ] futures = [] future_to_index = {} @@ -719,13 +933,22 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe executor = DaemonThreadPoolExecutor(max_workers=max_workers) abandon_executor = False try: - for submit_index, (i, tc, name, args) in enumerate(runnable_calls): + for submit_index, (i, tc, name, args, scope_block) in enumerate( + runnable_calls + ): # Propagate the agent turn's ContextVars (e.g. # _approval_session_key) AND thread-local approval/sudo # callbacks into the worker thread; clears callbacks on exit. try: f = executor.submit( - propagate_context_to_thread(_run_tool), i, tc, name, args, parsed_calls[i][3] + propagate_context_to_thread(_run_tool), + i, + tc, + name, + args, + parsed_calls[i][3], + scope_block, + submit_index, ) except RuntimeError as submit_error: if not _is_interpreter_shutdown_submit_error(submit_error): @@ -736,7 +959,13 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe "skipping %d unsubmitted tool(s)", len(skipped_calls), ) - for skipped_i, _tc, skipped_name, skipped_args in skipped_calls: + for ( + skipped_i, + _tc, + skipped_name, + skipped_args, + _scope_block, + ) in skipped_calls: if results[skipped_i] is None: middleware_trace = parsed_calls[skipped_i][3] result = ( @@ -766,7 +995,10 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe while True: wait_timeout = 5.0 if deadline is not None: - remaining = deadline - time.monotonic() + effective_deadline = ( + deadline + authorization_gate.excluded_seconds() + ) + remaining = effective_deadline - time.monotonic() if remaining <= 0: done, not_done = set(), { f for f in futures if not f.done() @@ -783,7 +1015,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe if not not_done: break - if deadline is not None and time.monotonic() >= deadline: + if ( + deadline is not None + and time.monotonic() + >= deadline + authorization_gate.excluded_seconds() + ): abandon_executor = True timed_out_indices = { future_to_index[f] @@ -863,7 +1099,9 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total") # ── Post-execution: display per-tool results ───────────────────── - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + for i, (tc, name, args, middleware_trace, _parse_error, _scope_block) in enumerate( + parsed_calls + ): r = results[i] blocked = False is_error = True @@ -923,6 +1161,8 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe tool_duration = 0.0 else: function_name, function_args, function_result, tool_duration, is_error, blocked, middleware_trace = r + name = function_name + args = function_args progress_function_name = function_name if blocked: effect_disposition = "none" @@ -1169,153 +1409,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe except Exception: pass - function_args, middleware_trace = _apply_tool_request_middleware_for_agent( - agent, - function_name=function_name, - function_args=function_args, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - ) - - # Check plugin hooks for a block directive before executing. - _block_msg: Optional[str] = None - _block_error_type = "plugin_block" - if _ts_scope_block is not None: - _block_msg = _ts_scope_block - _block_error_type = "tool_scope_block" - else: - try: - from hermes_cli.plugins import resolve_pre_tool_block - _block_msg = resolve_pre_tool_block( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=getattr(tool_call, "id", "") or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - middleware_trace=list(middleware_trace), - ) - except Exception: - pass - - _guardrail_block_decision: ToolGuardrailDecision | None = None - if _block_msg is None: - guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - _guardrail_block_decision = guardrail_decision - - _execution_blocked = _block_msg is not None or _guardrail_block_decision is not None - - if _execution_blocked: - # Tool blocked by plugin or guardrail policy — skip counters, - # callbacks, checkpointing, activity mutation, and real execution. - pass - # Reset nudge counters when the relevant tool is actually used - elif function_name == "memory": - agent._turns_since_memory = 0 - elif function_name == "skill_manage": - agent._iters_since_skill = 0 - - if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - args_str = json.dumps(display_args, ensure_ascii=False) - if agent.verbose_logging: - print(f" 📞 Tool {i}: {function_name}({list(display_args.keys())})") - print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())}) - {args_preview}") - - if not _execution_blocked: - agent._current_tool = function_name - agent._touch_activity(f"executing tool: {function_name}") - - # Set activity callback for long-running tool execution (terminal - # commands, etc.) so the gateway's inactivity monitor doesn't kill - # the agent while a command is running. - if not _execution_blocked: - try: - from tools.environments.base import set_activity_callback - set_activity_callback(agent._touch_activity) - except Exception: - pass - - if not _execution_blocked and agent.tool_progress_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - preview = _build_tool_preview(function_name, display_args) - agent.tool_progress_callback("tool.started", function_name, preview, display_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - if not _execution_blocked and agent.tool_start_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - agent.tool_start_callback(tool_call.id, function_name, display_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") - - # Checkpoint: snapshot working dir before file-mutating tools - if not _execution_blocked and function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: - try: - _ensure_file_checkpoint( - agent, - function_name, - function_args, - effective_task_id, - ) - except Exception: - pass # never block tool execution - - # Checkpoint before destructive terminal commands - if not _execution_blocked and function_name == "terminal" and agent._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - agent._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass # never block tool execution + middleware_trace: list[dict[str, Any]] = [] + _execution_blocked = False tool_start_time = time.time() - if _block_msg is not None: - # Tool blocked by plugin policy — return error without executing. - function_result = json.dumps({"error": _block_msg}, ensure_ascii=False) - tool_duration = 0.0 - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=function_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type=_block_error_type, - error_message=_block_msg, - middleware_trace=list(middleware_trace), - ) - elif _guardrail_block_decision is not None: - # Tool blocked by tool-loop guardrail — synthesize exactly one - # tool result for the original tool_call_id without executing. - function_result = agent._guardrail_block_result(_guardrail_block_decision) - tool_duration = 0.0 - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=function_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="guardrail_block", - error_message=getattr(_guardrail_block_decision, "message", None) or "Tool blocked by guardrail policy", - middleware_trace=list(middleware_trace), - ) - elif function_name == "todo": + if function_name == "todo": def _execute(next_args: dict) -> Any: from tools.todo_tool import todo_tool as _todo_tool return _todo_tool( @@ -1323,14 +1422,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe merge=next_args.get("merge", False), store=agent._todo_store, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") @@ -1352,14 +1453,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe db=session_db, current_session_id=agent.session_id, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('session_search', function_args, tool_duration, result=function_result)}") @@ -1389,14 +1492,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe ), ) return result - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('memory', function_args, tool_duration, result=function_result)}") @@ -1409,14 +1514,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe multi_select=next_args.get("multi_select", False), callback=agent.clarify_callback, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('clarify', function_args, tool_duration, result=function_result)}") @@ -1428,14 +1535,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe count=next_args.get("count"), callback=getattr(agent, "read_terminal_callback", None), ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}") @@ -1460,14 +1569,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe try: def _execute(next_args: dict) -> Any: return agent._dispatch_delegate_task(next_args) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _delegate_result = function_result finally: agent._delegate_spinner = None @@ -1491,14 +1602,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe try: def _execute(next_args: dict) -> Any: return agent.context_compressor.handle_tool_call(function_name, next_args, messages=messages) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _ce_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Context engine tool '{function_name}' failed: {tool_error}"}) @@ -1525,14 +1638,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe try: def _execute(next_args: dict) -> Any: return agent._memory_manager.handle_tool_call(function_name, next_args) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _mem_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Memory tool '{function_name}' failed: {tool_error}"}) @@ -1555,18 +1670,46 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe spinner.start() _spinner_result = None try: - function_result = _ra().handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict) -> Any: + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) + + ( + function_result, + function_args, + middleware_trace, + _execution_blocked, + ) = _managed_values( + _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + middleware_trace=middleware_trace, + ) ) _spinner_result = function_result except KeyboardInterrupt: @@ -1597,18 +1740,46 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe agent._vprint(f" {cute_msg}") else: try: - function_result = _ra().handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict) -> Any: + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) + + ( + function_result, + function_args, + middleware_trace, + _execution_blocked, + ) = _managed_values( + _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + middleware_trace=middleware_trace, + ) ) except KeyboardInterrupt: _emit_cancelled_terminal_post_tool_call( diff --git a/agent/turn_context.py b/agent/turn_context.py index ac9555d012e4..5a0068948879 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -436,7 +436,12 @@ def build_turn_context( # Generate unique task_id if not provided to isolate VMs between tasks. effective_task_id = task_id or str(uuid.uuid4()) agent._current_task_id = effective_task_id - turn_id = f"{agent.session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + turn_id = str(getattr(agent, "_relay_pending_turn_id", "") or "") + if not turn_id: + turn_id = ( + f"{agent.session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + ) + agent._relay_pending_turn_id = None agent._current_turn_id = turn_id agent._current_api_request_id = "" # Tripwire: warn (with both turn ids) when this turn starts before the @@ -1030,7 +1035,7 @@ def build_turn_context( # Plugin hook: pre_llm_call (context injected into user message, not system prompt). plugin_user_context = "" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _pre_results = _invoke_hook( "pre_llm_call", session_id=agent.session_id, @@ -1041,6 +1046,7 @@ def build_turn_context( is_first_turn=(not bool(conversation_history)), model=agent.model, platform=getattr(agent, "platform", None) or "", + parent_session_id=getattr(agent, "_parent_session_id", None) or "", sender_id=getattr(agent, "_user_id", None) or "", ) _ctx_parts: list[str] = [] diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 4e2d318b2e2c..0f636c1dd241 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -488,7 +488,7 @@ def finalize_turn( # First hook to return a string wins; None/empty return leaves text unchanged. if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _transform_results = _invoke_hook( "transform_llm_output", response_text=final_response, @@ -510,7 +510,7 @@ def finalize_turn( # to an external memory system). if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "post_llm_call", session_id=agent.session_id, @@ -669,14 +669,16 @@ def finalize_turn( # Fired at the very end of every run_conversation call. # Plugins can use this for cleanup, flushing buffers, etc. try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=agent.session_id, task_id=effective_task_id, turn_id=turn_id, completed=completed, + failed=failed, interrupted=interrupted, + turn_exit_reason=_turn_exit_reason, model=agent.model, platform=getattr(agent, "platform", None) or "", ) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 932e99929019..437b24c4db8c 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1465,6 +1465,22 @@ display: # hooks_auto_accept: false +# ============================================================================= +# Telemetry +# ============================================================================= +# Shared metrics are disabled by default. When enabled, Hermes writes only +# allowlisted aggregate counters and immutable JSON +# packages under $HERMES_HOME/telemetry/shared_metrics; it does not upload them. +# Packages include a random profile-scoped ID that stays stable until this +# directory is deleted. It is not derived from hardware, account, or host data. +# Successfully exported local history is retained for 30 days; pending deltas +# are retained until they can be exported. +# This profile-owned choice is not overridden by managed-scope configuration. +telemetry: + shared_metrics: + enabled: false + + # ============================================================================= # Update Behavior # ============================================================================= diff --git a/cli.py b/cli.py index d7042309a000..dfa438374664 100644 --- a/cli.py +++ b/cli.py @@ -1267,9 +1267,8 @@ def _notify_session_finalize( reason: str = "shutdown", ) -> None: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=session_id, platform=platform, reason=reason, @@ -1297,7 +1296,7 @@ def _emit_interrupted_session_end(cli, *, reason: str = "keyboard_interrupt") -> pass try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=session_id, @@ -7695,13 +7694,21 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): lifecycle point (shutdown, /new, /reset). """ try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - event_type, - session_id=self.agent.session_id if self.agent else None, - platform=getattr(self, "platform", None) or "cli", - reason="new_session" if event_type == "on_session_reset" else "session_boundary", - ) + from hermes_cli.lifecycle import finalize_session, invoke_hook + + context = { + "session_id": self.agent.session_id if self.agent else None, + "platform": getattr(self, "platform", None) or "cli", + "reason": ( + "new_session" + if event_type == "on_session_reset" + else "session_boundary" + ), + } + if event_type == "on_session_finalize": + finalize_session(**context) + else: + invoke_hook(event_type, **context) except Exception: pass @@ -16596,7 +16603,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # the exit occurred, meaning run_conversation's hook didn't fire. if self.agent and getattr(self, '_agent_running', False): try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=self.agent.session_id, diff --git a/contributors/emails/afournier@nvidia.com b/contributors/emails/afournier@nvidia.com new file mode 100644 index 000000000000..d4289ed44061 --- /dev/null +++ b/contributors/emails/afournier@nvidia.com @@ -0,0 +1 @@ +afourniernv diff --git a/docs/observability/README.md b/docs/observability/README.md index 800c7d73e958..14847f74969f 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -14,6 +14,10 @@ Behavior-changing request or execution wrappers are outside this observer contract. Observer hooks should report what happened; they should not replace provider requests, tool arguments, or execution callbacks. +Hermes also has a first-party NeMo Relay shared-metrics path. It uses these +lifecycle boundaries directly and does not require enabling an observability +plugin. See [Relay shared metrics](relay-shared-metrics.md). + ## Contract Plugins register observer callbacks from `register(ctx)`: diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md new file mode 100644 index 000000000000..8c87da044ff0 --- /dev/null +++ b/docs/observability/relay-shared-metrics.md @@ -0,0 +1,127 @@ +# NeMo Relay Shared Metrics + +Hermes includes NeMo Relay as a normal runtime dependency on platforms for +which Relay publishes a native wheel. The shared-metrics integration is built +into Hermes and does not require `hermes plugins enable +observability/nemo_relay`. Hermes remains importable without Relay on other +native targets. Those targets use an explicit reduced-capability no-op host: +Hermes execution remains available, while Relay scopes, middleware, plugins, +and subscribers are unavailable. The `hermes-agent[nemo-relay]` extra remains +as a no-op compatibility alias for existing installation commands. + +Hermes requires NeMo Relay 0.6.0 or later within the 0.6 release line. That +release establishes the lossless provider-codec contract used for Anthropic +Messages, OpenAI Chat Completions, and OpenAI Responses requests. + +## Runtime Dependency and Data Boundary + +Hermes installs the platform-specific `nemo-relay` native wheel from the +bounded `>=0.6.0,<0.7` dependency range. The published package is built from +the [NVIDIA NeMo Relay repository](https://github.com/NVIDIA/NeMo-Relay). +Unsupported platforms use the explicit no-op runtime described above rather +than downloading a different implementation. + +When Relay managed execution is active, the provider request and response pass +through that native module in the Hermes process so configured interceptors can +operate on the real call. This is separate from the shared-metrics data +contract. Shared-metrics mode installs no network exporter and its subscriber +accepts only the versioned, allowlisted projection described below. Enabling a +separately configured rich-observability or dynamic plugin can create a +different data path and requires its own policy review. + +Collection remains off unless Hermes policy enables it: + +```yaml +telemetry: + shared_metrics: + enabled: true +``` + +This choice is read from the profile's own `config.yaml`. A machine-managed +configuration overlay cannot enable or disable shared metrics on the profile's +behalf. + +The existing `observability/nemo_relay` plugin remains separate. Enable that +plugin only for its opt-in rich observability exporters, adaptive execution, +or dynamic Relay plugins. + +Hermes core owns one Relay host and one isolated Relay session scope per Hermes +session. Core lifecycle producers use +`hermes_cli.observability.relay_runtime` to obtain the shared session handle or +run Relay scope, LLM, tool, and mark APIs in that session context. New product +marks do not require Hermes plugin registration. Shared-metrics marks must +still contain only fields approved by the versioned allowlist; the hard +dependency does not change the collection or privacy policy. + +## Current Slices + +The current vertical slices record logical model calls and top-level task runs: + +```text +Hermes turn, API, and tool hooks + -> Relay session, task, and LLM lifecycle + -> Hermes shared-metrics subscriber + -> SQLite counters + -> immutable JSON delta package +``` + +Hermes sends an empty `LLMRequest` into the metrics-owned lifecycle. This does +not describe the separate managed-execution call through the native runtime +documented above. The terminal metrics event contains only bounded model +family, provider family, locality, call role, and outcome values. Prompts, +responses, exact model IDs, endpoints, errors, session IDs, task IDs, and +request IDs are not included in the metrics event or package. + +Each task run is a Relay `Function` scope named `hermes.task_run`, parented to +the owning Hermes session. The start counter contains only bounded execution +surface and entrypoint values. The terminal counter contains bounded outcome, +end reason, termination status, duration, logical model-call count, terminal +tool-call count, and provider-retry count buckets. Retries are additional +provider attempts for the same Hermes API request ID; they do not inflate the +logical model-call count. Tool calls are deduplicated by their Hermes tool-call +ID after a terminal tool result is observed. The outer `AIAgent` execution +boundary closes the task for normal returns, early returns, exceptions, and +cancellations. Active task ownership follows the task ID if Hermes rotates its +conversation session during context compression. + +Local state is written under: + +```text +$HERMES_HOME/telemetry/shared_metrics/metrics.sqlite3 +$HERMES_HOME/telemetry/shared_metrics/outbox/*.json +``` + +The database keeps transactional aggregate and package-outbox state. Package +files are immutable delta documents that conform to a closed JSON schema and +are written with atomic replacement. Fully packaged aggregate rows and +successfully exported package rows and files are retained locally for 30 days. +Pending package rows and counters with unexported deltas are never pruned. + +Each package contains an `install_id` generated as a random UUID. Despite the +schema field name, its current scope is one `HERMES_HOME`, so it is more +precisely a persistent pseudonymous profile identifier. It is not derived from +hardware, account, host, path, or credential data. It remains stable across +packages from that profile and can therefore link those local packages. +Deleting `$HERMES_HOME/telemetry/shared_metrics` resets the identifier together +with all aggregates and package files. + +This slice has no remote-delivery path. A future remote exporter must not reuse +the persistent local identifier by default. It requires a separate product and +privacy decision covering consent, identity scope, rotation or keyed +pseudonymization, reset behavior, retention, and deletion. + +## Smoke Test + +Run a real Hermes CLI turn against the deterministic local model server: + +```bash +./.venv/bin/python scripts/smoke_nemo_relay_shared_metrics.py +``` + +The script uses the installed `nemo-relay` dependency by default. Pass +`--relay-python ../nemo-relay/python` only when testing a locally built Relay +binding. + +The smoke verifies the model request reached the local server, model and task +counters were stored, one package was exported, and prompt, response, and +exact-model canaries are absent from the package. diff --git a/gateway/run.py b/gateway/run.py index 37dfd9d92cde..500ae8b66ce2 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -6868,9 +6868,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _e: logger.debug("Shutdown transcript flush failed: %s", _e) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=getattr(agent, "session_id", None), platform="gateway", reason="shutdown", @@ -9162,11 +9161,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew for key, entry in _expired_entries: try: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import finalize_session _parts = key.split(":") _platform = _parts[2] if len(_parts) > 2 else "" - _invoke_hook( - "on_session_finalize", + finalize_session( session_id=entry.session_id, platform=_platform, reason="session_expired", @@ -10983,7 +10981,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # (e.g. customer handover ingest) without triggering the pairing flow. if not is_internal: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _hook_results = _invoke_hook( "pre_gateway_dispatch", event=event, diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index cf1d44929cbc..a32e7c6a45bb 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -223,9 +223,8 @@ class GatewaySlashCommandsMixin: # Fire plugin on_session_finalize hook (session boundary) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=_old_sid, platform=source.platform.value if source.platform else "", reason="new_session", @@ -302,7 +301,7 @@ class GatewaySlashCommandsMixin: # Fire plugin on_session_reset hook (new session guaranteed to exist) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _new_sid = new_entry.session_id if new_entry else None _invoke_hook( "on_session_reset", diff --git a/hermes_cli/config.py b/hermes_cli/config.py index aab86489c4c6..53f5161262fe 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3418,6 +3418,14 @@ DEFAULT_CONFIG = { "profile_build": "ask", }, + # Privacy-safe aggregate metrics written only to this profile's local + # telemetry directory. Collection is opt-in and no remote sink exists. + "telemetry": { + "shared_metrics": { + "enabled": False, + }, + }, + # ``hermes update`` behaviour. "updates": { # Pre-update safety backup — ONE consolidated mechanism, three modes: diff --git a/hermes_cli/hooks.py b/hermes_cli/hooks.py index 58faf7b41df5..852bfa1c8428 100644 --- a/hermes_cli/hooks.py +++ b/hermes_cli/hooks.py @@ -149,7 +149,17 @@ _DEFAULT_PAYLOADS = { "changed_paths": ["src/app.tsx"], }, "on_session_start": {"session_id": "test-session"}, - "on_session_end": {"session_id": "test-session"}, + "on_session_end": { + "session_id": "test-session", + "task_id": "test-task", + "turn_id": "test-turn", + "completed": True, + "failed": False, + "interrupted": False, + "turn_exit_reason": "text_response(stop)", + "model": "gpt-4", + "platform": "cli", + }, "on_session_finalize": {"session_id": "test-session"}, "on_session_reset": {"session_id": "test-session"}, "pre_api_request": { diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index ca81157decb0..d32f85f81d3c 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -175,7 +175,7 @@ def _fire_kanban_lifecycle_hook(event: str, task_id: str, **fields: Any) -> None it through. """ try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook from hermes_cli.profiles import get_active_profile_name try: profile_name = get_active_profile_name() diff --git a/hermes_cli/lifecycle.py b/hermes_cli/lifecycle.py new file mode 100644 index 000000000000..350e3880aff9 --- /dev/null +++ b/hermes_cli/lifecycle.py @@ -0,0 +1,63 @@ +"""Hermes lifecycle dispatch for first-party observers and plugins.""" + +from __future__ import annotations + +import logging +from typing import Any, List + +logger = logging.getLogger(__name__) + + +def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]: + """Notify first-party observers, then invoke compatibility plugin hooks.""" + try: + from hermes_cli.observability import observe_lifecycle + + observe_lifecycle(hook_name, **kwargs) + except Exception: + logger.warning("Built-in observability hook failed", exc_info=True) + + from hermes_cli import plugins + + return plugins.invoke_hook(hook_name, **kwargs) + + +def has_hook(hook_name: str) -> bool: + """Return whether a first-party observer or plugin consumes a hook.""" + try: + from hermes_cli.observability import handles_hook + + if handles_hook(hook_name): + return True + except Exception: + logger.warning("Unable to inspect built-in observability hooks", exc_info=True) + + from hermes_cli import plugins + + return plugins.has_hook(hook_name) + + +def finalize_session(**kwargs: Any) -> List[Any]: + """Notify observers and hard-close one core-owned Relay conversation.""" + try: + from hermes_cli.observability import observe_lifecycle + + observe_lifecycle("on_session_finalize", **kwargs) + except Exception: + logger.warning("Built-in observability hook failed", exc_info=True) + + session_id = str(kwargs.get("session_id") or "") + if session_id: + try: + from agent import relay_runtime + + relay_runtime.SESSION_COORDINATOR.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=session_id, + ) + except Exception: + logger.warning("Core Relay session finalization failed", exc_info=True) + + from hermes_cli import plugins + + return plugins.invoke_hook("on_session_finalize", **kwargs) diff --git a/hermes_cli/middleware.py b/hermes_cli/middleware.py index 8795952a2b7d..a3b472a45636 100644 --- a/hermes_cli/middleware.py +++ b/hermes_cli/middleware.py @@ -127,18 +127,32 @@ def apply_tool_request_middleware( Middleware may return ``{"args": {...}}`` to replace the effective tool arguments before hooks, guardrails, approvals, and execution see them. """ - if not _has_middleware(TOOL_REQUEST_MIDDLEWARE): - return RequestMiddlewareResult( - payload=args, - original_payload=args, - changed=False, - trace=[], - ) - original_args = _safe_copy(args) current_args = _safe_copy(original_args) trace: List[Dict[str, Any]] = [] + session_id = str(context.get("session_id") or "") + skip_relay = bool(context.pop("skip_relay", False)) + if session_id and not skip_relay: + from agent import relay_runtime + + relay_args = relay_runtime.apply_tool_request_intercepts( + session_id=session_id, + tool_name=tool_name, + args=current_args, + ) + if relay_args != current_args: + current_args = _safe_copy(relay_args) + trace.append({"source": "nemo_relay"}) + + if not _has_middleware(TOOL_REQUEST_MIDDLEWARE): + return RequestMiddlewareResult( + payload=args if not trace else current_args, + original_payload=args, + changed=bool(trace), + trace=trace, + ) + for result in _invoke_middleware( TOOL_REQUEST_MIDDLEWARE, tool_name=tool_name, diff --git a/hermes_cli/observability/__init__.py b/hermes_cli/observability/__init__.py new file mode 100644 index 000000000000..1c4e70300c6f --- /dev/null +++ b/hermes_cli/observability/__init__.py @@ -0,0 +1,31 @@ +"""First-party Hermes observability integrations.""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: + """Dispatch a Hermes lifecycle event to built-in observability features.""" + from . import relay_shared_metrics + + _safe_observe(relay_shared_metrics.observe_lifecycle, hook_name, kwargs) + + +def handles_hook(hook_name: str) -> bool: + """Return whether any built-in observability feature handles a hook.""" + from . import relay_shared_metrics + + return relay_shared_metrics.handles_hook(hook_name) + + +def _safe_observe(callback: Any, hook_name: str, kwargs: dict[str, Any]) -> None: + try: + callback(hook_name, **kwargs) + except Exception: + logger.warning( + "Built-in observability hook failed: %s", hook_name, exc_info=True + ) diff --git a/hermes_cli/observability/relay_runtime.py b/hermes_cli/observability/relay_runtime.py new file mode 100644 index 000000000000..bd6de0b83796 --- /dev/null +++ b/hermes_cli/observability/relay_runtime.py @@ -0,0 +1,14 @@ +"""Compatibility alias for the core Hermes Relay runtime. + +New code should import :mod:`agent.relay_runtime`. This module remains an +alias, rather than a copy, so existing plugins and tests share the same +profile registry and test-reset state during the migration. +""" + +from __future__ import annotations + +import sys + +from agent import relay_runtime as _core_relay_runtime + +sys.modules[__name__] = _core_relay_runtime diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py new file mode 100644 index 000000000000..fb6f885b2f05 --- /dev/null +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -0,0 +1,817 @@ +"""Direct NeMo Relay integration for Hermes shared client metrics.""" + +from __future__ import annotations + +import atexit +import contextvars +import logging +import threading +from dataclasses import dataclass, field +from time import monotonic_ns +from typing import Any, Callable + +from agent import relay_runtime +from hermes_cli import __version__ + +from .shared_metrics import SharedMetricsStore +from .shared_metrics_contract import ( + MODEL_CALL_SCOPE, + SCHEMA_KEY, + SCHEMA_VERSION, + SUBSCRIBER_NAME, + TASK_SCOPE, + model_call_fields, + model_call_outcome, + task_start_fields, + task_terminal_fields, +) +from .shared_metrics_subscriber import SharedMetricsSubscriber + +logger = logging.getLogger(__name__) + +HANDLED_HOOKS = frozenset({ + "on_session_start", + "on_session_end", + "on_session_finalize", + "on_session_reset", + "pre_llm_call", + "pre_api_request", + "post_tool_call", + "post_api_request", + "api_request_error", + "subagent_stop", +}) + +_RUNTIME_FAILED = object() +_RUNTIMES: dict[str, _Runtime | object] = {} +_RUNTIME_LOCK = threading.RLock() + + +def _retry_ordinal(event: dict[str, Any]) -> int | None: + value = event.get("retry_count") + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return None + + +@dataclass +class _ModelCall: + handle: Any + task_id: str + fields: dict[str, str] + retry_ordinal: int | None = None + + +@dataclass +class _TaskRun: + handle: Any + context: contextvars.Context + started_ns: int + start_fields: dict[str, str] + model_call_ids: set[str] = field(default_factory=set) + tool_call_ids: set[str] = field(default_factory=set) + turn_ids: set[str] = field(default_factory=set) + unidentified_tool_calls: int = 0 + retry_count: int = 0 + + +@dataclass +class _MetricsSession: + session_id: str + relay_session: relay_runtime.RelaySession + lock: threading.RLock = field(default_factory=threading.RLock, repr=False) + closing: bool = False + model_calls: dict[str, _ModelCall] = field(default_factory=dict) + tasks: dict[str, _TaskRun] = field(default_factory=dict) + + +class _Runtime: + """Own shared-metrics state layered on the Hermes core Relay host.""" + + def __init__(self, host: relay_runtime.RelayRuntime | None = None) -> None: + resolved_host = host or relay_runtime.get_runtime() + if resolved_host is None: + raise RuntimeError("Hermes core Relay runtime is unavailable") + self.host: relay_runtime.RelayRuntime = resolved_host + self.relay = self.host.relay + self._sessions_lock = threading.RLock() + self._active = True + self._sessions: dict[str, _MetricsSession] = {} + self._task_creation_lock = threading.RLock() + self._task_sessions_lock = threading.RLock() + self._task_sessions: dict[tuple[str, str], _MetricsSession] = {} + self._turn_sessions: dict[tuple[str, str], _MetricsSession] = {} + self._subscriber_name = f"{SUBSCRIBER_NAME}.{self.host.runtime_id}" + self.subscriber = SharedMetricsSubscriber( + SharedMetricsStore(), + __version__, + runtime_id=self.host.runtime_id, + ) + self.relay.subscribers.register(self._subscriber_name, self.subscriber) + self.host.retain_managed_execution(self._subscriber_name) + self._registered = True + atexit.register(self.shutdown) + + def ensure_session(self, event: dict[str, Any]) -> _MetricsSession | None: + session_id = str(event.get("session_id") or "") + if not session_id: + return None + with self._sessions_lock: + if not self._active: + return None + relay_session = self.host.ensure_session(event) + if relay_session is None: + return None + session = self._sessions.get(session_id) + if session is None: + session = _MetricsSession( + session_id=session_id, + relay_session=relay_session, + ) + self._sessions[session_id] = session + with session.lock: + if session.closing: + return None + return session + + def _run_in_session( + self, + session: _MetricsSession, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> Any: + return self.host.run_in_session( + session.relay_session, + callback, + *args, + **kwargs, + ) + + def start_task(self, event: dict[str, Any]) -> _TaskRun | None: + """Open one Relay function scope for a Hermes task run.""" + task_key = self._task_key(event) + if task_key is None: + return None + _, task_id = task_key + with self._task_creation_lock: + owner = self._task_session(event) + if owner is not None: + with owner.lock: + if owner.closing: + return None + task = owner.tasks.get(task_id) + if task is not None: + self._remember_turn(owner, task, event) + return task + + session = self.ensure_session(event) + if session is None: + return None + with session.lock: + if session.closing or session.relay_session.context is None: + return None + task_context = session.relay_session.context.copy() + start_fields = task_start_fields(event) + active_turn = relay_runtime.active_turn(session.session_id) + parent_handle = session.relay_session.handle + if ( + active_turn is not None + and active_turn.lease.session_id == session.session_id + and active_turn.task_id == task_id + and active_turn.handle is not None + ): + parent_handle = active_turn.handle + + def push_task() -> Any: + self.relay.get_scope_stack() + return self.relay.scope.push( + TASK_SCOPE, + self.relay.ScopeType.Function, + handle=parent_handle, + input=start_fields, + metadata=self._event_metadata(), + ) + + handle = task_context.run(push_task) + task = _TaskRun( + handle=handle, + context=task_context, + started_ns=monotonic_ns(), + start_fields=start_fields, + ) + session.tasks[task_id] = task + with self._task_sessions_lock: + self._task_sessions[task_key] = session + self._remember_turn(session, task, event) + return task + + def _run_in_task( + self, + task: _TaskRun, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> Any: + def invoke() -> Any: + self.relay.get_scope_stack() + return callback(*args, **kwargs) + + return task.context.copy().run(invoke) + + def start_model_call(self, event: dict[str, Any]) -> None: + task_id = str(event.get("task_id") or "") + session = self._task_session(event, allow_task_id_fallback=True) + task = session.tasks.get(task_id) if session is not None else None + if task is None: + task = self.start_task(event) + session = self._task_session(event) if task is not None else None + if session is None: + session = self.ensure_session(event) + if session is None: + return + request_id = str(event.get("api_request_id") or "") + if not request_id: + return + fields = model_call_fields(event) + retry_ordinal = _retry_ordinal(event) + model_family = fields["model_family"] + with session.lock: + if session.closing: + return + if task is not None: + self._remember_turn(session, task, event) + existing = session.model_calls.get(request_id) + if existing is not None: + existing.fields = fields + if task is not None: + if retry_ordinal is None or existing.retry_ordinal is None: + task.retry_count += 1 + elif retry_ordinal > existing.retry_ordinal: + task.retry_count += retry_ordinal - existing.retry_ordinal + if retry_ordinal is not None: + existing.retry_ordinal = max( + existing.retry_ordinal or 0, + retry_ordinal, + ) + return + if task is not None: + task.model_call_ids.add(request_id) + if retry_ordinal is not None and retry_ordinal > 0: + # A real Hermes retry can advance api_request_id while + # carrying the retry ordinal. Count that physical attempt. + task.retry_count += 1 + handle = self._run_in_task( + task, + self.relay.llm.call, + MODEL_CALL_SCOPE, + self.relay.LLMRequest({}, {}), + handle=task.handle, + metadata=self._event_metadata(), + model_name=model_family, + ) + else: + handle = self._run_in_session( + session, + self.relay.llm.call, + MODEL_CALL_SCOPE, + self.relay.LLMRequest({}, {}), + handle=session.relay_session.handle, + metadata=self._event_metadata(), + model_name=model_family, + ) + session.model_calls[request_id] = _ModelCall( + handle=handle, + task_id=str(event.get("task_id") or ""), + fields=fields, + retry_ordinal=retry_ordinal, + ) + + def record_tool_call(self, event: dict[str, Any]) -> None: + """Count one unique tool invocation under its owning task.""" + task_id = str(event.get("task_id") or "") + session = self._task_session(event, allow_task_id_fallback=True) + task = session.tasks.get(task_id) if session is not None else None + if task is None: + task = self.start_task(event) + session = self._task_session(event) if task is not None else None + if session is None or task is None: + return + tool_call_id = str(event.get("tool_call_id") or "") + with session.lock: + if session.closing: + return + self._remember_turn(session, task, event) + if tool_call_id: + task.tool_call_ids.add(tool_call_id) + else: + task.unidentified_tool_calls += 1 + + def end_model_call(self, event: dict[str, Any], outcome: str | None = None) -> None: + session = self._task_session(event, allow_task_id_fallback=True) + if session is None: + session = self._session(event) + if session is None: + return + request_id = str(event.get("api_request_id") or "") + with session.lock: + if session.closing: + return + model_call = session.model_calls.get(request_id) + if model_call is None: + return + fields = model_call_fields(event) + model_call.fields = fields + self._finish_model_call( + session, + request_id, + outcome or model_call_outcome(event), + ) + + def end_pending_model_calls(self, event: dict[str, Any]) -> None: + session = self._task_session(event, allow_task_id_fallback=True) + if session is None: + session = self._session(event) + if session is None: + return + with session.lock: + if session.closing: + return + self._end_pending_model_calls(session, event) + + def finish_task(self, event: dict[str, Any]) -> None: + """Close one task scope exactly once with bounded terminal fields.""" + task_id = str(event.get("task_id") or "") + session = self._task_session( + event, + allow_task_id_fallback=True, + ) or self._session(event) + if session is None: + return + with session.lock: + if session.closing: + return + self._finish_task(session, task_id, event) + + def close_session(self, event: dict[str, Any]) -> None: + session = self._session(event) + if session is None: + return + failures: list[str] = [] + with session.lock: + if session.closing: + return + session.closing = True + for task_id in list(session.tasks): + self._finish_task( + session, + task_id, + { + **event, + "task_id": task_id, + "completed": False, + "failed": True, + "interrupted": False, + "turn_exit_reason": "system_aborted", + }, + ) + self._end_pending_model_calls(session, event) + try: + self.relay.subscribers.flush() + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + self._export() + with self._sessions_lock: + if self._sessions.get(session.session_id) is session: + self._sessions.pop(session.session_id, None) + if failures: + logger.warning( + "Hermes shared-metrics session %s closed with errors: %s", + session.session_id, + "; ".join(failures), + ) + + def shutdown(self) -> None: + with self._sessions_lock: + self._active = False + session_ids = list(self._sessions) + for session_id in session_ids: + self._safe(self.close_session, {"session_id": session_id}) + if not self._registered: + return + self._safe(self.relay.subscribers.flush) + self._export() + self._safe(self.relay.subscribers.deregister, self._subscriber_name) + self.host.release_managed_execution(self._subscriber_name) + self._registered = False + try: + atexit.unregister(self.shutdown) + except Exception: + pass + + def deactivate(self) -> None: + """Stop collection without exporting locally aggregated metrics.""" + with self._sessions_lock: + self._active = False + self.subscriber.deactivate() + if self._registered: + self._safe(self.relay.subscribers.deregister, self._subscriber_name) + self.host.release_managed_execution(self._subscriber_name) + self._registered = False + with self._sessions_lock: + sessions = list(self._sessions.values()) + for session in sessions: + with session.lock: + if session.closing: + continue + session.closing = True + for task_id in list(session.tasks): + self._finish_task( + session, + task_id, + { + "session_id": session.session_id, + "task_id": task_id, + "failed": True, + "turn_exit_reason": "system_aborted", + }, + ) + self._end_pending_model_calls(session, {}) + with self._sessions_lock: + self._sessions.clear() + with self._task_sessions_lock: + self._task_sessions.clear() + self._turn_sessions.clear() + try: + atexit.unregister(self.shutdown) + except Exception: + pass + + def _session(self, event: dict[str, Any]) -> _MetricsSession | None: + session_id = str(event.get("session_id") or "") + with self._sessions_lock: + return self._sessions.get(session_id) + + @staticmethod + def _task_key(event: dict[str, Any]) -> tuple[str, str] | None: + session_id = str(event.get("session_id") or "") + task_id = str(event.get("task_id") or "") + if not session_id or not task_id: + return None + return session_id, task_id + + def _task_session( + self, + event: dict[str, Any], + *, + allow_task_id_fallback: bool = False, + ) -> _MetricsSession | None: + task_key = self._task_key(event) + if task_key is None: + return None + turn_key = self._turn_key(event) + with self._task_sessions_lock: + if turn_key is not None: + owner = self._turn_sessions.get(turn_key) + if owner is not None: + return owner + owner = self._task_sessions.get(task_key) + if owner is not None or not allow_task_id_fallback: + return owner + task_id = task_key[1] + candidates: list[_MetricsSession] = [] + for (_, candidate_task_id), session in self._task_sessions.items(): + if candidate_task_id != task_id: + continue + if not any(candidate is session for candidate in candidates): + candidates.append(session) + return candidates[0] if len(candidates) == 1 else None + + @staticmethod + def _turn_key(event: dict[str, Any]) -> tuple[str, str] | None: + session_id = str(event.get("session_id") or "") + turn_id = str(event.get("turn_id") or "") + if not session_id or not turn_id: + return None + return session_id, turn_id + + def _remember_turn( + self, + session: _MetricsSession, + task: _TaskRun, + event: dict[str, Any], + ) -> None: + turn_id = str(event.get("turn_id") or "") + if not turn_id: + return + task.turn_ids.add(turn_id) + with self._task_sessions_lock: + self._turn_sessions[(session.session_id, turn_id)] = session + + def _finish_model_call( + self, + session: _MetricsSession, + request_id: str, + outcome: str, + ) -> None: + model_call = session.model_calls.pop(request_id, None) + if model_call is None: + return + try: + task = session.tasks.get(model_call.task_id) + if task is not None: + self._run_in_task( + task, + self.relay.llm.call_end, + model_call.handle, + {**model_call.fields, "outcome": outcome}, + metadata=self._event_metadata(), + ) + else: + self._run_in_session( + session, + self.relay.llm.call_end, + model_call.handle, + {**model_call.fields, "outcome": outcome}, + metadata=self._event_metadata(), + ) + except Exception: + logger.warning( + "Hermes shared-metrics model call close failed", exc_info=True + ) + + def _end_pending_model_calls( + self, + session: _MetricsSession, + event: dict[str, Any], + ) -> None: + task_id = str(event.get("task_id") or "") + request_ids = [ + request_id + for request_id, model_call in session.model_calls.items() + if not task_id or model_call.task_id == task_id + ] + outcome = "cancelled" if event.get("interrupted") else "failed" + for request_id in request_ids: + self._finish_model_call(session, request_id, outcome) + + def _finish_task( + self, + session: _MetricsSession, + task_id: str, + event: dict[str, Any], + ) -> None: + task = session.tasks.get(task_id) + if task is None: + return + self._end_pending_model_calls(session, {**event, "task_id": task_id}) + fields = task_terminal_fields( + {**task.start_fields, **event}, + duration_ms=max(0, (monotonic_ns() - task.started_ns) // 1_000_000), + model_call_count=len(task.model_call_ids), + tool_call_count=len(task.tool_call_ids) + task.unidentified_tool_calls, + retry_count=task.retry_count, + ) + try: + self._run_in_task( + task, + self.relay.scope.pop, + task.handle, + output=fields, + metadata=self._event_metadata(), + ) + except Exception: + logger.warning("Hermes shared-metrics task close failed", exc_info=True) + finally: + session.tasks.pop(task_id, None) + with self._task_sessions_lock: + task_key = (session.session_id, task_id) + if self._task_sessions.get(task_key) is session: + self._task_sessions.pop(task_key, None) + for turn_id in task.turn_ids: + turn_key = (session.session_id, turn_id) + if self._turn_sessions.get(turn_key) is session: + self._turn_sessions.pop(turn_key, None) + + def _export(self) -> None: + self._safe(self.subscriber.store.create_and_export_package) + + def _event_metadata(self) -> dict[str, str]: + return { + SCHEMA_KEY: SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: self.host.runtime_id, + } + + @staticmethod + def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return callback(*args, **kwargs) + except Exception: + logger.warning("Hermes shared metrics operation failed", exc_info=True) + return None + + +def enabled() -> bool: + """Return the shared-metrics policy for the active Hermes profile.""" + profile_key = relay_runtime.current_profile_key() + try: + from hermes_cli.config import read_raw_config + + # Collection consent is profile-owned. Managed config overlays may + # control runtime policy, but cannot opt a profile into or out of + # shared metrics. + config = read_raw_config() or {} + except Exception: + logger.debug("Unable to read Hermes shared-metrics policy", exc_info=True) + value = False + else: + telemetry = config.get("telemetry") if isinstance(config, dict) else None + shared_metrics = ( + telemetry.get("shared_metrics") if isinstance(telemetry, dict) else None + ) + value = ( + isinstance(shared_metrics, dict) + and shared_metrics.get("enabled") is True + ) + if value: + return True + with _RUNTIME_LOCK: + runtime = _RUNTIMES.pop(profile_key, None) + if isinstance(runtime, _Runtime): + runtime.deactivate() + return False + + +def handles_hook(hook_name: str) -> bool: + return hook_name in HANDLED_HOOKS and enabled() + + +def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: + """Project one Hermes lifecycle event into the core Relay integration.""" + if not handles_hook(hook_name): + return + runtime = _get_runtime() + if runtime is None: + return + try: + if hook_name == "on_session_start": + runtime.ensure_session(kwargs) + elif hook_name == "pre_llm_call": + runtime.start_task(kwargs) + elif hook_name == "pre_api_request": + runtime.start_model_call(kwargs) + elif hook_name == "post_tool_call": + runtime.record_tool_call(kwargs) + elif hook_name == "post_api_request": + runtime.end_model_call(kwargs, "success") + elif hook_name == "api_request_error": + if kwargs.get("retryable") is False: + runtime.end_model_call(kwargs, "failed") + elif hook_name == "on_session_end": + runtime.finish_task(kwargs) + elif hook_name == "subagent_stop": + child_session_id = str(kwargs.get("child_session_id") or "") + if child_session_id: + runtime.close_session({"session_id": child_session_id}) + elif hook_name in {"on_session_finalize", "on_session_reset"}: + runtime.close_session(kwargs) + except Exception: + logger.warning( + "Hermes shared metrics hook failed: %s", hook_name, exc_info=True + ) + + +def prepare_session_start() -> None: + """Register the subscriber before any producer opens the session scope.""" + if enabled(): + _get_runtime(retry_failed=True) + + +def _prepare_core_session( + host: relay_runtime.RelayRuntime, + context: dict[str, Any], +) -> None: + """Prepare the profile subscriber before the coordinator opens a scope.""" + del context + if host.profile_key == relay_runtime.current_profile_key(): + if enabled(): + _get_runtime(retry_failed=True, host=host) + + +def start_task_run( + *, + session_id: str, + task_id: str, + platform: str, + parent_session_id: str = "", +) -> None: + """Start task metrics at the outer Hermes execution boundary.""" + if not enabled(): + return + runtime = _get_runtime(retry_failed=True) + if runtime is None: + return + runtime._safe( + runtime.start_task, + { + "session_id": session_id, + "task_id": task_id, + "platform": platform, + "parent_session_id": parent_session_id, + }, + ) + + +def finish_task_run( + *, + session_id: str, + task_id: str, + platform: str, + result: dict[str, Any] | None = None, + error: BaseException | None = None, +) -> None: + """Finish task metrics for every return or exception path.""" + if not enabled(): + return + runtime = _get_runtime() + if runtime is None: + return + + terminal = result if isinstance(result, dict) else {} + interrupted = terminal.get("interrupted") is True + completed = terminal.get("completed") is True + failed = terminal.get("failed") is True + reason = str( + terminal.get("turn_exit_reason") or terminal.get("failure_reason") or "" + ) + if error is not None: + interrupted = isinstance(error, (KeyboardInterrupt, InterruptedError)) or ( + type(error).__name__ == "CancelledError" + ) + timed_out = isinstance(error, TimeoutError) + completed = False + failed = not interrupted + if interrupted: + reason = "interrupted_by_user" + elif timed_out: + reason = "timed_out" + else: + reason = "system_aborted" + elif not reason: + reason = "failed" if failed else "unknown" + + runtime._safe( + runtime.finish_task, + { + "session_id": session_id, + "task_id": task_id, + "platform": platform, + "completed": completed, + "failed": failed, + "interrupted": interrupted, + "turn_exit_reason": reason, + }, + ) + + +def _get_runtime( + *, + retry_failed: bool = False, + host: relay_runtime.RelayRuntime | None = None, +) -> _Runtime | None: + profile_key = relay_runtime.current_profile_key() + with _RUNTIME_LOCK: + runtime = _RUNTIMES.get(profile_key) + if isinstance(runtime, _Runtime): + if host is None or runtime.host is host: + return runtime + runtime.deactivate() + _RUNTIMES.pop(profile_key, None) + if runtime is _RUNTIME_FAILED and not retry_failed: + return None + if runtime is _RUNTIME_FAILED: + _RUNTIMES.pop(profile_key, None) + try: + runtime = _Runtime(host=host) + except Exception: + logger.warning("Hermes shared metrics initialization failed", exc_info=True) + _RUNTIMES[profile_key] = _RUNTIME_FAILED + return None + _RUNTIMES[profile_key] = runtime + return runtime + + +relay_runtime.SESSION_COORDINATOR.register_session_initializer( + SUBSCRIBER_NAME, + _prepare_core_session, +) + + +def _reset_for_tests() -> None: + """Reset all profile-scoped shared-metrics state for isolated tests.""" + with _RUNTIME_LOCK: + runtimes = list(_RUNTIMES.values()) + _RUNTIMES.clear() + for runtime in runtimes: + if isinstance(runtime, _Runtime): + runtime.shutdown() diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json new file mode 100644 index 000000000000..68496e5ab6f3 --- /dev/null +++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json @@ -0,0 +1,338 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:hermes-agent:schema:shared-metrics:v1", + "title": "Hermes Shared Metrics Package v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "package_id", + "install_id", + "period_start", + "period_end", + "generated_at", + "resource", + "metrics" + ], + "properties": { + "schema_version": { + "const": "hermes.shared_metrics.v1" + }, + "package_id": { + "$ref": "#/$defs/uuid" + }, + "install_id": { + "description": "Random persistent identifier scoped to one HERMES_HOME; local-only in schema v1.", + "$ref": "#/$defs/uuid" + }, + "period_start": { + "type": "string", + "format": "date-time" + }, + "period_end": { + "type": "string", + "format": "date-time" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "resource": { + "type": "object", + "additionalProperties": false, + "required": [ + "hermes_version" + ], + "properties": { + "hermes_version": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + } + }, + "metrics": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "$ref": "#/$defs/model_call_counter" + }, + { + "$ref": "#/$defs/task_started_counter" + }, + { + "$ref": "#/$defs/task_finished_counter" + } + ] + } + } + }, + "$defs": { + "uuid": { + "type": "string", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "model_call_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.model_call.count" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "call_role", + "locality", + "model_family", + "outcome", + "provider_family" + ], + "properties": { + "call_role": { + "const": "primary" + }, + "locality": { + "enum": [ + "local", + "remote", + "unknown" + ] + }, + "model_family": { + "enum": [ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "o1", + "o3", + "o4", + "qwen", + "step", + "trinity", + "unknown" + ] + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success" + ] + }, + "provider_family": { + "enum": [ + "aggregator", + "custom", + "direct", + "local", + "unknown" + ] + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "task_started_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.task_run.started" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "entrypoint", + "execution_surface" + ], + "properties": { + "entrypoint": { + "$ref": "#/$defs/task_entrypoint" + }, + "execution_surface": { + "$ref": "#/$defs/execution_surface" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "task_finished_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.task_run.finished" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "duration_bucket", + "end_reason", + "entrypoint", + "execution_surface", + "model_call_count_bucket", + "outcome", + "retry_count_bucket", + "termination", + "tool_call_count_bucket" + ], + "properties": { + "duration_bucket": { + "$ref": "#/$defs/duration_bucket" + }, + "end_reason": { + "enum": [ + "approval_denied", + "completed", + "failed", + "guardrail_blocked", + "iteration_limit", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled" + ] + }, + "entrypoint": { + "$ref": "#/$defs/task_entrypoint" + }, + "execution_surface": { + "$ref": "#/$defs/execution_surface" + }, + "model_call_count_bucket": { + "$ref": "#/$defs/count_bucket" + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success", + "timed_out", + "unknown" + ] + }, + "retry_count_bucket": { + "$ref": "#/$defs/count_bucket" + }, + "termination": { + "enum": [ + "none", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled" + ] + }, + "tool_call_count_bucket": { + "$ref": "#/$defs/count_bucket" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "execution_surface": { + "enum": [ + "api", + "batch", + "cli", + "desktop", + "gateway", + "other", + "python", + "scheduled_task", + "tui", + "unknown" + ] + }, + "task_entrypoint": { + "enum": [ + "api", + "background", + "batch", + "delegated", + "gateway_message", + "interactive", + "other", + "python", + "scheduled_task", + "unknown" + ] + }, + "duration_bucket": { + "enum": [ + "1s_to_5s", + "2m_to_10m", + "30s_to_2m", + "5s_to_30s", + "gte_10m", + "lt_1s" + ] + }, + "count_bucket": { + "enum": [ + "0", + "1", + "2", + "3_to_5", + "6_to_10", + "gte_11" + ] + } + } +} diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py new file mode 100644 index 000000000000..506fb4ab1314 --- /dev/null +++ b/hermes_cli/observability/shared_metrics.py @@ -0,0 +1,486 @@ +"""Durable aggregation and local export for Hermes shared metrics.""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from hermes_cli.sqlite_util import write_txn +from hermes_constants import get_hermes_home +from utils import atomic_json_write + +from .shared_metrics_contract import ( + COUNTER_METRICS, + MODEL_CALL_METRIC, + counter_dimensions_are_valid, +) + + +_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v1" +_STORE_SCHEMA_VERSION = "1" +_BUSY_TIMEOUT_MS = 250 +_SCHEMA_BUSY_TIMEOUT_MS = 5_000 +_LOCAL_HISTORY_RETENTION_DAYS = 30 + +logger = logging.getLogger(__name__) + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class SharedMetricsStore: + """Persist allowlisted counters and export immutable delta packages.""" + + def __init__( + self, + database_path: Path | None = None, + outbox_directory: Path | None = None, + ) -> None: + root = get_hermes_home() / "telemetry" / "shared_metrics" + self.database_path = database_path or root / "metrics.sqlite3" + self.outbox_directory = outbox_directory or root / "outbox" + self._ensure_private_directory(self.database_path.parent) + self._ensure_private_directory(self.outbox_directory) + self._ensure_private_file(self.database_path) + self._ensure_schema() + + def record_model_call( + self, + dimensions: dict[str, str], + hermes_version: str, + ) -> None: + """Increment the terminal model-call counter for the current UTC day.""" + self.record_counter(MODEL_CALL_METRIC, dimensions, hermes_version) + + def record_counter( + self, + metric_name: str, + dimensions: dict[str, str], + hermes_version: str, + ) -> None: + """Increment one allowlisted counter for the current UTC day.""" + if metric_name not in COUNTER_METRICS: + raise ValueError(f"Unsupported shared metric: {metric_name}") + if not counter_dimensions_are_valid(metric_name, dimensions): + raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}") + dimensions_json = json.dumps( + dimensions, + sort_keys=True, + separators=(",", ":"), + ) + period_start = _utc_now().date().isoformat() + with self._connection() as connection: + connection.execute( + """ + INSERT INTO counter_aggregates( + period_start, + metric_name, + hermes_version, + dimensions_json, + value, + packaged_value + ) VALUES (?, ?, ?, ?, 1, 0) + ON CONFLICT( + period_start, + metric_name, + hermes_version, + dimensions_json + ) + DO UPDATE SET value = value + 1 + """, + ( + period_start, + metric_name, + hermes_version or "unknown", + dimensions_json, + ), + ) + + def create_and_export_package(self) -> list[Path]: + """Commit one pending delta package, then atomically export the outbox.""" + pending_periods = self._pending_period_count() + for _ in range(pending_periods): + if self._create_package() is None: + break + exported = self._export_pending_packages() + try: + self._prune_expired_history() + except Exception: + logger.warning( + "Unable to prune expired shared-metrics history", + exc_info=True, + ) + return exported + + def counter_snapshot(self) -> list[dict[str, Any]]: + """Return cumulative counters for focused tests and local inspection.""" + with self._connection() as connection: + rows = connection.execute( + """ + SELECT + period_start, + metric_name, + hermes_version, + dimensions_json, + value, + packaged_value + FROM counter_aggregates + ORDER BY period_start, hermes_version, metric_name, dimensions_json + """ + ).fetchall() + return [ + { + "period_start": row["period_start"], + "metric_name": row["metric_name"], + "hermes_version": row["hermes_version"], + "dimensions": json.loads(row["dimensions_json"]), + "value": row["value"], + "packaged_value": row["packaged_value"], + } + for row in rows + ] + + @contextmanager + def _connection( + self, + *, + busy_timeout_ms: int = _BUSY_TIMEOUT_MS, + ) -> Iterator[sqlite3.Connection]: + connection = sqlite3.connect( + self.database_path, + timeout=busy_timeout_ms / 1000, + ) + try: + connection.row_factory = sqlite3.Row + connection.execute(f"PRAGMA busy_timeout={busy_timeout_ms}") + with connection: + yield connection + finally: + connection.close() + + @staticmethod + def _ensure_private_directory(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + path.chmod(0o700) + except OSError: + pass + + @staticmethod + def _ensure_private_file(path: Path) -> None: + path.touch(mode=0o600, exist_ok=True) + try: + path.chmod(0o600) + except OSError: + pass + + def _ensure_schema(self) -> None: + with self._connection(busy_timeout_ms=_SCHEMA_BUSY_TIMEOUT_MS) as connection: + # Serialize first-run creation and upgrades across Hermes processes. + with write_txn(connection): + self._ensure_schema_in_transaction(connection) + + @staticmethod + def _ensure_schema_in_transaction(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS telemetry_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + """ + ) + schema_row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'schema_version'" + ).fetchone() + if schema_row is not None and str(schema_row["value"]) != _STORE_SCHEMA_VERSION: + raise RuntimeError( + "Unsupported shared-metrics store schema version: " + f"{schema_row['value']}" + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS counter_aggregates ( + period_start TEXT NOT NULL, + metric_name TEXT NOT NULL, + hermes_version TEXT NOT NULL, + dimensions_json TEXT NOT NULL, + value INTEGER NOT NULL, + packaged_value INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY ( + period_start, + metric_name, + hermes_version, + dimensions_json + ) + ) + """ + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS package_outbox ( + package_id TEXT PRIMARY KEY, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL, + exported_at TEXT + ) + """ + ) + connection.execute( + """ + INSERT OR IGNORE INTO telemetry_state(key, value) + VALUES ('schema_version', ?) + """, + (_STORE_SCHEMA_VERSION,), + ) + + def _install_id(self, connection: sqlite3.Connection) -> str: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'install_id'" + ).fetchone() + if row is not None: + return str(row["value"]) + candidate = str(uuid.uuid4()) + connection.execute( + "INSERT OR IGNORE INTO telemetry_state(key, value) VALUES ('install_id', ?)", + (candidate,), + ) + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'install_id'" + ).fetchone() + if row is None: + raise RuntimeError("Unable to create the shared-metrics install identity") + return str(row["value"]) + + def _pending_period_count(self) -> int: + with self._connection() as connection: + row = connection.execute( + """ + SELECT COUNT(*) AS period_count + FROM ( + SELECT period_start, hermes_version + FROM counter_aggregates + WHERE value > packaged_value + GROUP BY period_start, hermes_version + ) + """ + ).fetchone() + return int(row["period_count"]) if row is not None else 0 + + def _create_package(self) -> dict[str, Any] | None: + now = _utc_now() + with self._connection() as connection: + with write_txn(connection): + return self._create_package_in_transaction(connection, now) + + def _create_package_in_transaction( + self, + connection: sqlite3.Connection, + now: datetime, + ) -> dict[str, Any] | None: + period_row = connection.execute( + """ + SELECT period_start, hermes_version + FROM counter_aggregates + WHERE value > packaged_value + ORDER BY period_start, hermes_version + LIMIT 1 + """ + ).fetchone() + period_value = period_row["period_start"] if period_row is not None else None + if not period_value: + return None + + rows = connection.execute( + """ + SELECT metric_name, dimensions_json, value, packaged_value + FROM counter_aggregates + WHERE period_start = ? + AND hermes_version = ? + AND value > packaged_value + ORDER BY metric_name, dimensions_json + """, + (period_value, period_row["hermes_version"]), + ).fetchall() + period_start = datetime.fromisoformat(str(period_value)).replace( + tzinfo=timezone.utc + ) + period_end = period_start + timedelta(days=1) + package_id = str(uuid.uuid4()) + payload = { + "schema_version": _PACKAGE_SCHEMA_VERSION, + "package_id": package_id, + "install_id": self._install_id(connection), + "period_start": _isoformat(period_start), + "period_end": _isoformat(period_end), + "generated_at": _isoformat(now), + "resource": {"hermes_version": period_row["hermes_version"]}, + "metrics": [self._package_metric(row) for row in rows], + } + payload_json = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ) + connection.execute( + """ + INSERT INTO package_outbox( + package_id, + period_start, + period_end, + payload_json, + created_at + ) VALUES (?, ?, ?, ?, ?) + """, + ( + package_id, + payload["period_start"], + payload["period_end"], + payload_json, + payload["generated_at"], + ), + ) + for row in rows: + connection.execute( + """ + UPDATE counter_aggregates + SET packaged_value = value + WHERE period_start = ? + AND metric_name = ? + AND hermes_version = ? + AND dimensions_json = ? + """, + ( + period_value, + row["metric_name"], + period_row["hermes_version"], + row["dimensions_json"], + ), + ) + return payload + + @staticmethod + def _package_metric(row: sqlite3.Row) -> dict[str, Any]: + metric_name = str(row["metric_name"]) + dimensions = json.loads(row["dimensions_json"]) + if not isinstance(dimensions, dict) or not counter_dimensions_are_valid( + metric_name, dimensions + ): + raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}") + return { + "name": metric_name, + "type": "counter", + "dimensions": dimensions, + "value": row["value"] - row["packaged_value"], + } + + def _export_pending_packages(self) -> list[Path]: + with self._connection() as connection: + rows = connection.execute( + """ + SELECT package_id, payload_json + FROM package_outbox + WHERE exported_at IS NULL + ORDER BY created_at, package_id + """ + ).fetchall() + + exported: list[Path] = [] + for row in rows: + package_id = str(row["package_id"]) + path = self.outbox_directory / f"{package_id}.json" + atomic_json_write( + path, + json.loads(row["payload_json"]), + indent=2, + sort_keys=True, + mode=0o600, + ) + with self._connection() as connection: + connection.execute( + """ + UPDATE package_outbox + SET exported_at = ? + WHERE package_id = ? AND exported_at IS NULL + """, + (_isoformat(_utc_now()), package_id), + ) + exported.append(path) + return exported + + def _prune_expired_history(self, *, now: datetime | None = None) -> None: + """Remove exported local history after the bounded retention window.""" + cutoff = (now or _utc_now()) - timedelta( + days=_LOCAL_HISTORY_RETENTION_DAYS + ) + cutoff_timestamp = _isoformat(cutoff) + cutoff_period = cutoff.date().isoformat() + with self._connection() as connection: + rows = connection.execute( + """ + SELECT package_id + FROM package_outbox + WHERE exported_at IS NOT NULL + AND exported_at < ? + ORDER BY exported_at, package_id + """, + (cutoff_timestamp,), + ).fetchall() + + removable_package_ids: list[str] = [] + for row in rows: + package_id = str(row["package_id"]) + try: + (self.outbox_directory / f"{package_id}.json").unlink( + missing_ok=True + ) + except OSError: + logger.warning( + "Unable to prune expired shared-metrics package %s", + package_id, + exc_info=True, + ) + continue + removable_package_ids.append(package_id) + + with self._connection() as connection: + with write_txn(connection): + for package_id in removable_package_ids: + connection.execute( + """ + DELETE FROM package_outbox + WHERE package_id = ? + AND exported_at IS NOT NULL + AND exported_at < ? + """, + (package_id, cutoff_timestamp), + ) + connection.execute( + """ + DELETE FROM counter_aggregates + WHERE period_start < ? + AND value = packaged_value + AND NOT EXISTS ( + SELECT 1 + FROM package_outbox + WHERE exported_at IS NULL + AND substr(package_outbox.period_start, 1, 10) + = counter_aggregates.period_start + ) + """, + (cutoff_period,), + ) diff --git a/hermes_cli/observability/shared_metrics_contract.py b/hermes_cli/observability/shared_metrics_contract.py new file mode 100644 index 000000000000..a8fefea40c55 --- /dev/null +++ b/hermes_cli/observability/shared_metrics_contract.py @@ -0,0 +1,516 @@ +"""Bounded product contract for the first Hermes shared-metrics slice.""" + +from __future__ import annotations + +import re +from functools import lru_cache +from typing import Any + +from agent.relay_runtime import RUNTIME_INSTANCE_KEY + +SCHEMA_KEY = "hermes.metrics.schema_version" +SCHEMA_VERSION = "hermes.metrics.event.v1" +MODEL_CALL_SCOPE = "hermes.model_call" +TASK_SCOPE = "hermes.task_run" +SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics" +PRIMARY_MODEL_CALL_ROLE = "primary" +MODEL_CALL_METRIC = "hermes.model_call.count" +TASK_STARTED_METRIC = "hermes.task_run.started" +TASK_FINISHED_METRIC = "hermes.task_run.finished" + +EXECUTION_SURFACES: frozenset[str] = frozenset({ + "api", + "batch", + "cli", + "desktop", + "gateway", + "python", + "scheduled_task", + "tui", + "other", + "unknown", +}) +PROVIDER_FAMILIES: frozenset[str] = frozenset({ + "aggregator", + "custom", + "direct", + "local", + "unknown", +}) +MODEL_LOCALITIES: frozenset[str] = frozenset({"local", "remote", "unknown"}) +MODEL_OUTCOMES: frozenset[str] = frozenset({"cancelled", "failed", "success"}) +TASK_OUTCOMES: frozenset[str] = frozenset({ + "cancelled", + "failed", + "success", + "timed_out", + "unknown", +}) +TASK_END_REASONS: frozenset[str] = frozenset({ + "approval_denied", + "completed", + "failed", + "guardrail_blocked", + "iteration_limit", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled", +}) +TASK_TERMINATIONS: frozenset[str] = frozenset({ + "none", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled", +}) +TASK_ENTRYPOINTS: frozenset[str] = frozenset({ + "api", + "background", + "batch", + "delegated", + "gateway_message", + "interactive", + "other", + "python", + "scheduled_task", + "unknown", +}) +DURATION_BUCKETS: frozenset[str] = frozenset({ + "1s_to_5s", + "2m_to_10m", + "30s_to_2m", + "5s_to_30s", + "gte_10m", + "lt_1s", +}) +COUNT_BUCKETS: frozenset[str] = frozenset({ + "0", + "1", + "2", + "3_to_5", + "6_to_10", + "gte_11", +}) + +# Shared metrics use an explicit family allowlist rather than raw model IDs or +# dynamically sourced catalog values. The latter would make the exported schema +# drift independently of this contract. +MODEL_FAMILIES: frozenset[str] = frozenset({ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "qwen", + "step", + "trinity", + "o1", + "o3", + "o4", + "unknown", +}) + +_COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = { + MODEL_CALL_METRIC: { + "call_role": frozenset({PRIMARY_MODEL_CALL_ROLE}), + "locality": MODEL_LOCALITIES, + "model_family": MODEL_FAMILIES, + "outcome": MODEL_OUTCOMES, + "provider_family": PROVIDER_FAMILIES, + }, + TASK_STARTED_METRIC: { + "entrypoint": TASK_ENTRYPOINTS, + "execution_surface": EXECUTION_SURFACES, + }, + TASK_FINISHED_METRIC: { + "duration_bucket": DURATION_BUCKETS, + "end_reason": TASK_END_REASONS, + "entrypoint": TASK_ENTRYPOINTS, + "execution_surface": EXECUTION_SURFACES, + "model_call_count_bucket": COUNT_BUCKETS, + "outcome": TASK_OUTCOMES, + "retry_count_bucket": COUNT_BUCKETS, + "termination": TASK_TERMINATIONS, + "tool_call_count_bucket": COUNT_BUCKETS, + }, +} +COUNTER_METRICS: frozenset[str] = frozenset(_COUNTER_DIMENSION_VALUES) + +_MODEL_FAMILY_PATTERN = re.compile( + r"(?:^|[/_.:-])(" + + "|".join( + re.escape(family) + for family in sorted( + MODEL_FAMILIES - {"unknown"}, + key=lambda value: len(value), + reverse=True, + ) + ) + + r")(?=$|[/_.:-]|\d)" +) + +# These providers route across model families but are not marked as aggregators +# in Hermes's execution metadata because that flag has narrower routing/catalog +# semantics there. +_TELEMETRY_AGGREGATOR_OVERRIDES = frozenset({ + "copilot-acp", + "github-copilot", + "moa", + "nous", +}) + +# Hermes intentionally resolves these local runtimes through the generic custom +# provider path, so canonical provider metadata cannot distinguish them alone. +_LOCAL_CUSTOM_PROVIDER_ALIASES = frozenset({"mlx", "ollama"}) + + +def counter_dimensions_are_valid( + metric_name: str, + dimensions: dict[str, Any], +) -> bool: + """Return whether dimensions match one closed shared-metric contract.""" + contract = _COUNTER_DIMENSION_VALUES.get(metric_name) + if contract is None or set(dimensions) != set(contract): + return False + return all( + isinstance(dimensions[field], str) + and dimensions[field] in allowed_values + for field, allowed_values in contract.items() + ) + + +def model_call_dimensions(event: Any) -> dict[str, str] | None: + """Return package dimensions for one valid primary model-call end event.""" + metadata = getattr(event, "metadata", None) + if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION: + return None + relay_metadata = set(metadata) - {SCHEMA_KEY, RUNTIME_INSTANCE_KEY} + if relay_metadata - {"otel.status_code"} or metadata.get( + "otel.status_code", "OK" + ) not in {"OK", "ERROR"}: + return None + if ( + str(getattr(event, "kind", "") or "") != "scope" + or str(getattr(event, "category", "") or "") != "llm" + or str(getattr(event, "name", "") or "") != MODEL_CALL_SCOPE + or str(getattr(event, "scope_category", "") or "") != "end" + ): + return None + category_profile = getattr(event, "category_profile", None) + if not isinstance(category_profile, dict) or set(category_profile) != { + "model_name" + }: + return None + event_model_family = category_profile.get("model_name") + if event_model_family not in MODEL_FAMILIES: + return None + data = getattr(event, "data", None) + expected_fields = { + "call_role", + "locality", + "model_family", + "outcome", + "provider_family", + } + if not isinstance(data, dict) or set(data) != expected_fields: + return None + dimensions = { + "call_role": data.get("call_role"), + "locality": data.get("locality"), + "model_family": data.get("model_family"), + "outcome": data.get("outcome"), + "provider_family": data.get("provider_family"), + } + if not counter_dimensions_are_valid(MODEL_CALL_METRIC, dimensions): + return None + return dimensions + + +def task_counter(event: Any) -> tuple[str, dict[str, str]] | None: + """Return one validated task counter from a task scope event.""" + metadata = getattr(event, "metadata", None) + if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION: + return None + relay_metadata = set(metadata) - {SCHEMA_KEY, RUNTIME_INSTANCE_KEY} + if relay_metadata - {"otel.status_code"} or metadata.get( + "otel.status_code", "OK" + ) not in {"OK", "ERROR"}: + return None + if ( + str(getattr(event, "kind", "") or "") != "scope" + or str(getattr(event, "category", "") or "") != "function" + or str(getattr(event, "name", "") or "") != TASK_SCOPE + ): + return None + if getattr(event, "category_profile", None) is not None: + return None + + scope_category = str(getattr(event, "scope_category", "") or "") + data = getattr(event, "data", None) + if scope_category == "start": + expected_fields = {"entrypoint", "execution_surface"} + if not isinstance(data, dict) or set(data) != expected_fields: + return None + dimensions = { + "entrypoint": data.get("entrypoint"), + "execution_surface": data.get("execution_surface"), + } + if not counter_dimensions_are_valid(TASK_STARTED_METRIC, dimensions): + return None + return TASK_STARTED_METRIC, dimensions + + expected_fields = { + "duration_bucket", + "end_reason", + "entrypoint", + "execution_surface", + "model_call_count_bucket", + "outcome", + "retry_count_bucket", + "termination", + "tool_call_count_bucket", + } + if ( + scope_category != "end" + or not isinstance(data, dict) + or set(data) != expected_fields + ): + return None + dimensions = {field: data.get(field) for field in sorted(expected_fields)} + if not counter_dimensions_are_valid(TASK_FINISHED_METRIC, dimensions): + return None + return TASK_FINISHED_METRIC, dimensions + + +def execution_surface(kwargs: dict[str, Any]) -> str: + """Normalize the safe session surface carried by the parent Relay scope.""" + value = ( + str(kwargs.get("execution_surface") or kwargs.get("platform") or "unknown") + .strip() + .lower() + ) + if value in EXECUTION_SURFACES: + return value + if value == "api_server": + return "api" + if value in {"cron", "scheduler", "scheduled"}: + return "scheduled_task" + try: + from hermes_cli.platforms import get_all_platforms + + if value in get_all_platforms(): + return "gateway" + except Exception: + pass + if value in {"discord", "email", "slack", "telegram", "teams", "whatsapp"}: + return "gateway" + return "unknown" if value == "unknown" else "other" + + +def task_start_fields(kwargs: dict[str, Any]) -> dict[str, str]: + """Build the bounded fields recorded on a task scope start event.""" + surface = execution_surface(kwargs) + return { + "entrypoint": task_entrypoint(kwargs, surface), + "execution_surface": surface, + } + + +def task_entrypoint(kwargs: dict[str, Any], surface: str | None = None) -> str: + """Normalize the task dispatch owner without exporting source strings.""" + declared = str(kwargs.get("entrypoint") or "").strip().lower() + if declared in TASK_ENTRYPOINTS: + return declared + resolved_surface = surface or execution_surface(kwargs) + if kwargs.get("parent_task_id") or kwargs.get("parent_session_id"): + return "delegated" + return { + "api": "api", + "batch": "batch", + "cli": "interactive", + "desktop": "interactive", + "gateway": "gateway_message", + "python": "python", + "scheduled_task": "scheduled_task", + "tui": "interactive", + "unknown": "unknown", + }.get(resolved_surface, "other") + + +def task_terminal_fields( + kwargs: dict[str, Any], + *, + duration_ms: int, + model_call_count: int, + tool_call_count: int, + retry_count: int, +) -> dict[str, str]: + """Build the bounded terminal payload for one task scope.""" + start_fields = task_start_fields(kwargs) + outcome, end_reason, termination = task_terminal_state(kwargs) + return { + **start_fields, + "duration_bucket": duration_bucket(duration_ms), + "end_reason": end_reason, + "model_call_count_bucket": count_bucket(model_call_count), + "outcome": outcome, + "retry_count_bucket": count_bucket(retry_count), + "termination": termination, + "tool_call_count_bucket": count_bucket(tool_call_count), + } + + +def task_terminal_state(kwargs: dict[str, Any]) -> tuple[str, str, str]: + """Map Hermes terminal state to bounded task outcome dimensions.""" + reason = str(kwargs.get("turn_exit_reason") or "").strip().lower() + if kwargs.get("interrupted") or "interrupt" in reason or "cancel" in reason: + return "cancelled", "user_cancelled", "user_cancelled" + if "timeout" in reason or "timed_out" in reason: + return "timed_out", "timed_out", "timed_out" + if "max_iterations" in reason or "budget_exhausted" in reason: + return "failed", "iteration_limit", "system_aborted" + if "approval" in reason and ("denied" in reason or "rejected" in reason): + return "failed", "approval_denied", "none" + if "guardrail" in reason: + return "failed", "guardrail_blocked", "system_aborted" + if reason == "system_aborted": + return "failed", "system_aborted", "system_aborted" + if kwargs.get("completed") is True: + return "success", "completed", "none" + if kwargs.get("failed") is True or (reason and reason != "unknown"): + return "failed", "failed", "none" + return "unknown", "unknown", "unknown" + + +def duration_bucket(duration_ms: int) -> str: + """Bucket a non-negative task duration into a fixed low-cardinality range.""" + value = max(0, int(duration_ms)) + if value < 1_000: + return "lt_1s" + if value < 5_000: + return "1s_to_5s" + if value < 30_000: + return "5s_to_30s" + if value < 120_000: + return "30s_to_2m" + if value < 600_000: + return "2m_to_10m" + return "gte_10m" + + +def count_bucket(count: int) -> str: + """Bucket a non-negative per-task count into a fixed range.""" + value = max(0, int(count)) + if value <= 2: + return str(value) + if value <= 5: + return "3_to_5" + if value <= 10: + return "6_to_10" + return "gte_11" + + +def provider_family(kwargs: dict[str, Any]) -> str: + """Map a Hermes provider to a bounded product category.""" + raw_provider = str(kwargs.get("provider") or "").strip().lower().replace("_", "-") + if not raw_provider: + return "unknown" + if raw_provider in _LOCAL_CUSTOM_PROVIDER_ALIASES: + return "local" + if raw_provider == "custom" or raw_provider.startswith(("custom-", "custom:")): + return "custom" + provider, is_aggregator, is_known = _provider_metadata(raw_provider) + if provider in {"lmstudio", "local"}: + return "local" + if is_aggregator or provider in _TELEMETRY_AGGREGATOR_OVERRIDES: + return "aggregator" + if provider == "custom": + return "custom" + return "direct" if is_known else "unknown" + + +def _provider_metadata(provider: str) -> tuple[str, bool, bool]: + """Resolve provider identity without refreshing remote provider metadata.""" + try: + from hermes_cli.models import normalize_provider as normalize_model_provider + from hermes_cli.providers import HERMES_OVERLAYS, normalize_provider + + canonical = normalize_provider(normalize_model_provider(provider)) + overlay = HERMES_OVERLAYS.get(canonical) + return ( + canonical, + bool(overlay and overlay.is_aggregator), + canonical in _known_provider_ids(), + ) + except Exception: + return provider, False, False + + +@lru_cache(maxsize=1) +def _known_provider_ids() -> frozenset[str]: + """Cache Hermes's static provider catalog for the process lifetime.""" + try: + from hermes_cli.provider_catalog import provider_catalog_by_slug + + return frozenset(provider_catalog_by_slug()) + except Exception: + return frozenset() + + +def model_locality(kwargs: dict[str, Any]) -> str: + """Classify local endpoints without exporting their URL.""" + return _model_locality(kwargs, provider_family(kwargs)) + + +def _model_locality(kwargs: dict[str, Any], provider_category: str) -> str: + base_url = kwargs.get("base_url") + if isinstance(base_url, str) and base_url: + try: + from agent.model_metadata import is_local_endpoint + + if is_local_endpoint(base_url): + return "local" + except Exception: + pass + if provider_category == "local": + return "local" + if provider_category in {"aggregator", "direct"}: + return "remote" + return "unknown" + + +def model_call_fields(kwargs: dict[str, Any]) -> dict[str, str]: + """Build the bounded producer fields for one logical model call.""" + provider_category = provider_family(kwargs) + return { + "call_role": PRIMARY_MODEL_CALL_ROLE, + "locality": _model_locality(kwargs, provider_category), + "model_family": model_family(kwargs), + "provider_family": provider_category, + } + + +def model_family(kwargs: dict[str, Any]) -> str: + """Map a raw model identifier to an allowlisted family.""" + declared_family = str(kwargs.get("model_family") or "").strip().lower() + if declared_family in MODEL_FAMILIES - {"unknown"}: + return declared_family + model = str(kwargs.get("response_model") or kwargs.get("model") or "").lower() + match = _MODEL_FAMILY_PATTERN.search(model) + return match.group(1) if match is not None else "unknown" + + +def model_call_outcome(kwargs: dict[str, Any]) -> str: + """Fail closed when a terminal model-call outcome is not recognized.""" + value = str(kwargs.get("outcome") or "").lower() + return value if value in MODEL_OUTCOMES else "failed" diff --git a/hermes_cli/observability/shared_metrics_subscriber.py b/hermes_cli/observability/shared_metrics_subscriber.py new file mode 100644 index 000000000000..677308503af8 --- /dev/null +++ b/hermes_cli/observability/shared_metrics_subscriber.py @@ -0,0 +1,67 @@ +"""Relay subscriber for the persisted Hermes shared-metrics slice.""" + +from __future__ import annotations + +import logging +import threading +from typing import Any + +from agent.relay_runtime import RUNTIME_INSTANCE_KEY + +from .shared_metrics import SharedMetricsStore +from .shared_metrics_contract import MODEL_CALL_METRIC, model_call_dimensions, task_counter + +logger = logging.getLogger(__name__) + + +class SharedMetricsSubscriber: + """Persist validated Hermes counters from Relay lifecycle events.""" + + def __init__( + self, + store: SharedMetricsStore, + hermes_version: str, + *, + runtime_id: str | None = None, + ) -> None: + self.store = store + self._hermes_version = hermes_version or "unknown" + self._runtime_id = runtime_id + self._active = True + self._lock = threading.RLock() + + def deactivate(self) -> None: + """Stop accepting events before telemetry is disabled or torn down.""" + with self._lock: + self._active = False + + def __call__(self, event: Any) -> None: + if self._runtime_id is not None: + metadata = getattr(event, "metadata", None) + if ( + not isinstance(metadata, dict) + or metadata.get(RUNTIME_INSTANCE_KEY) != self._runtime_id + ): + return + dimensions = model_call_dimensions(event) + metric_name = MODEL_CALL_METRIC + if dimensions is None: + task_metric = task_counter(event) + if task_metric is None: + return + metric_name, dimensions = task_metric + with self._lock: + if not self._active: + return + try: + self.store.record_counter( + metric_name, + dimensions, + self._hermes_version, + ) + except Exception: + logger.warning( + "Unable to persist the Hermes shared metric: %s", + metric_name, + exc_info=True, + ) diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index d706ca1a468b..8dd02a1bdf5e 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -2066,7 +2066,7 @@ def discover_plugins(force: bool = False) -> None: def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]: - """Invoke a lifecycle hook on all loaded plugins. + """Invoke a lifecycle hook on loaded plugins. Returns a list of non-``None`` return values from plugin callbacks. """ @@ -2091,7 +2091,7 @@ def has_middleware(kind: str) -> bool: def has_hook(hook_name: str) -> bool: - """Return True when a hook has registered callbacks.""" + """Return True when a loaded plugin handles a hook.""" return get_plugin_manager().has_hook(hook_name) diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 4763e35e9359..0f4d413d795b 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2221,6 +2221,37 @@ def setup_tools(config: dict, first_install: bool = False): tools_command(first_install=first_install, config=config) +# ============================================================================= +# Shared Metrics +# ============================================================================= + + +def setup_telemetry(config: dict): + """Configure the local, privacy-safe shared-metrics subscriber.""" + print_header("Shared Metrics") + print_info("Shared metrics contain only bounded counters and histograms.") + print_info("Packages stay under this Hermes profile and are not uploaded.") + + telemetry = config.get("telemetry") + if not isinstance(telemetry, dict): + telemetry = {} + config["telemetry"] = telemetry + shared_metrics = telemetry.get("shared_metrics") + if not isinstance(shared_metrics, dict): + shared_metrics = {} + telemetry["shared_metrics"] = shared_metrics + + current = shared_metrics.get("enabled") is True + shared_metrics["enabled"] = prompt_yes_no( + "Enable local shared metrics?", + default=current, + ) + if shared_metrics["enabled"]: + print_success("Local shared metrics enabled.") + else: + print_info("Local shared metrics disabled.") + + # ============================================================================= # Post-Migration Section Skip Logic # ============================================================================= @@ -2629,6 +2660,7 @@ SETUP_SECTIONS = [ ("terminal", "Terminal Backend", setup_terminal_backend), ("gateway", "Messaging Platforms (Gateway)", setup_gateway), ("tools", "Tools", setup_tools), + ("telemetry", "Shared Metrics", setup_telemetry), ("agent", "Agent Settings", setup_agent_settings), ] @@ -2727,6 +2759,7 @@ def run_setup_wizard(args): hermes setup terminal — just terminal backend hermes setup gateway — just messaging platforms hermes setup tools — just tool configuration + hermes setup telemetry — just local shared metrics hermes setup agent — just agent settings """ from hermes_cli.config import is_managed, managed_error diff --git a/hermes_cli/subcommands/setup.py b/hermes_cli/subcommands/setup.py index 406710a68875..fa0c0dc37c74 100644 --- a/hermes_cli/subcommands/setup.py +++ b/hermes_cli/subcommands/setup.py @@ -18,12 +18,21 @@ def build_setup_parser(subparsers, *, cmd_setup: Callable) -> None: "setup", help="Interactive setup wizard", description="Configure Hermes Agent with an interactive wizard. " - "Run a specific section: hermes setup model|tts|terminal|gateway|tools|agent", + "Run a specific section: " + "hermes setup model|tts|terminal|gateway|tools|telemetry|agent", ) setup_parser.add_argument( "section", nargs="?", - choices=["model", "tts", "terminal", "gateway", "tools", "agent"], + choices=[ + "model", + "tts", + "terminal", + "gateway", + "tools", + "telemetry", + "agent", + ], default=None, help="Run a specific setup section instead of the full wizard", ) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 4c0b105da046..56da56c70ada 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -967,6 +967,9 @@ _CATEGORY_MERGE: Dict[str, str] = { # field — fold it into the agent tab rather than spawning a one-field # orphan category. "computer_use": "agent", + # `telemetry.shared_metrics.enabled` is the only schema-surfaced telemetry + # field — fold it into security alongside the other privacy-posture toggles. + "telemetry": "security", } # Display order for tabs — unlisted categories sort alphabetically after these. diff --git a/model_tools.py b/model_tools.py index 7b5811e8fbfe..47f6ee18e74a 100644 --- a/model_tools.py +++ b/model_tools.py @@ -1056,7 +1056,7 @@ def _emit_post_tool_call_hook( listener will actually consume it). """ try: - from hermes_cli.plugins import has_hook, invoke_hook + from hermes_cli.lifecycle import has_hook, invoke_hook if not has_hook("post_tool_call"): return if status is None: @@ -1093,6 +1093,7 @@ def handle_function_call( enabled_tools: Optional[List[str]] = None, skip_pre_tool_call_hook: bool = False, skip_tool_request_middleware: bool = False, + skip_tool_execution_middleware: bool = False, tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None, enabled_toolsets: Optional[List[str]] = None, disabled_toolsets: Optional[List[str]] = None, @@ -1203,6 +1204,7 @@ def handle_function_call( enabled_tools=enabled_tools, skip_pre_tool_call_hook=skip_pre_tool_call_hook, skip_tool_request_middleware=skip_tool_request_middleware, + skip_tool_execution_middleware=skip_tool_execution_middleware, tool_request_middleware_trace=list(_tool_middleware_trace), enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, @@ -1341,19 +1343,22 @@ def handle_function_call( session_id=session_id, user_task=user_task, ) - from hermes_cli.middleware import run_tool_execution_middleware + if skip_tool_execution_middleware: + result = _dispatch(function_args) + else: + from hermes_cli.middleware import run_tool_execution_middleware - result = run_tool_execution_middleware( - function_name, - function_args, - _dispatch, - original_args=_tool_original_args, - task_id=task_id or "", - session_id=session_id or "", - tool_call_id=tool_call_id or "", - turn_id=turn_id or "", - api_request_id=api_request_id or "", - ) + result = run_tool_execution_middleware( + function_name, + function_args, + _dispatch, + original_args=_tool_original_args, + task_id=task_id or "", + session_id=session_id or "", + tool_call_id=tool_call_id or "", + turn_id=turn_id or "", + api_request_id=api_request_id or "", + ) finally: if _approval_tokens is not None and reset_current_observability_context is not None: try: @@ -1384,7 +1389,7 @@ def handle_function_call( # Gated on has_hook so the no-listener path skips both the result # field derivation and the payload dispatch. try: - from hermes_cli.plugins import has_hook, invoke_hook + from hermes_cli.lifecycle import has_hook, invoke_hook if has_hook("transform_tool_result"): status, error_type, error_message = _tool_result_observer_fields(result) hook_results = invoke_hook( diff --git a/plugins/observability/nemo_relay/README.md b/plugins/observability/nemo_relay/README.md index a1b90c4da4fa..85f7d039031f 100644 --- a/plugins/observability/nemo_relay/README.md +++ b/plugins/observability/nemo_relay/README.md @@ -1,18 +1,19 @@ # NeMo Relay Observability -Optional Hermes observability plugin that maps Hermes observer hooks to -NeMo Relay scopes, LLM spans, tool spans, marks, ATOF, and ATIF. +Optional Hermes observability plugin that configures exporters and maps +Hermes-specific observer hooks to NeMo Relay marks and ATIF state. Hermes core +owns Relay session, turn, LLM, and tool execution scopes. NeMo Relay is NVIDIA's runtime layer for agent execution boundaries. It does not replace Hermes Agent's planner, tools, memory, model provider routing, or -CLI UX. Instead, this plugin lets Hermes emit NeMo Relay lifecycle events for -the work Hermes already owns: sessions, turns, provider/API calls, tool calls, -approval prompts, and delegated subagents. +CLI UX. Hermes core emits NeMo Relay lifecycle events for provider and tool +execution, while this plugin enables rich exporters and observer marks for +sessions, turns, approval prompts, and delegated subagents. With this plugin enabled, Hermes Agent can: -- Preserve Hermes execution as NeMo Relay scopes, LLM spans, tool spans, and - mark events. +- Export the Relay scopes and LLM/tool lifecycles emitted by Hermes core. +- Add Hermes session, turn, approval, and subagent mark events. - Export raw lifecycle events as Agent Trajectory Observability Format (ATOF) JSONL for debugging and offline inspection. - Export Agent Trajectory Interchange Format (ATIF) trajectories for replay, @@ -73,27 +74,24 @@ checkout that contains this plugin. A globally installed older CLI will not see new bundled plugins from your working tree. ```bash -uv sync --extra nemo-relay +uv sync uv run hermes plugins enable observability/nemo_relay uv run hermes chat --query 'Reply exactly ok' --provider custom --model qwen3.6:35b ``` To ship the updated CLI into another environment, build and install a fresh -wheel from this checkout, then install the official NeMo Relay runtime extra: +wheel from this checkout. On platforms for which Relay publishes a native +wheel, Hermes installs its supported NeMo Relay runtime as a normal dependency: ```bash uv build --wheel python -m pip install --force-reinstall dist/hermes_agent-*.whl -python -m pip install "nemo-relay>=0.5,<1.0" hermes plugins enable observability/nemo_relay ``` -The plugin fails open when `nemo-relay` is not installed. Install a supported -NeMo Relay 0.x distribution beginning with 0.5: - -```bash -pip install "nemo-relay>=0.5,<1.0" -``` +The plugin remains opt-in even though the runtime dependency is installed by +default. Enabling this plugin controls rich observability and adaptive +behavior; it does not control Hermes shared client metrics. ## Export Configuration @@ -170,8 +168,9 @@ Relay owns exporter lifecycle through that config. The direct double-export trajectories on teardown. If `plugins.toml` initialization fails, Hermes keeps the direct env-var fallbacks active for that run. -To enable NeMo Relay managed execution intercepts for provider and tool calls, -include an adaptive component in the same `plugins.toml`: +Hermes core routes provider and tool execution through NeMo Relay managed APIs +regardless of whether this plugin is enabled. To install adaptive interceptors +on those boundaries, include an adaptive component in the same `plugins.toml`: ```toml [[components]] @@ -182,18 +181,15 @@ enabled = true mode = "observe_only" ``` -When the adaptive component is enabled and the installed NeMo Relay runtime -exposes `llm.execute(...)` / `tools.execute(...)`, Hermes routes LLM and tool -execution through those middleware boundaries. The observer hooks still emit -session, turn, approval, and subagent marks; the plugin skips its manual -`llm.call` and `tools.call` spans for executions that are already managed by -NeMo Relay. `tool_parallelism.mode = "observe_only"` keeps tool scheduling -observational while still wrapping the real execution boundary. +The observer hooks emit session, turn, approval, and subagent marks. They do not +create a second LLM or tool lifecycle. `tool_parallelism.mode = "observe_only"` +keeps tool scheduling observational while still intercepting the core-managed +execution boundary. ### Dynamic Plugins -Hermes feature-detects the dynamic-plugin activation API available in NeMo Relay -0.6 and later. Configure native or worker plugins with Hermes-owned +Hermes uses the dynamic-plugin activation API available in NeMo Relay 0.6 and +later. Configure native or worker plugins with Hermes-owned `[[dynamic_plugins]]` entries that match the Python binding's activation-spec fields: @@ -242,24 +238,15 @@ During shutdown it closes session exporters, flushes Relay subscribers, and then closes the activation so callbacks are removed before plugin code is unloaded. -NeMo Relay 0.5 does not expose dynamic activation through its Python binding. -When dynamic plugin configuration is present with a binding that lacks the -activation API, Hermes logs an actionable warning and continues with the -ordinary static component configuration, so ATOF and ATIF observability remain -available. No dynamic plugin is loaded in that degraded mode. - For the full generic Hermes middleware contract, see [`docs/middleware/README.md`](../../../docs/middleware/README.md). ## Canonical Local Examples -The observe-only examples in this section use a supported NeMo Relay 0.x -distribution beginning with 0.5 and a local Ollama model served through the -OpenAI-compatible API. +The observe-only examples in this section use the NeMo Relay runtime installed +with Hermes and a local Ollama model served through the OpenAI-compatible API. ```bash -pip install "nemo-relay>=0.5,<1.0" - export HERMES_HOME=/tmp/hermes-nemo-relay-docs/hermes-home mkdir -p "$HERMES_HOME" @@ -443,8 +430,8 @@ Sanitized ATIF excerpt: The plugin keeps NeMo Relay's native event model: - Hermes sessions map to `agent` scopes. -- Hermes API request hooks map to `llm` scope start/end events. -- Hermes tool hooks map to `tool` scope start/end events. +- Hermes core managed provider calls map to `llm` scope start/end events. +- Hermes core managed tool calls map to `tool` scope start/end events. - Turn, approval, subagent, and diagnostic fallback events map to `mark` events. @@ -454,11 +441,13 @@ subagent IDs, role/status fields when present, and derived stream lossless for later ATIF conversion that can compact subagents into separate trajectories. -## Adaptive Middleware Example +## Adaptive Execution Example -The `observability/nemo_relay` plugin uses Hermes execution middleware to hand -LLM and tool calls to NeMo Relay managed execution when an adaptive component is -enabled. +Hermes core owns the LLM and tool boundaries and enters NeMo Relay managed +execution while a Hermes-managed Relay consumer is active. With no shared +metrics subscriber or explicitly configured Relay plugin, Hermes calls the +provider or tool directly. The `observability/nemo_relay` plugin retains the +managed path while its adaptive components are installed on those boundaries. Minimal `plugins.toml`: @@ -479,33 +468,28 @@ Enable it for Hermes: export HERMES_NEMO_RELAY_PLUGINS_TOML=/tmp/hermes-middleware-test/plugins.toml ``` -When the adaptive component is enabled and the installed NeMo Relay runtime -exposes `llm.execute(...)` and `tools.execute(...)`, Hermes routes execution -through these boundaries: +Execution follows these boundaries with or without an adaptive component: ```text Hermes provider call - -> llm_execution middleware - -> nemo_relay.llm.execute(...) - -> Hermes provider adapter next_call(...) + -> nemo_relay.llm.execute(...) + -> Hermes provider adapter callback(...) Hermes tool call - -> tool_execution middleware - -> nemo_relay.tools.execute(...) - -> Hermes tool dispatcher next_call(...) + -> nemo_relay.tools.execute(...) + -> Hermes authorization and dispatch callback(...) ``` -The plugin still emits observer marks for sessions, turns, approvals, and -subagents. When adaptive managed execution is active, it skips manual -`llm.call` and `tools.call` observer spans to avoid duplicate LLM/tool events -for the same execution. +The plugin emits observer marks for sessions, turns, approvals, and subagents. +It does not register provider or tool lifecycle hooks, so each managed call +produces one Relay lifecycle. ### Local Adaptive E2E This example enables both NeMo Relay observability export and adaptive execution middleware for a local Hermes run. This path requires a NeMo Relay runtime that -supports `[components.config.tool_parallelism]`, as provided by the supported -0.x release range beginning with 0.5. +supports `[components.config.tool_parallelism]`, as provided by NeMo Relay 0.6 +and later. ```bash export HERMES_HOME=/tmp/hermes-middleware-test/hermes-home diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index 56c6140ca58c..13218ed5e4a6 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -15,34 +15,30 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional +from agent import relay_runtime + logger = logging.getLogger(__name__) _INIT_FAILED = object() _LOCK = threading.RLock() -_RUNTIME: "_Runtime | object | None" = None -_RELAY_LLM_SURFACE_BY_API_MODE = { - "anthropic_messages": "anthropic.messages", - "chat_completions": "openai.chat_completions", - "codex_responses": "openai.responses", -} +_RUNTIMES: dict[str, "_Runtime | object"] = {} +_SESSION_INITIALIZER_NAME = "hermes.nemo_relay.rich_observability" @dataclass class _SessionState: session_id: str + relay_session: relay_runtime.RelaySession | None = None handle: Any = None atif_exporter: Any = None atif_subscriber_name: str = "" is_embedded_subagent: bool = False parent_session_id: str = "" - llm_spans: dict[str, Any] = field(default_factory=dict) - tool_spans: dict[str, Any] = field(default_factory=dict) @dataclass -class _SubagentParent: +class _SubagentContext: parent_session_id: str - parent_handle: Any metadata: dict[str, Any] @@ -51,8 +47,6 @@ class _Settings: plugins_toml_path: str = "" plugins_config: dict[str, Any] | None = None dynamic_plugins: list[dict[str, Any]] = field(default_factory=list) - adaptive_enabled: bool = False - adaptive_mode: str = "observe_only" atof_enabled: bool = False atof_output_directory: str = "" atof_filename: str = "hermes-atof.jsonl" @@ -66,20 +60,179 @@ class _Settings: atif_model_name: str = "unknown" +class _ProcessPluginConfiguration: + """Own Relay's process-global plugin configuration across profile runtimes.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._key: str | None = None + self._plugin_mod: Any = None + self._activation: Any = None + self._owners: set[int] = set() + + def acquire( + self, + owner: "_Runtime", + plugin_mod: Any, + plugin_config: dict[str, Any], + dynamic_plugins: list[dict[str, Any]], + ) -> tuple[bool, Any]: + owner_id = id(owner) + key = _plugin_configuration_key(plugin_config, dynamic_plugins) + with self._lock: + if owner_id in self._owners: + return True, self._activation + if self._owners: + if self._plugin_mod is plugin_mod and self._key == key: + self._owners.add(owner_id) + return True, self._activation + logger.warning( + "NeMo Relay plugin configuration is already active for another " + "Hermes profile; keeping the existing process-global configuration " + "and using direct observability for this profile." + ) + return False, None + + activation = None + if dynamic_plugins: + initialize_dynamic = getattr( + plugin_mod, + "initialize_with_dynamic_plugins", + None, + ) + if callable(initialize_dynamic): + try: + activation = _resolve_awaitable( + initialize_dynamic(plugin_config, dynamic_plugins) + ) + except Exception as exc: + logger.warning( + "NeMo Relay dynamic plugin activation failed; continuing " + "with static observability only: %s", + exc, + ) + else: + logger.warning( + "NeMo Relay dynamic plugins require a binding that exposes " + "plugin.initialize_with_dynamic_plugins (available in NeMo " + "Relay 0.6+). Continuing with static observability only." + ) + + if activation is None: + initialize = getattr(plugin_mod, "initialize", None) + if not callable(initialize): + return False, None + try: + _resolve_awaitable(initialize(plugin_config)) + except Exception as exc: + logger.debug( + "NeMo Relay plugins.toml init failed: %s", + exc, + exc_info=True, + ) + return False, None + + self._key = key + self._plugin_mod = plugin_mod + self._activation = activation + self._owners.add(owner_id) + return True, activation + + def release(self, owner: "_Runtime", nemo_relay: Any) -> None: + owner_id = id(owner) + with self._lock: + if owner_id not in self._owners: + return + self._owners.remove(owner_id) + if self._owners: + return + + failures: list[str] = [] + activation = self._activation + plugin_mod = self._plugin_mod + try: + if activation is not None: + try: + _flush_relay_subscribers(nemo_relay) + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + close = getattr(activation, "close", None) + if callable(close): + try: + _resolve_awaitable(close()) + except Exception as exc: + failures.append( + f"dynamic plugin activation close failed: {exc}" + ) + else: + failures.append("dynamic plugin activation has no close method") + else: + clear = getattr(plugin_mod, "clear", None) + if callable(clear): + try: + _resolve_awaitable(clear()) + except Exception as exc: + failures.append( + f"static plugin configuration clear failed: {exc}" + ) + finally: + self._key = None + self._plugin_mod = None + self._activation = None + + if failures: + raise RuntimeError("; ".join(failures)) + + def reset_for_tests(self) -> None: + with self._lock: + self._key = None + self._plugin_mod = None + self._activation = None + self._owners.clear() + + +_PLUGIN_CONFIGURATION = _ProcessPluginConfiguration() + + class _Runtime: - def __init__(self, nemo_relay: Any, settings: _Settings) -> None: + def __init__( + self, + nemo_relay: Any, + settings: _Settings, + host: relay_runtime.RelayRuntime, + ) -> None: self.nemo_relay = nemo_relay self.settings = settings + self.host = host + self._sessions_lock = threading.RLock() self.sessions: dict[str, _SessionState] = {} - self.subagent_parents: dict[str, _SubagentParent] = {} + self.subagent_contexts: dict[str, _SubagentContext] = {} self.atof_exporter: Any = None - self._atof_subscriber_name = "hermes.nemo_relay.atof" + self._atof_subscriber_name = f"hermes.nemo_relay.atof.{self.host.runtime_id}" + self._execution_consumer_name = ( + f"hermes.nemo_relay.rich_observability.{self.host.runtime_id}" + ) + self._execution_consumer_retained = False self._plugin_activation: Any = None self._shutdown_registered = False self._plugin_config_initialized = self._configure_plugins_toml() self._plugin_config_needs_reinit = False if not self._plugin_config_initialized: self._activate_direct_fallbacks() + self._sync_managed_execution() + + def _sync_managed_execution(self) -> None: + required = bool( + self._plugin_config_initialized + or self.atof_exporter is not None + or self.settings.atif_enabled + ) + if required and not self._execution_consumer_retained: + self.host.retain_managed_execution(self._execution_consumer_name) + self._execution_consumer_retained = True + elif not required and self._execution_consumer_retained: + self.host.release_managed_execution(self._execution_consumer_name) + self._execution_consumer_retained = False def _configure_plugins_toml(self) -> bool: if not self.settings.plugins_config: @@ -88,38 +241,17 @@ class _Runtime: if plugin_mod is None: return False plugin_config = _static_plugin_config(self.settings.plugins_config) - if self.settings.dynamic_plugins: - activate_dynamic = getattr(plugin_mod, "activate_dynamic_plugins", None) - if callable(activate_dynamic): - try: - self._ensure_plugin_config_output_dirs(plugin_config) - self._plugin_activation = _resolve_awaitable( - activate_dynamic(plugin_config, self.settings.dynamic_plugins) - ) - self._ensure_shutdown_registered() - return True - except Exception as exc: - logger.warning( - "NeMo Relay dynamic plugin activation failed; continuing with static " - "observability only: %s", - exc, - ) - else: - logger.warning( - "NeMo Relay dynamic plugins require a binding that exposes " - "plugin.activate_dynamic_plugins (available in NeMo Relay 0.6+). " - "Continuing with static observability only." - ) - initialize = getattr(plugin_mod, "initialize", None) - if not callable(initialize): - return False - try: - self._ensure_plugin_config_output_dirs(plugin_config) - _resolve_awaitable(initialize(plugin_config)) - return True - except Exception as exc: - logger.debug("NeMo Relay plugins.toml init failed: %s", exc, exc_info=True) - return False + self._ensure_plugin_config_output_dirs(plugin_config) + initialized, activation = _PLUGIN_CONFIGURATION.acquire( + self, + plugin_mod, + plugin_config, + self.settings.dynamic_plugins, + ) + self._plugin_activation = activation + if activation is not None: + self._ensure_shutdown_registered() + return initialized def _ensure_shutdown_registered(self) -> None: if self._shutdown_registered: @@ -130,43 +262,12 @@ class _Runtime: def _clear_plugins_toml(self) -> None: if not self._plugin_config_initialized: return - failures: list[str] = [] - if self._plugin_activation is not None: - activation = self._plugin_activation - try: - _flush_relay_subscribers(self.nemo_relay) - except Exception as exc: - failures.append(f"subscriber flush failed: {exc}") - - close = getattr(activation, "close", None) - if callable(close): - try: - _resolve_awaitable(close()) - except Exception as exc: - failures.append(f"dynamic plugin activation close failed: {exc}") - finally: - # Retain the owned activation through the complete close - # attempt. The binding transitions it to a terminal state - # before its awaitable resolves, including error results. - self._plugin_activation = None - self._plugin_config_initialized = False - self._plugin_config_needs_reinit = bool(self.settings.plugins_config) - else: - failures.append("dynamic plugin activation has no close method") - else: - try: - plugin_mod = getattr(self.nemo_relay, "plugin", None) - clear = getattr(plugin_mod, "clear", None) - if callable(clear): - _resolve_awaitable(clear()) - except Exception as exc: - failures.append(f"static plugin configuration clear failed: {exc}") - finally: - self._plugin_config_initialized = False - self._plugin_config_needs_reinit = bool(self.settings.plugins_config) - - if failures: - raise RuntimeError("; ".join(failures)) + try: + _PLUGIN_CONFIGURATION.release(self, self.nemo_relay) + finally: + self._plugin_activation = None + self._plugin_config_initialized = False + self._plugin_config_needs_reinit = bool(self.settings.plugins_config) def _activate_direct_fallbacks(self) -> None: self._plugin_config_needs_reinit = False @@ -178,9 +279,11 @@ class _Runtime: self._plugin_config_initialized = self._configure_plugins_toml() if not self._plugin_config_initialized: self._activate_direct_fallbacks() + self._sync_managed_execution() return self._clear_atof() self._plugin_config_needs_reinit = False + self._sync_managed_execution() def _plugins_toml_owns_exporter(self, exporter_name: str) -> bool: return self._plugin_config_initialized and _observability_exporter_enabled( @@ -232,45 +335,76 @@ class _Runtime: except Exception: logger.debug("NeMo Relay ATOF deregister failed", exc_info=True) self.atof_exporter = None + self._sync_managed_execution() - def ensure_session(self, kwargs: dict[str, Any]) -> _SessionState: - self._maybe_reinitialize_plugins_toml() + def prepare_session(self, kwargs: dict[str, Any]) -> _SessionState: + """Register per-session subscribers without opening the core scope.""" session_id = _session_id(kwargs) - state = self.sessions.get(session_id) - if state is not None: + with self._sessions_lock: + self._maybe_reinitialize_plugins_toml() + state = self.sessions.get(session_id) + if state is not None: + return state + + state = _SessionState(session_id=session_id) + if self.settings.atif_enabled and not self._plugins_toml_owns_exporter("atif"): + state.atif_exporter = self.nemo_relay.AtifExporter( + session_id, + self.settings.atif_agent_name, + self.settings.atif_agent_version, + model_name=str(kwargs.get("model") or self.settings.atif_model_name), + extra={ + "source": "hermes-agent", + "plugin": "observability/nemo_relay", + }, + ) + state.atif_subscriber_name = ( + f"hermes.nemo_relay.atif.{self.host.runtime_id}.{session_id}" + ) + state.atif_exporter.register(state.atif_subscriber_name) + self.sessions[session_id] = state return state - state = _SessionState(session_id=session_id) - if self.settings.atif_enabled and not self._plugins_toml_owns_exporter("atif"): - state.atif_exporter = self.nemo_relay.AtifExporter( - session_id, - self.settings.atif_agent_name, - self.settings.atif_agent_version, - model_name=str(kwargs.get("model") or self.settings.atif_model_name), - extra={"source": "hermes-agent", "plugin": "observability/nemo_relay"}, - ) - state.atif_subscriber_name = f"hermes.nemo_relay.atif.{session_id}" - state.atif_exporter.register(state.atif_subscriber_name) + def ensure_session(self, kwargs: dict[str, Any]) -> _SessionState: + state = self.prepare_session(kwargs) + if state.relay_session is not None: + return state - subagent_parent = self.subagent_parents.get(session_id) - metadata = _metadata(kwargs) - parent_handle = None - if subagent_parent is not None: - parent_handle = subagent_parent.parent_handle - metadata = {**metadata, **subagent_parent.metadata} - state.is_embedded_subagent = True - state.parent_session_id = subagent_parent.parent_session_id - - state.handle = self.nemo_relay.scope.push( - f"hermes-session-{session_id}", - self.nemo_relay.ScopeType.Agent, - handle=parent_handle, - data={"session_id": session_id}, - metadata=metadata, + rich_metadata = _metadata(kwargs) + with self._sessions_lock: + subagent_context = self.subagent_contexts.get(state.session_id) + if subagent_context is not None: + rich_metadata = {**rich_metadata, **subagent_context.metadata} + relay_session = self.host.ensure_session( + kwargs, + data={"session_id": state.session_id}, + metadata=rich_metadata, ) - self.sessions[session_id] = state + if relay_session is None: + raise RuntimeError("Hermes core Relay session is unavailable") + state.relay_session = relay_session + state.handle = relay_session.handle + if subagent_context is not None: + state.is_embedded_subagent = True + state.parent_session_id = subagent_context.parent_session_id return state + def run_in_session( + self, + state: _SessionState, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> Any: + if state.relay_session is None: + raise RuntimeError("Hermes core Relay session is unavailable") + return self.host.run_in_session( + state.relay_session, + callback, + *args, + **kwargs, + ) + def export_atif(self, state: _SessionState) -> None: if not self.settings.atif_enabled or state.atif_exporter is None: return @@ -283,22 +417,24 @@ class _Runtime: filename = self.settings.atif_filename_template.format(session_id=state.session_id) Path(output_dir, filename).write_text(state.atif_exporter.export_json(), encoding="utf-8") - def close_session(self, kwargs: dict[str, Any]) -> None: + def close_session( + self, + kwargs: dict[str, Any], + *, + close_host: bool = True, + ) -> None: session_id = _session_id(kwargs) - self.subagent_parents.pop(session_id, None) - state = self.sessions.pop(session_id, None) + with self._sessions_lock: + self.subagent_contexts.pop(session_id, None) + state = self.sessions.pop(session_id, None) if state is None: return failures: list[str] = [] - if state.handle is not None: + if close_host: try: - self.nemo_relay.scope.pop(state.handle, output=_jsonable(kwargs)) + self.host.close_session(kwargs) except Exception as exc: - failures.append(f"session scope pop failed: {exc}") - try: - _flush_relay_subscribers(self.nemo_relay) - except Exception as exc: - failures.append(f"subscriber flush failed: {exc}") + failures.append(f"core session close failed: {exc}") try: self.export_atif(state) except Exception as exc: @@ -308,21 +444,22 @@ class _Runtime: state.atif_exporter.deregister(state.atif_subscriber_name) except Exception as exc: failures.append(f"ATIF deregister failed: {exc}") - if ( - self._plugin_config_initialized - and self._plugin_activation is None - and not self.sessions - ): - try: - self._clear_plugins_toml() - except Exception as exc: - failures.append(f"plugin configuration clear failed: {exc}") - elif ( - self.settings.plugins_config - and self._plugin_activation is None - and not self.sessions - ): - self._plugin_config_needs_reinit = True + with self._sessions_lock: + if ( + self._plugin_config_initialized + and self._plugin_activation is None + and not self.sessions + ): + try: + self._clear_plugins_toml() + except Exception as exc: + failures.append(f"plugin configuration clear failed: {exc}") + elif ( + self.settings.plugins_config + and self._plugin_activation is None + and not self.sessions + ): + self._plugin_config_needs_reinit = True if failures: logger.warning( "NeMo Relay session %s teardown completed with errors: %s", @@ -333,7 +470,9 @@ class _Runtime: def shutdown(self) -> None: """Close active sessions and the process-lifetime plugin activation.""" failures: list[str] = [] - for session_id in list(self.sessions): + with self._sessions_lock: + session_ids = list(self.sessions) + for session_id in session_ids: try: self.close_session({"session_id": session_id, "reason": "runtime_shutdown"}) except Exception as exc: @@ -344,6 +483,9 @@ class _Runtime: except Exception as exc: failures.append(f"plugin runtime close failed: {exc}") self._clear_atof() + if self._execution_consumer_retained: + self.host.release_managed_execution(self._execution_consumer_name) + self._execution_consumer_retained = False if self._shutdown_registered and self._plugin_activation is None: atexit.unregister(self.shutdown) self._shutdown_registered = False @@ -355,7 +497,9 @@ class _Runtime: def mark(self, name: str, kwargs: dict[str, Any]) -> None: state = self.ensure_session(kwargs) - self.nemo_relay.scope.event( + self.run_in_session( + state, + self.nemo_relay.scope.event, name, handle=state.handle, data=_jsonable(kwargs), @@ -367,12 +511,14 @@ class _Runtime: metadata = _metadata(kwargs) child_session_id = _child_session_id(kwargs) if child_session_id: - self.subagent_parents[child_session_id] = _SubagentParent( - parent_session_id=parent_state.session_id, - parent_handle=parent_state.handle, - metadata=_subagent_child_metadata(kwargs, metadata), - ) - self.nemo_relay.scope.event( + with self._sessions_lock: + self.subagent_contexts[child_session_id] = _SubagentContext( + parent_session_id=parent_state.session_id, + metadata=_subagent_child_metadata(kwargs, metadata), + ) + self.run_in_session( + parent_state, + self.nemo_relay.scope.event, "hermes.subagent.start", handle=parent_state.handle, data=_jsonable(kwargs), @@ -382,148 +528,19 @@ class _Runtime: def mark_subagent_stop(self, kwargs: dict[str, Any]) -> None: child_session_id = _child_session_id(kwargs) if child_session_id: - self.subagent_parents.pop(child_session_id, None) + self.close_session( + {"session_id": child_session_id}, + close_host=False, + ) + with self._sessions_lock: + self.subagent_contexts.pop(child_session_id, None) self.mark("hermes.subagent.stop", kwargs) - def managed_llm_enabled(self) -> bool: - return ( - (self.settings.adaptive_enabled or self._plugin_activation is not None) - and callable(getattr(getattr(self.nemo_relay, "llm", None), "execute", None)) - and callable(getattr(self.nemo_relay, "LLMRequest", None)) - ) - - def managed_tool_enabled(self) -> bool: - return ( - (self.settings.adaptive_enabled or self._plugin_activation is not None) - and callable(getattr(getattr(self.nemo_relay, "tools", None), "execute", None)) - ) - - def _run_managed_with_downstream_preservation( - self, - next_call: Callable[[Any], Any], - normalize_payload: Callable[[Any], Any], - shape_response: Callable[[Any], Any], - make_managed_execute: Callable[[Callable[[Any], Any]], Any], - *, - preserve_raw_response: bool, - ) -> Any: - # NeMo Relay's native managed execution may wrap a failing callback as an - # internal runtime error, hiding the real downstream provider/tool - # exception. Capture the original here and re-raise it after managed - # execution so Hermes retry classification still sees it. The LLM and tool - # paths share this scaffolding; they differ only in payload normalization, - # response shaping, and the Relay call itself. - raw_response: dict[str, Any] = {"set": False, "value": None, "normalized": None} - callback_error: Exception | None = None - downstream_error: BaseException | None = None - - def _impl(next_payload: Any) -> Any: - nonlocal callback_error, downstream_error - try: - raw = next_call(normalize_payload(next_payload)) - except Exception as exc: - callback_error = exc - downstream_error = _original_downstream_error(exc) - raise - raw_response["set"] = True - raw_response["value"] = raw - raw_response["normalized"] = shape_response(raw) - return raw_response["normalized"] - - try: - managed_result = _resolve_awaitable(make_managed_execute(_impl)) - except Exception as exc: - if downstream_error is not None and _is_relay_wrapped_callback_error(exc, callback_error): - raise downstream_error - raise - if ( - preserve_raw_response - and raw_response["set"] - and _json_semantically_equal(managed_result, raw_response["normalized"]) - ): - return raw_response["value"] - return managed_result - - def execute_llm(self, kwargs: dict[str, Any]) -> Any: - state = self.ensure_session(kwargs) - request_body = _jsonable(kwargs.get("request") or {}) - request = self.nemo_relay.LLMRequest({}, request_body) - next_call = kwargs.get("next_call") - if not callable(next_call): - return request_body - - def _normalize(next_request: Any) -> Any: - next_body = getattr(next_request, "content", next_request) - return next_body if isinstance(next_body, dict) else request_body - - def _make_managed(impl: Callable[[Any], Any]) -> Any: - async def _managed_execute() -> Any: - result = self.nemo_relay.llm.execute( - _relay_llm_surface(kwargs), - request, - impl, - handle=state.handle, - data=_jsonable( - { - "turn_id": kwargs.get("turn_id"), - "api_request_id": kwargs.get("api_request_id"), - "api_call_count": kwargs.get("api_call_count"), - "mode": self.settings.adaptive_mode, - } - ), - metadata=_metadata(kwargs), - model_name=str(kwargs.get("model") or ""), - ) - if inspect.isawaitable(result): - return await result - return result - - return _managed_execute() - - return self._run_managed_with_downstream_preservation( - next_call, _normalize, _llm_response_payload, _make_managed, preserve_raw_response=True - ) - - def execute_tool(self, kwargs: dict[str, Any]) -> Any: - state = self.ensure_session(kwargs) - tool_name = str(kwargs.get("tool_name") or "tool") - args = _jsonable(kwargs.get("args") or {}) - next_call = kwargs.get("next_call") - if not callable(next_call): - return args - - def _normalize(next_args: Any) -> Any: - return next_args if isinstance(next_args, dict) else args - - def _make_managed(impl: Callable[[Any], Any]) -> Any: - async def _managed_execute() -> Any: - result = self.nemo_relay.tools.execute( - tool_name, - args, - impl, - handle=state.handle, - data=_jsonable( - { - "turn_id": kwargs.get("turn_id"), - "api_request_id": kwargs.get("api_request_id"), - "tool_call_id": kwargs.get("tool_call_id"), - "mode": self.settings.adaptive_mode, - } - ), - metadata=_metadata(kwargs), - ) - if inspect.isawaitable(result): - return await result - return result - - return _managed_execute() - - return self._run_managed_with_downstream_preservation( - next_call, _normalize, _jsonable, _make_managed, preserve_raw_response=False - ) - - def register(ctx) -> None: + relay_runtime.SESSION_COORDINATOR.register_session_initializer( + _SESSION_INITIALIZER_NAME, + _prepare_core_session, + ) # Activate dynamic plugins before Hermes installs the managed execution # boundaries that invoke their interceptors. if _load_settings().dynamic_plugins: @@ -534,17 +551,10 @@ def register(ctx) -> None: ctx.register_hook("on_session_reset", on_session_reset) ctx.register_hook("pre_llm_call", on_pre_llm_call) ctx.register_hook("post_llm_call", on_post_llm_call) - ctx.register_hook("pre_api_request", on_pre_api_request) - ctx.register_hook("post_api_request", on_post_api_request) - ctx.register_hook("api_request_error", on_api_request_error) - ctx.register_hook("pre_tool_call", on_pre_tool_call) - ctx.register_hook("post_tool_call", on_post_tool_call) ctx.register_hook("pre_approval_request", on_pre_approval_request) ctx.register_hook("post_approval_response", on_post_approval_response) ctx.register_hook("subagent_start", on_subagent_start) ctx.register_hook("subagent_stop", on_subagent_stop) - ctx.register_middleware("llm_execution", on_llm_execution_middleware) - ctx.register_middleware("tool_execution", on_tool_execution_middleware) def on_session_start(**kwargs: Any) -> None: @@ -562,13 +572,13 @@ def on_session_end(**kwargs: Any) -> None: def on_session_finalize(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: - _safe(lambda: runtime.close_session(kwargs)) + _safe(lambda: runtime.close_session(kwargs, close_host=False)) def on_session_reset(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: - _safe(lambda: runtime.close_session(kwargs)) + _safe(lambda: runtime.close_session(kwargs, close_host=False)) def on_pre_llm_call(**kwargs: Any) -> None: @@ -583,122 +593,6 @@ def on_post_llm_call(**kwargs: Any) -> None: _safe(lambda: runtime.mark("hermes.turn.end", kwargs)) -def on_pre_api_request(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - request_payload = kwargs.get("request") - request_body = request_payload.get("body") if isinstance(request_payload, dict) else {} - request = runtime.nemo_relay.LLMRequest({}, _jsonable(request_body)) - span = runtime.nemo_relay.llm.call( - str(kwargs.get("provider") or "llm"), - request, - handle=state.handle, - data=_jsonable({"turn_id": kwargs.get("turn_id"), "api_request_id": kwargs.get("api_request_id")}), - metadata=_metadata(kwargs), - model_name=str(kwargs.get("model") or ""), - ) - state.llm_spans[_api_key(kwargs)] = span - - _safe(_record) - - -def on_post_api_request(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.llm_spans.pop(_api_key(kwargs), None) - if span is None: - runtime.mark("hermes.api.response.unmatched", kwargs) - return - runtime.nemo_relay.llm.call_end( - span, - _jsonable(kwargs.get("response") or {}), - data=_jsonable({"usage": kwargs.get("usage"), "finish_reason": kwargs.get("finish_reason")}), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - -def on_api_request_error(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.llm_spans.pop(_api_key(kwargs), None) - if span is None: - runtime.mark("hermes.api.error", kwargs) - return - runtime.nemo_relay.llm.call_end( - span, - {"error": _jsonable(kwargs.get("error") or {})}, - data=_jsonable(kwargs), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - -def on_pre_tool_call(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_tool_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = runtime.nemo_relay.tools.call( - str(kwargs.get("tool_name") or "tool"), - _jsonable(kwargs.get("args") or {}), - handle=state.handle, - data=_jsonable({"turn_id": kwargs.get("turn_id"), "api_request_id": kwargs.get("api_request_id")}), - metadata=_metadata(kwargs), - tool_call_id=str(kwargs.get("tool_call_id") or ""), - ) - state.tool_spans[_tool_key(kwargs)] = span - - _safe(_record) - - -def on_post_tool_call(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_tool_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.tool_spans.pop(_tool_key(kwargs), None) - if span is None: - runtime.mark("hermes.tool.response.unmatched", kwargs) - return - runtime.nemo_relay.tools.call_end( - span, - _jsonable(kwargs.get("result")), - data=_jsonable({"status": kwargs.get("status"), "duration_ms": kwargs.get("duration_ms")}), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - def on_pre_approval_request(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: @@ -723,60 +617,58 @@ def on_subagent_stop(**kwargs: Any) -> None: _safe(lambda: runtime.mark_subagent_stop(kwargs)) -def on_llm_execution_middleware(**kwargs: Any) -> Any: - runtime = _get_runtime() - next_call = kwargs.get("next_call") - request = kwargs.get("request") or {} - if runtime is not None and runtime.managed_llm_enabled(): - return runtime.execute_llm(kwargs) - if callable(next_call): - return next_call(request) - return request +def _prepare_core_session( + host: relay_runtime.RelayRuntime, + context: dict[str, Any], +) -> None: + """Register rich subscribers before core creates the conversation scope.""" + runtime = _get_runtime( + profile_key=str(context.get("profile_key") or host.profile_key), + host=host, + ) + if runtime is not None: + runtime.prepare_session(context) -def on_tool_execution_middleware(**kwargs: Any) -> Any: - runtime = _get_runtime() - next_call = kwargs.get("next_call") - args = kwargs.get("args") or {} - if runtime is not None and runtime.managed_tool_enabled(): - return runtime.execute_tool(kwargs) - if callable(next_call): - return next_call(args) - return args - - -def _get_runtime() -> Optional[_Runtime]: - global _RUNTIME +def _get_runtime( + *, + profile_key: str | None = None, + host: relay_runtime.RelayRuntime | None = None, +) -> Optional[_Runtime]: + profile_key = profile_key or relay_runtime.current_profile_key() with _LOCK: - if _RUNTIME is _INIT_FAILED: + runtime = _RUNTIMES.get(profile_key) + if runtime is _INIT_FAILED: return None - if isinstance(_RUNTIME, _Runtime): - return _RUNTIME + if isinstance(runtime, _Runtime): + if host is None or runtime.host is host: + return runtime + runtime.shutdown() + _RUNTIMES.pop(profile_key, None) try: - import nemo_relay as nemo_runtime - except Exception as exc: - logger.debug("NeMo Relay plugin disabled: import failed: %s", exc) - _RUNTIME = _INIT_FAILED - return None - try: - _RUNTIME = _Runtime(nemo_relay=nemo_runtime, settings=_load_settings()) + resolved_host = host or relay_runtime.get_runtime(profile_key=profile_key) + if resolved_host is None: + raise RuntimeError("Hermes core Relay runtime is unavailable") + runtime = _Runtime( + nemo_relay=resolved_host.relay, + settings=_load_settings(), + host=resolved_host, + ) except Exception as exc: logger.debug("NeMo Relay plugin disabled: init failed: %s", exc, exc_info=True) - _RUNTIME = _INIT_FAILED + _RUNTIMES[profile_key] = _INIT_FAILED return None - return _RUNTIME + _RUNTIMES[profile_key] = runtime + return runtime def _load_settings() -> _Settings: plugins_toml_path = _env("HERMES_NEMO_RELAY_PLUGINS_TOML") plugins_config = _load_plugins_config(plugins_toml_path) - adaptive_config = _enabled_component_config(plugins_config, "adaptive") return _Settings( plugins_toml_path=plugins_toml_path, plugins_config=plugins_config, dynamic_plugins=_dynamic_plugin_specs(plugins_config, plugins_toml_path), - adaptive_enabled=adaptive_config is not None, - adaptive_mode=_adaptive_mode(adaptive_config), atof_enabled=_env_bool("HERMES_NEMO_RELAY_ATOF_ENABLED"), atof_output_directory=_env("HERMES_NEMO_RELAY_ATOF_OUTPUT_DIRECTORY"), atof_filename=_env("HERMES_NEMO_RELAY_ATOF_FILENAME") or "hermes-atof.jsonl", @@ -800,6 +692,18 @@ def _static_plugin_config(plugins_config: dict[str, Any]) -> dict[str, Any]: } +def _plugin_configuration_key( + plugin_config: dict[str, Any], + dynamic_plugins: list[dict[str, Any]], +) -> str: + return json.dumps( + {"config": plugin_config, "dynamic_plugins": dynamic_plugins}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + def _dynamic_plugin_specs( plugins_config: dict[str, Any] | None, plugins_toml_path: str = "", @@ -950,20 +854,6 @@ def _enabled_component_config( return None -def _adaptive_mode(config: dict[str, Any] | None) -> str: - if not isinstance(config, dict): - return "observe_only" - tool_parallelism = config.get("tool_parallelism") - if isinstance(tool_parallelism, dict): - mode = tool_parallelism.get("mode") - if isinstance(mode, str) and mode.strip(): - return mode.strip() - mode = config.get("mode") - if isinstance(mode, str) and mode.strip(): - return mode.strip() - return "observe_only" - - def _observability_exporter_enabled( plugins_config: dict[str, Any] | None, exporter_name: str, @@ -998,7 +888,10 @@ def _child_session_id(kwargs: dict[str, Any]) -> str: return str(kwargs.get("child_session_id") or "") -def _subagent_child_metadata(kwargs: dict[str, Any], parent_metadata: dict[str, Any]) -> dict[str, Any]: +def _subagent_child_metadata( + kwargs: dict[str, Any], + parent_metadata: dict[str, Any], +) -> dict[str, Any]: child_session_id = _child_session_id(kwargs) metadata = { "session_id": child_session_id, @@ -1022,25 +915,6 @@ def _subagent_child_metadata(kwargs: dict[str, Any], parent_metadata: dict[str, return metadata -def _api_key(kwargs: dict[str, Any]) -> str: - return str(kwargs.get("api_request_id") or f"{_session_id(kwargs)}:{kwargs.get('api_call_count') or 'api'}") - - -def _tool_key(kwargs: dict[str, Any]) -> str: - return str( - kwargs.get("tool_call_id") - or f"{_session_id(kwargs)}:{kwargs.get('turn_id') or ''}:{kwargs.get('tool_name') or 'tool'}" - ) - - -def _relay_llm_surface(kwargs: dict[str, Any]) -> str: - api_mode = str(kwargs.get("api_mode") or "").strip().lower() - return _RELAY_LLM_SURFACE_BY_API_MODE.get( - api_mode, - str(kwargs.get("provider") or "llm"), - ) - - def _metadata(kwargs: dict[str, Any]) -> dict[str, Any]: keys = ( "telemetry_schema_version", @@ -1100,100 +974,6 @@ def _jsonable(value: Any) -> Any: return str(value) -def _json_semantically_equal(left: Any, right: Any) -> bool: - """Compare JSON-compatible values without conflating booleans and numbers.""" - try: - options = {"ensure_ascii": False, "sort_keys": True, "separators": (",", ":")} - return json.dumps(_jsonable(left), **options) == json.dumps(_jsonable(right), **options) - except (TypeError, ValueError): - return False - - -def _value(obj: Any, key: str, default: Any = None) -> Any: - if isinstance(obj, dict): - return obj.get(key, default) - return getattr(obj, key, default) - - -def _original_downstream_error(exc: Exception) -> BaseException: - # Hermes wraps downstream execution failures in a local/private exception - # class, so detect the wrapper by shape instead of importing it here. - original = getattr(exc, "original", None) - if exc.__class__.__name__ == "_DownstreamExecutionError" and isinstance(original, BaseException): - return original - return exc - - -def _is_relay_wrapped_callback_error(exc: Exception, callback_error: Exception | None) -> bool: - # NeMo Relay re-wraps a failing callback as ``RuntimeError("internal error: - # : ")``. Match by prefix rather than exact equality so a - # trailing traceback/suffix in a future Relay version doesn't silently defeat - # the unwrap; the class-name + message prefix still discriminates the real - # downstream failure from unrelated Relay-internal errors. If Relay drops the - # leading ``internal error:`` shape entirely, this returns False and Hermes - # falls back to surfacing Relay's error (the pre-fix behavior) rather than - # masking it. - if callback_error is None or not isinstance(exc, RuntimeError): - return False - expected = f"internal error: {callback_error.__class__.__name__}: {callback_error}" - return str(exc).startswith(expected) - - -def _llm_response_payload(response: Any) -> Any: - """Return the LLM response shape NeMo Relay's ATIF conversion expects.""" - payload = _jsonable(response) - if isinstance(payload, dict) and "assistant_message" in payload: - return payload - - choices = _value(response, "choices") - if choices is None and isinstance(payload, dict): - choices = payload.get("choices") - first_choice = choices[0] if isinstance(choices, list) and choices else None - message = _value(first_choice, "message") - finish_reason = _value(first_choice, "finish_reason") - - assistant_message: dict[str, Any] = {"role": "assistant", "content": ""} - if message is not None: - assistant_message["role"] = _value(message, "role", "assistant") or "assistant" - content = _value(message, "content") - if content is not None: - assistant_message["content"] = _jsonable(content) - tool_calls = _tool_calls_payload(_value(message, "tool_calls")) - if tool_calls: - assistant_message["tool_calls"] = tool_calls - reasoning = _value(message, "reasoning_content") - if reasoning is not None: - assistant_message["reasoning_content"] = _jsonable(reasoning) - elif isinstance(payload, dict): - assistant_message["content"] = payload.get("content") or payload.get("output_text") or "" - - return { - "model": _value(response, "model", payload.get("model") if isinstance(payload, dict) else None), - "assistant_message": assistant_message, - "finish_reason": finish_reason, - "usage": _jsonable(_value(response, "usage", payload.get("usage") if isinstance(payload, dict) else None)), - } - - -def _tool_calls_payload(tool_calls: Any) -> list[dict[str, Any]]: - if not isinstance(tool_calls, list): - return [] - normalized: list[dict[str, Any]] = [] - for call in tool_calls: - function = _value(call, "function") - normalized.append( - { - "id": _value(call, "id"), - "type": _value(call, "type", "function") or "function", - "function": { - "name": _value(function, "name"), - "arguments": _value(function, "arguments"), - }, - } - ) - return normalized - - def _safe(fn) -> None: try: fn() @@ -1231,8 +1011,13 @@ def _resolve_awaitable(value: Any) -> Any: def reset_for_tests() -> None: - global _RUNTIME + relay_runtime.SESSION_COORDINATOR.unregister_session_initializer( + _SESSION_INITIALIZER_NAME + ) with _LOCK: - if isinstance(_RUNTIME, _Runtime): - _RUNTIME.shutdown() - _RUNTIME = None + runtimes = list(_RUNTIMES.values()) + _RUNTIMES.clear() + for runtime in runtimes: + if isinstance(runtime, _Runtime): + runtime.shutdown() + _PLUGIN_CONFIGURATION.reset_for_tests() diff --git a/plugins/observability/nemo_relay/plugin.yaml b/plugins/observability/nemo_relay/plugin.yaml index b1b00f25d819..046d5d0d8510 100644 --- a/plugins/observability/nemo_relay/plugin.yaml +++ b/plugins/observability/nemo_relay/plugin.yaml @@ -9,11 +9,6 @@ hooks: - on_session_reset - pre_llm_call - post_llm_call - - pre_api_request - - post_api_request - - api_request_error - - pre_tool_call - - post_tool_call - pre_approval_request - post_approval_response - subagent_start diff --git a/pyproject.toml b/pyproject.toml index 02c00a096752..d623df04b9ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,11 @@ dependencies = [ # Hence the ``sys_platform == 'win32'`` marker: the dep (and its portalocker # / pywin32 tree) ships only where it's actually used. "concurrent-log-handler==0.9.29; sys_platform == 'win32'", + # First-party lifecycle and shared-metrics runtime. Relay 0.6 is the minimum + # lossless provider-codec contract. Managed calls pass request/response data + # through this native module in-process; shared metrics installs no network + # exporter and consumes only its bounded projection. + "nemo-relay>=0.6.0,<0.7; (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'linux' and platform_machine == 'aarch64') or (sys_platform == 'win32' and platform_machine == 'AMD64') or (sys_platform == 'win32' and platform_machine == 'ARM64')", ] [project.optional-dependencies] @@ -202,7 +207,9 @@ pty = [] # extra that exposes a Starlette-backed server surface so pip/uv can't resolve # a vulnerable pre-1.0.1 transitive. Bump in lockstep with uv.lock. mcp = ["mcp==1.26.0", "starlette==1.3.1"] # starlette: CVE-2026-48710 -nemo-relay = ["nemo-relay>=0.5,<1.0"] +# Backwards-compatible no-op alias. Relay is a core dependency on supported +# wheel targets and intentionally unavailable on other platforms. +nemo-relay = [] homeassistant = ["aiohttp==3.14.1"] sms = ["aiohttp==3.14.1"] teams = ["microsoft-teams-apps==2.0.13.4", "aiohttp==3.14.1"] # aiohttp 3.14.1: CVE-2026-34993(RCE)/47265 + 34513/34518/34519/34520/34525 @@ -319,6 +326,7 @@ py-modules = ["run_agent", "model_tools", "toolsets", "batch_runner", "trajector include = ["agent", "agent.*", "tools", "tools.*", "hermes_cli", "hermes_cli.*", "gateway", "gateway.*", "tui_gateway", "tui_gateway.*", "cron", "cron.*", "acp_adapter", "plugins", "plugins.*", "providers", "providers.*"] [tool.setuptools.package-data] +hermes_cli = ["observability/schemas/*.json"] # gateway/assets/ ships status_phrases.yaml and the Telegram BotFather # screenshot. Without this, sealed venvs (uv2nix) silently lose both — # status phrases fall back to the tiny hardcoded set and the Telegram diff --git a/run_agent.py b/run_agent.py index f4d501914fe5..d624656facd7 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2685,17 +2685,16 @@ class AIAgent: retryable: Optional[bool] = None, reason: Optional[str] = None, ) -> None: - # Lazy module import (not from-import) so tests that - # ``monkeypatch.setattr("hermes_cli.plugins.has_hook", ...)`` still - # take effect on this call site. After first call the import is a + # Lazy module import (not from-import) so tests can replace lifecycle + # dispatch at this call site. After first call the import is a # ``sys.modules`` dict lookup, so retries don't repay any real cost. try: - from hermes_cli import plugins as _plugins + from hermes_cli import lifecycle as _lifecycle - if not _plugins.has_hook("api_request_error"): + if not _lifecycle.has_hook("api_request_error"): return ended_at = time.time() - _plugins.invoke_hook( + _lifecycle.invoke_hook( "api_request_error", task_id=task_id, turn_id=turn_id, @@ -6749,7 +6748,8 @@ class AIAgent: tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False, skip_tool_request_middleware: bool = False, - tool_request_middleware_trace: Optional[list[dict[str, Any]]] = None) -> str: + tool_request_middleware_trace: Optional[list[dict[str, Any]]] = None, + skip_tool_execution_middleware: bool = False) -> str: """Forwarder — see ``agent.agent_runtime_helpers.invoke_tool``.""" from agent.agent_runtime_helpers import invoke_tool return invoke_tool( @@ -6762,6 +6762,7 @@ class AIAgent: pre_tool_block_checked, skip_tool_request_middleware, tool_request_middleware_trace, + skip_tool_execution_middleware, ) @staticmethod @@ -6849,48 +6850,140 @@ class AIAgent: reset_accounting_context, set_accounting_context, ) + from agent import relay_runtime from agent.conversation_loop import run_conversation from agent.portal_tags import ( reset_conversation_context, set_conversation_context, ) - from agent.subagent_lifecycle import bind_subagent_parent - - # Publish the conversation id for ambient Nous Portal tagging. Every - # LLM call made inside this turn — main loop, compression, vision, - # web_extract, session_search, MoA slots, background-review forks - # (which copy this Context into their thread) — inherits the - # ``conversation=`` tag with zero per-call-site plumbing. - token = set_conversation_context(self._conversation_root_id()) - # Publish the session accounting handles the same way so auxiliary - # calls record their token usage into session_model_usage (task - # dimension) — the fix for aux spend being invisible in analytics - # (issue #23270). - acct_token = set_accounting_context( - getattr(self, "_session_db", None), getattr(self, "session_id", None) + from hermes_cli.observability.relay_shared_metrics import ( + finish_task_run, + start_task_run, ) - from agent.auxiliary_client import scoped_runtime_main + from agent.subagent_lifecycle import bind_subagent_parent + effective_task_id = task_id or str(uuid.uuid4()) + session_id = str(getattr(self, "session_id", None) or "") + task_context = { + "session_id": session_id, + "task_id": effective_task_id, + "platform": getattr(self, "platform", None) or "", + } + relay_turn_id = ( + f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + ) + self._relay_pending_turn_id = relay_turn_id + relay_parent_session_id = ( + str(getattr(self, "_parent_session_id", None) or "") + if task_context["platform"] == "subagent" + else "" + ) + relay_lease = None + relay_turn = None + token = None + acct_token = None + task_started = False + task_finished = False + relay_outcome = "failed" + try: + relay_lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=task_context["session_id"], + platform=task_context["platform"], + parent_session_id=relay_parent_session_id, + model=str(getattr(self, "model", None) or ""), + ) + relay_turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + relay_lease, + turn_id=relay_turn_id, + task_id=effective_task_id, + ) + start_task_run( + **task_context, + parent_session_id=getattr(self, "_parent_session_id", None) or "", + ) + task_started = True + # Publish the conversation id for ambient Nous Portal tagging. Every + # LLM call made inside this turn — main loop, compression, vision, + # web_extract, session_search, MoA slots, background-review forks + # (which copy this Context into their thread) — inherits the + # ``conversation=`` tag with zero per-call-site plumbing. + token = set_conversation_context(self._conversation_root_id()) + # Publish the session accounting handles the same way so auxiliary + # calls record their token usage into session_model_usage (task + # dimension) — the fix for aux spend being invisible in analytics + # (issue #23270). + acct_token = set_accounting_context( + getattr(self, "_session_db", None), + getattr(self, "session_id", None), + ) + from agent.auxiliary_client import scoped_runtime_main - # The outer token restores the caller's Context even though turn setup - # replaces the value with the live runtime after fallback restoration. - # Keep the scope local instead of storing ContextVar tokens on the agent, - # which may be observed from another thread. - with bind_subagent_parent(self), scoped_runtime_main({}): - try: - return run_conversation( + # The outer token restores the caller's Context even though turn setup + # replaces the value with the live runtime after fallback restoration. + # Keep the scope local instead of storing ContextVar tokens on the agent, + # which may be observed from another thread. + with bind_subagent_parent(self), scoped_runtime_main({}): + result = run_conversation( self, user_message, system_message, conversation_history, - task_id, + effective_task_id, stream_callback, persist_user_message, persist_user_timestamp=persist_user_timestamp, moa_config=moa_config, ) + terminal = result if isinstance(result, dict) else {} + if terminal.get("interrupted") is True: + relay_outcome = "cancelled" + elif terminal.get("failed") is True: + relay_outcome = "failed" + else: + relay_outcome = "success" + relay_runtime.SESSION_COORDINATOR.finish_logical_calls( + relay_turn, + outcome=relay_outcome, + ) + task_finished = True + finish_task_run(**task_context, result=result) + return result + except BaseException as exc: + if isinstance(exc, (KeyboardInterrupt, InterruptedError)) or ( + type(exc).__name__ == "CancelledError" + ): + relay_outcome = "cancelled" + elif isinstance(exc, TimeoutError): + relay_outcome = "timed_out" + if relay_turn is not None: + relay_runtime.SESSION_COORDINATOR.finish_logical_calls( + relay_turn, + outcome=relay_outcome, + ) + if task_started and not task_finished: + task_finished = True + finish_task_run(**task_context, error=exc) + raise + finally: + try: + if relay_turn is not None: + relay_runtime.SESSION_COORDINATOR.end_turn( + relay_turn, + outcome=relay_outcome, + ) finally: - reset_accounting_context(acct_token) - reset_conversation_context(token) + try: + if relay_lease is not None: + relay_runtime.SESSION_COORDINATOR.release_conversation( + relay_lease + ) + finally: + if getattr(self, "_relay_pending_turn_id", None) == relay_turn_id: + self._relay_pending_turn_id = None + if acct_token is not None: + reset_accounting_context(acct_token) + if token is not None: + reset_conversation_context(token) def chat(self, message: str, stream_callback: Optional[callable] = None) -> str: """ diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.py new file mode 100644 index 000000000000..38a42641d1b1 --- /dev/null +++ b/scripts/smoke_nemo_relay_shared_metrics.py @@ -0,0 +1,443 @@ +"""Run a real Hermes CLI turn and validate the Relay shared-metrics output.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +PROMPT_CANARY = "relay-smoke-sensitive-prompt" +MODEL_CANARY = "gpt-relay-smoke-sensitive-model" +RESPONSE_CANARY = "relay-smoke-sensitive-response" + + +def _resolve_hermes_executable(hermes_repo: Path) -> Path: + for relative_path in ( + Path(".venv") / "bin" / "hermes", + Path(".venv") / "Scripts" / "hermes.exe", + ): + candidate = hermes_repo / relative_path + if candidate.is_file(): + return candidate + discovered = shutil.which("hermes") + if discovered: + return Path(discovered) + raise SystemExit( + "Hermes executable not found in the repository virtual environment " + "or on PATH" + ) + + +class _ModelHandler(BaseHTTPRequestHandler): + """Minimal OpenAI-compatible model server for one deterministic turn.""" + + protocol_version = "HTTP/1.1" + requests: list[dict[str, Any]] = [] + + def do_GET(self) -> None: # noqa: N802 + if self.path.rstrip("/") != "/v1/models": + self.send_error(404) + return + self._write_json({ + "object": "list", + "data": [ + { + "id": MODEL_CANARY, + "object": "model", + "created": 0, + "owned_by": "smoke-test", + } + ], + }) + + def do_POST(self) -> None: # noqa: N802 + if self.path.rstrip("/") != "/v1/chat/completions": + self.send_error(404) + return + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length) or b"{}") + type(self).requests.append(request) + if request.get("stream"): + self._write_stream() + else: + self._write_json({ + "id": "chatcmpl-relay-smoke", + "object": "chat.completion", + "created": int(time.time()), + "model": MODEL_CANARY, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": RESPONSE_CANARY, + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 1, + "total_tokens": 11, + }, + }) + + def log_message(self, format: str, *args: Any) -> None: + return + + def _write_json(self, payload: dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body) + self.close_connection = True + + def _write_stream(self) -> None: + now = int(time.time()) + chunks = [ + { + "id": "chatcmpl-relay-smoke", + "object": "chat.completion.chunk", + "created": now, + "model": MODEL_CANARY, + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": RESPONSE_CANARY, + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-relay-smoke", + "object": "chat.completion.chunk", + "created": now, + "model": MODEL_CANARY, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + "id": "chatcmpl-relay-smoke", + "object": "chat.completion.chunk", + "created": now, + "model": MODEL_CANARY, + "choices": [], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 1, + "total_tokens": 11, + }, + }, + ] + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + for chunk in chunks: + self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode("utf-8")) + self.wfile.flush() + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + self.close_connection = True + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--hermes-repo", + type=Path, + default=Path.cwd(), + help="Hermes source checkout containing .venv/bin/hermes", + ) + parser.add_argument( + "--relay-python", + type=Path, + default=None, + help="Optional NeMo Relay checkout's python directory", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help="Directory for the isolated HERMES_HOME and captured output", + ) + return parser.parse_args() + + +def _write_config(home: Path, port: int) -> None: + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text( + f"""model: + default: {MODEL_CANARY} + provider: custom + base_url: http://127.0.0.1:{port}/v1 + api_mode: chat_completions + api_key: no-key-required +security: + tirith_enabled: false +telemetry: + shared_metrics: + enabled: true +""", + encoding="utf-8", + ) + + +def _validate_store(database_path: Path) -> list[dict[str, Any]]: + if not database_path.is_file(): + raise AssertionError(f"Metrics database was not created: {database_path}") + with sqlite3.connect(database_path) as connection: + rows = connection.execute( + """ + SELECT metric_name, dimensions_json, value, packaged_value + FROM counter_aggregates + ORDER BY metric_name, dimensions_json + """ + ).fetchall() + counters = [ + { + "name": name, + "dimensions": json.loads(dimensions), + "value": value, + "packaged_value": packaged_value, + } + for name, dimensions, value, packaged_value in rows + ] + by_name = {counter["name"]: counter for counter in counters} + if set(by_name) != { + "hermes.model_call.count", + "hermes.task_run.finished", + "hermes.task_run.started", + }: + raise AssertionError( + f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}" + ) + expected_model = { + "name": "hermes.model_call.count", + "dimensions": { + "call_role": "primary", + "locality": "local", + "model_family": "gpt", + "outcome": "success", + "provider_family": "custom", + }, + "value": 1, + "packaged_value": 1, + } + if by_name["hermes.model_call.count"] != expected_model: + raise AssertionError( + f"Unexpected model counter: {by_name['hermes.model_call.count']}" + ) + expected_start = { + "name": "hermes.task_run.started", + "dimensions": { + "entrypoint": "interactive", + "execution_surface": "cli", + }, + "value": 1, + "packaged_value": 1, + } + if by_name["hermes.task_run.started"] != expected_start: + raise AssertionError( + f"Unexpected task start: {by_name['hermes.task_run.started']}" + ) + terminal = by_name["hermes.task_run.finished"] + expected_terminal_dimensions = { + "duration_bucket": terminal["dimensions"].get("duration_bucket"), + "end_reason": "completed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "success", + "retry_count_bucket": "0", + "termination": "none", + "tool_call_count_bucket": "0", + } + if ( + terminal["dimensions"] != expected_terminal_dimensions + or terminal["value"] != 1 + or terminal["packaged_value"] != 1 + ): + raise AssertionError(f"Unexpected task terminal counter: {terminal}") + return counters + + +def _validate_package(outbox: Path, schema_path: Path) -> tuple[Path, dict[str, Any]]: + packages = sorted(outbox.glob("*.json")) + if len(packages) != 1: + raise AssertionError(f"Expected one package in {outbox}, found {len(packages)}") + package_path = packages[0] + package = json.loads(package_path.read_text(encoding="utf-8")) + try: + import jsonschema + except ImportError as exc: + raise RuntimeError( + "The Hermes development environment requires jsonschema" + ) from exc + schema = json.loads(schema_path.read_text(encoding="utf-8")) + jsonschema.validate(package, schema) + + serialized = json.dumps(package) + for prohibited in (PROMPT_CANARY, MODEL_CANARY, RESPONSE_CANARY): + if prohibited in serialized: + raise AssertionError( + f"Exported package leaked prohibited value: {prohibited!r}" + ) + metrics = {metric["name"]: metric for metric in package.get("metrics", [])} + if set(metrics) != { + "hermes.model_call.count", + "hermes.task_run.finished", + "hermes.task_run.started", + }: + raise AssertionError( + f"Unexpected package metrics:\n{json.dumps(package.get('metrics'), indent=2)}" + ) + if metrics["hermes.model_call.count"]["dimensions"] != { + "call_role": "primary", + "locality": "local", + "model_family": "gpt", + "outcome": "success", + "provider_family": "custom", + }: + raise AssertionError( + f"Unexpected model metric: {metrics['hermes.model_call.count']}" + ) + terminal = metrics["hermes.task_run.finished"] + if terminal["dimensions"] != { + "duration_bucket": terminal["dimensions"].get("duration_bucket"), + "end_reason": "completed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "success", + "retry_count_bucket": "0", + "termination": "none", + "tool_call_count_bucket": "0", + }: + raise AssertionError(f"Unexpected task terminal metric: {terminal}") + return package_path, package + + +def main() -> int: + args = _arguments() + hermes_repo = args.hermes_repo.resolve() + relay_python = args.relay_python.resolve() if args.relay_python else None + hermes = _resolve_hermes_executable(hermes_repo) + if relay_python is not None and not any( + (relay_python / "nemo_relay").glob("_native.*") + ): + raise SystemExit( + "Built NeMo Relay Python binding not found under " + f"{relay_python}; run the Relay Python build first" + ) + + if args.output_dir: + root = args.output_dir.resolve() + if root.exists(): + raise SystemExit(f"Refusing to replace existing output directory: {root}") + root.mkdir(parents=True) + else: + root = Path(tempfile.mkdtemp(prefix="hermes-relay-shared-metrics-")) + home = root / "hermes-home" + workdir = root / "workspace" + workdir.mkdir() + home.mkdir() + (home / ".no-bundled-skills").touch() + + _ModelHandler.requests = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _ModelHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + _write_config(home, server.server_port) + env = os.environ.copy() + env["HERMES_HOME"] = str(home) + if relay_python is not None: + env["PYTHONPATH"] = os.pathsep.join([ + str(relay_python), + env.get("PYTHONPATH", ""), + ]).rstrip(os.pathsep) + result = subprocess.run( + [ + str(hermes), + "chat", + "--query", + PROMPT_CANARY, + "--provider", + "custom", + "--model", + MODEL_CANARY, + "--quiet", + "--ignore-rules", + "--toolsets", + "search", + "--max-turns", + "2", + ], + cwd=workdir, + env=env, + text=True, + capture_output=True, + timeout=120, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + (root / "hermes.stdout.txt").write_text(result.stdout, encoding="utf-8") + (root / "hermes.stderr.txt").write_text(result.stderr, encoding="utf-8") + if result.returncode != 0: + raise AssertionError( + f"Hermes exited with {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if not _ModelHandler.requests: + raise AssertionError("Hermes did not call the local model endpoint") + request = _ModelHandler.requests[0] + if request.get("model") != MODEL_CANARY: + raise AssertionError(f"Unexpected model request: {request.get('model')!r}") + if PROMPT_CANARY not in json.dumps(request.get("messages", [])): + raise AssertionError("Hermes model request did not contain the prompt canary") + if RESPONSE_CANARY not in result.stdout: + raise AssertionError("Hermes did not print the mock model response") + + telemetry = home / "telemetry" / "shared_metrics" + counters = _validate_store(telemetry / "metrics.sqlite3") + package_path, package = _validate_package( + telemetry / "outbox", + hermes_repo + / "hermes_cli" + / "observability" + / "schemas" + / "hermes.shared_metrics.v1.schema.json", + ) + + print("Hermes -> NeMo Relay shared-metrics smoke test passed") + print(f"Artifact directory: {root}") + print(f"Model requests: {len(_ModelHandler.requests)}") + print(f"SQLite counters: {json.dumps(counters, indent=2)}") + print(f"Export package: {package_path}") + print(json.dumps(package, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/agent/test_auxiliary_relay.py b/tests/agent/test_auxiliary_relay.py new file mode 100644 index 000000000000..0902f0fb91c3 --- /dev/null +++ b/tests/agent/test_auxiliary_relay.py @@ -0,0 +1,445 @@ +from types import SimpleNamespace + +import pytest + +pytest.importorskip("nemo_relay") + +from agent import auxiliary_client, relay_llm, relay_runtime + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + try: + yield lease.host.relay, turn + finally: + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_auxiliary_retries_share_logical_relay_identity(monkeypatch): + attempts = [] + logical_completions = [] + responses = iter([ + SimpleNamespace(choices=[]), + SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ), + ]) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: next(responses), + ) + ) + ) + + def execute_current(request, callback, **kwargs): + attempts.append(kwargs) + return callback(request) + + monkeypatch.setattr(relay_llm, "execute_current", execute_current) + monkeypatch.setattr( + relay_llm, + "complete_logical_call", + lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + result = run("compression") + + assert result.choices[0].message.content == "ok" + assert attempts[0]["metadata"]["api_request_id"] == ( + attempts[1]["metadata"]["api_request_id"] + ) + assert [attempt["metadata"]["retry_count"] for attempt in attempts] == [0, 1] + assert attempts[0]["metadata"]["call_role"] == "auxiliary:compression" + assert all(attempt["defer_logical_completion"] is True for attempt in attempts) + assert logical_completions == [ + (attempts[0]["metadata"]["api_request_id"], "success") + ] + + +@pytest.mark.asyncio +async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch): + captured = {} + logical_completions = [] + + async def create(**kwargs): + return SimpleNamespace( + request=kwargs, + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))], + ) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + async def execute_current_async(request, callback, **kwargs): + captured.update(kwargs) + return await callback(request) + + monkeypatch.setattr( + relay_llm, + "execute_current_async", + execute_current_async, + ) + monkeypatch.setattr( + relay_llm, + "complete_logical_call", + lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ) + + @auxiliary_client._relay_auxiliary_call_async + async def run(task): + auxiliary_client._set_relay_auxiliary_route( + "anthropic", + "claude-test", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ), + task, + ) + + result = await run("title_generation") + + assert result.request["model"] == "claude-test" + assert captured["name"] == "anthropic" + assert captured["metadata"]["call_role"] == "auxiliary:title_generation" + assert captured["defer_logical_completion"] is True + assert logical_completions == [ + (captured["metadata"]["api_request_id"], "success") + ] + + +def test_terminal_auxiliary_failure_stays_failed_when_caller_catches_it( + relay_turn, monkeypatch +): + _relay, turn = relay_turn + consumer = "test.terminal-auxiliary-failure" + turn.lease.host.retain_managed_execution(consumer) + outcomes = [] + original_pop = turn.lease.host.relay.scope.pop + + def record_pop(*args, **kwargs): + outcomes.append((kwargs.get("output") or {}).get("outcome")) + return original_pop(*args, **kwargs) + + monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: SimpleNamespace(choices=[]), + ) + ) + ) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + assert len(turn.logical_llm_calls) == 1 + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + try: + with pytest.raises(RuntimeError, match="invalid response"): + run("compression") + + assert outcomes == ["failed"] + assert turn.logical_llm_calls == {} + + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + + assert outcomes == ["failed", "success"] + finally: + turn.lease.host.release_managed_execution(consumer) + + +@pytest.mark.asyncio +async def test_async_terminal_auxiliary_failure_closes_logical_call(relay_turn): + _relay, turn = relay_turn + consumer = "test.async-terminal-auxiliary-failure" + turn.lease.host.retain_managed_execution(consumer) + + async def create(**_kwargs): + return SimpleNamespace(choices=[]) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + @auxiliary_client._relay_auxiliary_call_async + async def run(task): + auxiliary_client._set_relay_auxiliary_route( + "anthropic", + "claude-test", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ), + task, + ) + assert len(turn.logical_llm_calls) == 1 + return auxiliary_client._validate_llm_response( + await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ), + task, + ) + + try: + with pytest.raises(RuntimeError, match="invalid response"): + await run("title_generation") + + assert turn.logical_llm_calls == {} + finally: + turn.lease.host.release_managed_execution(consumer) + + +def test_auxiliary_stream_uses_streaming_relay_primitive(monkeypatch): + captured = {} + raw_stream = iter([{"delta": "one"}, {"delta": "two"}]) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace(create=lambda **_kwargs: raw_stream) + ) + ) + + def stream_current(request, stream_factory, **kwargs): + captured.update(kwargs) + return stream_factory(request) + + monkeypatch.setattr(relay_llm, "stream_current", stream_current) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "moa-model", + "chat_completions", + ) + return auxiliary_client._relay_sync_stream( + client, + {"model": "moa-model", "messages": [], "stream": True}, + ) + + assert list(run("moa")) == [{"delta": "one"}, {"delta": "two"}] + assert captured["metadata"]["call_role"] == "auxiliary:moa" + + +def test_partial_auxiliary_stream_failure_closes_before_recovery( + relay_turn, monkeypatch +): + _relay, turn = relay_turn + consumer = "test.partial-auxiliary-stream-failure" + turn.lease.host.retain_managed_execution(consumer) + outcomes = [] + original_pop = turn.lease.host.relay.scope.pop + + def record_pop(*args, **kwargs): + outcomes.append((kwargs.get("output") or {}).get("outcome")) + return original_pop(*args, **kwargs) + + monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop) + + class ProviderError(Exception): + pass + + provider_error = ProviderError("stream failed") + partial_chunk = SimpleNamespace( + model="test-model", + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="partial", tool_calls=None), + finish_reason=None, + ) + ], + usage=None, + ) + + def partial_stream(): + yield partial_chunk + raise provider_error + + stream_client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: partial_stream(), + ) + ) + ) + recovery_client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: SimpleNamespace( + choices=[ + SimpleNamespace(message=SimpleNamespace(content="recovered")) + ] + ), + ) + ) + ) + + @auxiliary_client._relay_auxiliary_call + def start_stream(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._relay_sync_stream( + stream_client, + {"model": "test-model", "messages": [], "stream": True}, + ) + + @auxiliary_client._relay_auxiliary_call + def recover(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + recovery_client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + try: + stream = start_stream("moa") + assert next(stream) is partial_chunk + + with pytest.raises(ProviderError) as caught: + next(stream) + + assert caught.value is provider_error + assert outcomes == ["failed"] + assert turn.logical_llm_calls == {} + + result = recover("moa") + + assert result.choices[0].message.content == "recovered" + assert outcomes == ["failed", "success"] + assert turn.logical_llm_calls == {} + finally: + turn.lease.host.release_managed_execution(consumer) + + +def test_auxiliary_attempt_uses_real_relay_request_intercepts(relay_turn): + relay, turn = relay_turn + consumer = "test.auxiliary-request-intercept" + turn.lease.host.retain_managed_execution(consumer) + captured_requests = [] + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **kwargs: captured_requests.append(kwargs) + or SimpleNamespace( + choices=[ + SimpleNamespace(message=SimpleNamespace(content="ok")) + ] + ), + ) + ) + ) + + def rewrite_request(_name, request, annotated): + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + relay.intercepts.register_llm_request( + "hermes-auxiliary-request", + 1, + False, + rewrite_request, + ) + try: + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + result = run("compression") + finally: + relay.intercepts.deregister_llm_request("hermes-auxiliary-request") + turn.lease.host.release_managed_execution(consumer) + + assert result.choices[0].message.content == "ok" + assert captured_requests[0]["temperature"] == 0.25 + assert turn.logical_llm_calls == {} diff --git a/tests/agent/test_bedrock_interrupt_post_worker.py b/tests/agent/test_bedrock_interrupt_post_worker.py index 0ee5e4fce33d..52c6877d1f48 100644 --- a/tests/agent/test_bedrock_interrupt_post_worker.py +++ b/tests/agent/test_bedrock_interrupt_post_worker.py @@ -86,7 +86,21 @@ def test_bedrock_stream_returns_normally_when_not_interrupted(): agent._interrupt_requested = False resp = SimpleNamespace(choices=[], usage=None, stop_reason="end_turn") - fake_client = SimpleNamespace(converse_stream=lambda **kw: {"stream": []}) + + class ProviderStream: + def __init__(self): + self.closed = False + + def __iter__(self): + return iter(()) + + def close(self): + self.closed = True + + provider_stream = ProviderStream() + fake_client = SimpleNamespace( + converse_stream=lambda **kw: {"stream": provider_stream} + ) with patch("agent.bedrock_adapter._get_bedrock_runtime_client", return_value=fake_client), \ patch("agent.bedrock_adapter.stream_converse_with_callbacks", return_value=resp), \ @@ -97,3 +111,4 @@ def test_bedrock_stream_returns_normally_when_not_interrupted(): api_kwargs = {"__bedrock_region__": "us-east-1", "__bedrock_converse__": True} out = cch.interruptible_streaming_api_call(agent, api_kwargs) assert out is resp + assert provider_stream.closed is True diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py new file mode 100644 index 000000000000..0266d21bd2f2 --- /dev/null +++ b/tests/agent/test_relay_llm.py @@ -0,0 +1,1498 @@ +"""Tests for the core Relay-managed physical LLM attempt adapter.""" + +from __future__ import annotations + +import asyncio +import contextvars +import json +from types import SimpleNamespace + +import pytest + +pytest.importorskip("nemo_relay") + +from agent import relay_llm, relay_runtime + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + lease.host.retain_managed_execution("test.relay_llm") + try: + yield lease.host.relay, turn + finally: + lease.host.release_managed_execution("test.relay_llm") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn): + relay, turn = relay_turn + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + content = {**request.content, "temperature": 0.25} + return relay.LLMRequestInterceptOutcome( + relay.LLMRequest(request.headers, content), + annotated, + ) + + def rewrite_stream(request, next_call): + async def generate(): + upstream = await next_call(request) + async for chunk in upstream: + updated = dict(chunk) + choices = [dict(choice) for choice in updated.get("choices", [])] + if choices: + delta = dict(choices[0].get("delta") or {}) + if delta.get("content"): + delta["content"] = delta["content"].upper() + choices[0]["delta"] = delta + updated["choices"] = choices + yield updated + + return generate() + + def raw_stream(request): + captured_requests.append(request) + return iter([ + SimpleNamespace( + model="test-model", + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="hello", tool_calls=None), + finish_reason=None, + ) + ], + usage=None, + ), + SimpleNamespace( + model="test-model", + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content=None, tool_calls=None), + finish_reason="stop", + ) + ], + usage=None, + ), + ]) + + relay.intercepts.register_llm_request( + "hermes-test-request", + 1, + False, + rewrite_request, + ) + relay.intercepts.register_llm_stream_execution( + "hermes-test-stream", + 1, + rewrite_stream, + ) + try: + stream = relay_llm.stream( + { + "model": "test-model", + "messages": [], + "extra_headers": {"authorization": "Bearer provider-token"}, + }, + raw_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: { + "model": "test-model", + "choices": [ + { + "message": {"role": "assistant", "content": "HELLO"}, + "finish_reason": "stop", + } + ], + }, + metadata={ + "api_mode": "custom", + "api_request_id": "request-1", + "call_role": "primary", + }, + ) + chunks = list(stream) + finally: + relay.intercepts.deregister_llm_stream_execution("hermes-test-stream") + relay.intercepts.deregister_llm_request("hermes-test-request") + + assert captured_requests[0]["temperature"] == 0.25 + assert captured_requests[0]["extra_headers"] == { + "authorization": "Bearer provider-token" + } + assert chunks[0].choices[0].delta.content == "HELLO" + assert stream.output_modified is True + assert turn.logical_llm_calls == {} + + +def test_deferred_stream_preserves_provider_error_and_logical_scope_for_retry( + relay_turn, +): + _relay, turn = relay_turn + + class ProviderError(Exception): + pass + + provider_error = ProviderError("provider failed") + + def failing_stream(_request): + def generate(): + raise provider_error + yield # pragma: no cover + + return generate() + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + failing_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-2", + }, + defer_logical_completion=True, + ) + + with pytest.raises(ProviderError) as caught: + list(stream) + + assert caught.value is provider_error + assert "request-2" in turn.logical_llm_calls + + +def test_non_deferred_partial_stream_close_cancels_logical_call( + relay_turn, + monkeypatch, +): + relay, turn = relay_turn + original_pop = relay.scope.pop + terminal_outputs = [] + + def record_pop(handle, *args, **kwargs): + terminal_outputs.append(kwargs.get("output")) + return original_pop(handle, *args, **kwargs) + + monkeypatch.setattr(relay.scope, "pop", record_pop) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "partial"}, {"delta": "unused"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "partial"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-partial-close", + }, + ) + + assert next(stream) == {"delta": "partial"} + assert "request-partial-close" in turn.logical_llm_calls + + stream.close() + + assert "request-partial-close" not in turn.logical_llm_calls + assert {"outcome": "cancelled"} in terminal_outputs + + +def test_direct_stream_close_reaches_original_provider_resource(monkeypatch): + class ProviderStream: + def __init__(self): + self.closed = False + + def __iter__(self): + return iter([{"delta": "partial"}, {"delta": "unused"}]) + + def close(self): + self.closed = True + + provider_stream = ProviderStream() + monkeypatch.setattr( + relay_runtime, + "resolve_execution_context", + lambda _session_id: (None, None, None), + ) + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: provider_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + ) + + assert next(stream) == {"delta": "partial"} + stream.close() + + assert provider_stream.closed is True + + +def test_anthropic_stream_accumulator_merges_terminal_usage(): + accumulator = relay_llm.AnthropicStreamAccumulator() + accumulator.observe({ + "type": "message_start", + "message": { + "id": "message-1", + "type": "message", + "role": "assistant", + "model": "claude-test", + "usage": { + "input_tokens": 100, + "output_tokens": 1, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 30, + }, + }, + }) + accumulator.observe({ + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 12}, + }) + + response = accumulator.finalize() + + assert response["usage"] == { + "input_tokens": 100, + "output_tokens": 12, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 30, + } + + +def test_anthropic_stream_accumulator_merges_plain_provider_object(): + accumulator = relay_llm.AnthropicStreamAccumulator() + accumulator.observe({ + "type": "message_start", + "message": { + "id": "message-1", + "type": "message", + "role": "assistant", + "model": "claude-test", + "usage": {"input_tokens": 10}, + }, + }) + accumulator.observe({ + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": "hello"}, + }) + + response = accumulator.response( + SimpleNamespace( + id="message-1", + type="message", + role="assistant", + model="claude-test", + content=[], + stop_reason=None, + usage={"input_tokens": 10}, + ) + ) + + assert response.id == "message-1" + assert response.content[0].text == "hello" + assert response.usage.input_tokens == 10 + + +def test_jsonable_does_not_probe_dynamic_attributes(): + class DynamicProviderObject: + def __getattr__(self, name): + raise AssertionError(f"unexpected dynamic attribute lookup: {name}") + + def __str__(self): + return "opaque-provider-object" + + assert relay_llm._jsonable(DynamicProviderObject()) == "opaque-provider-object" + + +def test_non_stream_preserves_raw_provider_response_identity(relay_turn): + _relay, _turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-raw"}, + ) + + assert result is raw_response + + +def test_non_stream_provider_callback_preserves_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar("llm_caller_value", default="default") + caller_value.set("caller") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"caller_value": caller_value.get()}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-context"}, + ) + + assert result == {"caller_value": "caller"} + + +@pytest.mark.asyncio +async def test_async_provider_callback_preserves_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar( + "async_llm_caller_value", + default="default", + ) + caller_value.set("caller") + + async def provider(_request): + await asyncio.sleep(0) + return {"caller_value": caller_value.get()} + + result = await relay_llm.execute_async( + {"model": "test-model", "messages": []}, + provider, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-async-context", + }, + ) + + assert result == {"caller_value": "caller"} + + +def test_stream_provider_callbacks_preserve_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar( + "stream_llm_caller_value", + default="default", + ) + caller_value.set("caller") + observed = [] + + def stream_factory(_request): + observed.append(("factory", caller_value.get())) + + def generate(): + observed.append(("next", caller_value.get())) + yield {"delta": "hello"} + + return generate() + + def on_chunk(_chunk): + observed.append(("chunk", caller_value.get())) + + def finalizer(): + observed.append(("finalizer", caller_value.get())) + return {"content": "hello"} + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + stream_factory, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=finalizer, + on_chunk=on_chunk, + metadata={ + "api_mode": "custom", + "api_request_id": "request-stream-context", + }, + ) + + assert list(stream) == [{"delta": "hello"}] + assert observed == [ + ("factory", "caller"), + ("next", "caller"), + ("chunk", "caller"), + ("finalizer", "caller"), + ] + + +def test_non_stream_does_not_forward_relay_session_headers(relay_turn): + _relay, _turn = relay_turn + captured_requests = [] + + relay_llm.execute( + { + "model": "test-model", + "messages": [], + "extra_headers": {"x-provider-header": "provider-value"}, + }, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-headers"}, + ) + + assert captured_requests[0]["extra_headers"] == { + "x-provider-header": "provider-value" + } + + +def test_non_stream_defers_logical_success_and_reuses_scope_for_retry(relay_turn): + _relay, turn = relay_turn + metadata = {"api_mode": "custom", "api_request_id": "request-retry"} + + first = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "invalid"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata=metadata, + defer_logical_completion=True, + ) + first_handle = turn.logical_llm_calls["request-retry"] + + second = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "valid"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata=metadata, + defer_logical_completion=True, + ) + + assert first == {"content": "invalid"} + assert second == {"content": "valid"} + assert turn.logical_llm_calls == {"request-retry": first_handle} + + relay_llm.complete_logical_call("request-retry", outcome="success") + + assert turn.logical_llm_calls == {} + + +def test_non_stream_result_survives_logical_scope_close_failure( + relay_turn, monkeypatch +): + relay, turn = relay_turn + original_pop = relay.scope.pop + pop_calls = 0 + + def fail_first_pop(*args, **kwargs): + nonlocal pop_calls + pop_calls += 1 + if pop_calls == 1: + raise RuntimeError("simulated logical scope close failure") + return original_pop(*args, **kwargs) + + monkeypatch.setattr(relay.scope, "pop", fail_first_pop) + raw_response = SimpleNamespace(model="test-model", content="raw") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-close"}, + ) + + assert result is raw_response + assert "request-close" in turn.logical_llm_calls + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + assert turn.logical_llm_calls == {} + + +def test_non_stream_returns_provider_response_after_relay_post_processing_failure( + relay_turn, monkeypatch, caplog +): + relay, turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def fail_after_callback(_name, request, callback, **_kwargs): + callback(request) + raise RuntimeError("simulated Relay post-processing failure") + + monkeypatch.setattr(relay.llm, "execute", fail_after_callback) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-post-failure", + }, + ) + + assert result is raw_response + assert turn.logical_llm_calls == {} + assert "returning the provider response" in caplog.text + + +def test_non_stream_does_not_swallow_interrupt_after_provider_success( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + async def interrupt_after_callback(_name, request, callback, **_kwargs): + callback(request) + raise KeyboardInterrupt + + monkeypatch.setattr(relay.llm, "execute", interrupt_after_callback) + + with pytest.raises(KeyboardInterrupt): + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "already returned"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-post-interrupt", + }, + ) + + assert "request-post-interrupt" in turn.logical_llm_calls + relay_llm.complete_logical_call("request-post-interrupt", outcome="cancelled") + + +@pytest.mark.asyncio +async def test_async_non_stream_preserves_raw_provider_response_identity(relay_turn): + _relay, _turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def provider(_request): + return raw_response + + result = await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async"}, + ) + + assert result is raw_response + + +@pytest.mark.asyncio +async def test_async_non_stream_returns_provider_response_after_relay_failure( + relay_turn, monkeypatch, caplog +): + relay, turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def provider(_request): + return raw_response + + async def fail_after_callback(_name, request, callback, **_kwargs): + await callback(request) + raise RuntimeError("simulated Relay post-processing failure") + + monkeypatch.setattr(relay.llm, "execute", fail_after_callback) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + result = await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-async-post-failure", + }, + ) + + assert result is raw_response + assert turn.logical_llm_calls == {} + assert "returning the provider response" in caplog.text + + +@pytest.mark.asyncio +async def test_async_non_stream_does_not_swallow_cancellation_after_provider_success( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + async def provider(_request): + return {"content": "already returned"} + + async def cancel_after_callback(_name, request, callback, **_kwargs): + await callback(request) + raise asyncio.CancelledError + + monkeypatch.setattr(relay.llm, "execute", cancel_after_callback) + + with pytest.raises(asyncio.CancelledError): + await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-async-post-cancel", + }, + ) + + assert "request-async-post-cancel" in turn.logical_llm_calls + relay_llm.complete_logical_call( + "request-async-post-cancel", + outcome="cancelled", + ) + + +@pytest.mark.asyncio +async def test_async_non_stream_defers_logical_success_for_validation(relay_turn): + _relay, turn = relay_turn + + async def provider(_request): + return {"content": "pending-validation"} + + await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async-defer"}, + defer_logical_completion=True, + ) + + assert "request-async-defer" in turn.logical_llm_calls + + relay_llm.complete_logical_call("request-async-defer", outcome="success") + + assert turn.logical_llm_calls == {} + + +def test_stream_finishes_after_relay_post_processing_failure( + relay_turn, monkeypatch, caplog +): + relay, turn = relay_turn + + async def fail_after_stream( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + async for chunk in upstream: + observe_chunk(chunk) + yield chunk + finalizer() + raise RuntimeError("simulated Relay post-processing failure") + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", fail_after_stream) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "complete"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-stream-post-failure", + }, + ) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + chunks = list(stream) + + assert chunks == [{"delta": "complete"}] + assert turn.logical_llm_calls == {} + assert "preserving the provider result" in caplog.text + + +def test_stream_flushes_buffered_provider_chunks_after_relay_failure( + relay_turn, monkeypatch +): + relay, turn = relay_turn + raw_chunks = [{"delta": "first"}, {"delta": "second"}] + + async def fail_with_buffered_chunk( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + first = await anext(upstream) + observe_chunk(first) + yield first + second = await anext(upstream) + observe_chunk(second) + with pytest.raises(StopAsyncIteration): + await anext(upstream) + finalizer() + raise RuntimeError("simulated buffered Relay failure") + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", fail_with_buffered_chunk) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter(raw_chunks), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-buffered-failure", + }, + ) + + assert list(stream) == raw_chunks + assert turn.logical_llm_calls == {} + + +def test_stream_constructor_flushes_provider_chunks_after_relay_failure( + relay_turn, monkeypatch +): + relay, turn = relay_turn + raw_chunks = [{"delta": "first"}, {"delta": "second"}] + + async def fail_during_stream_setup( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + upstream = callback(request) + async for chunk in upstream: + observe_chunk(chunk) + finalizer() + raise RuntimeError("simulated Relay setup failure") + + monkeypatch.setattr(relay.llm, "stream_execute", fail_during_stream_setup) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter(raw_chunks), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-setup-failure", + }, + ) + + assert list(stream) == raw_chunks + assert turn.logical_llm_calls == {} + + +def test_stream_does_not_swallow_interrupt_after_provider_success( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + async def interrupt_after_stream( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + async for chunk in upstream: + observe_chunk(chunk) + yield chunk + finalizer() + raise KeyboardInterrupt + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", interrupt_after_stream) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "complete"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-stream-post-interrupt", + }, + ) + + assert next(stream) == {"delta": "complete"} + with pytest.raises(KeyboardInterrupt): + next(stream) + assert turn.logical_llm_calls == {} + + +def test_stream_does_not_swallow_hermes_finalizer_failure(relay_turn, monkeypatch): + relay, _turn = relay_turn + finalizer_error = RuntimeError("Hermes finalizer failed") + + def fail_finalizer(): + raise finalizer_error + + async def execute_stream( + _name, + request, + callback, + _observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + async for chunk in upstream: + yield chunk + finalizer() + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", execute_stream) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "complete"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=fail_finalizer, + metadata={ + "api_mode": "custom", + "api_request_id": "request-finalizer-failure", + }, + ) + + with pytest.raises(Exception) as caught: + list(stream) + + assert caught.value is finalizer_error + + +def test_stream_defers_logical_success_for_response_validation(relay_turn): + _relay, turn = relay_turn + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "pending-validation"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "pending-validation"}, + metadata={"api_mode": "custom", "api_request_id": "request-stream-defer"}, + defer_logical_completion=True, + ) + + assert list(stream) == [{"delta": "pending-validation"}] + assert stream.output_modified is False + assert "request-stream-defer" in turn.logical_llm_calls + + relay_llm.complete_logical_call("request-stream-defer", outcome="success") + + assert turn.logical_llm_calls == {} + + +def test_current_attempt_bypasses_relay_without_an_active_turn(monkeypatch): + monkeypatch.setattr(relay_runtime, "current_turn", lambda: None) + request = {"model": "test-model", "messages": []} + + result = relay_llm.execute_current( + request, + lambda value: value, + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_non_stream_bypasses_relay_without_an_active_consumer(relay_turn, monkeypatch): + relay, turn = relay_turn + turn.lease.host.release_managed_execution("test.relay_llm") + request = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]} + + monkeypatch.setattr( + relay.llm, + "execute", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not manage the provider call") + ), + ) + + result = relay_llm.execute( + request, + lambda value: value, + session_id="session-1", + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_stream_bypasses_relay_without_an_active_consumer(relay_turn, monkeypatch): + relay, turn = relay_turn + turn.lease.host.release_managed_execution("test.relay_llm") + request = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]} + observed = [] + + monkeypatch.setattr( + relay.llm, + "stream_execute", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not manage the provider stream") + ), + ) + + stream = relay_llm.stream( + request, + lambda value: (observed.append(value), iter([{"delta": "ok"}]))[1], + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + ) + + assert list(stream) == [{"delta": "ok"}] + assert observed == [request] + + +def test_bypassed_stream_still_honors_chunk_acceptance(relay_turn): + _relay, turn = relay_turn + turn.lease.host.release_managed_execution("test.relay_llm") + provider_closed = [] + + def provider_stream(_request): + try: + yield {"delta": "accepted"} + yield {"delta": "rejected"} + yield {"delta": "unreachable"} + finally: + provider_closed.append(True) + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + provider_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + accept_chunk=lambda chunk: chunk["delta"] != "rejected", + ) + + assert list(stream) == [{"delta": "accepted"}] + assert provider_closed == [True] + + +def test_anthropic_codec_preserves_tool_history_and_cached_system_blocks(relay_turn): + _relay, _turn = relay_turn + request = { + "model": "claude-sonnet-4-5", + "max_tokens": 512, + "system": [ + { + "type": "text", + "text": "You are Hermes.", + "cache_control": {"type": "ephemeral"}, + } + ], + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Run pwd"}]}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "terminal", + "input": {"command": "pwd"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": "/tmp/worktree"}], + } + ], + }, + ], + } + original_wire = json.dumps(request, ensure_ascii=False, separators=(",", ":")) + observed_body_wire = "" + + def provider(final_request): + nonlocal observed_body_wire + provider_body = { + key: value for key, value in final_request.items() if key != "extra_headers" + } + observed_body_wire = json.dumps( + provider_body, + ensure_ascii=False, + separators=(",", ":"), + ) + return { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + + relay_llm.execute( + request, + provider, + session_id="session-1", + name="anthropic", + model_name="claude-sonnet-4-5", + metadata={ + "api_mode": "anthropic_messages", + "api_request_id": "request-anthropic", + }, + ) + + assert observed_body_wire == original_wire + + +def test_current_attempt_bypasses_a_closed_turn_from_a_copied_context( + relay_turn, + monkeypatch, +): + _relay, turn = relay_turn + stale_context = contextvars.copy_context() + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + request = {"model": "test-model", "messages": []} + + def fail_execute(*_args, **_kwargs): + raise AssertionError("a closed turn must not manage later provider work") + + monkeypatch.setattr(relay_llm, "execute", fail_execute) + + result = stale_context.run( + relay_llm.execute_current, + request, + lambda value: value, + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_non_stream_returns_post_execution_interceptor_result(relay_turn, monkeypatch): + relay, _turn = relay_turn + + async def post_execute(_name, request, callback, **_kwargs): + response = callback(request) + return {**response, "post_interceptor": True} + + monkeypatch.setattr(relay.llm, "execute", post_execute) + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "raw"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-post"}, + ) + + assert result.content == "raw" + assert result.post_interceptor is True + + +@pytest.mark.asyncio +async def test_async_non_stream_returns_namespaced_interceptor_result( + relay_turn, + monkeypatch, +): + relay, _turn = relay_turn + + async def post_execute(_name, request, callback, **_kwargs): + response = await callback(request) + return { + **response, + "post_interceptor": True, + "usage": {"input_tokens": 10}, + } + + monkeypatch.setattr(relay.llm, "execute", post_execute) + + async def provider(_request): + return {"content": "raw"} + + result = await relay_llm.execute_async( + {"model": "test-model", "messages": []}, + provider, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async-post"}, + ) + + assert result.content == "raw" + assert result.post_interceptor is True + assert result.usage.input_tokens == 10 + + +def test_non_stream_preserves_provider_error_from_relay_wrapper_suffix( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + class ProviderError(Exception): + pass + + provider_error = ProviderError("provider failed") + + async def wrapping_execute(_name, request, callback, **_kwargs): + try: + return callback(request) + except Exception as exc: + raise RuntimeError( + f"internal error: {type(exc).__name__}: {exc} (retried 3x)" + ) from None + + monkeypatch.setattr(relay.llm, "execute", wrapping_execute) + + with pytest.raises(ProviderError) as caught: + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: (_ for _ in ()).throw(provider_error), + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-error"}, + ) + + assert caught.value is provider_error + assert "request-error" in turn.logical_llm_calls + + +def test_non_stream_does_not_mask_relay_error_after_callback_failure( + relay_turn, monkeypatch +): + relay, _turn = relay_turn + provider_error = RuntimeError("provider failed") + relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") + + async def translating_execute(_name, request, callback, **_kwargs): + try: + callback(request) + except Exception: + raise relay_error + + monkeypatch.setattr(relay.llm, "execute", translating_execute) + + with pytest.raises(RuntimeError) as caught: + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: (_ for _ in ()).throw(provider_error), + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-policy"}, + ) + + assert caught.value is relay_error + + +def test_chat_codec_preserves_provider_message_extensions_after_rewrite(relay_turn): + relay, _turn = relay_turn + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + def provider(request): + captured_requests.append(request) + return { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + } + + relay.intercepts.register_llm_request( + "hermes-provider-extension-request", + 1, + False, + rewrite_request, + ) + try: + relay_llm.execute( + { + "model": "test-model", + "messages": [ + { + "role": "assistant", + "content": "", + "reasoning_content": "provider scratchpad", + } + ], + }, + provider, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-3", + }, + ) + finally: + relay.intercepts.deregister_llm_request( + "hermes-provider-extension-request" + ) + + assert captured_requests[0]["temperature"] == 0.25 + assert captured_requests[0]["messages"][0]["reasoning_content"] == ( + "provider scratchpad" + ) + + +def test_request_rewrite_preserves_unmodified_provider_objects(relay_turn): + relay, _turn = relay_turn + timeout = object() + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + relay.intercepts.register_llm_request( + "hermes-provider-object-request", + 1, + False, + rewrite_request, + ) + try: + relay_llm.execute( + {"model": "test-model", "messages": [], "timeout": timeout}, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-provider-object", + }, + ) + finally: + relay.intercepts.deregister_llm_request( + "hermes-provider-object-request" + ) + + assert captured_requests[0]["timeout"] is timeout + assert captured_requests[0]["temperature"] == 0.25 + + +def test_request_rewrite_preserves_fields_dropped_by_codec(relay_turn, monkeypatch): + relay, _turn = relay_turn + captured_requests = [] + vendor_body = { + "routing": {"provider": "nim", "region": "us-west-2"}, + "trace_vendor_request": False, + } + + async def lossy_execute(_name, request, callback, **_kwargs): + content = { + key: value + for key, value in request.content.items() + if key != "extra_body" + } + content["temperature"] = 0.25 + return callback(relay.LLMRequest(request.headers, content)) + + monkeypatch.setattr(relay.llm, "execute", lossy_execute) + monkeypatch.setattr( + relay_llm, + "_codec_round_trip_request_body", + lambda *_args, relay_request_body, **_kwargs: { + key: value + for key, value in relay_request_body.items() + if key != "extra_body" + }, + ) + + relay_llm.execute( + { + "model": "test-model", + "messages": [], + "temperature": 0.0, + "extra_body": vendor_body, + }, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-lossy-codec", + }, + ) + + assert captured_requests == [ + { + "model": "test-model", + "messages": [], + "temperature": 0.25, + "extra_body": vendor_body, + } + ] + + +def test_request_rewrite_is_ignored_when_codec_baseline_fails( + relay_turn, monkeypatch +): + relay, _turn = relay_turn + captured_requests = [] + original = { + "model": "test-model", + "messages": [], + "temperature": 0.0, + "extra_body": {"routing": {"provider": "nim"}}, + } + + async def lossy_execute(_name, request, callback, **_kwargs): + rewritten = { + key: value + for key, value in request.content.items() + if key != "extra_body" + } + rewritten["temperature"] = 0.25 + return callback(relay.LLMRequest(request.headers, rewritten)) + + monkeypatch.setattr(relay.llm, "execute", lossy_execute) + monkeypatch.setattr( + relay_llm, + "_codec_round_trip_request_body", + lambda *_args, **_kwargs: None, + ) + + relay_llm.execute( + original, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-codec-failure", + }, + ) + + assert captured_requests == [original] + + +def test_codec_baseline_failure_is_explicit(relay_turn, monkeypatch, caplog): + relay, _turn = relay_turn + request_body = {"model": "test-model", "messages": []} + request = relay.LLMRequest({}, request_body) + + class FailingCodec: + def decode(self, _request): + raise RuntimeError("simulated codec failure") + + monkeypatch.setattr(relay_llm, "_codec", lambda *_args, **_kwargs: FailingCodec()) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + baseline = relay_llm._codec_round_trip_request_body( + relay, + request, + relay_request_body=request_body, + metadata={"api_mode": "chat_completions"}, + ) + + assert baseline is None + assert "ignoring request rewrites" in caplog.text + + +def test_request_rewrite_can_remove_codec_represented_field(relay_turn, monkeypatch): + relay, _turn = relay_turn + captured_requests = [] + + async def remove_temperature(_name, request, callback, **_kwargs): + content = dict(request.content) + content.pop("temperature") + return callback(relay.LLMRequest(request.headers, content)) + + monkeypatch.setattr(relay.llm, "execute", remove_temperature) + + relay_llm.execute( + { + "model": "test-model", + "messages": [], + "temperature": 0.25, + "extra_body": {"routing": {"provider": "nim"}}, + }, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-remove-field", + }, + ) + + assert captured_requests == [ + { + "model": "test-model", + "messages": [], + "extra_body": {"routing": {"provider": "nim"}}, + } + ] diff --git a/tests/agent/test_relay_tools.py b/tests/agent/test_relay_tools.py new file mode 100644 index 000000000000..8f605dfd4d9d --- /dev/null +++ b/tests/agent/test_relay_tools.py @@ -0,0 +1,262 @@ +"""Tests for the core Relay-managed Hermes tool adapter.""" + +from __future__ import annotations + +import contextvars +import json + +import pytest + +pytest.importorskip("nemo_relay") + +from agent import relay_runtime, relay_tools + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + lease.host.retain_managed_execution("test.relay_tools") + try: + yield lease.host.relay + finally: + lease.host.release_managed_execution("test.relay_tools") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_tool_adapter_bypasses_relay_without_an_active_consumer( + relay_turn, monkeypatch +): + relay = relay_turn + runtime = relay_runtime.get_runtime() + assert runtime is not None + runtime.release_managed_execution("test.relay_tools") + args = {"command": "pwd"} + + monkeypatch.setattr( + relay.tools, + "execute", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not manage the tool call") + ), + ) + + result, final_args = relay_tools.execute( + "terminal", + args, + lambda value: value, + session_id="session-1", + ) + + assert result is args + assert final_args is args + + +def test_tool_request_intercepts_bypass_relay_without_an_active_consumer( + relay_turn, monkeypatch +): + relay = relay_turn + runtime = relay_runtime.get_runtime() + assert runtime is not None + runtime.release_managed_execution("test.relay_tools") + args = {"command": "pwd"} + + monkeypatch.setattr( + relay.tools, + "request_intercepts", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not run tool request intercepts") + ), + ) + + final_args = runtime.apply_tool_request_intercepts( + session_id="session-1", + tool_name="terminal", + args=args, + ) + + assert final_args is args + + +def test_request_rewrite_reaches_authorized_callback_once(relay_turn): + relay = relay_turn + callback_args = [] + + def rewrite_request(_name, args): + return {**args, "path": "/approved/path"} + + async def wrap_execution(_name, args, next_call): + result = await next_call(args) + return relay.ToolExecutionInterceptOutcome({**result, "wrapped": True}) + + relay.intercepts.register_tool_request( + "hermes-test-tool-request", 1, False, rewrite_request + ) + relay.intercepts.register_tool_execution( + "hermes-test-tool-execution", 1, wrap_execution + ) + try: + result, observed_args = relay_tools.execute( + "write_file", + {"path": "/original/path"}, + lambda args: callback_args.append(args) or {"ok": True}, + session_id="session-1", + metadata={"tool_call_id": "call-1"}, + ) + finally: + relay.intercepts.deregister_tool_execution("hermes-test-tool-execution") + relay.intercepts.deregister_tool_request("hermes-test-tool-request") + + assert callback_args == [{"path": "/approved/path"}] + assert observed_args == {"path": "/approved/path"} + assert isinstance(result, str) + assert json.loads(result) == {"ok": True, "wrapped": True} + + +def test_authorized_tool_callback_preserves_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar("tool_caller_value", default="default") + caller_value.set("caller") + + result, _observed_args = relay_tools.execute( + "terminal", + {"command": "true"}, + lambda _args: caller_value.get(), + session_id="session-1", + ) + + assert result == "caller" + + +def test_provider_error_identity_is_preserved(relay_turn): + del relay_turn + + class ToolError(Exception): + pass + + tool_error = ToolError("dispatch failed") + + def fail(_args): + raise tool_error + + with pytest.raises(ToolError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + fail, + session_id="session-1", + ) + + assert caught.value is tool_error + + +def test_tool_error_is_preserved_from_relay_wrapper_suffix(relay_turn, monkeypatch): + relay = relay_turn + + class ToolError(Exception): + pass + + tool_error = ToolError("dispatch failed") + + async def wrapping_execute(_name, args, callback, **_kwargs): + try: + return callback(args) + except Exception as exc: + raise RuntimeError( + f"internal error: {type(exc).__name__}: {exc} (worker trace)" + ) from None + + monkeypatch.setattr(relay.tools, "execute", wrapping_execute) + + with pytest.raises(ToolError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + lambda _args: (_ for _ in ()).throw(tool_error), + session_id="session-1", + ) + + assert caught.value is tool_error + + +def test_tool_adapter_does_not_mask_relay_error_after_callback_failure( + relay_turn, monkeypatch +): + relay = relay_turn + tool_error = RuntimeError("dispatch failed") + relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") + + async def translating_execute(_name, args, callback, **_kwargs): + try: + callback(args) + except Exception: + raise relay_error + + monkeypatch.setattr(relay.tools, "execute", translating_execute) + + with pytest.raises(RuntimeError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + lambda _args: (_ for _ in ()).throw(tool_error), + session_id="session-1", + ) + + assert caught.value is relay_error + + +def test_tool_adapter_returns_dispatch_result_after_relay_post_processing_failure( + relay_turn, monkeypatch, caplog +): + relay = relay_turn + raw_result = '{"ok":true}' + + async def fail_after_callback(_name, args, callback, **_kwargs): + callback(args) + raise RuntimeError("simulated Relay post-processing failure") + + monkeypatch.setattr(relay.tools, "execute", fail_after_callback) + + with caplog.at_level("WARNING", logger="agent.relay_tools"): + result, observed_args = relay_tools.execute( + "terminal", + {"command": "pwd"}, + lambda _args: raw_result, + session_id="session-1", + ) + + assert result is raw_result + assert observed_args == {"command": "pwd"} + assert "returning the Hermes tool result" in caplog.text + + +def test_tool_adapter_does_not_swallow_interrupt_after_dispatch_success( + relay_turn, monkeypatch +): + relay = relay_turn + + async def interrupt_after_callback(_name, args, callback, **_kwargs): + callback(args) + raise KeyboardInterrupt + + monkeypatch.setattr(relay.tools, "execute", interrupt_after_callback) + + with pytest.raises(KeyboardInterrupt): + relay_tools.execute( + "terminal", + {"command": "pwd"}, + lambda _args: '{"ok":true}', + session_id="session-1", + ) diff --git a/tests/cli/test_session_boundary_hooks.py b/tests/cli/test_session_boundary_hooks.py index 52c64c01c2d8..0a78ad1f88c5 100644 --- a/tests/cli/test_session_boundary_hooks.py +++ b/tests/cli/test_session_boundary_hooks.py @@ -10,8 +10,12 @@ def test_session_hooks_in_valid_hooks(): assert "on_session_reset" in VALID_HOOKS -@patch("hermes_cli.plugins.invoke_hook") -def test_session_finalize_on_reset(mock_invoke_hook): +# These tests pin CLI ownership of the finalize request. The end-to-end +# built-in/core/plugin dispatch order is exercised by +# tests/hermes_cli/test_lifecycle.py::test_finalize_session_closes_core_before_plugin_export. +@patch("hermes_cli.lifecycle.invoke_hook") +@patch("hermes_cli.lifecycle.finalize_session") +def test_session_finalize_on_reset(mock_finalize_session, mock_invoke_hook): """Verify on_session_finalize fires when /new or /reset is used.""" cli = HermesCLI() cli.agent = MagicMock() @@ -22,10 +26,10 @@ def test_session_finalize_on_reset(mock_invoke_hook): # Check if on_session_finalize was called for the old session assert any( - c.args == ("on_session_finalize",) + not c.args and c.kwargs["session_id"] == "test-session-id" and c.kwargs["platform"] == "cli" - for c in mock_invoke_hook.call_args_list + for c in mock_finalize_session.call_args_list ) # Check if on_session_reset was called for the new session assert any( @@ -36,8 +40,8 @@ def test_session_finalize_on_reset(mock_invoke_hook): ) -@patch("hermes_cli.plugins.invoke_hook") -def test_session_finalize_on_cleanup(mock_invoke_hook): +@patch("hermes_cli.lifecycle.finalize_session") +def test_session_finalize_on_cleanup(mock_finalize_session): """Verify on_session_finalize fires during CLI exit cleanup.""" import cli as cli_mod @@ -49,15 +53,15 @@ def test_session_finalize_on_cleanup(mock_invoke_hook): cli_mod._run_cleanup() assert any( - c.args == ("on_session_finalize",) + not c.args and c.kwargs["session_id"] == "cleanup-session-id" and c.kwargs["platform"] == "cli" and c.kwargs["reason"] == "shutdown" - for c in mock_invoke_hook.call_args_list + for c in mock_finalize_session.call_args_list ) -@patch("hermes_cli.plugins.invoke_hook") +@patch("hermes_cli.lifecycle.invoke_hook") def test_interrupted_session_end_helper_emits_observer_shape(mock_invoke_hook): """Verify quiet single-query interruption emits a correlated session end.""" import cli as cli_mod diff --git a/tests/hermes_cli/test_lifecycle.py b/tests/hermes_cli/test_lifecycle.py new file mode 100644 index 000000000000..999d6057b3bf --- /dev/null +++ b/tests/hermes_cli/test_lifecycle.py @@ -0,0 +1,60 @@ +from types import SimpleNamespace + +from agent import relay_runtime +from hermes_cli import lifecycle, observability, plugins + + +def test_invoke_hook_notifies_builtin_observers_before_plugins(monkeypatch): + calls = [] + manager = SimpleNamespace( + invoke_hook=lambda name, **kwargs: calls.append(("plugin", name, kwargs)) or ["ok"] + ) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda name, **kwargs: calls.append(("builtin", name, kwargs)), + ) + monkeypatch.setattr(plugins, "invoke_hook", manager.invoke_hook) + + result = lifecycle.invoke_hook("on_session_start", session_id="session-1") + + assert result == ["ok"] + assert [call[0] for call in calls] == ["builtin", "plugin"] + + +def test_finalize_session_closes_core_before_plugin_export(monkeypatch): + calls = [] + manager = SimpleNamespace( + invoke_hook=lambda name, **kwargs: calls.append(("plugin", name, kwargs)) or [] + ) + coordinator = SimpleNamespace( + finalize_conversation=lambda **kwargs: calls.append(("core", kwargs)) + ) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda name, **kwargs: calls.append(("builtin", name, kwargs)), + ) + monkeypatch.setattr(plugins, "invoke_hook", manager.invoke_hook) + monkeypatch.setattr(relay_runtime, "SESSION_COORDINATOR", coordinator) + monkeypatch.setattr(relay_runtime, "current_profile_key", lambda: "profile-1") + + lifecycle.finalize_session(session_id="session-1", platform="cli") + + assert [call[0] for call in calls] == ["builtin", "core", "plugin"] + assert calls[1][1] == { + "profile_key": "profile-1", + "session_id": "session-1", + } + + +def test_plugin_only_dispatch_does_not_reenter_builtin_observers(monkeypatch): + manager = SimpleNamespace(invoke_hook=lambda name, **kwargs: [name, kwargs]) + monkeypatch.setattr(plugins, "get_plugin_manager", lambda: manager) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unexpected")), + ) + + assert plugins.invoke_hook("custom", value=1) == ["custom", {"value": 1}] diff --git a/tests/hermes_cli/test_plugins.py b/tests/hermes_cli/test_plugins.py index ee9746a34de9..2a6de58ac702 100644 --- a/tests/hermes_cli/test_plugins.py +++ b/tests/hermes_cli/test_plugins.py @@ -189,6 +189,32 @@ class TestPluginDiscovery: assert result.changed is True assert result.trace == [{"source": "same-payload"}] + def test_tool_request_middleware_hides_internal_skip_relay_flag( + self, + monkeypatch, + ): + observed = [] + + def middleware(**kwargs): + observed.append(kwargs) + return {"args": kwargs["args"]} + + manager = types.SimpleNamespace( + _middleware={"tool_request": [middleware]}, + invoke_middleware=lambda kind, **kwargs: [middleware(**kwargs)], + ) + monkeypatch.setattr("hermes_cli.plugins.get_plugin_manager", lambda: manager) + + apply_tool_request_middleware( + "read_file", + {"path": "README.md"}, + session_id="", + skip_relay=True, + ) + + assert len(observed) == 1 + assert "skip_relay" not in observed[0] + def test_execution_middleware_post_next_call_error_does_not_retry(self, monkeypatch): calls = [] diff --git a/tests/hermes_cli/test_relay_shared_metrics.py b/tests/hermes_cli/test_relay_shared_metrics.py new file mode 100644 index 000000000000..058e07db7bc7 --- /dev/null +++ b/tests/hermes_cli/test_relay_shared_metrics.py @@ -0,0 +1,839 @@ +"""Focused tests for the Hermes shared-metrics durable store.""" + +from __future__ import annotations + +import json +import multiprocessing as mp +import os +import sqlite3 +import stat +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from hermes_cli.observability.shared_metrics import SharedMetricsStore +from hermes_cli.observability.shared_metrics_contract import ( + COUNT_BUCKETS, + DURATION_BUCKETS, + EXECUTION_SURFACES, + MODEL_FAMILIES, + MODEL_LOCALITIES, + MODEL_OUTCOMES, + PRIMARY_MODEL_CALL_ROLE, + PROVIDER_FAMILIES, + TASK_END_REASONS, + TASK_ENTRYPOINTS, + TASK_OUTCOMES, + TASK_TERMINATIONS, + count_bucket, + duration_bucket, + execution_surface, + model_call_outcome, + model_call_dimensions, + model_family, + model_locality, + provider_family, + task_counter, + task_start_fields, + task_terminal_fields, + task_terminal_state, +) + + +SCHEMA_PATH = ( + Path(__file__).resolve().parents[2] + / "hermes_cli" + / "observability" + / "schemas" + / "hermes.shared_metrics.v1.schema.json" +) + + +def _schema_validator(): + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(schema) + return jsonschema.Draft202012Validator( + schema, + format_checker=jsonschema.FormatChecker(), + ) + + +def _package_dimension_schema() -> dict[str, object]: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + return schema["$defs"]["model_call_counter"]["properties"]["dimensions"] + + +def _task_dimension_schema(kind: str) -> dict[str, object]: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + return schema["$defs"][kind]["properties"]["dimensions"] + + +def _dimensions() -> dict[str, str]: + return { + "call_role": PRIMARY_MODEL_CALL_ROLE, + "locality": "remote", + "model_family": "claude", + "outcome": "success", + "provider_family": "direct", + } + + +def _record_model_calls_in_process( + database_path: str, + outbox_directory: str, + count: int, + start_barrier: Any | None = None, +) -> None: + if start_barrier is not None: + start_barrier.wait() + store = SharedMetricsStore(Path(database_path), Path(outbox_directory)) + for _ in range(count): + store.record_model_call(_dimensions(), "test-version") + + +def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), "test-version") + + first_paths = store.create_and_export_package() + + assert len(first_paths) == 1 + first_package = json.loads(first_paths[0].read_text(encoding="utf-8")) + _schema_validator().validate(first_package) + uuid.UUID(first_package["package_id"]) + uuid.UUID(first_package["install_id"]) + assert first_package["schema_version"] == "hermes.shared_metrics.v1" + assert first_package["resource"] == {"hermes_version": "test-version"} + assert first_package["metrics"] == [ + { + "name": "hermes.model_call.count", + "type": "counter", + "dimensions": _dimensions(), + "value": 2, + } + ] + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 2 + assert restarted.counter_snapshot()[0]["packaged_value"] == 2 + assert restarted.create_and_export_package() == [] + assert len(list(outbox_directory.glob("*.json"))) == 1 + + restarted.record_model_call(_dimensions(), "test-version") + second_paths = restarted.create_and_export_package() + + assert len(second_paths) == 1 + second_package = json.loads(second_paths[0].read_text(encoding="utf-8")) + assert second_package["package_id"] != first_package["package_id"] + assert second_package["install_id"] == first_package["install_id"] + assert second_package["metrics"][0]["value"] == 1 + assert restarted.counter_snapshot()[0]["value"] == 3 + assert restarted.counter_snapshot()[0]["packaged_value"] == 3 + + +def test_package_schema_matches_the_model_call_contract(): + properties = _package_dimension_schema()["properties"] + + assert properties["call_role"] == {"const": PRIMARY_MODEL_CALL_ROLE} + assert set(properties["locality"]["enum"]) == MODEL_LOCALITIES + assert set(properties["model_family"]["enum"]) == MODEL_FAMILIES + assert set(properties["outcome"]["enum"]) == MODEL_OUTCOMES + assert set(properties["provider_family"]["enum"]) == PROVIDER_FAMILIES + + +def test_package_schema_matches_the_task_contract(): + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + start = _task_dimension_schema("task_started_counter")["properties"] + terminal = _task_dimension_schema("task_finished_counter")["properties"] + + assert set(schema["$defs"]["execution_surface"]["enum"]) == EXECUTION_SURFACES + assert set(schema["$defs"]["task_entrypoint"]["enum"]) == TASK_ENTRYPOINTS + assert set(schema["$defs"]["duration_bucket"]["enum"]) == DURATION_BUCKETS + assert set(schema["$defs"]["count_bucket"]["enum"]) == COUNT_BUCKETS + assert start["entrypoint"] == {"$ref": "#/$defs/task_entrypoint"} + assert set(terminal["end_reason"]["enum"]) == TASK_END_REASONS + assert set(terminal["outcome"]["enum"]) == TASK_OUTCOMES + assert set(terminal["termination"]["enum"]) == TASK_TERMINATIONS + + +@pytest.mark.parametrize( + ("provider", "expected"), + [ + ("", "unknown"), + ("not-a-hermes-provider", "unknown"), + ("custom", "custom"), + ("custom-local", "custom"), + ("custom:private-endpoint", "custom"), + ("lmstudio", "local"), + ("lm_studio", "local"), + ("ollama", "local"), + ("nous", "aggregator"), + ("openrouter", "aggregator"), + ("kilo", "aggregator"), + ("copilot-acp", "aggregator"), + ("huggingface", "aggregator"), + ("novita", "aggregator"), + ("anthropic", "direct"), + ("google", "direct"), + ("openai-api", "direct"), + ], +) +def test_provider_family_uses_bounded_product_categories(provider, expected): + assert provider_family({"provider": provider}) == expected + + +def test_provider_family_does_not_resolve_live_provider_metadata(monkeypatch): + def fail_live_lookup(_provider): + raise AssertionError("telemetry must not refresh provider metadata") + + monkeypatch.setattr("hermes_cli.providers.get_provider", fail_live_lookup) + assert provider_family({"provider": "anthropic"}) == "direct" + + +def test_locality_uses_the_endpoint_only_for_local_classification(): + kwargs = { + "provider": "custom", + "base_url": "http://127.0.0.1:11434/v1", + } + + assert provider_family(kwargs) == "custom" + assert model_locality(kwargs) == "local" + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("google/gemma-3", "gemma"), + ("x-ai/grok-4", "grok"), + ("minimax/minimax-m2.5", "minimax"), + ("xiaomi/mimo-v2", "mimo"), + ("amazon/nova-pro", "nova"), + ("stepfun/step-3.5", "step"), + ("arcee-ai/trinity-large", "trinity"), + ], +) +def test_model_family_covers_families_evidenced_by_the_hermes_catalog(model, expected): + assert model_family({"model": model}) == expected + + +@pytest.mark.parametrize( + "model", + [ + "private-gptish-model", + "innovation-private", + "mimosa-private", + "stepstone-private", + "supernova-private", + ], +) +def test_model_family_requires_identifier_boundaries(model): + assert model_family({"model": model}) == "unknown" + + +def test_model_family_accepts_only_allowlisted_declared_metadata(): + assert model_family({"model": "private", "model_family": "qwen"}) == "qwen" + assert model_family({"model": "private", "model_family": "private"}) == "unknown" + + +def test_model_family_prefers_the_provider_reported_terminal_model(): + assert ( + model_family({"model": "gpt-5", "response_model": "claude-sonnet"}) == "claude" + ) + + +@pytest.mark.parametrize( + ("platform", "expected"), + [ + ("", "unknown"), + ("cli", "cli"), + ("api_server", "api"), + ("cron", "scheduled_task"), + ("whatsapp_cloud", "gateway"), + ("private-surface", "other"), + ], +) +def test_execution_surface_uses_the_hermes_platform_registry(platform, expected): + assert execution_surface({"platform": platform}) == expected + + +@pytest.mark.parametrize( + ("platform", "expected"), + [ + ("cli", "interactive"), + ("tui", "interactive"), + ("whatsapp_cloud", "gateway_message"), + ("cron", "scheduled_task"), + ("api_server", "api"), + ("private-surface", "other"), + ], +) +def test_task_start_fields_use_bounded_surface_and_entrypoint(platform, expected): + fields = task_start_fields({"platform": platform}) + + assert fields["entrypoint"] == expected + assert fields["execution_surface"] in EXECUTION_SURFACES + + +def test_task_start_fields_identify_delegated_work_without_exporting_parent_id(): + fields = task_start_fields({ + "platform": "cli", + "parent_session_id": "private-parent-session", + }) + + assert fields == { + "entrypoint": "delegated", + "execution_surface": "cli", + } + assert "private-parent-session" not in json.dumps(fields) + + +@pytest.mark.parametrize( + ("duration_ms", "expected"), + [ + (0, "lt_1s"), + (999, "lt_1s"), + (1_000, "1s_to_5s"), + (5_000, "5s_to_30s"), + (30_000, "30s_to_2m"), + (120_000, "2m_to_10m"), + (600_000, "gte_10m"), + ], +) +def test_duration_bucket_boundaries(duration_ms, expected): + assert duration_bucket(duration_ms) == expected + + +@pytest.mark.parametrize( + ("count", "expected"), + [ + (0, "0"), + (1, "1"), + (2, "2"), + (3, "3_to_5"), + (6, "6_to_10"), + (11, "gte_11"), + ], +) +def test_count_bucket_boundaries(count, expected): + assert count_bucket(count) == expected + + +@pytest.mark.parametrize( + ("event", "expected"), + [ + ( + {"completed": True, "turn_exit_reason": "text_response(stop)"}, + ("success", "completed", "none"), + ), + ( + {"failed": True, "turn_exit_reason": "all_retries_exhausted_no_response"}, + ("failed", "failed", "none"), + ), + ( + {"interrupted": True, "turn_exit_reason": "interrupted_by_user"}, + ("cancelled", "user_cancelled", "user_cancelled"), + ), + ( + {"turn_exit_reason": "budget_exhausted"}, + ("failed", "iteration_limit", "system_aborted"), + ), + ( + {"turn_exit_reason": "guardrail_halt"}, + ("failed", "guardrail_blocked", "system_aborted"), + ), + ( + {"failed": True, "turn_exit_reason": "provider_timeout"}, + ("timed_out", "timed_out", "timed_out"), + ), + ( + {"failed": True, "turn_exit_reason": "approval_denied"}, + ("failed", "approval_denied", "none"), + ), + ], +) +def test_task_terminal_state_is_bounded(event, expected): + assert task_terminal_state(event) == expected + + +def test_model_outcome_fails_closed_to_a_bounded_value(): + assert model_call_outcome({"outcome": "private"}) == "failed" + + +def test_unlisted_model_collapses_to_a_bounded_value(): + assert model_family({"model": "private-model-name"}) == "unknown" + + +def test_subscriber_contract_rejects_unknown_fields_and_dimension_values(): + event = SimpleNamespace( + kind="scope", + category="llm", + category_profile={"model_name": "gpt"}, + name="hermes.model_call", + scope_category="end", + metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"}, + data={ + "call_role": "primary", + "locality": "remote", + "model_family": "gpt", + "outcome": "success", + "provider_family": "direct", + }, + ) + + assert model_call_dimensions(event) == { + "call_role": "primary", + "locality": "remote", + "model_family": "gpt", + "outcome": "success", + "provider_family": "direct", + } + event.category_profile["model_name"] = "private-model-name" + assert model_call_dimensions(event) is None + event.category_profile["model_name"] = "gpt" + event.data["prompt"] = "must-not-pass" + assert model_call_dimensions(event) is None + event.data.pop("prompt") + event.metadata["prompt"] = "must-not-pass" + assert model_call_dimensions(event) is None + event.metadata.pop("prompt") + event.category_profile["private"] = "must-not-pass" + assert model_call_dimensions(event) is None + event.category_profile.pop("private") + event.category = "function" + assert model_call_dimensions(event) is None + + +def test_task_subscriber_contract_accepts_only_bounded_scope_events(): + start = SimpleNamespace( + kind="scope", + category="function", + category_profile=None, + name="hermes.task_run", + scope_category="start", + metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"}, + data={"entrypoint": "interactive", "execution_surface": "cli"}, + ) + assert task_counter(start) == ( + "hermes.task_run.started", + {"entrypoint": "interactive", "execution_surface": "cli"}, + ) + + terminal_fields = task_terminal_fields( + { + "platform": "cli", + "completed": True, + "turn_exit_reason": "text_response(stop)", + }, + duration_ms=6_000, + model_call_count=2, + tool_call_count=3, + retry_count=1, + ) + end = SimpleNamespace(**{ + **start.__dict__, + "scope_category": "end", + "data": terminal_fields, + }) + assert task_counter(end) == ( + "hermes.task_run.finished", + terminal_fields, + ) + + end.data["task_id"] = "must-not-pass" + assert task_counter(end) is None + end.data.pop("task_id") + end.data["outcome"] = "private" + assert task_counter(end) is None + end.data["outcome"] = "success" + end.metadata["prompt"] = "must-not-pass" + assert task_counter(end) is None + + +def test_store_rejects_an_unsupported_schema_version(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + connection.execute( + "INSERT INTO telemetry_state(key, value) VALUES ('schema_version', '999')" + ) + + with pytest.raises(RuntimeError, match="Unsupported shared-metrics store schema"): + SharedMetricsStore(database_path, tmp_path / "outbox") + + with sqlite3.connect(database_path) as connection: + [schema_version] = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'schema_version'" + ).fetchone() + assert schema_version == "999" + + +def test_pending_metrics_keep_the_version_recorded_at_event_time(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_model_call(_dimensions(), "version-a") + store.record_model_call(_dimensions(), "version-b") + + packages = [ + json.loads(path.read_text(encoding="utf-8")) + for path in store.create_and_export_package() + ] + + assert {package["resource"]["hermes_version"] for package in packages} == { + "version-a", + "version-b", + } + assert all(package["metrics"][0]["value"] == 1 for package in packages) + + +def test_store_exports_task_started_and_terminal_counters(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_counter( + "hermes.task_run.started", + {"entrypoint": "interactive", "execution_surface": "cli"}, + "test-version", + ) + terminal = task_terminal_fields( + { + "platform": "cli", + "completed": True, + "turn_exit_reason": "text_response(stop)", + }, + duration_ms=2_000, + model_call_count=1, + tool_call_count=2, + retry_count=0, + ) + store.record_counter("hermes.task_run.finished", terminal, "test-version") + + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + _schema_validator().validate(package) + + assert {metric["name"] for metric in package["metrics"]} == { + "hermes.task_run.finished", + "hermes.task_run.started", + } + + +def test_package_schema_rejects_unknown_fields(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_model_call(_dimensions(), "test-version") + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + invalid_package = deepcopy(package) + invalid_package["prompt"] = "must-not-be-accepted" + + jsonschema = pytest.importorskip("jsonschema") + with pytest.raises(jsonschema.ValidationError): + _schema_validator().validate(invalid_package) + + +def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + + with pytest.raises(ValueError, match="Unsupported dimensions"): + store.record_counter( + "hermes.model_call.count", + {"prompt": "must-not-be-persisted"}, + "test-version", + ) + + assert store.counter_snapshot() == [] + + +def test_package_builder_rejects_tampered_dimensions(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + with sqlite3.connect(database_path) as connection: + connection.execute( + "UPDATE counter_aggregates SET dimensions_json = ?", + (json.dumps({"prompt": "must-not-be-exported"}),), + ) + + with pytest.raises(ValueError, match="Unsupported dimensions"): + store.create_and_export_package() + + assert list(outbox_directory.glob("*.json")) == [] + + +def test_pending_package_retry_reuses_the_same_package_and_file(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + [package_path] = store.create_and_export_package() + original_payload = package_path.read_bytes() + + with sqlite3.connect(database_path) as connection: + connection.execute("UPDATE package_outbox SET exported_at = NULL") + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.create_and_export_package() == [package_path] + assert package_path.read_bytes() == original_payload + assert list(outbox_directory.glob("*.json")) == [package_path] + + +def test_retention_prunes_only_expired_exported_history(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + + store.record_model_call(_dimensions(), "expired-version") + [expired_path] = store.create_and_export_package() + store.record_model_call(_dimensions(), "current-version") + [current_path] = store.create_and_export_package() + store.record_model_call(_dimensions(), "pending-version") + pending_package = store._create_package() + assert pending_package is not None + + with sqlite3.connect(database_path) as connection: + connection.execute( + """ + UPDATE counter_aggregates + SET period_start = '2026-05-01' + WHERE hermes_version = 'expired-version' + """ + ) + connection.execute( + """ + UPDATE package_outbox + SET period_start = '2026-05-01T00:00:00Z', + period_end = '2026-05-02T00:00:00Z', + exported_at = '2026-05-02T00:00:00Z' + WHERE package_id = ? + """, + (expired_path.stem,), + ) + + store._prune_expired_history( + now=datetime(2026, 7, 23, tzinfo=timezone.utc) + ) + + assert not expired_path.exists() + assert current_path.exists() + assert not (outbox_directory / f"{pending_package['package_id']}.json").exists() + with sqlite3.connect(database_path) as connection: + outbox_rows = connection.execute( + "SELECT package_id, exported_at FROM package_outbox ORDER BY package_id" + ).fetchall() + aggregate_versions = { + row[0] + for row in connection.execute( + "SELECT hermes_version FROM counter_aggregates" + ).fetchall() + } + assert {row[0] for row in outbox_rows} == { + current_path.stem, + pending_package["package_id"], + } + assert next( + row[1] for row in outbox_rows if row[0] == pending_package["package_id"] + ) is None + assert aggregate_versions == {"current-version", "pending-version"} + + +def test_retention_failure_does_not_fail_a_committed_export(tmp_path, monkeypatch): + store = SharedMetricsStore( + tmp_path / "metrics.sqlite3", + tmp_path / "outbox", + ) + store.record_model_call(_dimensions(), "test-version") + + def fail_pruning(): + raise OSError("retention unavailable") + + monkeypatch.setattr(store, "_prune_expired_history", fail_pruning) + + [package_path] = store.create_and_export_package() + + assert package_path.exists() + assert store.counter_snapshot()[0]["packaged_value"] == 1 + + +def test_file_export_failure_retries_committed_outbox_without_duplicate_delta( + tmp_path, monkeypatch +): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + + def fail_write(*_args, **_kwargs): + raise OSError("simulated atomic export failure") + + module_globals = SharedMetricsStore._export_pending_packages.__globals__ + original_write = module_globals["atomic_json_write"] + monkeypatch.setitem(module_globals, "atomic_json_write", fail_write) + with pytest.raises(OSError, match="simulated atomic export failure"): + store.create_and_export_package() + + with sqlite3.connect(database_path) as connection: + package_id, exported_at = connection.execute( + "SELECT package_id, exported_at FROM package_outbox" + ).fetchone() + assert exported_at is None + assert store.counter_snapshot()[0]["packaged_value"] == 1 + assert list(outbox_directory.glob("*.json")) == [] + + monkeypatch.setitem(module_globals, "atomic_json_write", original_write) + assert store.create_and_export_package() == [ + outbox_directory / f"{package_id}.json" + ] + assert len(list(outbox_directory.glob("*.json"))) == 1 + assert store.create_and_export_package() == [] + + +def test_package_export_does_not_chase_concurrent_updates(tmp_path, monkeypatch): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + original_create = store._create_package + create_calls = 0 + + def create_and_record_another(): + nonlocal create_calls + create_calls += 1 + package = original_create() + if create_calls == 1: + store.record_model_call(_dimensions(), "test-version") + return package + + monkeypatch.setattr(store, "_create_package", create_and_record_another) + first_paths = store.create_and_export_package() + + assert create_calls == 1 + assert len(first_paths) == 1 + [counter] = store.counter_snapshot() + assert counter["metric_name"] == "hermes.model_call.count" + assert counter["dimensions"] == _dimensions() + assert counter["value"] == 2 + assert counter["packaged_value"] == 1 + + second_paths = store.create_and_export_package() + assert len(second_paths) == 1 + assert store.counter_snapshot()[0]["packaged_value"] == 2 + + +def test_concurrent_package_builders_commit_one_delta(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + ready = threading.Barrier(2) + + def export() -> list[Path]: + worker_store = SharedMetricsStore(database_path, outbox_directory) + ready.wait(timeout=5) + return worker_store.create_and_export_package() + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(export) for _ in range(2)] + for future in futures: + future.result() + + with sqlite3.connect(database_path) as connection: + [outbox_count] = connection.execute( + "SELECT COUNT(*) FROM package_outbox" + ).fetchone() + [package_path] = list(outbox_directory.glob("*.json")) + package = json.loads(package_path.read_text(encoding="utf-8")) + + assert outbox_count == 1 + assert package["metrics"][0]["value"] == 1 + assert store.counter_snapshot()[0]["packaged_value"] == 1 + + +def test_concurrent_model_call_updates_are_transactional(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + SharedMetricsStore(database_path, outbox_directory) + + def record_calls(count: int) -> None: + store = SharedMetricsStore(database_path, outbox_directory) + for _ in range(count): + store.record_model_call(_dimensions(), "test-version") + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(record_calls, 10) for _ in range(2)] + for future in futures: + future.result() + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 20 + + +def test_cross_process_model_call_updates_are_transactional(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + context = mp.get_context("spawn") + start_barrier = context.Barrier(2) + processes = [ + context.Process( + target=_record_model_calls_in_process, + args=(str(database_path), str(outbox_directory), 10, start_barrier), + ) + for _ in range(2) + ] + + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert not process.is_alive() + assert process.exitcode == 0 + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 20 + + +def test_schema_initialization_waits_for_an_existing_writer(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + database_path.touch() + blocker = sqlite3.connect(database_path) + blocker.execute("BEGIN IMMEDIATE") + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + SharedMetricsStore, + database_path, + outbox_directory, + ) + try: + time.sleep(0.4) + assert not future.done() + finally: + blocker.rollback() + blocker.close() + store = future.result(timeout=2) + + assert store.counter_snapshot() == [] + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX permission modes are unavailable") +def test_store_and_export_are_owner_only(tmp_path): + database_path = tmp_path / "private-store" / "metrics.sqlite3" + outbox_directory = tmp_path / "private-outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + [package_path] = store.create_and_export_package() + + assert stat.S_IMODE(database_path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(outbox_directory.stat().st_mode) == 0o700 + assert stat.S_IMODE(database_path.stat().st_mode) == 0o600 + assert stat.S_IMODE(package_path.stat().st_mode) == 0o600 diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py new file mode 100644 index 000000000000..23258def3385 --- /dev/null +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -0,0 +1,2403 @@ +"""Tests for the direct Hermes-to-Relay shared-metrics runtime.""" + +from __future__ import annotations + +import contextvars +import asyncio +import json +import threading +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from hermes_cli import lifecycle, plugins +from hermes_cli.observability import relay_runtime, relay_shared_metrics +from hermes_cli.plugins import PluginManager + + +class _Request: + def __init__(self, headers: dict[str, Any], content: dict[str, Any]) -> None: + self.headers = headers + self.content = content + + +class _Relay: + def __init__(self) -> None: + self.events: list[tuple[Any, ...]] = [] + self._callbacks: dict[str, Any] = {} + self._starts: dict[Any, dict[str, Any]] = {} + self._scope_starts: dict[Any, dict[str, Any]] = {} + self._scope = contextvars.ContextVar("relay_scope", default=None) + self._scope_serial = 0 + self.ScopeType = SimpleNamespace(Agent="agent", Function="function") + self.LLMRequest = _Request + self.scope = SimpleNamespace( + push=self._scope_push, + pop=self._scope_pop, + event=self._scope_event, + ) + self.llm = SimpleNamespace(call=self._llm_call, call_end=self._llm_call_end) + self.subscribers = SimpleNamespace( + register=self._register, + deregister=self._deregister, + flush=self._flush, + ) + self.get_scope_stack = self._get_scope_stack + + def _scope_push(self, name: str, scope_type: Any, **kwargs: Any) -> Any: + self._scope_serial += 1 + handle = ("scope", name, self._scope_serial) + self._scope.set(handle) + self.events.append(("scope.push", name, scope_type, kwargs)) + if scope_type == self.ScopeType.Function: + self._scope_starts[handle] = kwargs + event = SimpleNamespace( + kind="scope", + category="function", + name=name, + scope_category="start", + category_profile=None, + metadata=kwargs.get("metadata"), + data=kwargs.get("input"), + ) + for callback in list(self._callbacks.values()): + callback(event) + return handle + + def _scope_pop(self, handle: Any, **kwargs: Any) -> None: + self.events.append(("scope.pop", handle, kwargs)) + start = self._scope_starts.pop(handle, None) + if start is not None: + event = SimpleNamespace( + kind="scope", + category="function", + name=handle[1], + scope_category="end", + category_profile=None, + metadata={ + **(start.get("metadata") or {}), + **(kwargs.get("metadata") or {}), + }, + data=kwargs.get("output"), + ) + for callback in list(self._callbacks.values()): + callback(event) + + def _scope_event(self, name: str, **kwargs: Any) -> None: + self.events.append(("scope.event", name, kwargs)) + + def _get_scope_stack(self) -> Any: + current = self._scope.get() + self.events.append(("scope.sync", current)) + return current + + def _llm_call( + self, + name: str, + request: _Request, + **kwargs: Any, + ) -> Any: + handle = ("llm", name, len(self._starts)) + self._starts[handle] = kwargs + self.events.append(("llm.call", name, request.content, kwargs)) + return handle + + def _llm_call_end( + self, + handle: Any, + response: dict[str, Any], + **kwargs: Any, + ) -> None: + start = self._starts.pop(handle) + self.events.append(("llm.call_end", handle, response, kwargs)) + event = SimpleNamespace( + kind="scope", + category="llm", + name=handle[1], + scope_category="end", + category_profile={"model_name": start["model_name"]}, + metadata={ + **start["metadata"], + **kwargs["metadata"], + "otel.status_code": "OK", + }, + data=response, + ) + for callback in list(self._callbacks.values()): + callback(event) + + def _register(self, name: str, callback: Any) -> None: + self._callbacks[name] = callback + self.events.append(("subscribers.register", name)) + + def _deregister(self, name: str) -> None: + self._callbacks.pop(name, None) + self.events.append(("subscribers.deregister", name)) + + def _flush(self) -> None: + self.events.append(("subscribers.flush",)) + + +@pytest.fixture +def direct_runtime(tmp_path, monkeypatch): + fake = _Relay() + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + yield fake + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +@pytest.fixture +def real_binding_runtime(tmp_path, monkeypatch): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + yield relay + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_path): + base = { + "session_id": "sensitive-session", + "task_id": "task-1", + "api_request_id": "request-1", + "platform": "cli", + "provider": "custom", + "model": "gpt-sensitive-model-id", + "base_url": "http://127.0.0.1:11434/v1", + } + + assert lifecycle.has_hook("pre_api_request") + lifecycle.invoke_hook("on_session_start", **base) + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook( + "pre_api_request", + **base, + request={"body": {"messages": ["sensitive-prompt"]}}, + ) + lifecycle.invoke_hook( + "post_tool_call", + **base, + tool_call_id="sensitive-tool-call", + tool_name="terminal", + args={"command": "sensitive-command"}, + result={"output": "sensitive-tool-result"}, + status="ok", + ) + lifecycle.invoke_hook( + "api_request_error", + **base, + retryable=True, + error={"message": "sensitive-error"}, + ) + lifecycle.invoke_hook( + "pre_api_request", + **{ + **base, + "provider": "anthropic", + "model": "claude-sonnet", + "base_url": "https://api.anthropic.com", + }, + request={"body": {"messages": ["sensitive-prompt"]}}, + ) + lifecycle.invoke_hook( + "post_api_request", + **{ + **base, + "provider": "anthropic", + "model": "claude-sonnet", + "base_url": "https://api.anthropic.com", + }, + response={"content": "sensitive-response"}, + ) + lifecycle.invoke_hook( + "on_session_end", + **base, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id=base["session_id"]) + + starts = [event for event in direct_runtime.events if event[0] == "llm.call"] + ends = [event for event in direct_runtime.events if event[0] == "llm.call_end"] + scope_starts = [ + event for event in direct_runtime.events if event[0] == "scope.push" + ] + assert len(scope_starts) == 2 + assert scope_starts[0][2] == direct_runtime.ScopeType.Agent + assert scope_starts[1][1] == "hermes.task_run" + assert scope_starts[1][2] == direct_runtime.ScopeType.Function + assert scope_starts[1][3]["handle"][1] == relay_runtime.SESSION_SCOPE + assert scope_starts[1][3]["input"] == { + "entrypoint": "interactive", + "execution_surface": "cli", + } + assert len(starts) == 1 + assert len(ends) == 1 + assert starts[0][2] == {} + assert starts[0][3]["model_name"] == "gpt" + assert ends[0][2] == { + "call_role": "primary", + "locality": "remote", + "model_family": "claude", + "outcome": "success", + "provider_family": "direct", + } + serialized_events = json.dumps(direct_runtime.events) + assert "sensitive-prompt" not in serialized_events + assert "sensitive-response" not in serialized_events + assert "sensitive-error" not in serialized_events + assert "sensitive-command" not in serialized_events + assert "sensitive-tool-result" not in serialized_events + assert "sensitive-tool-call" not in serialized_events + assert "gpt-sensitive-model-id" not in serialized_events + assert plugins.get_plugin_manager().list_plugins() == [] + + root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" + packages = list((root / "outbox").glob("*.json")) + assert len(packages) == 1 + package = json.loads(packages[0].read_text(encoding="utf-8")) + metrics = {metric["name"]: metric for metric in package["metrics"]} + assert set(metrics) == { + "hermes.model_call.count", + "hermes.task_run.finished", + "hermes.task_run.started", + } + assert metrics["hermes.model_call.count"]["dimensions"]["model_family"] == "claude" + assert metrics["hermes.model_call.count"]["value"] == 1 + assert metrics["hermes.task_run.started"] == { + "name": "hermes.task_run.started", + "type": "counter", + "dimensions": { + "entrypoint": "interactive", + "execution_surface": "cli", + }, + "value": 1, + } + terminal = metrics["hermes.task_run.finished"]["dimensions"] + assert terminal["duration_bucket"] in { + "lt_1s", + "1s_to_5s", + "5s_to_30s", + "30s_to_2m", + "2m_to_10m", + "gte_10m", + } + assert { + key: value for key, value in terminal.items() if key != "duration_bucket" + } == { + "end_reason": "completed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "success", + "retry_count_bucket": "1", + "termination": "none", + "tool_call_count_bucket": "1", + } + + +def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot( + real_binding_runtime, + tmp_path, +): + assert real_binding_runtime._native is not None + prompt_canary = "real-relay-sensitive-prompt" + response_canary = "real-relay-sensitive-response" + model_canary = "gpt-real-relay-sensitive-model" + tool_canary = "real-relay-sensitive-tool-result" + + def base(index: int) -> dict[str, Any]: + return { + "session_id": f"sensitive-session-{index}", + "task_id": f"sensitive-task-{index}", + "turn_id": f"sensitive-turn-{index}", + "api_request_id": f"sensitive-request-{index}", + "platform": "cli", + "provider": "custom", + "model": model_canary, + "base_url": "http://127.0.0.1:11434/v1", + } + + success = base(1) + lifecycle.invoke_hook("on_session_start", **success) + lifecycle.invoke_hook("pre_llm_call", **success, messages=[prompt_canary]) + lifecycle.invoke_hook("pre_api_request", **success, retry_count=0) + lifecycle.invoke_hook( + "api_request_error", + **success, + retry_count=0, + retryable=True, + error={"message": prompt_canary}, + ) + lifecycle.invoke_hook("pre_api_request", **success, retry_count=1) + lifecycle.invoke_hook( + "post_tool_call", + **success, + tool_call_id="sensitive-tool-call", + tool_name="terminal", + args={"command": prompt_canary}, + result={"output": tool_canary}, + status="ok", + ) + lifecycle.invoke_hook( + "post_api_request", + **success, + retry_count=1, + response={"content": response_canary}, + ) + lifecycle.invoke_hook( + "on_session_end", + **success, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id=success["session_id"]) + + failed = base(2) + lifecycle.invoke_hook("on_session_start", **failed) + lifecycle.invoke_hook("pre_llm_call", **failed, messages=[prompt_canary]) + lifecycle.invoke_hook("pre_api_request", **failed, retry_count=0) + lifecycle.invoke_hook( + "api_request_error", + **failed, + retry_count=0, + retryable=False, + error={"message": response_canary}, + ) + lifecycle.invoke_hook( + "on_session_end", + **failed, + completed=False, + failed=True, + interrupted=False, + turn_exit_reason="system_aborted", + ) + lifecycle.finalize_session(session_id=failed["session_id"]) + + cancelled = base(3) + lifecycle.invoke_hook("on_session_start", **cancelled) + lifecycle.invoke_hook("pre_llm_call", **cancelled, messages=[prompt_canary]) + lifecycle.invoke_hook("pre_api_request", **cancelled, retry_count=0) + lifecycle.invoke_hook( + "on_session_end", + **cancelled, + completed=False, + failed=False, + interrupted=True, + turn_exit_reason="interrupted_by_user", + ) + lifecycle.finalize_session(session_id=cancelled["session_id"]) + + from hermes_cli.observability.shared_metrics import SharedMetricsStore + + root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" + store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") + snapshot = store.counter_snapshot() + by_metric: dict[str, list[dict[str, Any]]] = {} + for counter in snapshot: + by_metric.setdefault(counter["metric_name"], []).append(counter) + + assert len(by_metric["hermes.task_run.started"]) == 1 + assert by_metric["hermes.task_run.started"][0]["value"] == 3 + assert { + counter["dimensions"]["outcome"] + for counter in by_metric["hermes.model_call.count"] + } == {"success", "failed", "cancelled"} + terminal_by_outcome = { + counter["dimensions"]["outcome"]: counter + for counter in by_metric["hermes.task_run.finished"] + } + assert set(terminal_by_outcome) == {"success", "failed", "cancelled"} + assert terminal_by_outcome["success"]["dimensions"]["retry_count_bucket"] == "1" + assert terminal_by_outcome["success"]["dimensions"]["tool_call_count_bucket"] == "1" + assert terminal_by_outcome["failed"]["dimensions"]["end_reason"] == ( + "system_aborted" + ) + assert terminal_by_outcome["cancelled"]["dimensions"]["termination"] == ( + "user_cancelled" + ) + assert all(counter["packaged_value"] == counter["value"] for counter in snapshot) + + snapshot_values = { + ( + counter["metric_name"], + tuple(sorted(counter["dimensions"].items())), + ): counter["value"] + for counter in snapshot + } + package_values: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {} + packages = sorted((root / "outbox").glob("*.json")) + assert len(packages) == 3 + package_payloads = [ + json.loads(package.read_text(encoding="utf-8")) for package in packages + ] + for package in package_payloads: + assert package["schema_version"] == "hermes.shared_metrics.v1" + for metric in package["metrics"]: + key = (metric["name"], tuple(sorted(metric["dimensions"].items()))) + package_values[key] = package_values.get(key, 0) + metric["value"] + assert package_values == snapshot_values + + serialized_analytics = json.dumps({ + "snapshot": snapshot, + "packages": package_payloads, + }) + for canary in ( + prompt_canary, + response_canary, + model_canary, + tool_canary, + "sensitive-session", + "sensitive-task", + "sensitive-request", + "sensitive-tool-call", + ): + assert canary not in serialized_analytics + + +def test_direct_runtime_is_disabled_by_default(tmp_path, monkeypatch): + fake = _Relay() + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {}) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + + assert not plugins.has_hook("pre_api_request") + lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") + lifecycle.finalize_session(session_id="s1") + + assert fake.events == [] + assert not (tmp_path / "hermes-home" / "telemetry").exists() + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_tool_intercept_bypass_does_not_create_relay_host(monkeypatch): + relay_runtime._reset_for_tests() + imports = [] + + def load_relay(): + imports.append("nemo_relay") + raise AssertionError("disabled helper created Relay host") + + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) + args = {"command": "true"} + + assert ( + relay_runtime.apply_tool_request_intercepts( + session_id="s1", + tool_name="terminal", + args=args, + ) + is args + ) + assert relay_runtime.get_host(create=False) is None + assert imports == [] + + +def test_execution_adapters_do_not_create_relay_host_without_a_consumer( + monkeypatch, +): + from agent import relay_llm, relay_tools + + relay_runtime._reset_for_tests() + imports = [] + + def load_relay(): + imports.append("nemo_relay") + raise AssertionError("disabled execution adapter created Relay host") + + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) + request = {"model": "test-model", "messages": []} + response = object() + tool_args = {"command": "true"} + tool_result = object() + + assert ( + relay_llm.execute( + request, + lambda observed: response if observed is request else None, + session_id="llm-session", + name="test-provider", + model_name="test-model", + ) + is response + ) + result, observed_args = relay_tools.execute( + "terminal", + tool_args, + lambda observed: tool_result if observed is tool_args else None, + session_id="tool-session", + ) + + assert result is tool_result + assert observed_args is tool_args + assert relay_runtime.get_host(create=False) is None + assert imports == [] + + +def test_profile_key_caches_absolute_path_resolution(monkeypatch): + relay_runtime._reset_for_tests() + + class Home: + def __init__(self): + self.resolve_calls = 0 + + def expanduser(self): + return self + + def is_absolute(self): + return True + + def resolve(self): + self.resolve_calls += 1 + return self + + def __str__(self): + return "/profiles/cached" + + home = Home() + monkeypatch.setattr(relay_runtime, "get_hermes_home", lambda: home) + + assert relay_runtime.current_profile_key() == "/profiles/cached" + assert relay_runtime.current_profile_key() == "/profiles/cached" + assert home.resolve_calls == 1 + + +def test_host_registry_reads_existing_host_without_lock(): + registry = relay_runtime.RelayHostRegistry() + host = relay_runtime.NoopRelayRuntime("profile", "test") + registry._hosts["profile"] = host + + class UnexpectedLock: + def __enter__(self): + raise AssertionError("registry read acquired the write lock") + + def __exit__(self, *_args): + return False + + registry._lock = UnexpectedLock() + + assert registry.for_profile("profile", create=False) is host + assert registry.for_profile("missing", create=False) is None + + +def test_core_runtime_is_fail_open_without_a_published_binding(monkeypatch, caplog): + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + def missing_relay(name: str): + assert name == "nemo_relay" + raise ModuleNotFoundError(name) + + monkeypatch.setattr(relay_runtime.importlib, "import_module", missing_relay) + + assert relay_runtime.get_runtime() is None + host = relay_runtime.get_host() + assert isinstance(host, relay_runtime.NoopRelayRuntime) + assert host.profile_key == relay_runtime.current_profile_key() + assert "nemo_relay" in host.reason + assert host.apply_tool_request_intercepts( + session_id="s1", + tool_name="terminal", + args={"command": "true"}, + ) == {"command": "true"} + assert not relay_runtime.emit_mark("hermes.probe", session_id="s1") + assert "Hermes Relay runtime initialization failed" in caplog.text + relay_runtime._reset_for_tests() + + +def test_core_mark_uses_the_shared_session_handle_without_a_plugin(direct_runtime): + lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") + + handle = relay_runtime.get_session_handle("s1") + assert handle is not None + assert relay_runtime.emit_mark( + "hermes.skill.created", + session_id="s1", + data={"provenance": "agent_created"}, + metadata={"data_schema": "hermes.skill.lifecycle.v1"}, + ) + + [mark] = [event for event in direct_runtime.events if event[0] == "scope.event"] + assert mark[1] == "hermes.skill.created" + assert mark[2]["handle"] == handle + assert plugins.get_plugin_manager().list_plugins() == [] + + +def test_core_task_instrumentation_preserves_prompt_history_and_tool_schema( + direct_runtime, + monkeypatch, +): + from run_agent import AIAgent + + agent = object.__new__(AIAgent) + agent.session_id = "cache-stable-session" + agent.platform = "cli" + agent._parent_session_id = None + agent._session_db = None + agent._cached_system_prompt = "byte-stable-system-prompt\nwith exact spacing" + agent.tools = [ + { + "type": "function", + "function": { + "name": "probe", + "parameters": { + "type": "object", + "properties": {"value": {"type": "string"}}, + }, + }, + } + ] + history = [{"role": "user", "content": "sensitive-history-canary"}] + prompt_before = agent._cached_system_prompt.encode("utf-8") + history_before = json.dumps(history, ensure_ascii=False, sort_keys=True) + tools_before = json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) + + def fake_run_conversation( + active_agent, + user_message, + system_message, + conversation_history, + task_id, + stream_callback, + persist_user_message, + **kwargs, + ): + del ( + user_message, + system_message, + task_id, + stream_callback, + persist_user_message, + kwargs, + ) + assert active_agent is agent + assert conversation_history is history + return {"final_response": "ok", "completed": True} + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + fake_run_conversation, + ) + + for task_id in ("cache-task-1", "cache-task-2"): + result = AIAgent.run_conversation( + agent, + "hello", + conversation_history=history, + task_id=task_id, + ) + assert result["final_response"] == "ok" + + assert agent._cached_system_prompt.encode("utf-8") == prompt_before + assert json.dumps(history, ensure_ascii=False, sort_keys=True) == history_before + assert json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) == tools_before + + +def test_session_coordinator_separates_turn_release_from_hard_finalize( + direct_runtime, +): + profile_key = relay_runtime.current_profile_key() + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="coordinated-session", + platform="cli", + ) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + + assert relay_runtime.current_turn() is turn + assert turn.handle is not None + session_handle = lease.session.handle + turn_push = next( + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == relay_runtime.TURN_SCOPE + ) + assert turn_push[3]["handle"] == session_handle + + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + + assert relay_runtime.current_turn() is None + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + assert runtime.get_session("coordinated-session") is not None + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="coordinated-session", + ) + assert runtime.get_session("coordinated-session") is None + + +def test_turn_cleanup_can_be_retried_from_its_originating_context(direct_runtime): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="cross-context-session", + platform="cli", + ) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + + contextvars.Context().run(coordinator.end_turn, turn, outcome="success") + + assert turn.closed + assert relay_runtime.active_turn("cross-context-session") is None + assert relay_runtime.current_turn() is turn + + coordinator.end_turn(turn, outcome="success") + assert relay_runtime.current_turn() is None + coordinator.release_conversation(lease) + + +def test_session_coordinator_prepares_subscribers_before_opening_scope( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + initializer_name = "test.pre_scope_subscriber" + + def prepare(host, context): + assert host.profile_key == relay_runtime.current_profile_key() + direct_runtime.events.append(("session.prepare", dict(context))) + + coordinator.register_session_initializer(initializer_name, prepare) + try: + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="prepared-session", + platform="cli", + model="demo-model", + ) + finally: + coordinator.unregister_session_initializer(initializer_name) + + prepared = next( + index + for index, event in enumerate(direct_runtime.events) + if event[0] == "session.prepare" + ) + opened = next( + index + for index, event in enumerate(direct_runtime.events) + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ) + assert prepared < opened + assert direct_runtime.events[prepared][1] == { + "profile_key": relay_runtime.current_profile_key(), + "session_id": "prepared-session", + "platform": "cli", + "parent_session_id": "", + "model": "demo-model", + } + coordinator.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=lease.session_id, + ) + + +def test_session_initializer_failure_does_not_block_conversation( + direct_runtime, + caplog, +): + coordinator = relay_runtime.SESSION_COORDINATOR + initializer_name = "test.failing_subscriber" + + def fail(_host, _context): + raise RuntimeError("subscriber setup failed") + + coordinator.register_session_initializer(initializer_name, fail) + try: + with caplog.at_level("WARNING"): + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="initializer-failure", + platform="cli", + ) + finally: + coordinator.unregister_session_initializer(initializer_name) + + assert lease.session is not None + assert "Hermes Relay session initializer failed" in caplog.text + coordinator.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=lease.session_id, + ) + + +def test_profile_host_recreation_rebinds_shared_metrics_subscriber( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + first_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="before-restart", + platform="cli", + ) + first_metrics = relay_shared_metrics._get_runtime() + assert first_metrics is not None + first_host = first_lease.host + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=first_lease.session_id, + ) + coordinator.shutdown_profile(profile_key) + + second_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="after-restart", + platform="cli", + ) + second_metrics = relay_shared_metrics._get_runtime() + + assert second_metrics is not None + assert second_lease.host is not first_host + assert second_metrics is not first_metrics + assert second_metrics.host is second_lease.host + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=second_lease.session_id, + ) + + +def test_core_mark_does_not_start_relay_without_metrics_or_a_plugin( + tmp_path, + monkeypatch, +): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + imports = [] + + def load_relay(): + imports.append("nemo_relay") + return _Relay() + + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) + monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {}) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + + assert not relay_runtime.emit_mark( + "hermes.skill.created", + session_id="s1", + data={"provenance": "agent_created"}, + ) + + assert imports == [] + assert relay_runtime.get_host(create=False) is None + assert not (tmp_path / "hermes-home" / "telemetry").exists() + relay_runtime._reset_for_tests() + + +def test_core_runtime_creates_one_session_under_concurrent_access(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + ready = threading.Barrier(8) + sessions: list[Any] = [] + + def ensure() -> None: + ready.wait(timeout=5) + sessions.append(runtime.ensure_session({"session_id": "shared"})) + + threads = [threading.Thread(target=ensure) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + + assert all(not thread.is_alive() for thread in threads) + assert len({id(session) for session in sessions}) == 1 + assert ( + len([event for event in direct_runtime.events if event[0] == "scope.push"]) == 1 + ) + + +def test_core_runtime_isolates_same_session_id_by_profile(direct_runtime, tmp_path): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + + token = set_hermes_home_override(profile_a) + try: + runtime_a = relay_runtime.get_runtime() + session_a = runtime_a.ensure_session({"session_id": "shared"}) + finally: + reset_hermes_home_override(token) + + token = set_hermes_home_override(profile_b) + try: + runtime_b = relay_runtime.get_runtime() + session_b = runtime_b.ensure_session({"session_id": "shared"}) + finally: + reset_hermes_home_override(token) + + assert runtime_a is not None + assert runtime_b is not None + assert runtime_a is not runtime_b + assert runtime_a.profile_key == str(profile_a.resolve()) + assert runtime_b.profile_key == str(profile_b.resolve()) + assert session_a is not session_b + assert session_a.handle != session_b.handle + + +@pytest.mark.parametrize( + ("profile_enabled", "managed_enabled"), + ((None, True), (False, True), (True, False)), +) +def test_managed_config_cannot_override_shared_metrics_consent( + tmp_path, + monkeypatch, + profile_enabled, + managed_enabled, +): + from hermes_cli import config, managed_scope + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + profile = tmp_path / "profile" + managed = tmp_path / "managed" + profile.mkdir() + managed.mkdir() + profile_config = "{}\n" + if profile_enabled is not None: + profile_config = ( + "telemetry:\n" + " shared_metrics:\n" + f" enabled: {str(profile_enabled).lower()}\n" + ) + (profile / "config.yaml").write_text(profile_config, encoding="utf-8") + (managed / "config.yaml").write_text( + "telemetry:\n" + " shared_metrics:\n" + f" enabled: {str(managed_enabled).lower()}\n", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed)) + config._LOAD_CONFIG_CACHE.clear() + config._RAW_CONFIG_CACHE.clear() + managed_scope.invalidate_managed_cache() + + token = set_hermes_home_override(profile) + try: + assert ( + config.load_config_readonly()["telemetry"]["shared_metrics"]["enabled"] + is managed_enabled + ) + assert relay_shared_metrics.enabled() is (profile_enabled is True) + finally: + reset_hermes_home_override(token) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + managed_scope.invalidate_managed_cache() + + +def test_shared_metrics_policy_and_store_are_profile_scoped(tmp_path, monkeypatch): + from hermes_constants import ( + get_hermes_home, + reset_hermes_home_override, + set_hermes_home_override, + ) + + fake = _Relay() + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: { + "telemetry": { + "shared_metrics": {"enabled": get_hermes_home() == profile_a} + } + }, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + token = set_hermes_home_override(profile_a) + try: + assert relay_shared_metrics.enabled() + relay_shared_metrics.start_task_run( + session_id="shared", + task_id="task-a", + platform="cli", + ) + relay_shared_metrics.finish_task_run( + session_id="shared", + task_id="task-a", + platform="cli", + result={"completed": True}, + ) + relay_shared_metrics._get_runtime().close_session({"session_id": "shared"}) + finally: + reset_hermes_home_override(token) + + token = set_hermes_home_override(profile_b) + try: + assert not relay_shared_metrics.enabled() + relay_shared_metrics.start_task_run( + session_id="shared", + task_id="task-b", + platform="cli", + ) + finally: + reset_hermes_home_override(token) + + assert list((profile_a / "telemetry" / "shared_metrics" / "outbox").glob("*.json")) + assert not (profile_b / "telemetry").exists() + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_shared_metrics_subscribers_isolate_two_enabled_profiles(tmp_path, monkeypatch): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + fake = _Relay() + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + for profile, task_id in ((profile_a, "task-a"), (profile_b, "task-b")): + token = set_hermes_home_override(profile) + try: + relay_shared_metrics.start_task_run( + session_id="shared", + task_id=task_id, + platform="cli", + ) + relay_shared_metrics.finish_task_run( + session_id="shared", + task_id=task_id, + platform="cli", + result={"completed": True}, + ) + relay_shared_metrics._get_runtime().close_session( + {"session_id": "shared"} + ) + finally: + reset_hermes_home_override(token) + + for profile in (profile_a, profile_b): + packages = list( + (profile / "telemetry" / "shared_metrics" / "outbox").glob("*.json") + ) + assert len(packages) == 1 + package = json.loads(packages[0].read_text(encoding="utf-8")) + metrics = {metric["name"]: metric for metric in package["metrics"]} + assert metrics["hermes.task_run.started"]["value"] == 1 + assert metrics["hermes.task_run.finished"]["value"] == 1 + + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_shared_metrics_isolates_same_task_id_across_sessions(direct_runtime): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + + task_a = runtime.start_task({ + "session_id": "session-a", + "task_id": "shared-task", + "platform": "cli", + }) + task_b = runtime.start_task({ + "session_id": "session-b", + "task_id": "shared-task", + "platform": "gateway", + }) + + assert task_a is not None + assert task_b is not None + assert task_a is not task_b + assert task_a.handle != task_b.handle + + runtime.finish_task({ + "session_id": "session-a", + "task_id": "shared-task", + "platform": "cli", + "completed": True, + }) + runtime.finish_task({ + "session_id": "session-b", + "task_id": "shared-task", + "platform": "gateway", + "completed": True, + }) + + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + task_ends = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert len(task_starts) == 2 + assert len(task_ends) == 2 + + +def test_shared_metrics_keys_turn_ownership_by_session(direct_runtime): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + event_a = { + "session_id": "session-a", + "task_id": "task-a", + "turn_id": "shared-turn", + "platform": "cli", + } + event_b = { + "session_id": "session-b", + "task_id": "task-b", + "turn_id": "shared-turn", + "platform": "gateway", + } + task_a = runtime.start_task(event_a) + task_b = runtime.start_task(event_b) + + runtime.start_model_call({ + **event_a, + "api_request_id": "request-a", + "provider": "anthropic", + "model": "claude-sonnet", + }) + + session_a = runtime._session(event_a) + session_b = runtime._session(event_b) + assert task_a is not None + assert task_b is not None + assert session_a is not None + assert session_b is not None + assert "request-a" in session_a.model_calls + assert "request-a" not in session_b.model_calls + [model_start] = [ + event for event in direct_runtime.events if event[0] == "llm.call" + ] + assert model_start[3]["handle"] == task_a.handle + + +def test_disabling_shared_metrics_stops_collection_and_shutdown_export( + tmp_path, monkeypatch +): + from hermes_cli.observability.shared_metrics import SharedMetricsStore + + fake = _Relay() + profile = tmp_path / "profile" + policy = {"enabled": True} + monkeypatch.setenv("HERMES_HOME", str(profile)) + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": dict(policy)}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + relay_shared_metrics.start_task_run( + session_id="session", + task_id="task", + platform="cli", + ) + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + policy["enabled"] = False + + assert not relay_shared_metrics.enabled() + counters_before_stale_event = runtime.subscriber.store.counter_snapshot() + runtime.subscriber(SimpleNamespace( + kind="scope", + category="function", + category_profile=None, + name="hermes.task_run", + scope_category="start", + metadata={ + "hermes.metrics.schema_version": "hermes.metrics.event.v1", + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.host.runtime_id, + }, + data={"entrypoint": "interactive", "execution_surface": "cli"}, + )) + assert runtime.subscriber.store.counter_snapshot() == counters_before_stale_event + assert runtime.start_task({ + "session_id": "session", + "task_id": "stale-runtime-task", + "platform": "cli", + }) is None + relay_shared_metrics.finish_task_run( + session_id="session", + task_id="task", + platform="cli", + result={"completed": True}, + ) + relay_shared_metrics._reset_for_tests() + + root = profile / "telemetry" / "shared_metrics" + store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") + assert [row["metric_name"] for row in store.counter_snapshot()] == [ + "hermes.task_run.started" + ] + assert list((root / "outbox").glob("*.json")) == [] + relay_runtime._reset_for_tests() + + +def test_shared_metrics_retries_transient_initialization_failure( + direct_runtime, monkeypatch +): + real_store = relay_shared_metrics.SharedMetricsStore + attempts = 0 + + def flaky_store(): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError("transient store failure") + return real_store() + + monkeypatch.setattr(relay_shared_metrics, "SharedMetricsStore", flaky_store) + + relay_shared_metrics.start_task_run( + session_id="session", + task_id="first", + platform="cli", + ) + relay_shared_metrics.start_task_run( + session_id="session", + task_id="second", + platform="cli", + ) + + assert attempts == 2 + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + assert len(task_starts) == 1 + + +def test_async_session_runner_awaits_inside_saved_relay_context(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "async-session"}) + assert session is not None + + async def probe() -> Any: + await asyncio.sleep(0) + return direct_runtime._scope.get() + + result = asyncio.run(runtime.run_in_session_async(session, probe)) + + assert result == session.handle + + +def test_session_runners_preserve_caller_context_and_profile_override( + direct_runtime, + tmp_path, +): + from hermes_constants import ( + get_hermes_home_override, + reset_hermes_home_override, + set_hermes_home_override, + ) + + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "caller-context-session"}) + assert session is not None + caller_value = contextvars.ContextVar("caller_value", default="default") + caller_value.set("caller") + profile = tmp_path / "caller-profile" + profile_token = set_hermes_home_override(profile) + + def sync_probe() -> tuple[Any, str, str | None]: + return ( + direct_runtime._scope.get(), + caller_value.get(), + get_hermes_home_override(), + ) + + async def async_probe() -> tuple[Any, str, str | None]: + await asyncio.sleep(0) + return sync_probe() + + try: + sync_result = runtime.run_in_session(session, sync_probe) + async_result = asyncio.run(runtime.run_in_session_async(session, async_probe)) + finally: + reset_hermes_home_override(profile_token) + + expected = (session.handle, "caller", str(profile)) + assert sync_result == expected + assert async_result == expected + + +@pytest.mark.asyncio +async def test_async_session_runner_isolates_concurrent_caller_contexts( + direct_runtime, +): + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "concurrent-context-session"}) + assert session is not None + caller_value = contextvars.ContextVar("concurrent_caller", default="default") + ready = asyncio.Event() + entered = 0 + + async def run(value: str) -> tuple[Any, str]: + nonlocal entered + caller_value.set(value) + + async def probe() -> tuple[Any, str]: + nonlocal entered + entered += 1 + if entered == 2: + ready.set() + await ready.wait() + return direct_runtime._scope.get(), caller_value.get() + + return await runtime.run_in_session_async(session, probe) + + results = await asyncio.gather(run("first"), run("second")) + + assert results == [ + (session.handle, "first"), + (session.handle, "second"), + ] + + +def test_sync_session_runner_releases_lock_before_callback(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "sync-session"}) + assert session is not None + acquired = threading.Event() + contender = None + + def probe() -> Any: + nonlocal contender + + def acquire_session_lock() -> None: + with session.lock: + acquired.set() + + contender = threading.Thread(target=acquire_session_lock) + contender.start() + assert acquired.wait(timeout=1) + return direct_runtime._scope.get() + + result = runtime.run_in_session(session, probe) + assert contender is not None + contender.join(timeout=1) + + assert result == session.handle + assert contender.is_alive() is False + + +def test_active_turn_requires_matching_session_and_profile( + direct_runtime, + tmp_path, + monkeypatch, +): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="session-1", + platform="cli", + ) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + + assert relay_runtime.active_turn("session-1") is turn + assert relay_runtime.active_turn("session-2") is None + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "other-profile")) + assert relay_runtime.active_turn("session-1") is None + + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + assert relay_runtime.active_turn("session-1") is None + + +def test_turn_cleanup_drains_logical_calls_after_turn_scope_start_failure( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="session-1", + platform="cli", + ) + assert lease.session is not None + original_push = direct_runtime.scope.push + + def fail_turn_push(name, *args, **kwargs): + if name == relay_runtime.TURN_SCOPE: + raise RuntimeError("simulated turn scope failure") + return original_push(name, *args, **kwargs) + + direct_runtime.scope.push = fail_turn_push + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + direct_runtime.scope.push = original_push + assert turn.handle is None + + runtime = lease.host + logical_handle = runtime.run_in_session( + lease.session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=lease.session.handle, + input={}, + ) + turn.logical_llm_calls["request-1"] = logical_handle + + coordinator.end_turn(turn, outcome="failed") + coordinator.release_conversation(lease) + + logical_closes = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1] == logical_handle + ] + assert len(logical_closes) == 1 + assert turn.logical_llm_calls == {} + + +def test_shared_metrics_creates_one_task_under_concurrent_access(direct_runtime): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + ready = threading.Barrier(8) + tasks: list[Any] = [] + + def start() -> None: + ready.wait(timeout=5) + tasks.append( + runtime.start_task({"session_id": "s1", "task_id": "t1", "platform": "cli"}) + ) + + threads = [threading.Thread(target=start) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + + assert all(not thread.is_alive() for thread in threads) + assert len({id(task) for task in tasks}) == 1 + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + assert len(task_starts) == 1 + + +def test_core_runtime_parents_subagent_session_without_exposing_ids( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + parent_turn = coordinator.begin_turn( + parent_lease, + turn_id="parent-turn", + task_id="parent-task", + ) + child_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="sensitive-child", + platform="subagent", + parent_session_id="parent", + ) + + runtime = relay_runtime.get_runtime() + assert runtime is not None + child = runtime.get_session("sensitive-child") + assert child is not None + assert child.parent_session_id == "parent" + session_pushes = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ] + assert len(session_pushes) == 2 + child_kwargs = session_pushes[1][3] + assert child_kwargs["handle"] == parent_turn.handle + assert child_kwargs["metadata"] == { + "hermes.execution_surface": "subagent", + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id, + "nemo_relay_scope_role": "subagent", + } + assert "sensitive-child" not in json.dumps(session_pushes) + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="sensitive-child", + ) + coordinator.release_conversation(child_lease) + coordinator.end_turn(parent_turn, outcome="success") + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_subagent_stop_hook_does_not_own_child_session_lifetime(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + child = runtime.register_subagent( + { + "parent_session_id": "parent", + "child_session_id": "child", + } + ) + assert child is not None + + lifecycle.invoke_hook( + "subagent_stop", + parent_session_id="parent", + child_session_id="child", + child_status="completed", + ) + + assert runtime.get_session("child") is child + + runtime.unregister_subagent({"child_session_id": "child"}) + + assert runtime.get_session("child") is None + child_closes = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1] == child.handle + ] + assert len(child_closes) == 1 + + +@pytest.mark.parametrize( + "terminal", + ["return", "exception", "cancelled", "timeout"], +) +def test_subagent_agent_boundary_closes_its_own_scope( + direct_runtime, + monkeypatch, + terminal, +): + from run_agent import AIAgent + + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + parent_turn = coordinator.begin_turn( + parent_lease, + turn_id="parent-turn", + task_id="parent-task", + ) + child_agent = SimpleNamespace( + session_id="child", + platform="subagent", + _parent_session_id="parent", + _session_db=None, + _conversation_root_id=lambda: "parent", + ) + + if terminal == "return": + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + lambda *_args, **_kwargs: { + "final_response": "done", + "completed": True, + "interrupted": False, + }, + ) + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + elif terminal == "exception": + def fail(*_args, **_kwargs): + raise RuntimeError("child failed") + + monkeypatch.setattr("agent.conversation_loop.run_conversation", fail) + with pytest.raises(RuntimeError, match="child failed"): + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + elif terminal == "cancelled": + def cancel(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr("agent.conversation_loop.run_conversation", cancel) + with pytest.raises(KeyboardInterrupt): + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + else: + def time_out(*_args, **_kwargs): + raise TimeoutError("child timed out") + + monkeypatch.setattr("agent.conversation_loop.run_conversation", time_out) + with pytest.raises(TimeoutError, match="child timed out"): + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + assert runtime.get_session("child") is None + child_push = next( + event + for event in direct_runtime.events + if event[0] == "scope.push" + and event[1] == relay_runtime.SESSION_SCOPE + and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent" + ) + assert child_push[3]["handle"] == parent_turn.handle + child_closes = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == relay_runtime.SESSION_SCOPE + ] + assert len(child_closes) == 1 + assert relay_runtime.current_turn() is parent_turn + + coordinator.end_turn(parent_turn, outcome="success") + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_concurrent_subagents_inherit_parent_turn_and_close_independently( + direct_runtime, +): + from concurrent.futures import ThreadPoolExecutor + + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + parent_turn = coordinator.begin_turn( + parent_lease, + turn_id="parent-turn", + task_id="parent-task", + ) + + def run_child(child_id): + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id=child_id, + platform="subagent", + parent_session_id="parent", + ) + turn = coordinator.begin_turn( + lease, + turn_id=f"{child_id}-turn", + task_id=f"{child_id}-task", + ) + coordinator.end_turn(turn, outcome="success") + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=child_id, + ) + coordinator.release_conversation(lease) + + contexts = [contextvars.copy_context(), contextvars.copy_context()] + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(context.run, run_child, f"child-{index}") + for index, context in enumerate(contexts) + ] + for future in futures: + future.result() + + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + assert runtime.get_session("child-0") is None + assert runtime.get_session("child-1") is None + child_pushes = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" + and event[1] == relay_runtime.SESSION_SCOPE + and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent" + ] + assert len(child_pushes) == 2 + assert all(event[3]["handle"] == parent_turn.handle for event in child_pushes) + + coordinator.end_turn(parent_turn, outcome="success") + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_coordinator_tracks_active_turns_across_threads(direct_runtime): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="cross-thread-child", + platform="subagent", + ) + + assert not coordinator.has_active_turn( + profile_key=profile_key, + session_id="cross-thread-child", + ) + + turn = coordinator.begin_turn( + lease, + turn_id="child-turn", + task_id="child-task", + ) + + observed = [] + thread = threading.Thread( + target=lambda: observed.append( + coordinator.has_active_turn( + profile_key=profile_key, + session_id="cross-thread-child", + ) + ) + ) + thread.start() + thread.join(timeout=5) + + assert observed == [True] + + coordinator.end_turn(turn, outcome="success") + + assert not coordinator.has_active_turn( + profile_key=profile_key, + session_id="cross-thread-child", + ) + coordinator.release_conversation(lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="cross-thread-child", + ) + + +def test_child_session_closes_before_active_turn_guard_is_released( + direct_runtime, + monkeypatch, +): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + child_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="child", + platform="subagent", + parent_session_id="parent", + ) + child_turn = coordinator.begin_turn( + child_lease, + turn_id="child-turn", + task_id="child-task", + ) + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + close_observations = [] + original_close = runtime.close_session + + def observe_close(event): + close_observations.append( + coordinator.has_active_turn( + profile_key=profile_key, + session_id="child", + ) + ) + original_close(event) + + monkeypatch.setattr(runtime, "close_session", observe_close) + + coordinator.end_turn(child_turn, outcome="success") + + assert close_observations == [True] + assert runtime.get_session("child") is None + assert not coordinator.has_active_turn( + profile_key=profile_key, + session_id="child", + ) + coordinator.release_conversation(child_lease) + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_core_runtime_ignores_self_parenting_subagent_event(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + + runtime.register_subagent({"parent_session_id": "same", "child_session_id": "same"}) + session = runtime.ensure_session({"session_id": "same"}) + + assert session is not None + assert session.parent_session_id == "" + + +def test_terminal_model_error_is_counted_as_failed(direct_runtime): + base = { + "session_id": "s1", + "task_id": "t1", + "api_request_id": "r1", + "provider": "anthropic", + "model": "claude-sonnet", + } + + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=False) + lifecycle.finalize_session(session_id="s1") + + [end] = [event for event in direct_runtime.events if event[0] == "llm.call_end"] + assert end[2]["outcome"] == "failed" + + +def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runtime): + base = { + "session_id": "s1", + "task_id": "t1", + "api_request_id": "r1", + "platform": "cli", + "provider": "nvidia", + "model": "nvidia/nemotron-3-super-120b-a12b", + } + + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=True) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=True) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=False) + for tool_call_id in ("tool-1", "tool-1", "tool-2"): + lifecycle.invoke_hook( + "post_tool_call", + **base, + tool_call_id=tool_call_id, + tool_name="terminal", + result={"output": "private"}, + status="ok", + ) + lifecycle.invoke_hook( + "on_session_end", + **base, + completed=False, + failed=True, + interrupted=False, + turn_exit_reason="all_retries_exhausted_no_response", + ) + lifecycle.finalize_session(session_id="s1") + + model_starts = [event for event in direct_runtime.events if event[0] == "llm.call"] + model_ends = [ + event for event in direct_runtime.events if event[0] == "llm.call_end" + ] + assert len(model_starts) == 1 + assert len(model_ends) == 1 + assert model_ends[0][2]["outcome"] == "failed" + [task_end] = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end[2]["output"] == { + "duration_bucket": task_end[2]["output"]["duration_bucket"], + "end_reason": "failed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "failed", + "retry_count_bucket": "2", + "termination": "none", + "tool_call_count_bucket": "2", + } + + +def test_task_terminal_counts_explicit_retry_with_new_request_id(direct_runtime): + base = { + "session_id": "s1", + "task_id": "t1", + "platform": "cli", + "provider": "nvidia", + "model": "nvidia/nemotron-3-super-120b-a12b", + } + + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook( + "pre_api_request", + **base, + api_request_id="r1", + retry_count=0, + ) + lifecycle.invoke_hook( + "api_request_error", + **base, + api_request_id="r1", + retryable=True, + ) + lifecycle.invoke_hook( + "pre_api_request", + **base, + api_request_id="r2", + retry_count=1, + ) + lifecycle.invoke_hook( + "post_api_request", + **base, + api_request_id="r2", + ) + lifecycle.invoke_hook( + "on_session_end", + **base, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id="s1") + + [task_end] = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end[2]["output"]["model_call_count_bucket"] == "2" + assert task_end[2]["output"]["retry_count_bucket"] == "1" + + +def test_outer_agent_boundary_closes_early_returns_and_exceptions( + direct_runtime, + monkeypatch, +): + from run_agent import AIAgent + + agent = SimpleNamespace( + session_id="s1", + platform="cli", + _parent_session_id=None, + _session_db=None, + _conversation_root_id=lambda: "s1", + ) + + def early_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return { + "final_response": "private failure detail", + "completed": False, + "failed": True, + "interrupted": False, + } + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + early_failure, + ) + result = AIAgent.run_conversation(agent, "private prompt", task_id="early") + assert result["failed"] is True + + def raise_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise RuntimeError("private exception detail") + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + raise_failure, + ) + with pytest.raises(RuntimeError, match="private exception detail"): + AIAgent.run_conversation(agent, "private prompt", task_id="exception") + + def raise_interrupt(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise KeyboardInterrupt + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + raise_interrupt, + ) + with pytest.raises(KeyboardInterrupt): + AIAgent.run_conversation(agent, "private prompt", task_id="cancelled") + + def raise_timeout(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise TimeoutError("private timeout detail") + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + raise_timeout, + ) + with pytest.raises(TimeoutError, match="private timeout detail"): + AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") + + lifecycle.finalize_session(session_id="s1") + + task_ends = [ + event[2]["output"] + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert len(task_ends) == 4 + assert task_ends[0]["outcome"] == "failed" + assert task_ends[0]["end_reason"] == "failed" + assert task_ends[0]["termination"] == "none" + assert task_ends[1]["outcome"] == "failed" + assert task_ends[1]["end_reason"] == "system_aborted" + assert task_ends[1]["termination"] == "system_aborted" + assert task_ends[2]["outcome"] == "cancelled" + assert task_ends[2]["end_reason"] == "user_cancelled" + assert task_ends[2]["termination"] == "user_cancelled" + assert task_ends[3]["outcome"] == "timed_out" + assert task_ends[3]["end_reason"] == "timed_out" + assert task_ends[3]["termination"] == "timed_out" + serialized = json.dumps(direct_runtime.events) + assert "private prompt" not in serialized + assert "private failure detail" not in serialized + assert "private exception detail" not in serialized + assert "private timeout detail" not in serialized + + +def test_outer_agent_boundary_preserves_a_returned_timeout_reason( + direct_runtime, + monkeypatch, +): + from run_agent import AIAgent + + agent = SimpleNamespace( + session_id="s1", + platform="cli", + _parent_session_id=None, + _session_db=None, + _conversation_root_id=lambda: "s1", + ) + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + lambda *_args, **_kwargs: { + "final_response": "private timeout response", + "completed": False, + "failed": True, + "failure_reason": "timeout", + }, + ) + AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") + + [task_end] = [ + event[2]["output"] + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end["outcome"] == "timed_out" + assert task_end["end_reason"] == "timed_out" + assert task_end["termination"] == "timed_out" + serialized = json.dumps(direct_runtime.events) + assert "private prompt" not in serialized + assert "private timeout response" not in serialized + + +def test_session_finalize_closes_a_pending_task_as_system_aborted(direct_runtime): + lifecycle.invoke_hook( + "pre_llm_call", + session_id="s1", + task_id="t1", + platform="cli", + ) + + lifecycle.finalize_session(session_id="s1") + + [task_end] = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end[2]["output"] == { + "duration_bucket": task_end[2]["output"]["duration_bucket"], + "end_reason": "system_aborted", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "0", + "outcome": "failed", + "retry_count_bucket": "0", + "termination": "system_aborted", + "tool_call_count_bucket": "0", + } + + +def test_sequential_tasks_in_one_session_aggregate_once_each(direct_runtime, tmp_path): + for task_id in ("t1", "t2"): + lifecycle.invoke_hook( + "pre_llm_call", + session_id="s1", + task_id=task_id, + platform="cli", + ) + lifecycle.invoke_hook( + "on_session_end", + session_id="s1", + task_id=task_id, + platform="cli", + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id="s1") + + outbox = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" / "outbox" + [package_path] = list(outbox.glob("*.json")) + package = json.loads(package_path.read_text(encoding="utf-8")) + metrics = {metric["name"]: metric for metric in package["metrics"]} + assert metrics["hermes.task_run.started"]["value"] == 2 + assert metrics["hermes.task_run.finished"]["value"] == 2 + + +def test_task_ownership_survives_session_id_rotation(direct_runtime): + lifecycle.invoke_hook( + "pre_llm_call", + session_id="before-compression", + task_id="t1", + platform="cli", + ) + lifecycle.invoke_hook( + "pre_api_request", + session_id="after-compression", + task_id="t1", + api_request_id="r1", + platform="cli", + provider="nvidia", + model="nvidia/nemotron-3-super-120b-a12b", + ) + lifecycle.invoke_hook( + "post_api_request", + session_id="after-compression", + task_id="t1", + api_request_id="r1", + platform="cli", + provider="nvidia", + model="nvidia/nemotron-3-super-120b-a12b", + ) + lifecycle.invoke_hook( + "on_session_end", + session_id="after-compression", + task_id="t1", + platform="cli", + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id="before-compression") + + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + task_ends = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + model_ends = [ + event for event in direct_runtime.events if event[0] == "llm.call_end" + ] + assert len(task_starts) == 1 + assert len(task_ends) == 1 + assert len(model_ends) == 1 + assert model_ends[0][2]["outcome"] == "success" + assert task_ends[0][2]["output"]["model_call_count_bucket"] == "1" + assert task_ends[0][2]["output"]["outcome"] == "success" + + +def test_gateway_and_delegated_entrypoints_flow_through_relay(direct_runtime): + tasks = [ + { + "session_id": "gateway-session", + "task_id": "gateway-task", + "platform": "whatsapp_cloud", + }, + { + "session_id": "child-session", + "task_id": "delegated-task", + "platform": "cli", + "parent_session_id": "private-parent-session", + }, + ] + for task in tasks: + lifecycle.invoke_hook("pre_llm_call", **task) + lifecycle.invoke_hook( + "on_session_end", + **task, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + + starts = [ + event[3]["input"] + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + assert starts == [ + {"entrypoint": "gateway_message", "execution_surface": "gateway"}, + {"entrypoint": "delegated", "execution_surface": "cli"}, + ] + assert "private-parent-session" not in json.dumps(direct_runtime.events) + + +def test_persistence_failure_does_not_escape_the_hook( + direct_runtime, + monkeypatch, + caplog, +): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + + def fail_record(*_args: Any, **_kwargs: Any) -> None: + raise OSError("store unavailable") + + monkeypatch.setattr(runtime.subscriber.store, "record_counter", fail_record) + lifecycle.invoke_hook( + "pre_api_request", + session_id="s1", + task_id="t1", + api_request_id="r1", + provider="openai", + model="gpt-5", + ) + lifecycle.invoke_hook( + "post_api_request", + session_id="s1", + task_id="t1", + api_request_id="r1", + provider="openai", + model="gpt-5", + ) + + assert "Unable to persist the Hermes shared metric" in caplog.text + + +def test_close_does_not_reopen_a_session_after_scope_start_failure( + direct_runtime, + monkeypatch, +): + runtime = relay_runtime.get_runtime() + assert runtime is not None + original_push = direct_runtime.scope.push + push_attempts = 0 + + def fail_first_push(*args: Any, **kwargs: Any) -> Any: + nonlocal push_attempts + push_attempts += 1 + if push_attempts == 1: + raise RuntimeError("simulated scope failure") + return original_push(*args, **kwargs) + + direct_runtime.scope.push = fail_first_push + with pytest.raises(RuntimeError, match="simulated scope failure"): + runtime.ensure_session({"session_id": "s1"}) + + close_started = threading.Event() + allow_close = threading.Event() + original_flush = direct_runtime.subscribers.flush + + def block_flush(): + session = runtime._sessions["s1"] + assert session.closing is True + close_started.set() + assert allow_close.wait(timeout=5) + original_flush() + + direct_runtime.subscribers.flush = block_flush + close_thread = threading.Thread( + target=runtime.close_session, + args=({"session_id": "s1"},), + ) + close_thread.start() + assert close_started.wait(timeout=5) + + ensure_thread = threading.Thread( + target=runtime.ensure_session, + args=({"session_id": "s1"},), + ) + ensure_thread.start() + allow_close.set() + close_thread.join(timeout=5) + ensure_thread.join(timeout=5) + + assert not close_thread.is_alive() + assert not ensure_thread.is_alive() + assert push_attempts == 1 + assert "s1" not in runtime._sessions diff --git a/tests/hermes_cli/test_setup_telemetry.py b/tests/hermes_cli/test_setup_telemetry.py new file mode 100644 index 000000000000..e6ebcb428c45 --- /dev/null +++ b/tests/hermes_cli/test_setup_telemetry.py @@ -0,0 +1,37 @@ +"""Tests for shared-metrics configuration discovery and setup.""" + +from __future__ import annotations + +import argparse + +from hermes_cli.config import DEFAULT_CONFIG +from hermes_cli.setup import setup_telemetry +from hermes_cli.subcommands.setup import build_setup_parser + + +def test_shared_metrics_are_registered_disabled_by_default(): + assert DEFAULT_CONFIG["telemetry"]["shared_metrics"]["enabled"] is False + + +def test_setup_telemetry_enables_shared_metrics(monkeypatch): + config = {} + monkeypatch.setattr( + "hermes_cli.setup.prompt_yes_no", + lambda _question, default: not default, + ) + + setup_telemetry(config) + + assert config["telemetry"]["shared_metrics"]["enabled"] is True + + +def test_setup_parser_accepts_telemetry_section(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + handler = object() + build_setup_parser(subparsers, cmd_setup=handler) + + args = parser.parse_args(["setup", "telemetry"]) + + assert args.section == "telemetry" + assert args.func is handler diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 61958194e585..6081f0660f73 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio -import builtins +import contextvars import gc import importlib import json @@ -15,6 +15,8 @@ from types import SimpleNamespace import pytest import yaml +from hermes_cli import lifecycle, plugins as plugin_api +from hermes_cli.observability import relay_runtime, relay_shared_metrics from hermes_cli.plugins import PluginManager @@ -25,7 +27,13 @@ PLUGIN_DIR = REPO_ROOT / "plugins" / "observability" / "nemo_relay" class _FakeNemoRelay: def __init__(self): self.events = [] - self.ScopeType = SimpleNamespace(Agent="agent") + self._callbacks = {} + self._llm_starts = {} + self._scope_serial = 0 + self._scope_context = contextvars.ContextVar( + "fake_nemo_relay_scope", default=None + ) + self.ScopeType = SimpleNamespace(Agent="agent", Function="function") self.scope = SimpleNamespace( push=self._scope_push, pop=self._scope_pop, @@ -40,21 +48,29 @@ class _FakeNemoRelay: call=self._tool_call, call_end=self._tool_call_end, execute=self._tool_execute, + request_intercepts=self._tool_request_intercepts, ) self.plugin = SimpleNamespace( initialize=self._plugin_initialize, clear=self._plugin_clear, - activate_dynamic_plugins=self._plugin_activate_dynamic, + initialize_with_dynamic_plugins=self._plugin_initialize_with_dynamic, + ) + self.subscribers = SimpleNamespace( + register=self._register_subscriber, + deregister=self._deregister_subscriber, + flush=self._flush_subscribers, ) - self.subscribers = SimpleNamespace(flush=self._flush_subscribers) self.LLMRequest = _FakeLLMRequest self.AtofExporterConfig = _FakeAtofExporterConfig self.AtofExporterMode = SimpleNamespace(Append="append", Overwrite="overwrite") self.AtofExporter = self._make_atof_exporter self.AtifExporter = self._make_atif_exporter + self.get_scope_stack = self._get_scope_stack def _scope_push(self, name, scope_type, **kwargs): - handle = ("scope", name) + self._scope_serial += 1 + handle = ("scope", name, self._scope_serial) + self._scope_context.set(handle) self.events.append(("scope.push", name, scope_type, kwargs)) return handle @@ -64,17 +80,45 @@ class _FakeNemoRelay: def _scope_event(self, name, **kwargs): self.events.append(("scope.event", name, kwargs)) + def _get_scope_stack(self): + current = self._scope_context.get() + self.events.append(("scope.sync", current)) + return current + def _llm_call(self, name, request, **kwargs): handle = ("llm", name) + self._llm_starts[handle] = kwargs self.events.append(("llm.call", name, request.content, kwargs)) return handle def _llm_call_end(self, handle, response, **kwargs): self.events.append(("llm.call_end", handle, response, kwargs)) + start = self._llm_starts.pop(handle, {}) + event = SimpleNamespace( + kind="scope", + category="llm", + name=handle[1], + scope_category="end", + category_profile={"model_name": start.get("model_name")}, + metadata={ + **(start.get("metadata") or {}), + **(kwargs.get("metadata") or {}), + "otel.status_code": "OK", + }, + data=response, + ) + for callback in list(self._callbacks.values()): + callback(event) def _llm_execute(self, name, request, func, **kwargs): self.events.append(("llm.execute.start", name, request.content, kwargs)) + handle = self._llm_call(name, request, **kwargs) result = func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) + self._llm_call_end( + handle, + result, + **{key: value for key, value in kwargs.items() if key != "handle"}, + ) self.events.append(("llm.execute.end", name, result, kwargs)) return result @@ -88,10 +132,20 @@ class _FakeNemoRelay: def _tool_execute(self, name, args, func, **kwargs): self.events.append(("tool.execute.start", name, args, kwargs)) - result = func({"intercepted": True, **args}) + handle = self._tool_call(name, args, **kwargs) + result = func(args) + self._tool_call_end( + handle, + result, + **{key: value for key, value in kwargs.items() if key != "handle"}, + ) self.events.append(("tool.execute.end", name, result, kwargs)) return result + def _tool_request_intercepts(self, name, args): + self.events.append(("tool.request_intercepts", name, args)) + return {"intercepted": True, **args} + def _make_atof_exporter(self, config): return _FakeAtofExporter(self.events, config) @@ -105,10 +159,18 @@ class _FakeNemoRelay: async def _plugin_clear(self): self.events.append(("plugin.clear",)) - async def _plugin_activate_dynamic(self, config, dynamic_plugins): + async def _plugin_initialize_with_dynamic(self, config, dynamic_plugins): self.events.append(("plugin.activate_dynamic", config, dynamic_plugins)) return _FakePluginActivation(self.events) + def _register_subscriber(self, name, callback): + self._callbacks[name] = callback + self.events.append(("subscribers.register", name)) + + def _deregister_subscriber(self, name): + self._callbacks.pop(name, None) + self.events.append(("subscribers.deregister", name)) + def _flush_subscribers(self): self.events.append(("subscribers.flush",)) @@ -169,6 +231,12 @@ class _FakeAtifExporter: def _fresh_plugin(monkeypatch, fake): + existing = sys.modules.get("plugins.observability.nemo_relay") + if existing is not None: + existing.reset_for_tests() + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setitem(sys.modules, "nemo_relay", fake) sys.modules.pop("plugins.observability.nemo_relay", None) plugin = importlib.import_module("plugins.observability.nemo_relay") @@ -176,33 +244,6 @@ def _fresh_plugin(monkeypatch, fake): return plugin -def _wrapped_downstream_error(original): - class _DownstreamExecutionError(Exception): - def __init__(self, original): - super().__init__(str(original)) - self.original = original - - return _DownstreamExecutionError(original) - - -def _enable_adaptive_plugin(tmp_path, monkeypatch) -> None: - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - def _enable_dynamic_plugin(tmp_path, monkeypatch) -> Path: plugins_toml = tmp_path / "plugins.toml" plugins_toml.write_text( @@ -233,11 +274,6 @@ def test_manifest_fields(): "on_session_reset", "pre_llm_call", "post_llm_call", - "pre_api_request", - "post_api_request", - "api_request_error", - "pre_tool_call", - "post_tool_call", "pre_approval_request", "post_approval_response", "subagent_start", @@ -266,7 +302,12 @@ def test_nemo_relay_plugin_uses_nemo_relay_runtime(monkeypatch): assert any(event[0] == "scope.push" for event in fake_relay.events) -def test_nemo_relay_plugin_emits_llm_tool_and_exports_atif(tmp_path, monkeypatch): +def test_nemo_relay_plugin_exports_core_managed_llm_and_tool_events( + tmp_path, + monkeypatch, +): + from agent import relay_llm, relay_tools + fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) monkeypatch.setenv("HERMES_NEMO_RELAY_ATOF_ENABLED", "1") @@ -281,64 +322,153 @@ def test_nemo_relay_plugin_emits_llm_tool_and_exports_atif(tmp_path, monkeypatch "telemetry_schema_version": "hermes.observer.v1", } plugin.on_session_start(**base, model="demo-model", platform="cli") - plugin.on_pre_api_request( - **base, - api_request_id="api-1", - provider="openai", - model="demo-model", - request={"method": "POST", "body": {"messages": [{"role": "user", "content": "hi"}]}}, + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + platform="cli", ) - plugin.on_post_api_request( - **base, - api_request_id="api-1", - response={"assistant_message": {"role": "assistant", "content": "hello"}}, + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="t1", ) - plugin.on_pre_tool_call(**base, tool_name="read_file", tool_call_id="tool-1", args={"path": "x"}) - plugin.on_post_tool_call(**base, tool_name="read_file", tool_call_id="tool-1", result='{"ok": true}', status="ok") + relay_llm.execute( + {"messages": [{"role": "user", "content": "hi"}]}, + lambda request: { + "assistant_message": {"role": "assistant", "content": "hello"}, + "request": request, + }, + session_id="s1", + name="openai", + model_name="demo-model", + metadata={"api_request_id": "api-1", "api_mode": "custom"}, + ) + relay_tools.execute( + "read_file", + {"path": "x"}, + lambda _args: {"ok": True}, + session_id="s1", + metadata={"tool_call_id": "tool-1"}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) plugin.on_session_end(**base, completed=True, interrupted=False) plugin.on_session_finalize(**base, reason="shutdown") event_names = [event[0] for event in fake.events] assert "atof.register" in event_names assert "atif.register" in event_names - assert "llm.call" in event_names - assert "llm.call_end" in event_names - assert "tool.call" in event_names - assert "tool.call_end" in event_names + assert event_names.count("llm.call") == 1 + assert event_names.count("llm.call_end") == 1 + assert event_names.count("tool.call") == 1 + assert event_names.count("tool.call_end") == 1 assert "scope.pop" in event_names assert (tmp_path / "atif" / "hermes-atif-s1.json").exists() -def test_nemo_relay_plugin_closes_api_span_on_error(monkeypatch): +def test_shared_metrics_and_rich_plugin_share_one_core_session( + tmp_path, + monkeypatch, +): + from agent import relay_llm + fake = _FakeNemoRelay() + hermes_home = tmp_path / "hermes-home" + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1") + monkeypatch.setenv( + "HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "atif") + ) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) plugin = _fresh_plugin(monkeypatch, fake) - base = { + manager = PluginManager() + + class _Context: + def register_hook(self, name, callback): + manager._hooks.setdefault(name, []).append(callback) + + plugin.register(_Context()) + monkeypatch.setattr(plugin_api, "_plugin_manager", manager) + + event = { "session_id": "s1", "task_id": "t1", - "turn_id": "turn-1", - "telemetry_schema_version": "hermes.observer.v1", + "api_request_id": "api-1", + "provider": "anthropic", + "model": "claude-sonnet", + "platform": "cli", } - - plugin.on_pre_api_request( - **base, - api_request_id="api-err", - provider="openai", - model="demo-model", + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + platform="cli", + model=event["model"], + ) + lifecycle.invoke_hook("on_session_start", **event) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="t1", + ) + lifecycle.invoke_hook( + "pre_api_request", + **event, request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, ) - plugin.on_api_request_error( - **base, - api_request_id="api-err", - error={"type": "RateLimitError", "message": "rate limited"}, - retryable=True, - reason="rate_limit", + relay_llm.execute( + {"messages": [{"role": "user", "content": "hi"}]}, + lambda _request: { + "assistant_message": {"role": "assistant", "content": "hello"} + }, + session_id="s1", + name="anthropic", + model_name="claude-sonnet", + metadata={"api_request_id": "api-1", "api_mode": "custom"}, ) + lifecycle.invoke_hook( + "post_api_request", + **event, + response={"assistant_message": {"role": "assistant", "content": "hello"}}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + lifecycle.finalize_session(session_id="s1") - call_end = next(event for event in fake.events if event[0] == "llm.call_end") - assert call_end[1] == ("llm", "openai") - assert call_end[2] == {"error": {"type": "RateLimitError", "message": "rate limited"}} - assert call_end[3]["data"]["reason"] == "rate_limit" - assert not plugin._get_runtime().sessions["s1"].llm_spans + session_pushes = [ + item + for item in fake.events + if item[0] == "scope.push" and item[1] == relay_runtime.SESSION_SCOPE + ] + assert len(session_pushes) == 1 + register_metrics = next( + index + for index, item in enumerate(fake.events) + if item[0] == "subscribers.register" + and item[1].startswith("hermes.nemo_relay.shared_metrics.") + ) + register_atif = next( + index for index, item in enumerate(fake.events) if item[0] == "atif.register" + ) + open_session = fake.events.index(session_pushes[0]) + assert register_metrics < register_atif < open_session + + plugin_runtime = plugin._get_runtime() + assert plugin_runtime is not None + assert not plugin_runtime.sessions + assert relay_runtime.get_session_handle("s1") is None + packages = list( + (hermes_home / "telemetry" / "shared_metrics" / "outbox").glob("*.json") + ) + assert len(packages) == 1 + package = json.loads(packages[0].read_text(encoding="utf-8")) + assert package["metrics"][0]["name"] == "hermes.model_call.count" + assert package["metrics"][0]["value"] == 1 + assert (tmp_path / "atif" / "hermes-atif-s1.json").exists() def test_nemo_relay_plugin_emits_approval_marks(monkeypatch): @@ -353,24 +483,6 @@ def test_nemo_relay_plugin_emits_approval_marks(monkeypatch): assert "hermes.approval.response" in mark_names -def test_nemo_relay_plugin_emits_unmatched_fallback_marks(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - plugin.on_post_api_request(session_id="s1", api_request_id="missing-api", response={"ok": True}) - plugin.on_api_request_error( - session_id="s1", - api_request_id="missing-api", - error={"type": "TimeoutError", "message": "timed out"}, - ) - plugin.on_post_tool_call(session_id="s1", tool_call_id="missing-tool", result={"ok": True}) - - mark_names = [event[1] for event in fake.events if event[0] == "scope.event"] - assert "hermes.api.response.unmatched" in mark_names - assert "hermes.api.error" in mark_names - assert "hermes.tool.response.unmatched" in mark_names - - def test_nemo_relay_plugin_metadata_promotes_trajectory_and_subagent_ids(monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -417,7 +529,7 @@ def test_nemo_relay_plugin_metadata_promotes_trajectory_and_subagent_ids(monkeyp assert stop_mark[2]["metadata"]["child_status"] == "completed" -def test_nemo_relay_plugin_reparents_child_session_scope_for_embedded_atif(monkeypatch): +def test_nemo_relay_plugin_reuses_core_parented_child_scope_for_embedded_atif(monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -430,20 +542,52 @@ def test_nemo_relay_plugin_reparents_child_session_scope_for_embedded_atif(monke child_role="leaf", telemetry_schema_version="hermes.observer.v1", ) + runtime = plugin._get_runtime() + assert runtime is not None + assert runtime.host.get_session("child-session") is None + runtime.host.register_subagent( + { + "parent_session_id": "parent-session", + "child_session_id": "child-session", + } + ) plugin.on_session_start(session_id="child-session") - child_push = next( + session_pushes = [ event for event in fake.events - if event[0] == "scope.push" and event[1] == "hermes-session-child-session" - ) + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ] + assert len(session_pushes) == 2 + child_push = session_pushes[1] child_kwargs = child_push[3] - assert child_kwargs["handle"] == ("scope", "hermes-session-parent-session") - assert child_kwargs["metadata"]["session_id"] == "child-session" - assert child_kwargs["metadata"]["trajectory_id"] == "child-session" + assert child_kwargs["handle"] == runtime.sessions["parent-session"].handle assert child_kwargs["metadata"]["nemo_relay_scope_role"] == "subagent" - assert child_kwargs["metadata"]["subagent_id"] == "child-sa" - assert child_kwargs["metadata"]["parent_session_id"] == "parent-session" + assert "session_id" not in child_kwargs["metadata"] + assert "subagent_id" not in child_kwargs["metadata"] + assert runtime.sessions["child-session"].parent_session_id == "parent-session" + + runtime.host.unregister_subagent({"child_session_id": "child-session"}) + assert runtime.host.get_session("child-session") is None + child_close_index = max( + index + for index, event in enumerate(fake.events) + if event[0] == "scope.pop" and event[1] == runtime.sessions["child-session"].handle + ) + plugin.on_subagent_stop( + parent_session_id="parent-session", + child_session_id="child-session", + child_status="completed", + ) + + assert "child-session" not in runtime.sessions + assert runtime.host.get_session("child-session") is None + stop_mark_index = next( + index + for index, event in enumerate(fake.events) + if event[0] == "scope.event" and event[1] == "hermes.subagent.stop" + ) + assert child_close_index < stop_mark_index def test_nemo_relay_plugin_skips_embedded_child_atif_file_by_default(tmp_path, monkeypatch): @@ -551,6 +695,8 @@ enabled = true def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypatch): + from agent import relay_llm, relay_tools + fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -561,21 +707,42 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa runtime = plugin._get_runtime() assert runtime is not None assert runtime._plugin_activation is not None - llm_result = plugin.on_llm_execution_middleware( + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), session_id="s1", - provider="openai", - model="fixture", - request={"messages": []}, - next_call=lambda request: {"request": request}, + platform="cli", ) - tool_result = plugin.on_tool_execution_middleware( + turn = coordinator.begin_turn(lease, turn_id="turn-1", task_id="task-1") + llm_result = relay_llm.execute( + {"messages": []}, + lambda request: {"request": request}, + session_id="s1", + name="openai", + model_name="fixture", + metadata={"api_mode": "custom", "api_request_id": "api-1"}, + ) + tool_args = relay_runtime.apply_tool_request_intercepts( session_id="s1", tool_name="fixture-tool", args={"value": 1}, - next_call=lambda args: {"args": args}, ) + tool_result, final_args = relay_tools.execute( + "fixture-tool", + tool_args, + lambda args: {"args": args}, + session_id="s1", + metadata={"tool_call_id": "tool-1"}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) assert llm_result["request"]["intercepted"] is True assert tool_result["args"]["intercepted"] is True + assert final_args["intercepted"] is True + relay_runtime.SESSION_COORDINATOR.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + ) plugin.on_session_finalize(session_id="s1", reason="shutdown") assert runtime._plugin_activation is not None assert not any(event[0] == "plugin.activation.close" for event in fake.events) @@ -603,6 +770,190 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa assert event_names.index("atif.deregister") < event_names.index("plugin.activation.close") +def test_nemo_relay_plugin_uses_real_0_6_dynamic_activation_api( + tmp_path, + monkeypatch, +): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + plugin = _fresh_plugin(monkeypatch, relay) + _enable_dynamic_plugin(tmp_path, monkeypatch) + calls = [] + + class _NativeActivation: + def __init__(self): + self.report = {"diagnostics": []} + self.is_active = True + + async def close(self): + self.is_active = False + + async def _initialize_with_dynamic_plugins(config, dynamic_plugins): + calls.append((config, dynamic_plugins)) + return _NativeActivation() + + monkeypatch.setattr( + relay.plugin, + "_initialize_with_dynamic_plugins", + _initialize_with_dynamic_plugins, + ) + + plugin.on_session_start(session_id="s1") + runtime = plugin._get_runtime() + + assert runtime is not None + assert isinstance(runtime._plugin_activation, relay.plugin.PluginHostActivation) + assert calls == [ + ( + {"version": 1}, + [ + { + "plugin_id": "fixture", + "kind": "rust_dynamic", + "manifest_ref": str(tmp_path / "fixture" / "relay-plugin.toml"), + "config": {"mode": "test"}, + } + ], + ) + ] + + activation = runtime._plugin_activation + runtime.shutdown() + assert activation.is_active is False + + +def test_real_binding_shares_plugin_configuration_across_two_profiles( + tmp_path, + monkeypatch, +): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + plugin = _fresh_plugin(monkeypatch, relay) + original_initialize = relay.plugin.initialize + original_clear = relay.plugin.clear + original_clear() + initialize_calls = [] + clear_calls = 0 + + async def _initialize(config): + initialize_calls.append(config) + return await original_initialize(config) + + def _clear(): + nonlocal clear_calls + clear_calls += 1 + return original_clear() + + monkeypatch.setattr(relay.plugin, "initialize", _initialize) + monkeypatch.setattr(relay.plugin, "clear", _clear) + monkeypatch.setattr( + plugin, + "_load_settings", + lambda: plugin._Settings(plugins_config={"version": 1}), + ) + profile_a = str(tmp_path / "profile-a") + profile_b = str(tmp_path / "profile-b") + host_a = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_a) + host_b = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_b) + + try: + runtime_a = plugin._get_runtime(profile_key=profile_a, host=host_a) + runtime_b = plugin._get_runtime(profile_key=profile_b, host=host_b) + assert runtime_a is not None + assert runtime_b is not None + runtime_a.ensure_session({"session_id": "session-a"}) + runtime_b.ensure_session({"session_id": "session-b"}) + + assert initialize_calls == [{"version": 1}] + assert relay.plugin.report() is not None + + runtime_a.close_session({"session_id": "session-a"}) + + assert clear_calls == 0 + assert relay.plugin.report() is not None + assert runtime_b.host.get_session("session-b") is not None + + runtime_b.close_session({"session_id": "session-b"}) + + assert clear_calls == 1 + assert relay.plugin.report() is None + finally: + plugin.reset_for_tests() + host_a.shutdown() + host_b.shutdown() + original_clear() + + +def test_real_binding_does_not_replace_another_profiles_plugin_configuration( + tmp_path, + monkeypatch, +): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + plugin = _fresh_plugin(monkeypatch, relay) + original_initialize = relay.plugin.initialize + original_clear = relay.plugin.clear + original_clear() + initialize_calls = [] + clear_calls = 0 + + async def _initialize(config): + initialize_calls.append(config) + return await original_initialize(config) + + def _clear(): + nonlocal clear_calls + clear_calls += 1 + return original_clear() + + settings = iter(( + plugin._Settings( + plugins_config={"version": 1, "policy": {"unsupported": "warn"}} + ), + plugin._Settings( + plugins_config={"version": 1, "policy": {"unsupported": "ignore"}} + ), + )) + monkeypatch.setattr(relay.plugin, "initialize", _initialize) + monkeypatch.setattr(relay.plugin, "clear", _clear) + monkeypatch.setattr(plugin, "_load_settings", lambda: next(settings)) + profile_a = str(tmp_path / "profile-a") + profile_b = str(tmp_path / "profile-b") + host_a = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_a) + host_b = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_b) + + try: + runtime_a = plugin._get_runtime(profile_key=profile_a, host=host_a) + runtime_b = plugin._get_runtime(profile_key=profile_b, host=host_b) + assert runtime_a is not None + assert runtime_b is not None + + assert initialize_calls == [ + {"version": 1, "policy": {"unsupported": "warn"}} + ] + assert runtime_a._plugin_config_initialized is True + assert runtime_b._plugin_config_initialized is False + assert relay.plugin.report() is not None + + runtime_b.shutdown() + + assert clear_calls == 0 + assert relay.plugin.report() is not None + + runtime_a.shutdown() + + assert clear_calls == 1 + assert relay.plugin.report() is None + finally: + plugin.reset_for_tests() + host_a.shutdown() + host_b.shutdown() + original_clear() + + def test_nemo_relay_rejects_gateway_dynamic_config_with_actionable_diagnostic( tmp_path, monkeypatch, caplog ): @@ -670,136 +1021,40 @@ environment_ref = "../environments/worker-fixture" runtime.shutdown() -@pytest.mark.parametrize( - ("provider", "api_mode", "expected_surface", "should_rewrite"), - [ - ("custom", "chat_completions", "openai.chat_completions", True), - ("openai-codex", "codex_responses", "openai.responses", True), - ("anthropic", "anthropic_messages", "anthropic.messages", True), - ("custom", "anthropic_messages", "anthropic.messages", True), - ("bedrock", "bedrock_converse", "bedrock", False), - ], -) -def test_nemo_relay_managed_llm_uses_wire_protocol_for_interceptor_dispatch( +def test_relay_tool_request_rewrite_precedes_hermes_authorization_boundary( tmp_path, monkeypatch, - provider, - api_mode, - expected_surface, - should_rewrite, ): - fake = _FakeNemoRelay() - supported_surfaces = { - "anthropic.messages", - "openai.chat_completions", - "openai.responses", - } - - def execute(name, request, func, **kwargs): - fake.events.append(("llm.execute.start", name, request.content, kwargs)) - content = dict(request.content) - if name in supported_surfaces: - content["rewritten_for"] = name - result = func(_FakeLLMRequest(request.headers, content)) - fake.events.append(("llm.execute.end", name, result, kwargs)) - return result - - fake.llm.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_llm_execution_middleware( - session_id="s1", - provider=provider, - api_mode=api_mode, - model="fixture", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: request, - ) - - execute_start = next( - event for event in fake.events if event[0] == "llm.execute.start" - ) - assert execute_start[1] == expected_surface - assert execute_start[3]["metadata"]["provider"] == provider - assert execute_start[3]["metadata"]["api_mode"] == api_mode - if should_rewrite: - assert result["rewritten_for"] == expected_surface - else: - assert "rewritten_for" not in result - - -def test_nemo_relay_managed_llm_returns_post_next_interceptor_result(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - raw_response = SimpleNamespace( - model="fixture", - choices=[ - SimpleNamespace( - message=SimpleNamespace(role="assistant", content="raw", tool_calls=[]), - finish_reason="stop", - ) - ], - usage=None, - ) - - def execute(name, request, func, **kwargs): - del name, kwargs - normalized = func(_FakeLLMRequest(request.headers, request.content)) - return {**normalized, "post_next_interceptor": True} - - fake.llm.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_llm_execution_middleware( - session_id="s1", - provider="openai", - api_mode="chat_completions", - model="fixture", - request={"messages": []}, - next_call=lambda request: raw_response, - ) - - assert result["post_next_interceptor"] is True - assert result["assistant_message"]["content"] == "raw" - assert result is not raw_response - - -def test_nemo_relay_managed_tool_returns_post_interceptor_result(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def execute(name, args, func, **kwargs): - fake.events.append(("tool.execute.start", name, args, kwargs)) - raw = func({"intercepted": True, **args}) - result = {"compressed": True, "raw": raw} - fake.events.append(("tool.execute.end", name, result, kwargs)) - return result - - fake.tools.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="fixture-tool", - args={"value": 1}, - next_call=lambda args: {"tool_output": args}, - ) - - assert result == { - "compressed": True, - "raw": {"tool_output": {"intercepted": True, "value": 1}}, - } - - -def test_nemo_relay_plugin_activates_before_registering_managed_middleware(tmp_path, monkeypatch): + from hermes_cli.middleware import apply_tool_request_middleware + + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + plugin.on_session_start(session_id="s1") + + result = apply_tool_request_middleware( + "fixture-tool", + {"value": 1}, + session_id="s1", + tool_call_id="tool-1", + ) + + assert result.payload == {"intercepted": True, "value": 1} + assert result.trace[0] == {"source": "nemo_relay"} + + +def test_nemo_relay_plugin_activates_without_duplicate_execution_hooks( + tmp_path, monkeypatch +): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) + registered_hooks = [] class _Context: def register_hook(self, name, callback): - del name, callback + del callback + registered_hooks.append(name) def register_middleware(self, name, callback): del callback @@ -808,9 +1063,15 @@ def test_nemo_relay_plugin_activates_before_registering_managed_middleware(tmp_p plugin.register(_Context()) event_names = [event[0] for event in fake.events] - assert event_names.index("plugin.activate_dynamic") < event_names.index( - "hermes.register_middleware" - ) + assert "plugin.activate_dynamic" in event_names + assert "hermes.register_middleware" not in event_names + assert not { + "pre_api_request", + "post_api_request", + "api_request_error", + "pre_tool_call", + "post_tool_call", + }.intersection(registered_hooks) runtime = plugin._get_runtime() assert runtime is not None runtime.shutdown() @@ -820,7 +1081,7 @@ def test_nemo_relay_plugin_degrades_to_static_config_on_relay_0_5( tmp_path, monkeypatch, caplog ): fake = _FakeNemoRelay() - delattr(fake.plugin, "activate_dynamic_plugins") + delattr(fake.plugin, "initialize_with_dynamic_plugins") plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -899,7 +1160,7 @@ def test_nemo_relay_plugin_registers_shutdown_after_dynamic_retry(tmp_path, monk raise RuntimeError("temporary activation failure") return _FakePluginActivation(fake.events) - fake.plugin.activate_dynamic_plugins = _flaky_activate + fake.plugin.initialize_with_dynamic_plugins = _flaky_activate plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -1034,7 +1295,7 @@ enabled = true assert runtime is not None assert runtime._plugin_config_initialized is True scope_push_names = [event[1] for event in fake.events if event[0] == "scope.push"] - assert "hermes-session-s2" in scope_push_names + assert relay_runtime.SESSION_SCOPE in scope_push_names def test_nemo_relay_plugin_retries_plugins_toml_after_clear_failure(tmp_path, monkeypatch): @@ -1075,7 +1336,7 @@ enabled = true assert event_names.count("plugin.initialize.attempt") == 2 assert event_names.count("plugin.clear.failed") == 1 scope_push_names = [event[1] for event in fake.events if event[0] == "scope.push"] - assert "hermes-session-s2" in scope_push_names + assert relay_runtime.SESSION_SCOPE in scope_push_names def test_nemo_relay_plugin_disables_direct_atif_when_plugins_toml_owns_atif(tmp_path, monkeypatch): @@ -1195,652 +1456,3 @@ output_directory = "{(tmp_path / "managed-atof").as_posix()}" assert event_names.count("plugin.initialize.attempt") == 2 assert event_names.count("atof.register") == 1 assert event_names.count("atof.deregister") == 1 - - -def test_nemo_relay_adaptive_llm_execution_middleware_preserves_raw_response(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - seen_request = {} - raw_choice = SimpleNamespace( - message=SimpleNamespace( - role="assistant", - content=None, - tool_calls=[ - SimpleNamespace( - id="tool-1", - type="function", - function=SimpleNamespace(name="terminal", arguments='{"command":"pwd"}'), - ) - ], - reasoning_content="need a tool", - ), - finish_reason="tool_calls", - ) - raw_response = SimpleNamespace( - id="resp-1", - model="demo-model", - choices=[raw_choice], - usage=SimpleNamespace(prompt_tokens=3, completion_tokens=5, total_tokens=8), - ) - - def next_call(request): - seen_request.update(request) - return raw_response - - response = plugin.on_llm_execution_middleware( - session_id="s1", - task_id="t1", - turn_id="turn-1", - api_request_id="api-1", - provider="anthropic", - model="demo-model", - api_call_count=1, - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert response is raw_response - assert response.model == "demo-model" - assert response.choices == [raw_choice] - assert seen_request["intercepted"] is True - execute_start = next(event for event in fake.events if event[0] == "llm.execute.start") - assert execute_start[3]["data"]["mode"] == "observe_only" - execute_end = next(event for event in fake.events if event[0] == "llm.execute.end") - assert execute_end[2] == { - "model": "demo-model", - "assistant_message": { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "tool-1", - "type": "function", - "function": {"name": "terminal", "arguments": '{"command":"pwd"}'}, - } - ], - "reasoning_content": "need a tool", - }, - "finish_reason": "tool_calls", - "usage": {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}, - } - - -def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def native_like_execute(name, request, func, **kwargs): - fake.events.append(("llm.execute.start", name, request.content, kwargs)) - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None - - fake.llm.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ProviderAuthError(Exception): - status_code = 403 - - provider_error = ProviderAuthError("provider auth failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(ProviderAuthError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error_with_relay_suffix( - tmp_path, monkeypatch -): - # Guards the startswith (vs exact ==) match in _is_relay_wrapped_callback_error: - # Relay re-wraps the callback failure with its canonical prefix but APPENDS a - # trailing suffix. Exact equality would miss this and surface Relay's wrapper; - # prefix matching must still recover the original downstream error. - fake = _FakeNemoRelay() - - def native_like_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc} (retried 3x)") from None - - fake.llm.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ProviderAuthError(Exception): - status_code = 403 - - provider_error = ProviderAuthError("provider auth failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(ProviderAuthError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_llm_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - relay_error = RuntimeError("internal error: relay setup failed") - - def internal_error_execute(name, request, func, **kwargs): - raise relay_error - - fake.llm.execute = internal_error_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - with pytest.raises(RuntimeError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_llm_execution_keeps_wrapped_relay_error_after_downstream_failure( - tmp_path, monkeypatch -): - fake = _FakeNemoRelay() - relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream") - - def translated_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception: - raise relay_error - - fake.llm.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - def next_call(request): - raise _wrapped_downstream_error(RuntimeError("provider failed")) - - with pytest.raises(RuntimeError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_llm_execution_keeps_relay_translated_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - class RelayPolicyError(Exception): - pass - - relay_error = RelayPolicyError("relay policy blocked") - - def translated_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception: - raise relay_error - - fake.llm.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - provider_error = RuntimeError("provider failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(RelayPolicyError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_downstream_unwrap_matches_real_middleware_wrapper_shape(monkeypatch): - # Regression guard against core/plugin drift. The synthetic tests above model - # the downstream-error wrapper with a local class, so they keep passing even - # if core middleware renames its private ``_DownstreamExecutionError`` or drops - # ``.original`` -- the exact shape the plugin matches by name at - # ``_original_downstream_error``. Capture the wrapper the REAL - # ``hermes_cli.middleware._run_execution_chain`` hands to a middleware - # callback's ``next_call`` and assert the plugin's detector unwraps it to the - # original exception. If core middleware changes the wrapper shape, this fails - # here instead of silently defeating the unwrap in production. - from hermes_cli import middleware - - from plugins.observability.nemo_relay import _original_downstream_error - - class ProviderError(Exception): - status_code = 403 - - provider_error = ProviderError("provider auth failed") - captured: dict[str, Exception] = {} - - def terminal_call(payload): - raise provider_error - - def capturing_callback(**kwargs): - next_call = kwargs["next_call"] - try: - return next_call(kwargs.get("request")) - except Exception as exc: - captured["wrapper"] = exc - # Surface the original so the chain unwinds without re-wrapping noise. - raise _original_downstream_error(exc) from None - - with pytest.raises(ProviderError) as caught: - middleware._run_execution_chain( - "llm", - [capturing_callback], - terminal_call, - request={"messages": []}, - ) - - wrapper = captured["wrapper"] - # The wrapper the plugin sees must match what _original_downstream_error keys on. - assert wrapper.__class__.__name__ == "_DownstreamExecutionError" - assert isinstance(getattr(wrapper, "original", None), BaseException) - assert _original_downstream_error(wrapper) is provider_error - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def _adaptive_llm_execute_mode(tmp_path, monkeypatch, plugins_toml_text: str) -> str: - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text(plugins_toml_text, encoding="utf-8") - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - - execute_start = next(event for event in fake.events if event[0] == "llm.execute.start") - return execute_start[3]["data"]["mode"] - - -def test_nemo_relay_adaptive_llm_execution_middleware_defaults_to_observe_only_when_mode_is_unset( - tmp_path, monkeypatch -): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -version = 1 -""", - ) - assert mode == "observe_only" - - -def test_nemo_relay_adaptive_llm_execution_middleware_accepts_legacy_top_level_mode(tmp_path, monkeypatch): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -mode = "route" -""", - ) - assert mode == "route" - - -def test_nemo_relay_adaptive_llm_execution_middleware_prefers_tool_parallelism_mode(tmp_path, monkeypatch): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -mode = "route" - -[components.config.tool_parallelism] -mode = "schedule" -""", - ) - assert mode == "schedule" - - -def test_nemo_relay_llm_execution_middleware_calls_through_without_adaptive(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - response = plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": []}, - next_call=lambda request: {"raw": request}, - ) - - assert response == {"raw": {"messages": []}} - assert not any(event[0] == "llm.execute.start" for event in fake.events) - - -def test_nemo_relay_adaptive_tool_execution_middleware_preserves_raw_response(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - seen_args = {} - - def next_call(args): - seen_args.update(args) - return {"raw": True, "args": args} - - response = plugin.on_tool_execution_middleware( - session_id="s1", - task_id="t1", - turn_id="turn-1", - api_request_id="api-1", - tool_name="terminal", - tool_call_id="tool-1", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert response == {"raw": True, "args": {"command": "pwd", "intercepted": True}} - assert seen_args["intercepted"] is True - execute_start = next(event for event in fake.events if event[0] == "tool.execute.start") - assert execute_start[3]["data"]["mode"] == "observe_only" - assert execute_start[3]["data"]["tool_call_id"] == "tool-1" - - -def test_nemo_relay_adaptive_tool_execution_preserves_downstream_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def native_like_execute(name, args, func, **kwargs): - fake.events.append(("tool.execute.start", name, args, kwargs)) - try: - return func({"intercepted": True, **args}) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None - - fake.tools.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ToolAuthError(Exception): - status_code = 403 - - tool_error = ToolAuthError("tool auth failed") - - def next_call(args): - raise _wrapped_downstream_error(tool_error) - - with pytest.raises(ToolAuthError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is tool_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_tool_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - relay_error = RuntimeError("internal error: relay setup failed") - - def internal_error_execute(name, args, func, **kwargs): - raise relay_error - - fake.tools.execute = internal_error_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - with pytest.raises(RuntimeError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_tool_execution_keeps_wrapped_relay_error_after_downstream_failure( - tmp_path, monkeypatch -): - fake = _FakeNemoRelay() - relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream") - - def translated_execute(name, args, func, **kwargs): - try: - return func({"intercepted": True, **args}) - except Exception: - raise relay_error - - fake.tools.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - def next_call(args): - raise _wrapped_downstream_error(RuntimeError("tool failed")) - - with pytest.raises(RuntimeError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_tool_execution_keeps_relay_translated_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - class RelayPolicyError(Exception): - pass - - relay_error = RelayPolicyError("relay policy blocked") - - def translated_execute(name, args, func, **kwargs): - try: - return func({"intercepted": True, **args}) - except Exception: - raise relay_error - - fake.tools.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - tool_error = RuntimeError("tool failed") - - def next_call(args): - raise _wrapped_downstream_error(tool_error) - - with pytest.raises(RelayPolicyError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_tool_execution_middleware_calls_through_without_adaptive(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - response = plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - assert response == {"raw": {"command": "pwd"}} - assert not any(event[0] == "tool.execute.start" for event in fake.events) - - -def test_nemo_relay_adaptive_execution_skips_duplicate_observer_spans(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - base = { - "session_id": "s1", - "task_id": "t1", - "turn_id": "turn-1", - "api_request_id": "api-1", - } - plugin.on_pre_api_request( - **base, - provider="anthropic", - model="demo-model", - request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, - ) - plugin.on_post_api_request(**base, response={"ok": True}) - plugin.on_pre_tool_call(**base, tool_name="terminal", tool_call_id="tool-1", args={"command": "pwd"}) - plugin.on_post_tool_call(**base, tool_name="terminal", tool_call_id="tool-1", result={"ok": True}) - - plugin.on_llm_execution_middleware( - **base, - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - plugin.on_tool_execution_middleware( - **base, - tool_name="terminal", - tool_call_id="tool-1", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - event_names = [event[0] for event in fake.events] - assert "llm.call" not in event_names - assert "llm.call_end" not in event_names - assert "tool.call" not in event_names - assert "tool.call_end" not in event_names - assert "llm.execute.start" in event_names - assert "tool.execute.start" in event_names - - -def test_nemo_relay_plugin_noops_without_dependency(monkeypatch): - monkeypatch.delitem(sys.modules, "nemo_relay", raising=False) - sys.modules.pop("plugins.observability.nemo_relay", None) - plugin = importlib.import_module("plugins.observability.nemo_relay") - plugin.reset_for_tests() - - real_import = builtins.__import__ - - def blocked_import(name, *args, **kwargs): - if name == "nemo_relay": - raise ModuleNotFoundError(f"No module named {name!r}") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", blocked_import) - - plugin.on_pre_api_request(session_id="s1", api_request_id="api-1") - plugin.on_post_api_request(session_id="s1", api_request_id="api-1") diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index d6c5887d2706..c2b6db704eac 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2445,8 +2445,8 @@ class TestExecuteToolCalls: hook_calls.append((hook_name, kwargs)) return [] - monkeypatch.setattr("hermes_cli.plugins.invoke_hook", _capture_hook) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook", _capture_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with ( patch("run_agent.handle_function_call", side_effect=KeyboardInterrupt), @@ -3040,6 +3040,69 @@ class TestConcurrentToolExecution: assert "real-a" in messages[0]["content"] assert "real-b" in messages[1]["content"] + def test_concurrent_serializes_post_rewrite_authorization(self, agent, monkeypatch): + tc1 = _mock_tool_call( + name="web_search", arguments='{"q": "a"}', call_id="c1" + ) + tc2 = _mock_tool_call( + name="web_search", arguments='{"q": "b"}', call_id="c2" + ) + mock_msg = _mock_assistant_msg(content="", tool_calls=[tc1, tc2]) + messages = [] + state_lock = threading.Lock() + active = 0 + max_active = 0 + + def authorize(*_args, **_kwargs): + nonlocal active, max_active + with state_lock: + active += 1 + max_active = max(max_active, active) + try: + time.sleep(0.05) + return None + finally: + with state_lock: + active -= 1 + + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + authorize, + ) + + with patch( + "run_agent.handle_function_call", + side_effect=lambda _name, args, _task_id, **_kwargs: f"result-{args['q']}", + ): + agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") + + assert max_active == 1 + assert [message["tool_call_id"] for message in messages] == ["c1", "c2"] + + def test_concurrent_timeout_excludes_authorization_wait(self, agent, monkeypatch): + monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "0.05") + tool_call = _mock_tool_call( + name="web_search", arguments='{"q": "approved"}', call_id="c1" + ) + mock_msg = _mock_assistant_msg(content="", tool_calls=[tool_call]) + messages = [] + + def authorize(*_args, **_kwargs): + time.sleep(0.15) + return None + + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + authorize, + ) + + with patch("run_agent.handle_function_call", return_value="approved-result"): + agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") + + assert len(messages) == 1 + assert "approved-result" in messages[0]["content"] + assert "timed out after" not in messages[0]["content"] + def test_concurrent_interrupt_before_start(self, agent): """If interrupt is requested before concurrent execution, all tools are skipped.""" tc1 = _mock_tool_call(name="web_search", arguments='{}', call_id="c1") @@ -3107,6 +3170,55 @@ class TestConcurrentToolExecution: assert starts == [("c1", "web_search", {"query": "hello"})] assert completes == [("c1", "web_search", {"query": "hello"}, '{"success": true}')] + @pytest.mark.parametrize("quiet_mode", [True, False]) + def test_sequential_registry_tool_forwards_request_middleware_trace( + self, + agent, + monkeypatch, + quiet_mode, + ): + from hermes_cli.middleware import RequestMiddlewareResult + + trace = [{"source": "test-middleware"}] + observed = [] + agent.quiet_mode = quiet_mode + tool_call = _mock_tool_call( + name="web_search", + arguments='{"query":"hello"}', + call_id="c1", + ) + mock_msg = _mock_assistant_msg(content="", tool_calls=[tool_call]) + monkeypatch.setattr( + "hermes_cli.middleware.apply_tool_request_middleware", + lambda _name, args, **_kwargs: RequestMiddlewareResult( + payload=args, + original_payload=args, + changed=True, + trace=trace, + ), + ) + monkeypatch.setattr( + "hermes_cli.middleware.run_tool_execution_middleware", + lambda _name, args, callback, **_kwargs: callback(args), + ) + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + "agent.tool_executor._begin_tool_execution", + lambda *_args, **_kwargs: None, + ) + + def handle_function_call(*_args, **kwargs): + observed.append(kwargs) + return '{"success": true}' + + with patch("run_agent.handle_function_call", side_effect=handle_function_call): + agent._execute_tool_calls_sequential(mock_msg, [], "task-1") + + assert observed[0]["tool_request_middleware_trace"] == trace + def test_sequential_browser_type_callbacks_redact_api_key(self, agent): secret = "sk-proj-ABCD1234567890EFGH" tool_call = _mock_tool_call( @@ -3191,10 +3303,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: result = agent._invoke_tool("todo", {"todos": []}, "task-1", tool_call_id="todo-1") @@ -3274,10 +3386,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: "Blocked by policy", ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("run_agent.handle_function_call", side_effect=AssertionError("should not run")): agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3301,10 +3413,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3348,10 +3460,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3386,10 +3498,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}'): agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") @@ -3606,134 +3718,212 @@ class TestConcurrentToolExecution: # Second (allowed) write must checkpoint even though first was blocked. cp_mock.assert_called_once() + def test_managed_tool_pipeline_rejects_second_dispatch(self, agent, monkeypatch): + from agent import relay_tools, tool_executor + + dispatched = [] + duplicate_errors = [] + monkeypatch.setattr( + "hermes_cli.middleware.apply_tool_request_middleware", + lambda _name, args, **_kwargs: SimpleNamespace( + payload=args, + trace=[], + ), + ) + monkeypatch.setattr( + "hermes_cli.middleware.run_tool_execution_middleware", + lambda _name, args, callback, **_kwargs: callback(args), + ) + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr(tool_executor, "_begin_tool_execution", lambda *_a, **_k: None) + + def invoke_twice(name, args, callback, **kwargs): + del name, kwargs + result = callback(args) + try: + callback(args) + except RuntimeError as exc: + duplicate_errors.append(str(exc)) + return result, args + + monkeypatch.setattr(relay_tools, "execute", invoke_twice) + + outcome = tool_executor._run_agent_tool_execution_middleware( + agent, + function_name="terminal", + function_args={"command": "true"}, + effective_task_id="task-1", + tool_call_id="call-1", + execute=lambda args: dispatched.append(args) or "ok", + ) + + assert outcome.result == "ok" + assert dispatched == [{"command": "true"}] + assert duplicate_errors == [ + "Hermes tool execution callback invoked more than once" + ] + assert outcome.blocked is False + + def test_managed_tool_pipeline_allows_one_concurrent_dispatch( + self, + agent, + monkeypatch, + ): + from agent import relay_tools, tool_executor + + dispatched = [] + results = [] + errors = [] + barrier = threading.Barrier(2) + monkeypatch.setattr( + "hermes_cli.middleware.apply_tool_request_middleware", + lambda _name, args, **_kwargs: SimpleNamespace( + payload=args, + trace=[], + ), + ) + monkeypatch.setattr( + "hermes_cli.middleware.run_tool_execution_middleware", + lambda _name, args, callback, **_kwargs: callback(args), + ) + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr(tool_executor, "_begin_tool_execution", lambda *_a, **_k: None) + + def invoke_concurrently(name, args, callback, **kwargs): + del name, kwargs + + def invoke(): + barrier.wait(timeout=2) + try: + results.append(callback(args)) + except RuntimeError as exc: + errors.append(str(exc)) + + threads = [threading.Thread(target=invoke) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=2) + return results[0], args + + monkeypatch.setattr(relay_tools, "execute", invoke_concurrently) + + outcome = tool_executor._run_agent_tool_execution_middleware( + agent, + function_name="terminal", + function_args={"command": "true"}, + effective_task_id="task-1", + tool_call_id="call-1", + execute=lambda args: dispatched.append(args) or "ok", + ) + + assert outcome.result == "ok" + assert dispatched == [{"command": "true"}] + assert errors == ["Hermes tool execution callback invoked more than once"] + assert outcome.blocked is False + class TestAgentRuntimePostHookOwnershipSync: - """Pin the inline-dispatch tool list against the post-hook ownership set. + """Exercise post-hook ownership through both agent-runtime tool paths.""" - The post_tool_call hook fires from two places: the inline dispatcher in - agent/tool_executor.py:execute_tool_calls_sequential (for agent-runtime - tools that never reach handle_function_call) and - model_tools.handle_function_call itself (for registry-dispatched tools). - To prevent the executor from silently dropping or double-emitting, - AGENT_RUNTIME_POST_HOOK_TOOL_NAMES has to match exactly the static - `function_name == "..."` branches in the inline dispatch chain. + _CASES = ( + ("todo", {"todos": []}), + ("session_search", {"query": "needle"}), + ("memory", {"action": "view", "target": "memory"}), + ("clarify", {"question": "Continue?"}), + ("read_terminal", {}), + ("delegate_task", {"goal": "Check the child path"}), + ) - The chain is the if/elif tower anchored on `_block_msg is not None`. - Pre-dispatch `function_name == "..."` checks (counter resets, checkpoint - triggers) live outside the dispatch chain and are explicitly skipped. - """ - - _DISPATCH_ANCHOR_LEFT = "_block_msg" - - @classmethod - def _is_dispatch_anchor(cls, test_node) -> bool: - # Looking for `_block_msg is not None`. - if not isinstance(test_node, ast.Compare): - return False - if not (isinstance(test_node.left, ast.Name) and test_node.left.id == cls._DISPATCH_ANCHOR_LEFT): - return False - if not (len(test_node.ops) == 1 and isinstance(test_node.ops[0], ast.IsNot)): - return False - comparator = test_node.comparators[0] - return isinstance(comparator, ast.Constant) and comparator.value is None - - @staticmethod - def _function_name_literal(test_node) -> str | None: - """Return the string literal X for `function_name == "X"`, else None.""" - if not isinstance(test_node, ast.Compare): - return None - if not (isinstance(test_node.left, ast.Name) and test_node.left.id == "function_name"): - return None - if not (len(test_node.ops) == 1 and isinstance(test_node.ops[0], ast.Eq)): - return None - comparator = test_node.comparators[0] - if isinstance(comparator, ast.Constant) and isinstance(comparator.value, str): - return comparator.value - return None - - @classmethod - def _extract_dispatch_chain_names(cls, func) -> set[str]: - """Find the if/elif chain anchored on `_block_msg is not None`, return its - `function_name == "..."` literals.""" - source = inspect.cleandoc("\n" + inspect.getsource(func)) - tree = ast.parse(source) - names: set[str] = set() - for node in ast.walk(tree): - if not isinstance(node, ast.If): - continue - if not cls._is_dispatch_anchor(node.test): - continue - current = node - while current is not None: - literal = cls._function_name_literal(current.test) - if literal is not None: - names.add(literal) - if current.orelse and len(current.orelse) == 1 and isinstance(current.orelse[0], ast.If): - current = current.orelse[0] - else: - current = None - break - return names - - @classmethod - def _extract_invoke_tool_names(cls, func) -> set[str]: - """invoke_tool uses a flat if/elif on function_name directly; walk every - Compare in the function body (no other static `function_name == "..."` - checks live there).""" - source = inspect.cleandoc("\n" + inspect.getsource(func)) - tree = ast.parse(source) - names: set[str] = set() - for node in ast.walk(tree): - literal = cls._function_name_literal(node) - if literal is not None: - names.add(literal) - return names - - def test_frozenset_matches_inline_dispatch_chain(self): - from agent import tool_executor + @pytest.mark.parametrize(("tool_name", "tool_args"), _CASES) + def test_agent_runtime_tools_emit_once_per_executor_path( + self, + agent, + monkeypatch, + tool_name, + tool_args, + ): from agent.agent_runtime_helpers import AGENT_RUNTIME_POST_HOOK_TOOL_NAMES - inline_names = self._extract_dispatch_chain_names( - tool_executor.execute_tool_calls_sequential + hook_calls = [] + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *args, **kwargs: None, ) - assert inline_names, ( - "Could not find the dispatch chain (anchored on " - "`_block_msg is not None`) in execute_tool_calls_sequential. " - "If the dispatcher was refactored, update _DISPATCH_ANCHOR_LEFT " - "and the walker in this test." + monkeypatch.setattr( + "hermes_cli.lifecycle.invoke_hook", + lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - assert inline_names == set(AGENT_RUNTIME_POST_HOOK_TOOL_NAMES), ( - "Inline dispatch chain in " - "agent/tool_executor.py:execute_tool_calls_sequential has drifted " - "from AGENT_RUNTIME_POST_HOOK_TOOL_NAMES in " - "agent/agent_runtime_helpers.py.\n" - f" Inline branches: {sorted(inline_names)}\n" - f" Ownership frozenset: {sorted(AGENT_RUNTIME_POST_HOOK_TOOL_NAMES)}\n" - "Update both together so post_tool_call fires exactly once per " - "tool execution." + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) + monkeypatch.setattr( + "tools.todo_tool.todo_tool", + lambda **kwargs: '{"ok":true}', ) + monkeypatch.setattr( + "tools.memory_tool.memory_tool", + lambda **kwargs: '{"ok":true}', + ) + monkeypatch.setattr( + "tools.clarify_tool.clarify_tool", + lambda **kwargs: '{"ok":true}', + ) + monkeypatch.setattr( + "tools.read_terminal_tool.read_terminal_tool", + lambda **kwargs: '{"ok":true}', + ) + monkeypatch.setattr(agent, "_get_session_db_for_recall", lambda: None) + monkeypatch.setattr( + agent, + "_dispatch_delegate_task", + lambda args: '{"ok":true}', + ) + agent._memory_manager = None - def test_invoke_tool_dispatch_matches_inline_dispatch_chain(self): - """invoke_tool (concurrent path) and the inline dispatcher (sequential - path) must cover the same set of agent-runtime tools — otherwise - post_tool_call fires inconsistently depending on which executor ran - the tool.""" - from agent import agent_runtime_helpers, tool_executor + assert tool_name in AGENT_RUNTIME_POST_HOOK_TOOL_NAMES + with patch( + "run_agent.handle_function_call", + side_effect=AssertionError("agent-runtime tools must stay inline"), + ): + agent._invoke_tool( + tool_name, + dict(tool_args), + "task-concurrent", + tool_call_id=f"{tool_name}-concurrent", + ) + tool_call = _mock_tool_call( + name=tool_name, + arguments=json.dumps(tool_args), + call_id=f"{tool_name}-sequential", + ) + agent._execute_tool_calls_sequential( + _mock_assistant_msg(content="", tool_calls=[tool_call]), + [], + "task-sequential", + ) - invoke_tool_names = self._extract_invoke_tool_names( - agent_runtime_helpers.invoke_tool - ) - inline_names = self._extract_dispatch_chain_names( - tool_executor.execute_tool_calls_sequential - ) - assert invoke_tool_names == inline_names, ( - "Static `function_name == \"...\"` branches diverged between " - "agent/agent_runtime_helpers.py:invoke_tool (concurrent path) " - "and agent/tool_executor.py:execute_tool_calls_sequential " - "(sequential path).\n" - f" invoke_tool: {sorted(invoke_tool_names)}\n" - f" execute_tool_calls_sequential: {sorted(inline_names)}" - ) + post_calls = [ + kwargs + for hook_name, kwargs in hook_calls + if hook_name == "post_tool_call" + ] + assert [call["tool_call_id"] for call in post_calls] == [ + f"{tool_name}-concurrent", + f"{tool_name}-sequential", + ] + assert all(call["tool_name"] == tool_name for call in post_calls) + + def test_post_hook_ownership_contract_lists_exercised_tools(self): + from agent.agent_runtime_helpers import AGENT_RUNTIME_POST_HOOK_TOOL_NAMES + + assert AGENT_RUNTIME_POST_HOOK_TOOL_NAMES == { + tool_name for tool_name, _ in self._CASES + } class TestPathsOverlap: @@ -3876,14 +4066,50 @@ class TestHandleMaxIterations: assert len(result) > 0 assert "summary" in result.lower() + def test_summary_retries_share_relay_identity(self, agent): + agent.client.chat.completions.create.side_effect = [ + _mock_response(content=""), + _mock_response(content="Summary"), + ] + agent._cached_system_prompt = "You are helpful." + relay_calls = [] + + def execute_current(request, callback, **kwargs): + relay_calls.append(kwargs) + return callback(request) + + with ( + patch("agent.relay_llm.execute_current", side_effect=execute_current), + patch("agent.relay_llm.complete_logical_call") as complete_logical, + ): + result = agent._handle_max_iterations( + [{"role": "user", "content": "do stuff"}], + 60, + ) + + assert result == "Summary" + assert [call["metadata"]["retry_count"] for call in relay_calls] == [0, 1] + assert relay_calls[0]["metadata"]["api_request_id"] == ( + relay_calls[1]["metadata"]["api_request_id"] + ) + assert relay_calls[0]["metadata"]["call_role"] == "iteration_summary" + assert all(call["defer_logical_completion"] is True for call in relay_calls) + complete_logical.assert_called_once_with( + relay_calls[0]["metadata"]["api_request_id"], + outcome="success", + ) + def test_api_failure_returns_error(self, agent): agent.client.chat.completions.create.side_effect = Exception("API down") agent._cached_system_prompt = "You are helpful." messages = [{"role": "user", "content": "do stuff"}] - result = agent._handle_max_iterations(messages, 60) + with patch("agent.relay_llm.complete_logical_call") as complete_logical: + result = agent._handle_max_iterations(messages, 60) assert isinstance(result, str) assert "error" in result.lower() assert "API down" in result + complete_logical.assert_called_once() + assert complete_logical.call_args.kwargs == {"outcome": "failed"} def test_summary_skips_reasoning_for_unsupported_openrouter_model(self, agent): agent.base_url = "https://openrouter.ai/api/v1" @@ -4197,6 +4423,50 @@ class TestRunConversation: agent.compression_enabled = False agent.save_trajectories = False + def test_task_start_failure_closes_relay_turn_and_lease(self, agent): + relay_lease = SimpleNamespace( + parent_session_id="", + profile_key="/profile", + session_id=agent.session_id or "", + ) + relay_turn = object() + coordinator = MagicMock() + coordinator.acquire_conversation.return_value = relay_lease + coordinator.begin_turn.return_value = relay_turn + start_error = RuntimeError("task metrics start failed") + + with ( + patch("agent.relay_runtime.SESSION_COORDINATOR", coordinator), + patch( + "agent.relay_runtime.current_profile_key", + return_value="/profile", + ), + patch( + "hermes_cli.observability.relay_shared_metrics.start_task_run", + side_effect=start_error, + ), + patch( + "hermes_cli.observability.relay_shared_metrics.finish_task_run" + ) as finish_task_run, + patch("agent.conversation_loop.run_conversation") as run_conversation, + ): + with pytest.raises(RuntimeError) as caught: + agent.run_conversation("hello", task_id="task-1") + + assert caught.value is start_error + run_conversation.assert_not_called() + finish_task_run.assert_not_called() + coordinator.finish_logical_calls.assert_called_once_with( + relay_turn, + outcome="failed", + ) + coordinator.end_turn.assert_called_once_with( + relay_turn, + outcome="failed", + ) + coordinator.release_conversation.assert_called_once_with(relay_lease) + assert agent._relay_pending_turn_id is None + def test_stop_finish_reason_returns_response(self, agent): self._setup_agent(agent) resp = _mock_response(content="Final answer", finish_reason="stop") @@ -4280,6 +4550,7 @@ class TestRunConversation: usage=None, ) hook_events = [] + logical_completions = [] def _fake_activate(reason=None): agent._fallback_index = len(agent._fallback_chain) @@ -4291,6 +4562,12 @@ class TestRunConversation: patch.object(agent, "_run_codex_stream", side_effect=[content_filter_response, fallback_response]) as mock_run_codex_stream, patch.object(agent, "_try_activate_fallback", side_effect=_fake_activate) as mock_try_activate_fallback, patch.object(agent, "_invoke_api_request_error_hook", side_effect=lambda **kw: hook_events.append(kw)), + patch( + "agent.relay_llm.complete_logical_call", + side_effect=lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -4304,6 +4581,9 @@ class TestRunConversation: assert hook_events[0]["error_type"] == "ContentPolicyBlocked" assert hook_events[0]["retryable"] is False assert hook_events[0]["reason"] == FailoverReason.content_policy_blocked.value + assert logical_completions == [ + (hook_events[0]["api_request_id"], "success") + ] def test_ollama_small_runtime_context_fails_before_api_call(self, agent, caplog): self._setup_agent(agent) @@ -4383,10 +4663,10 @@ class TestRunConversation: with ( patch("run_agent.handle_function_call", return_value="search result"), patch( - "hermes_cli.plugins.has_hook", + "hermes_cli.lifecycle.has_hook", side_effect=lambda name: name in {"pre_api_request", "post_api_request"}, ), - patch("hermes_cli.plugins.invoke_hook", side_effect=_record_hook), + patch("hermes_cli.lifecycle.invoke_hook", side_effect=_record_hook), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -4399,6 +4679,7 @@ class TestRunConversation: assert len(pre_request_calls) == 2 assert len(post_request_calls) == 2 assert [call["api_call_count"] for call in pre_request_calls] == [1, 2] + assert [call["retry_count"] for call in pre_request_calls] == [0, 0] assert [call["api_call_count"] for call in post_request_calls] == [1, 2] assert all(call["session_id"] == agent.session_id for call in pre_request_calls) assert all(call["turn_id"] == pre_request_calls[0]["turn_id"] for call in pre_request_calls + post_request_calls) @@ -4411,6 +4692,41 @@ class TestRunConversation: assert all("usage" in c and "response" in c for c in post_request_calls) assert all("assistant_message" in c["response"] for c in post_request_calls) + def test_terminal_task_closes_logical_calls_before_metrics_scope(self, agent): + from agent import relay_runtime + + order = [] + failed_result = { + "final_response": "provider failed", + "messages": [], + "completed": False, + "failed": True, + "interrupted": False, + } + + with ( + patch( + "agent.conversation_loop.run_conversation", + return_value=failed_result, + ), + patch( + "hermes_cli.observability.relay_shared_metrics.start_task_run", + ), + patch( + "hermes_cli.observability.relay_shared_metrics.finish_task_run", + side_effect=lambda **_kwargs: order.append("metrics"), + ), + patch.object( + relay_runtime.SESSION_COORDINATOR, + "finish_logical_calls", + side_effect=lambda *_args, **_kwargs: order.append("logical"), + ), + ): + result = agent.run_conversation("private prompt") + + assert result is failed_result + assert order == ["logical", "metrics"] + def test_api_request_error_hook_skips_payload_work_without_listener(self, agent, monkeypatch): payload_built = False hook_called = False @@ -4425,8 +4741,8 @@ class TestRunConversation: hook_called = True return [] - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: False) - monkeypatch.setattr("hermes_cli.plugins.invoke_hook", _invoke_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: False) + monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook", _invoke_hook) monkeypatch.setattr(agent, "_api_request_payload_for_hook", _payload_for_hook) agent._invoke_api_request_error_hook( @@ -4465,12 +4781,12 @@ class TestRunConversation: payload_counts["response"] += 1 return {} - monkeypatch.setattr("hermes_cli.plugins.has_hook", _has_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", _has_hook) monkeypatch.setattr(agent, "_api_request_payload_for_hook", _request_payload) monkeypatch.setattr(agent, "_api_response_payload_for_hook", _response_payload) with ( - patch("hermes_cli.plugins.invoke_hook", return_value=[]), + patch("hermes_cli.lifecycle.invoke_hook", return_value=[]), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -6436,6 +6752,50 @@ class TestRetryExhaustion: assert "Invalid API response" in result["error"] assert result.get("final_response") == result["error"] + def test_invalid_response_retry_completes_one_logical_call(self, agent): + self._setup_agent(agent) + agent.client.chat.completions.create.side_effect = [ + SimpleNamespace(choices=[], model="test/model", usage=None), + _mock_response(content="recovered"), + ] + relay_attempts = [] + logical_completions = [] + + def execute(request, callback, **kwargs): + relay_attempts.append(kwargs) + return callback(request) + + from agent import conversation_loop as _conv_loop + + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch("run_agent.time", self._make_fast_time_mock()), + patch.object(_conv_loop, "time", self._make_fast_time_mock()), + patch.object(_conv_loop, "jittered_backoff", lambda *a, **k: 0.0), + patch("agent.relay_llm.execute", side_effect=execute), + patch( + "agent.relay_llm.complete_logical_call", + side_effect=lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ), + ): + result = agent.run_conversation("hello") + + assert result["completed"] is True + assert len(relay_attempts) == 2 + assert all( + attempt["defer_logical_completion"] is True + for attempt in relay_attempts + ) + request_ids = { + attempt["metadata"]["api_request_id"] for attempt in relay_attempts + } + assert len(request_ids) == 1 + assert logical_completions == [(request_ids.pop(), "success")] + def test_content_filter_refusal_surfaced_not_retried(self, agent): """A model refusal must be surfaced immediately, NOT laundered into the empty-response retry loop and reported as "rate limited" / "no diff --git a/tests/run_agent/test_stream_single_writer_65991.py b/tests/run_agent/test_stream_single_writer_65991.py index b1972f969f7e..4fbf74ae2514 100644 --- a/tests/run_agent/test_stream_single_writer_65991.py +++ b/tests/run_agent/test_stream_single_writer_65991.py @@ -137,6 +137,21 @@ class TestSingleWriterLoop: assert "".join(delivered) == "first" assert "-stale-tail" not in "".join(delivered) + def test_chat_parser_failure_closes_managed_stream(self): + agent = _make_agent() + managed_stream = MagicMock() + managed_stream.__iter__.return_value = iter([object()]) + managed_stream.final_response = None + + with patch( + "agent.relay_llm.stream", + return_value=managed_stream, + ): + with pytest.raises(AttributeError): + agent._interruptible_streaming_api_call({}) + + managed_stream.close.assert_called_once() + class TestCodexSingleWriter: """The codex_responses path claims the sink and stops when superseded, @@ -208,6 +223,55 @@ class TestCodexSingleWriter: assert "".join(delivered) == "hello world" + def test_codex_interrupt_closes_stream_without_draining_provider(self): + from agent.codex_runtime import run_codex_stream + + agent = _make_agent() + agent.api_mode = "codex_responses" + produced = [] + stream_closed = threading.Event() + + def interrupt_after_first_delta(_text): + agent._interrupt_requested = True + + agent.stream_delta_callback = interrupt_after_first_delta + agent._stream_callback = None + + def event_gen(): + try: + produced.append("first") + yield self._codex_event( + "response.output_text.delta", + delta="first", + item_id="i1", + ) + produced.append("lookahead") + yield self._codex_event( + "response.output_text.delta", + delta="-unused", + item_id="i1", + ) + produced.append("terminal") + yield self._codex_event( + "response.completed", + response=SimpleNamespace( + id="r1", + status="completed", + output=[], + usage=None, + ), + ) + finally: + stream_closed.set() + + mock_client = MagicMock() + mock_client.responses.create.return_value = event_gen() + + run_codex_stream(agent, {"model": "gpt-5.3-codex"}, client=mock_client) + + assert produced == ["first", "lookahead"] + assert stream_closed.is_set() + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/run_agent/test_streaming.py b/tests/run_agent/test_streaming.py index b613220df202..0f2fc27a5158 100644 --- a/tests/run_agent/test_streaming.py +++ b/tests/run_agent/test_streaming.py @@ -96,6 +96,51 @@ class TestStreamingAccumulator: assert response.usage is not None assert response.usage.completion_tokens == 3 + @patch("run_agent.AIAgent._create_request_openai_client") + @patch("run_agent.AIAgent._close_request_openai_client") + def test_chat_stream_closes_original_provider_resource( + self, + mock_close, + mock_create, + ): + from run_agent import AIAgent + + class ProviderStream: + def __init__(self): + self.closed = False + + def __iter__(self): + return iter([ + _make_stream_chunk( + content="Hello", + finish_reason="stop", + model="test-model", + ) + ]) + + def close(self): + self.closed = True + + provider_stream = ProviderStream() + mock_client = MagicMock() + mock_client.chat.completions.create.return_value = provider_stream + mock_create.return_value = mock_client + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "chat_completions" + agent._interrupt_requested = False + + response = agent._interruptible_streaming_api_call({}) + + assert response.choices[0].message.content == "Hello" + assert provider_stream.closed is True + @patch("run_agent.AIAgent._create_request_openai_client") @patch("run_agent.AIAgent._close_request_openai_client") def test_native_gemini_endpoint_omits_stream_options(self, mock_close, mock_create): @@ -1266,6 +1311,7 @@ class TestAnthropicStreamCallbacks: agent._interruptible_streaming_api_call({}) assert touch_calls.count("receiving stream response") == len(events) + mock_stream.close.assert_called_once() @patch("run_agent.AIAgent._rebuild_anthropic_client") @patch("run_agent.AIAgent._replace_primary_openai_client") diff --git a/tests/run_agent/test_tool_call_guardrail_runtime.py b/tests/run_agent/test_tool_call_guardrail_runtime.py index 36ced43c7956..3bc8a83ab592 100644 --- a/tests/run_agent/test_tool_call_guardrail_runtime.py +++ b/tests/run_agent/test_tool_call_guardrail_runtime.py @@ -220,6 +220,109 @@ def test_config_enabled_hard_stop_concurrent_path_does_not_submit_blocked_calls_ assert completed_events[0][1] == "web_search" +def test_relay_rewrite_precedes_sequential_policy_approval_checkpoint_and_dispatch(): + agent = _make_agent("write_file") + original_args = {"path": "/original/path", "content": "old"} + final_args = {"path": "/approved/path", "content": "new"} + tc = _mock_tool_call("write_file", json.dumps(original_args), "c-rewrite") + msg = SimpleNamespace(content="", tool_calls=[tc]) + messages = [] + observed = { + "plugin": [], + "guardrail": [], + "approval": [], + "checkpoint": [], + "start": [], + "dispatch": [], + } + + original_before_call = agent._tool_guardrails.before_call + + def observe_guardrail(name, args): + observed["guardrail"].append((name, dict(args))) + return original_before_call(name, args) + + def relay_execute(name, args, callback, **kwargs): + del name, args, kwargs + return callback(dict(final_args)), dict(final_args) + + def observe_plugin(name, args, **kwargs): + del kwargs + observed["plugin"].append((name, dict(args))) + return None + + def observe_approval(name, args): + observed["approval"].append((name, dict(args))) + return None + + def dispatch(name, args, task_id, **kwargs): + del task_id, kwargs + observed["dispatch"].append((name, dict(args))) + return json.dumps({"ok": True}) + + agent._checkpoint_mgr = SimpleNamespace( + enabled=True, + get_working_dir_for_path=lambda path: path, + ensure_checkpoint=lambda path, reason: observed["checkpoint"].append( + (path, reason) + ), + ) + agent.tool_start_callback = lambda _call_id, name, args: observed["start"].append( + (name, dict(args)) + ) + + with ( + patch("agent.relay_tools.execute", side_effect=relay_execute), + patch( + "hermes_cli.plugins.resolve_pre_tool_block", + side_effect=observe_plugin, + ), + patch.object(agent._tool_guardrails, "before_call", side_effect=observe_guardrail), + patch( + "acp_adapter.edit_approval.maybe_require_edit_approval", + side_effect=observe_approval, + ), + patch("model_tools.registry.dispatch", side_effect=dispatch), + ): + agent._execute_tool_calls_sequential(msg, messages, "task-1") + + expected = [("write_file", final_args)] + assert observed["plugin"] == expected + assert observed["guardrail"] == expected + assert observed["approval"] == expected + assert observed["start"] == expected + assert observed["dispatch"] == expected + assert observed["checkpoint"] == [ + ("/approved/path", "before write_file") + ] + + +def test_relay_rewrite_is_guarded_before_dispatch_in_concurrent_path(): + agent = _make_agent("web_search", config=_hard_stop_config()) + original_args = {"query": "original"} + blocked_args = {"query": "blocked"} + _seed_exact_failures(agent, "web_search", blocked_args) + tc = _mock_tool_call("web_search", json.dumps(original_args), "c-rewrite-block") + msg = SimpleNamespace(content="", tool_calls=[tc]) + messages = [] + starts = [] + + def relay_execute(name, args, callback, **kwargs): + del name, args, kwargs + return callback(dict(blocked_args)), dict(blocked_args) + + agent.tool_start_callback = lambda *args: starts.append(args) + with ( + patch("agent.relay_tools.execute", side_effect=relay_execute), + patch("run_agent.handle_function_call", return_value="SHOULD_NOT_RUN") as dispatch, + ): + agent._execute_tool_calls_concurrent(msg, messages, "task-1") + + dispatch.assert_not_called() + assert starts == [] + assert "repeated_exact_failure_block" in messages[0]["content"] + + def test_plugin_pre_tool_block_wins_without_counting_as_toolguard_block(): agent = _make_agent("web_search") args = {"query": "same"} diff --git a/tests/scripts/test_smoke_nemo_relay_shared_metrics.py b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py new file mode 100644 index 000000000000..d31a90bb4395 --- /dev/null +++ b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py @@ -0,0 +1,41 @@ +"""Tests for the shared-metrics smoke artifact.""" + +from pathlib import Path + +import pytest + +from scripts import smoke_nemo_relay_shared_metrics as smoke + + +@pytest.mark.parametrize( + "relative_path", + [ + Path(".venv") / "bin" / "hermes", + Path(".venv") / "Scripts" / "hermes.exe", + ], +) +def test_resolve_hermes_executable_from_repository_venv( + tmp_path, + monkeypatch, + relative_path, +): + executable = tmp_path / relative_path + executable.parent.mkdir(parents=True) + executable.touch() + monkeypatch.setattr(smoke.shutil, "which", lambda _name: None) + + assert smoke._resolve_hermes_executable(tmp_path) == executable + + +def test_resolve_hermes_executable_falls_back_to_path(tmp_path, monkeypatch): + executable = tmp_path / "bin" / "hermes" + monkeypatch.setattr(smoke.shutil, "which", lambda _name: str(executable)) + + assert smoke._resolve_hermes_executable(tmp_path / "repo") == executable + + +def test_resolve_hermes_executable_reports_missing_binary(tmp_path, monkeypatch): + monkeypatch.setattr(smoke.shutil, "which", lambda _name: None) + + with pytest.raises(SystemExit, match="or on PATH"): + smoke._resolve_hermes_executable(tmp_path) diff --git a/tests/test_model_tools.py b/tests/test_model_tools.py index 469b8a6921e9..e6b0bfd51f33 100644 --- a/tests/test_model_tools.py +++ b/tests/test_model_tools.py @@ -326,6 +326,40 @@ class TestPreToolCallBlocking: assert "post_tool_call" in hook_calls assert "transform_tool_result" in hook_calls + def test_relay_rewrite_is_visible_to_pre_tool_authorization(self, monkeypatch): + observed = {} + + def rewrite(**kwargs): + assert kwargs["tool_name"] == "read_file" + return {**kwargs["args"], "path": "approved.txt"} + + def fake_invoke_hook(hook_name, **kwargs): + if hook_name == "pre_tool_call": + observed["pre_tool_args"] = kwargs["args"] + return [] + + def dispatch(_name, args, **_kwargs): + observed["dispatch_args"] = args + return json.dumps({"ok": True}) + + monkeypatch.setattr( + "hermes_cli.observability.relay_runtime.apply_tool_request_intercepts", + rewrite, + ) + monkeypatch.setattr("hermes_cli.plugins.invoke_hook", fake_invoke_hook) + monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("model_tools.registry.dispatch", dispatch) + + handle_function_call( + "read_file", + {"path": "original.txt"}, + task_id="t1", + session_id="s1", + ) + + assert observed["pre_tool_args"]["path"] == "approved.txt" + assert observed["dispatch_args"]["path"] == "approved.txt" + def test_run_agent_pattern_fires_pre_tool_call_exactly_once(self, monkeypatch): """End-to-end regression for the double-fire bug. diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index 0b2257426299..4d685011ccf2 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -3,7 +3,6 @@ from pathlib import Path import tomllib - def _load_optional_dependencies(): pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" with pyproject_path.open("rb") as handle: @@ -11,6 +10,13 @@ def _load_optional_dependencies(): return project["optional-dependencies"] +def _load_package_data(): + pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" + with pyproject_path.open("rb") as handle: + tool = tomllib.load(handle)["tool"] + return tool["setuptools"]["package-data"] + + def test_matrix_extra_not_in_all(): """The [matrix] extra pulls `mautrix[encryption]` -> `python-olm`, which has Linux-only wheels and no native build path on Windows or @@ -215,14 +221,10 @@ def test_feishu_extra_includes_qrcode_for_qr_login(): assert any(dep.startswith("qrcode") for dep in feishu_extra) -def test_nemo_relay_extra_uses_supported_official_distribution_range(): - optional_dependencies = _load_optional_dependencies() +def test_shared_metrics_schema_is_packaged(): + package_data = _load_package_data() - assert optional_dependencies["nemo-relay"] == ["nemo-relay>=0.5,<1.0"] - assert not any( - spec == "hermes-agent[nemo-relay]" - for spec in optional_dependencies["all"] - ) + assert "observability/schemas/*.json" in package_data["hermes_cli"] def _uv_lock_version(package: str) -> str: diff --git a/tests/tools/test_zombie_process_cleanup.py b/tests/tools/test_zombie_process_cleanup.py index a8b745f541a9..b4679ffbe3ac 100644 --- a/tests/tools/test_zombie_process_cleanup.py +++ b/tests/tools/test_zombie_process_cleanup.py @@ -317,9 +317,15 @@ class TestGatewayCleanupWiring: class TestDelegationCleanup: """Verify subagent delegation cleans up child agents.""" - def test_run_single_child_calls_close(self): + def test_run_single_child_calls_close(self, monkeypatch, tmp_path): """_run_single_child finally block should call close() on child.""" from unittest.mock import MagicMock + from hermes_constants import ( + get_hermes_home, + reset_hermes_home_override, + set_hermes_home_override, + ) + from hermes_cli.observability import relay_runtime from tools.delegate_tool import _run_single_child parent = MagicMock() @@ -327,18 +333,155 @@ class TestDelegationCleanup: parent._active_children_lock = threading.Lock() child = MagicMock() + child.session_id = "child-session" child._delegate_saved_tool_names = ["tool1"] - child.run_conversation.side_effect = RuntimeError("test abort") + observed = {} + + def run_conversation(**_kwargs): + observed["hermes_home"] = get_hermes_home() + raise RuntimeError("test abort") + + child.run_conversation.side_effect = run_conversation + relay_host = MagicMock() + monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host) parent._active_children.append(child) + profile_home = tmp_path / "profile-a" + token = set_hermes_home_override(profile_home) + try: + result = _run_single_child( + task_index=0, + goal="test goal", + child=child, + parent_agent=parent, + ) + finally: + reset_hermes_home_override(token) + + child.close.assert_called_once() + assert observed["hermes_home"] == profile_home + relay_host.unregister_subagent.assert_called_once_with( + {"child_session_id": "child-session"} + ) + assert child not in parent._active_children + assert result["status"] == "error" + + def test_active_child_turn_owns_relay_scope_cleanup(self, monkeypatch): + from unittest.mock import MagicMock + + from hermes_cli.observability import relay_runtime + from tools.delegate_tool import _run_single_child + + parent = MagicMock() + parent._active_children = [] + parent._active_children_lock = threading.Lock() + child = MagicMock() + child.session_id = "active-child-session" + child._delegate_saved_tool_names = ["tool1"] + child.run_conversation.side_effect = RuntimeError("test abort") + parent._active_children.append(child) + relay_host = MagicMock() + monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host) + monkeypatch.setattr( + relay_runtime.SESSION_COORDINATOR, + "has_active_turn", + lambda **_kwargs: True, + ) + result = _run_single_child( task_index=0, - goal="test goal", + goal="test active turn cleanup", child=child, parent_agent=parent, ) - child.close.assert_called_once() - assert child not in parent._active_children assert result["status"] == "error" + relay_host.unregister_subagent.assert_not_called() + + def test_timed_out_child_keeps_relay_session_until_its_turn_exits( + self, monkeypatch, tmp_path + ): + from unittest.mock import MagicMock + + from agent import relay_runtime + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + from tools.delegate_tool import _run_single_child + + relay_runtime._reset_for_tests() + profile_home = tmp_path / "profile-timeout" + profile_token = set_hermes_home_override(profile_home) + child_started = threading.Event() + release_child = threading.Event() + child_finished = threading.Event() + parent = MagicMock() + parent._active_children = [] + parent._active_children_lock = threading.Lock() + child = MagicMock() + child.session_id = "timed-out-child" + child._delegate_saved_tool_names = ["tool1"] + child.get_activity_summary.return_value = {"api_call_count": 1} + parent._active_children.append(child) + relay_host = MagicMock() + monkeypatch.setattr(relay_runtime, "get_runtime", lambda **_kwargs: relay_host) + monkeypatch.setattr("tools.delegate_tool._get_child_timeout", lambda: 0.1) + + def run_conversation(**kwargs): + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=child.session_id, + platform="subagent", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="timed-out-child-turn", + task_id=kwargs["task_id"], + ) + child_started.set() + try: + release_child.wait(timeout=5) + return { + "final_response": "late result", + "completed": True, + "interrupted": False, + "api_calls": 1, + "messages": [], + } + finally: + relay_runtime.SESSION_COORDINATOR.end_turn( + turn, + outcome="cancelled", + ) + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + child_finished.set() + + child.run_conversation.side_effect = run_conversation + try: + result = _run_single_child( + task_index=0, + goal="test timed-out turn cleanup", + child=child, + parent_agent=parent, + ) + + assert child_started.is_set() + assert result["status"] == "timeout" + assert relay_runtime.SESSION_COORDINATOR.has_active_turn( + profile_key=str(profile_home), + session_id=child.session_id, + ) + relay_host.unregister_subagent.assert_not_called() + + release_child.set() + assert child_finished.wait(timeout=5) + assert not relay_runtime.SESSION_COORDINATOR.has_active_turn( + profile_key=str(profile_home), + session_id=child.session_id, + ) + finally: + release_child.set() + reset_hermes_home_override(profile_token) + relay_runtime._reset_for_tests() diff --git a/tools/approval.py b/tools/approval.py index 24c08f76916a..57300f0cb630 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -104,7 +104,7 @@ def _fire_approval_hook(hook_name: str, **kwargs) -> None: pre_approval_request, post_approval_response. """ try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook except Exception: # Plugin system not available in this execution context # (e.g. bare tool-only imports, minimal test environments). diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 7894a6af9bfc..b1d32e33d146 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -18,6 +18,7 @@ never the child's intermediate tool calls or reasoning. """ import enum +import contextvars import json import logging @@ -1592,7 +1593,7 @@ def _build_child_agent( logger.debug("spawn_requested relay failed: %s", exc) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "subagent_start", parent_session_id=getattr(parent_agent, "session_id", None), @@ -2183,7 +2184,11 @@ def _run_single_child( stream_callback=_relay_child_text, ) - _child_future = _timeout_executor.submit(_run_with_thread_capture) + _child_context = contextvars.copy_context() + _child_future = _timeout_executor.submit( + _child_context.run, + _run_with_thread_capture, + ) try: result = _child_future.result(timeout=child_timeout) except Exception as _timeout_exc: @@ -2586,6 +2591,23 @@ def _run_single_child( except Exception: logger.debug("Failed to close child agent after delegation") + # The AIAgent turn boundary normally closes the child scope itself. This + # fallback covers failures before that boundary starts, but must not pop + # a scope while a timed-out child worker is still unwinding. + try: + from agent import relay_runtime + + runtime = relay_runtime.get_runtime(create=False) + child_session_id = str(getattr(child, "session_id", "") or "") + child_turn_is_active = relay_runtime.SESSION_COORDINATOR.has_active_turn( + profile_key=relay_runtime.current_profile_key(), + session_id=child_session_id, + ) + if runtime is not None and child_session_id and not child_turn_is_active: + runtime.unregister_subagent({"child_session_id": child_session_id}) + except Exception: + logger.debug("Failed to close child Relay session after delegation") + _PARENT_FINALIZATION_LOCK_GUARD = threading.Lock() _PARENT_FINALIZATION_FALLBACK_LOCK = threading.RLock() @@ -2984,7 +3006,9 @@ def delegate_task( with DaemonThreadPoolExecutor(max_workers=max_children) as executor: futures = {} for i, t, child in children: + child_context = contextvars.copy_context() future = executor.submit( + child_context.run, _run_single_child, task_index=i, goal=t["goal"], diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 859d30a76497..476582d03e1f 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -2798,7 +2798,7 @@ def terminal_tool( # still subject to the final output limit below. # The hook is fail-open, and the first valid string return wins. try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook hook_results = invoke_hook( "transform_terminal_output", command=command, diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 1e15d788b0a3..e7b73107438f 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -473,13 +473,19 @@ def _notify_session_boundary( ) -> None: """Fire session lifecycle hooks with CLI parity.""" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import finalize_session, invoke_hook - _invoke_hook( - event_type, - session_id=session_id, - platform=_resolve_agent_platform(platform), - ) + if event_type == "on_session_finalize": + finalize_session( + session_id=session_id, + platform=_resolve_agent_platform(platform), + ) + else: + invoke_hook( + event_type, + session_id=session_id, + platform=_resolve_agent_platform(platform), + ) except Exception: pass @@ -658,7 +664,7 @@ def _finalize_session(session: dict | None, end_reason: str = "tui_close") -> No # the user Ctrl‑C's mid‑turn. if agent is not None: try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook invoke_hook( "on_session_end", diff --git a/uv.lock b/uv.lock index f0b075a08ee3..dee539ca975d 100644 --- a/uv.lock +++ b/uv.lock @@ -1530,6 +1530,7 @@ dependencies = [ { name = "httpx", extra = ["socks"] }, { name = "jinja2" }, { name = "markdown" }, + { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')" }, { name = "openai" }, { name = "packaging" }, { name = "pathspec" }, @@ -1665,9 +1666,6 @@ mistral = [ modal = [ { name = "modal" }, ] -nemo-relay = [ - { name = "nemo-relay" }, -] parallel-web = [ { name = "parallel-web" }, ] @@ -1807,7 +1805,7 @@ requires-dist = [ { name = "microsoft-teams-apps", marker = "extra == 'teams'", specifier = "==2.0.13.4" }, { name = "mistralai", marker = "extra == 'mistral'", specifier = "==2.4.8" }, { name = "modal", marker = "extra == 'modal'", specifier = "==1.3.4" }, - { name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5,<1.0" }, + { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')", specifier = ">=0.6.0,<0.7" }, { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "openai", specifier = "==2.24.0" }, { name = "packaging", specifier = "==26.0" }, @@ -2603,14 +2601,14 @@ wheels = [ [[package]] name = "nemo-relay" -version = "0.5.0" +version = "0.6.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/fc/6892859ffbfb60f8cc09a4181c8d407b574d49490e35cd563e7f0a61bf54/nemo_relay-0.5.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:203409e0f392fc4f40f52277347ed75ba9969b2af28ddb3b6ae53f789185a0ed", size = 7769721, upload-time = "2026-07-08T18:43:57.745Z" }, - { url = "https://files.pythonhosted.org/packages/dd/13/25c4fcf6fb979af07c9562238819183588a910c2ae09767603500d6948c2/nemo_relay-0.5.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de61ad0644c5e8c300de814f6884c5de7ae51db14694f5057f938b4da54bfaaf", size = 7037083, upload-time = "2026-07-08T18:44:00.135Z" }, - { url = "https://files.pythonhosted.org/packages/49/97/bd4c77e975d50f7d09f8bfcb74d7e704e05a8b2205f555bf557f84edc5fe/nemo_relay-0.5.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb323569f104506f4f1a0e2104b22d74c9a0bbf3280ec4d58ad3d768de76fb15", size = 7430817, upload-time = "2026-07-08T18:44:02.165Z" }, - { url = "https://files.pythonhosted.org/packages/da/8b/5681f3430e6d25bf7419dbe576da14599e1cfadc083edb74e691a0c3a980/nemo_relay-0.5.0-cp311-abi3-win_amd64.whl", hash = "sha256:da1bd1e1dec79b7bda3984dd8c59b32ba97abf96c37e85cd1f38d208a29f42b9", size = 7359048, upload-time = "2026-07-08T18:44:04.169Z" }, - { url = "https://files.pythonhosted.org/packages/4e/f2/65c3ef0435e671c829063d4872897f9054f369f0aeacab12c5aefe486705/nemo_relay-0.5.0-cp311-abi3-win_arm64.whl", hash = "sha256:87462585f17b40ce82c54347acef5ace96ee11de3015f234e0b902f23b3793fb", size = 6934235, upload-time = "2026-07-08T18:44:06.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/65/d320016505457cc30971f575e8dadffb923b7cfc780ab8bb25a4ce9d305c/nemo_relay-0.6.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:ad5dae6febf6532d7b113abc2a404679c8feffc499df3034b93d9a078185d2bb", size = 9917779, upload-time = "2026-07-22T20:07:48.961Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c0/f33250e71c4206da1b339072893f9a1e39295fe1aceb9a2fef4b8620a0f2/nemo_relay-0.6.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0cd9570f64c6956fe3bfb82af1cdb3ee70cb50b51098cdb0de831c3f9b4e904", size = 8888375, upload-time = "2026-07-22T20:07:51.049Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/d1dfaed022da0f6f14765a122867f976a69cc520fe1faaf99757f5719d1f/nemo_relay-0.6.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:849daa9e45158ac581e54506e0fcc7a24f557d1ed06dbdc074f5de7a00393cbc", size = 9336372, upload-time = "2026-07-22T20:07:53.224Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b0/908d77f75b9054e1e403da78f7d9430249a45939070d4298abbb41a04b5b/nemo_relay-0.6.0-cp311-abi3-win_amd64.whl", hash = "sha256:bfbbedfd130fa95c9b8c04643c30910df850e0ed3500beeab82b00ca2d94e7ea", size = 9613425, upload-time = "2026-07-22T20:07:55.175Z" }, + { url = "https://files.pythonhosted.org/packages/cd/71/c438b9d746303ff7f270d99f13b250bf947cdac3e52de2a83fd132cbca0b/nemo_relay-0.6.0-cp311-abi3-win_arm64.whl", hash = "sha256:82fe132943399d89e6ec34dc28df0be7bbe41b84f6698c545928b8816b6010f6", size = 9034810, upload-time = "2026-07-22T20:07:57.467Z" }, ] [[package]]