From 14bed44c8cfad8e51fab9283555fb4c53a37c998 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 27 Jul 2026 21:10:51 -0700 Subject: [PATCH 001/156] Reapply "feat(observability): integrate NeMo Relay runtime and shared metrics" Signed-off-by: Alex Fournier --- agent/agent_runtime_helpers.py | 17 +- agent/auxiliary_client.py | 387 ++- agent/chat_completion_helpers.py | 697 +++-- agent/codex_runtime.py | 116 +- agent/conversation_loop.py | 39 +- agent/relay_llm.py | 1108 ++++++++ agent/relay_runtime.py | 991 +++++++ agent/relay_tools.py | 123 + agent/tool_executor.py | 955 ++++--- agent/turn_context.py | 10 +- agent/turn_finalizer.py | 8 +- cli-config.yaml.example | 16 + cli.py | 31 +- contributors/emails/afournier@nvidia.com | 1 + docs/observability/README.md | 4 + docs/observability/relay-shared-metrics.md | 127 + gateway/run.py | 12 +- gateway/slash_commands.py | 7 +- hermes_cli/config.py | 8 + hermes_cli/hooks.py | 12 +- hermes_cli/kanban_db.py | 2 +- hermes_cli/lifecycle.py | 63 + hermes_cli/middleware.py | 30 +- hermes_cli/observability/__init__.py | 31 + hermes_cli/observability/relay_runtime.py | 14 + .../observability/relay_shared_metrics.py | 817 ++++++ .../hermes.shared_metrics.v1.schema.json | 338 +++ hermes_cli/observability/shared_metrics.py | 486 ++++ .../observability/shared_metrics_contract.py | 516 ++++ .../shared_metrics_subscriber.py | 67 + hermes_cli/plugins.py | 4 +- hermes_cli/setup.py | 33 + hermes_cli/subcommands/setup.py | 13 +- hermes_cli/web_server.py | 3 + model_tools.py | 33 +- plugins/observability/nemo_relay/README.md | 102 +- plugins/observability/nemo_relay/__init__.py | 947 +++---- plugins/observability/nemo_relay/plugin.yaml | 5 - pyproject.toml | 10 +- run_agent.py | 157 +- scripts/smoke_nemo_relay_shared_metrics.py | 443 +++ tests/agent/test_auxiliary_relay.py | 445 +++ .../test_bedrock_interrupt_post_worker.py | 17 +- tests/agent/test_relay_llm.py | 1498 ++++++++++ tests/agent/test_relay_tools.py | 262 ++ tests/cli/test_session_boundary_hooks.py | 22 +- tests/hermes_cli/test_lifecycle.py | 60 + tests/hermes_cli/test_plugins.py | 26 + tests/hermes_cli/test_relay_shared_metrics.py | 839 ++++++ .../test_relay_shared_metrics_runtime.py | 2403 +++++++++++++++++ tests/hermes_cli/test_setup_telemetry.py | 37 + tests/plugins/test_nemo_relay_plugin.py | 1388 ++++------ tests/run_agent/test_run_agent.py | 632 ++++- .../test_stream_single_writer_65991.py | 64 + tests/run_agent/test_streaming.py | 46 + .../test_tool_call_guardrail_runtime.py | 103 + .../test_smoke_nemo_relay_shared_metrics.py | 41 + tests/test_model_tools.py | 34 + tests/test_project_metadata.py | 18 +- tests/tools/test_zombie_process_cleanup.py | 153 +- tools/approval.py | 2 +- tools/delegate_tool.py | 28 +- tools/terminal_tool.py | 2 +- tui_gateway/server.py | 20 +- uv.lock | 18 +- 65 files changed, 14427 insertions(+), 2514 deletions(-) create mode 100644 agent/relay_llm.py create mode 100644 agent/relay_runtime.py create mode 100644 agent/relay_tools.py create mode 100644 contributors/emails/afournier@nvidia.com create mode 100644 docs/observability/relay-shared-metrics.md create mode 100644 hermes_cli/lifecycle.py create mode 100644 hermes_cli/observability/__init__.py create mode 100644 hermes_cli/observability/relay_runtime.py create mode 100644 hermes_cli/observability/relay_shared_metrics.py create mode 100644 hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json create mode 100644 hermes_cli/observability/shared_metrics.py create mode 100644 hermes_cli/observability/shared_metrics_contract.py create mode 100644 hermes_cli/observability/shared_metrics_subscriber.py create mode 100644 scripts/smoke_nemo_relay_shared_metrics.py create mode 100644 tests/agent/test_auxiliary_relay.py create mode 100644 tests/agent/test_relay_llm.py create mode 100644 tests/agent/test_relay_tools.py create mode 100644 tests/hermes_cli/test_lifecycle.py create mode 100644 tests/hermes_cli/test_relay_shared_metrics.py create mode 100644 tests/hermes_cli/test_relay_shared_metrics_runtime.py create mode 100644 tests/hermes_cli/test_setup_telemetry.py create mode 100644 tests/scripts/test_smoke_nemo_relay_shared_metrics.py diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index b5706cadfec3..97f78b9143cb 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -2475,7 +2475,8 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False, skip_tool_request_middleware: bool = False, - tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None) -> str: + tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None, + skip_tool_execution_middleware: bool = False) -> str: """Invoke a single tool and return the result string. No display logic. Handles both agent-level tools (todo, memory, etc.) and registry-dispatched @@ -2654,8 +2655,7 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i return _finish_agent_tool(agent._dispatch_delegate_task(next_args), next_args) else: def _execute(next_args: dict) -> Any: - return _ra().handle_function_call( - function_name, next_args, effective_task_id, + dispatch_kwargs = dict( tool_call_id=tool_call_id, session_id=agent.session_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", @@ -2667,6 +2667,17 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i disabled_toolsets=getattr(agent, "disabled_toolsets", None), tool_request_middleware_trace=list(_tool_middleware_trace), ) + if skip_tool_execution_middleware: + dispatch_kwargs["skip_tool_execution_middleware"] = True + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + **dispatch_kwargs, + ) + + if skip_tool_execution_middleware: + return _execute(function_args) from hermes_cli.middleware import run_tool_execution_middleware diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index fa5bc6b80d7e..da85f03659e8 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -42,6 +42,7 @@ Payment / credit exhaustion fallback: import contextlib import contextvars +import functools import hashlib import inspect import json @@ -50,9 +51,10 @@ import os import re import threading import time +import uuid from pathlib import Path # noqa: F401 — used by test mocks from types import SimpleNamespace -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse, parse_qs, urlunparse # NOTE: `from openai import OpenAI` is deliberately NOT at module top — the @@ -2492,6 +2494,167 @@ _RUNTIME_MAIN_AUTH_MODE: str = "" _RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( contextvars.ContextVar("auxiliary_runtime_main", default=None) ) + +_RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( + contextvars.ContextVar("auxiliary_relay_call", default=None) +) + + +def _relay_auxiliary_call(callback): + """Give every physical retry in one auxiliary call a shared Relay identity.""" + + @functools.wraps(callback) + def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return callback(*args, **kwargs) + except BaseException: + _fail_relay_auxiliary_call() + raise + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _relay_auxiliary_call_async(callback): + """Async counterpart to :func:`_relay_auxiliary_call`.""" + + @functools.wraps(callback) + async def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return await callback(*args, **kwargs) + except BaseException: + _fail_relay_auxiliary_call() + raise + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _set_relay_auxiliary_route( + provider: str | None, + model: str | None, + api_mode: str | None, +) -> None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + context["provider"] = str(provider or "auxiliary") + context["model"] = str(model or "unknown") + context["api_mode"] = str(api_mode or "chat_completions") + + +def _relay_auxiliary_metadata( + *, + provider: str | None = None, + api_mode: str | None = None, +) -> tuple[str, str, dict[str, Any]] | None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return None + attempt_count = int(context.get("attempt_count") or 0) + context["attempt_count"] = attempt_count + 1 + provider_name = str(provider or context.get("provider") or "auxiliary") + model_name = str(context.get("model") or "unknown") + return provider_name, model_name, { + "api_mode": str(api_mode or context.get("api_mode") or "chat_completions"), + "api_request_id": str(context["request_id"]), + "call_role": f"auxiliary:{context['task']}", + "retry_count": attempt_count, + "auxiliary_task": str(context["task"]), + } + + +def _relay_sync_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return callback(kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.execute_current( + kwargs, + callback, + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + defer_logical_completion=True, + ) + + +async def _relay_async_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return await callback(kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return await relay_llm.execute_current_async( + kwargs, + callback, + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + defer_logical_completion=True, + ) + + +def _relay_sync_stream( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, +) -> Any: + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return client.chat.completions.create(**kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.stream_current( + kwargs, + lambda request: client.chat.completions.create(**request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + finalizer=dict, + metadata=metadata, + ) _RUNTIME_MAIN_COMPAT_SNAPSHOT: Tuple[Any, ...] = ("", "", "", "", "", "") _RUNTIME_MAIN_COMPAT_LOCK = threading.Lock() @@ -3801,7 +3964,13 @@ def _retry_same_provider_sync( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task, + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -3866,7 +4035,13 @@ async def _retry_same_provider_async( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task, + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -4071,7 +4246,7 @@ def _call_fallback_candidate_sync( base_url=fb_base, task=task) try: return _validate_llm_response( - fb_client.chat.completions.create(**fb_kwargs), task) + _relay_sync_completion(fb_client, fb_kwargs, provider=fb_label), task) except Exception as fb_err: if not _is_auth_error(fb_err): raise @@ -4088,7 +4263,13 @@ def _call_fallback_candidate_sync( base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=fb_provider, + ), + task, + ) except Exception as retry_err: if not _is_auth_error(retry_err): raise @@ -4137,7 +4318,13 @@ async def _call_fallback_candidate_async( base_url=fb_base, task=task) try: return _validate_llm_response( - await fb_client.chat.completions.create(**fb_kwargs), task) + await _relay_async_completion( + fb_client, + fb_kwargs, + provider=fb_label, + ), + task, + ) except Exception as fb_err: if not _is_auth_error(fb_err): raise @@ -4155,7 +4342,13 @@ async def _call_fallback_candidate_async( base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=fb_provider, + ), + task, + ) except Exception as retry_err: if not _is_auth_error(retry_err): raise @@ -7270,6 +7463,7 @@ def _validate_llm_response( except (AttributeError, TypeError, IndexError) as exc: recovered = _recover_aux_response_message(response) if recovered is not None: + _complete_relay_auxiliary_call() return recovered response_type = type(response).__name__ response_preview = str(response)[:120] @@ -7279,9 +7473,34 @@ def _validate_llm_response( f"Expected object with .choices[0].message — check provider " f"adapter or custom endpoint compatibility." ) from exc + _complete_relay_auxiliary_call() return response +def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None: + """Close one auxiliary logical call after acceptance or terminal failure.""" + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + from agent import relay_llm + + relay_llm.complete_logical_call( + str(context.get("request_id") or ""), + outcome=outcome, + ) + + +def _fail_relay_auxiliary_call() -> None: + """Close a terminally failed call without replacing its original error.""" + try: + _complete_relay_auxiliary_call(outcome="failed") + except Exception: + logger.warning( + "Relay auxiliary failure finalization failed", + exc_info=True, + ) + + def _recover_aux_response_message(response: Any) -> Optional[Any]: """Synthesize chat-completions shape from Responses-style text fields. @@ -7680,6 +7899,7 @@ async def _acreate_with_stream( ) +@_relay_auxiliary_call def call_llm( task: str = None, *, @@ -7820,6 +8040,11 @@ def call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Log what we're about to do — makes auxiliary operations visible _base_info = str(getattr(client, "base_url", resolved_base_url) or "") @@ -7857,7 +8082,12 @@ def call_llm( kwargs["stream"] = True if stream_options: kwargs["stream_options"] = stream_options - return client.chat.completions.create(**kwargs) + return _relay_sync_stream( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ) # Handle unsupported temperature, max_tokens vs max_completion_tokens retry, # then payment fallback. @@ -7880,10 +8110,18 @@ def call_llm( # for the transient retry every auxiliary task shares. (PR #16587) try: return _validate_llm_response( - _create_with_progress( - client, kwargs, task, - force_stream=_provider_requires_stream( - resolved_provider, _base_info or resolved_base_url, + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, _base_info or resolved_base_url, + ), ), ), task, @@ -7919,10 +8157,19 @@ def call_llm( time.sleep(_backoff) try: return _validate_llm_response( - _create_with_progress( - client, kwargs, task, - force_stream=_provider_requires_stream( - resolved_provider, _base_info or resolved_base_url, + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, + _base_info or resolved_base_url, + ), ), ), task) @@ -7942,7 +8189,12 @@ def call_llm( ) try: return _validate_llm_response( - client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) # If retry still fails, fall through to the max_tokens / @@ -7980,7 +8232,12 @@ def call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. @@ -8010,7 +8267,12 @@ def call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -8043,7 +8305,12 @@ def call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -8071,7 +8338,12 @@ def call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry ─────────────────────────────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -8121,7 +8393,12 @@ def call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise @@ -8379,6 +8656,7 @@ def extract_content_or_reasoning(response) -> str: return "" +@_relay_auxiliary_call_async async def async_call_llm( task: str = None, *, @@ -8470,6 +8748,11 @@ async def async_call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Pass the client's actual base_url (not just resolved_base_url) so # endpoint-specific temperature overrides can distinguish @@ -8508,7 +8791,14 @@ async def async_call_llm( try: return _validate_llm_response( - await _acreate(kwargs), task, + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task, provider=resolved_provider, base_url=_client_base) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -8529,7 +8819,14 @@ async def async_call_llm( task or "call", transient_err, ) return _validate_llm_response( - await _acreate(kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task) except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) @@ -8540,7 +8837,12 @@ async def async_call_llm( ) try: return _validate_llm_response( - await client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) if not ( @@ -8574,7 +8876,12 @@ async def async_call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. @@ -8603,7 +8910,12 @@ async def async_call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -8635,7 +8947,12 @@ async def async_call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -8662,7 +8979,12 @@ async def async_call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry (mirrors sync call_llm) ─────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -8706,7 +9028,12 @@ async def async_call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 5fecd0e26cfe..7ce136652125 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -15,6 +15,7 @@ sites unchanged. Symbols that tests patch on ``run_agent`` (e.g. from __future__ import annotations +import contextvars import json import logging import math @@ -57,6 +58,12 @@ _OPENROUTER_PROVIDER_SORT_VALUES = {"throughput", "latency", "price"} _FALLBACK_EXHAUSTED_COOLDOWN_S = 5.0 +def _context_thread_target(callback): + """Bind a no-argument thread target to the caller's ContextVars.""" + context = contextvars.copy_context() + return lambda: context.run(callback) + + def _ra(): """Lazy ``run_agent`` reference. @@ -810,7 +817,7 @@ def interruptible_api_call(agent, api_kwargs: dict): _call_start = time.time() agent._touch_activity("waiting for non-streaming API response") - t = threading.Thread(target=_call, daemon=True) + t = threading.Thread(target=_context_thread_target(_call), daemon=True) t.start() _poll_count = 0 while t.is_alive(): @@ -1989,6 +1996,28 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: """Request a summary when max iterations are reached. Returns the final response text.""" print(f"⚠️ Reached maximum iterations ({agent.max_iterations}). Requesting summary...") + summary_api_request_id = f"iteration-summary:{uuid.uuid4()}" + summary_call_outcome = "failed" + + def _managed_summary_call(request, callback, *, retry_count: int): + from agent import relay_llm + + return relay_llm.execute_current( + request, + callback, + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + metadata={ + "api_mode": str( + getattr(agent, "api_mode", "") or "chat_completions" + ), + "api_request_id": summary_api_request_id, + "call_role": "iteration_summary", + "retry_count": retry_count, + }, + defer_logical_completion=True, + ) + summary_request = ( "You've reached the maximum number of tool-calling iterations allowed. " "Please provide a final response summarizing what you've found and accomplished so far, " @@ -2174,17 +2203,33 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if agent.api_mode == "anthropic_messages": _tsum = agent._get_transport() - _ant_kw = _tsum.build_kwargs(model=agent.model, messages=api_messages, tools=None, - max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - is_oauth=agent._is_anthropic_oauth, - preserve_dots=agent._anthropic_preserve_dots(), - base_url=getattr(agent, "_anthropic_base_url", None)) + _ant_kw = _tsum.build_kwargs( + model=agent.model, + messages=api_messages, + tools=None, + max_tokens=agent.max_tokens, + reasoning_config=agent.reasoning_config, + is_oauth=agent._is_anthropic_oauth, + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None), + ) _ant_kw = _merge_nous_portal_messages_extra_body(agent, _ant_kw) - summary_response = agent._anthropic_messages_create(_ant_kw) + summary_response = _managed_summary_call( + _ant_kw, + agent._anthropic_messages_create, + retry_count=0, + ) _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_summary_result.content or "").strip() else: - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=0, + ) _summary_result = agent._get_transport().normalize_response(summary_response) final_response = (_summary_result.content or "").strip() @@ -2192,6 +2237,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if "" in final_response: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: + summary_call_outcome = "success" messages.append({"role": "assistant", "content": final_response}) else: final_response = "I reached the iteration limit and couldn't generate a summary." @@ -2206,13 +2252,22 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: final_response = (_cnr_retry.content or "").strip() elif agent.api_mode == "anthropic_messages": _tretry = agent._get_transport() - _ant_kw2 = _tretry.build_kwargs(model=agent.model, messages=api_messages, tools=None, - is_oauth=agent._is_anthropic_oauth, - max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - preserve_dots=agent._anthropic_preserve_dots(), - base_url=getattr(agent, "_anthropic_base_url", None)) + _ant_kw2 = _tretry.build_kwargs( + model=agent.model, + messages=api_messages, + tools=None, + is_oauth=agent._is_anthropic_oauth, + max_tokens=agent.max_tokens, + reasoning_config=agent.reasoning_config, + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None), + ) _ant_kw2 = _merge_nous_portal_messages_extra_body(agent, _ant_kw2) - retry_response = agent._anthropic_messages_create(_ant_kw2) + retry_response = _managed_summary_call( + _ant_kw2, + agent._anthropic_messages_create, + retry_count=1, + ) _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_retry_result.content or "").strip() else: @@ -2229,7 +2284,14 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if summary_extra_body: summary_kwargs["extra_body"] = summary_extra_body - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary_retry").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary_retry" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=1, + ) _retry_result = agent._get_transport().normalize_response(summary_response) final_response = (_retry_result.content or "").strip() @@ -2237,6 +2299,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if "" in final_response: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: + summary_call_outcome = "success" messages.append({"role": "assistant", "content": final_response}) else: final_response = "I reached the iteration limit and couldn't generate a summary." @@ -2246,6 +2309,13 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: except Exception as e: logger.warning(f"Failed to get summary response: {e}") final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" + finally: + from agent import relay_llm + + relay_llm.complete_logical_call( + summary_api_request_id, + outcome=summary_call_outcome, + ) return final_response @@ -2404,7 +2474,9 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= pass def _bedrock_call(): + stream = None try: + from agent import relay_llm from agent.bedrock_adapter import ( _get_bedrock_runtime_client, invalidate_runtime_client, @@ -2413,44 +2485,40 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= normalize_converse_response, stream_converse_with_callbacks, ) - region = api_kwargs.pop("__bedrock_region__", "us-east-1") - api_kwargs.pop("__bedrock_converse__", None) - client = _get_bedrock_runtime_client(region) - try: - raw_response = client.converse_stream(**api_kwargs) - except Exception as _bedrock_exc: - # IAM policies scoped to bedrock:InvokeModel only (no - # InvokeModelWithResponseStream) reject converse_stream() - # with AccessDeniedException. That denial is permanent for - # the session — fall back to the non-streaming converse() - # inline (it maps to bedrock:InvokeModel) and disable - # streaming for subsequent calls so we don't re-fail every - # turn. - if is_streaming_access_denied_error(_bedrock_exc): - agent._disable_streaming = True - agent._safe_print( - "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — " - "falling back to non-streaming InvokeModel.\n" - " Grant that action to restore streaming output.\n" - ) - logger.info( - "bedrock: converse_stream denied by IAM (%s) — " - "using non-streaming converse() for this session.", - type(_bedrock_exc).__name__, - ) - result["response"] = normalize_converse_response( - client.converse(**api_kwargs) - ) - return - # Evict the cached client on stale-connection failures - # so the outer retry loop builds a fresh client/pool. - if is_stale_connection_error(_bedrock_exc): - invalidate_runtime_client(region) - raise + intercepted_events = [] + writer_token = {"value": None} - # Claim the delta sink for this bedrock stream (#65991) so a - # superseded attempt's callbacks are fenced by the sink guard. - claim_stream_writer(agent) + def _open_bedrock_stream(next_api_kwargs: dict[str, Any]): + final_kwargs = dict(next_api_kwargs) + region = final_kwargs.pop("__bedrock_region__", "us-east-1") + final_kwargs.pop("__bedrock_converse__", None) + client = _get_bedrock_runtime_client(region) + try: + raw_response = client.converse_stream(**final_kwargs) + except Exception as _bedrock_exc: + # InvokeModel-only policies cannot open a stream. Keep + # the fallback inside the same managed Relay attempt so + # the real provider request and terminal response still + # share one lifecycle boundary. + if is_streaming_access_denied_error(_bedrock_exc): + agent._disable_streaming = True + agent._safe_print( + "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — " + "falling back to non-streaming InvokeModel.\n" + " Grant that action to restore streaming output.\n" + ) + logger.info( + "bedrock: converse_stream denied by IAM (%s) — " + "using non-streaming converse() for this session.", + type(_bedrock_exc).__name__, + ) + return normalize_converse_response( + client.converse(**final_kwargs) + ) + if is_stale_connection_error(_bedrock_exc): + invalidate_runtime_client(region) + raise + return raw_response.get("stream", []) def _on_text(text): _fire_first() @@ -2465,18 +2533,65 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= _fire_first() agent._fire_reasoning_delta(text) - result["response"] = stream_converse_with_callbacks( - raw_response, + def _finalize_bedrock_stream(): + return stream_converse_with_callbacks( + {"stream": list(intercepted_events)} + ) + + def _bedrock_stream_created(_stream: Any) -> None: + writer_token["value"] = claim_stream_writer(agent) + + def _accept_bedrock_event(_event: Any) -> bool: + token = writer_token["value"] + return token is None or stream_writer_is_current(agent, token) + + stream = relay_llm.stream( + dict(api_kwargs), + _open_bedrock_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "bedrock"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=_finalize_bedrock_stream, + on_stream_created=_bedrock_stream_created, + on_chunk=intercepted_events.append, + chunk_adapter=lambda chunk: chunk, + accept_chunk=_accept_bedrock_event, + completed_response_predicate=lambda response: bool( + getattr(response, "choices", None) + ), + metadata={ + "api_mode": "custom", + "api_request_id": getattr( + agent, "_current_api_request_id", None + ), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + streamed_response = stream_converse_with_callbacks( + {"stream": stream}, on_text_delta=_on_text if agent._has_stream_consumers() else None, on_tool_start=_on_tool, on_reasoning_delta=_on_reasoning if agent.reasoning_callback or agent.stream_delta_callback else None, on_interrupt_check=lambda: agent._interrupt_requested, on_event=lambda: _bedrock_last_event.__setitem__("t", time.time()), ) + result["response"] = stream.final_response or streamed_response except Exception as e: result["error"] = e + finally: + if stream is not None: + stream.close() - t = threading.Thread(target=_bedrock_call, daemon=True) + t = threading.Thread( + target=_context_thread_target(_bedrock_call), daemon=True + ) t.start() while t.is_alive(): t.join(timeout=0.3) @@ -2661,6 +2776,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= first_delta_fired = {"done": False} deltas_were_sent = {"yes": False} # Track if any deltas were fired (for fallback) + provider_tool_in_flight = {"yes": False} # Wall-clock timestamp of the last real streaming chunk. The outer # poll loop uses this to detect stale connections that keep receiving # SSE keep-alive pings but no actual data. @@ -2680,11 +2796,29 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= "discarded_chunks": 0, "discarded_bytes": 0, } + managed_stream_holder = {"stream": None} + + def _set_managed_stream(stream: Any) -> Any: + managed_stream_holder["stream"] = stream + return stream + + def _close_managed_stream() -> None: + stream = managed_stream_holder.pop("stream", None) + if stream is None: + return + close = getattr(stream, "close", None) + if callable(close): + try: + close() + except Exception: + logger.debug("Managed provider stream cleanup failed", exc_info=True) def _start_stream_attempt() -> int: with stream_attempt_lock: stream_attempt_state["current"] += 1 - return int(stream_attempt_state["current"]) + attempt_id = int(stream_attempt_state["current"]) + provider_tool_in_flight["yes"] = False + return attempt_id def _cancel_current_stream_attempt(reason: str) -> None: with stream_attempt_lock: @@ -2794,107 +2928,6 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # Cap connect/pool at 60s even when provider timeout is higher. # connect/pool cover TCP handshake, not model inference. _conn_cap = min(_base_timeout, 60.0) if _provider_timeout_cfg is not None else 30.0 - stream_kwargs = { - **api_kwargs, - "stream": True, - "timeout": _httpx.Timeout( - connect=_conn_cap, - read=_stream_read_timeout, - write=_base_timeout, - pool=_conn_cap, - ), - } - # OpenAI's `stream_options={"include_usage": True}` drives usage - # accounting on OpenAI-compatible endpoints (incl. the Gemini OpenAI - # compat shim and aggregators like OpenRouter). Google's *native* - # Gemini REST endpoint rejects the keyword outright - # (`Completions.create() got an unexpected keyword argument - # 'stream_options'`), so omit it only for that endpoint. - if not is_native_gemini_base_url(agent.base_url): - stream_kwargs["stream_options"] = {"include_usage": True} - request_client = _set_request_client( - agent._create_request_openai_client( - reason="chat_completion_stream_request", - api_kwargs=stream_kwargs, - ) - ) - # Reset stale-stream timer so the detector measures from this - # attempt's start, not a previous attempt's last chunk. - last_chunk_time["t"] = time.time() - agent._touch_activity("waiting for provider response (streaming)") - # Initialize per-attempt stream diagnostics so the retry block can - # reach for them after the stream dies. Lives on - # ``request_client_holder["diag"]`` for closure access. - _diag = agent._stream_diag_init() - request_client_holder["diag"] = _diag - stream = request_client.chat.completions.create(**stream_kwargs) - if agent.provider == "moa": - # The MoA facade is a shared singleton — abort/close of the - # registered client is a no-op, so register the stream handle - # itself for interrupt teardown (#57354). - stream = _set_request_stream_handle(stream) - # Claim the delta sink for THIS attempt (#65991). If a prior attempt's - # stream is somehow still alive (a stale-stream reconnect whose socket - # abort raced), this claim supersedes it so its late chunks are fenced - # out of the turn instead of interleaving with ours. - _writer_token = claim_stream_writer(agent) - - # Some OpenAI-compatible adapters (for example copilot-acp, and the MoA - # openai-codex aggregator) accept stream=True but still return a - # completed response object rather than an iterator of chunks. Treat - # that as "streaming unsupported" for the rest of this session instead - # of crashing on ``for chunk in stream`` with ``'types.SimpleNamespace' - # object is not iterable`` (#11732, #55933). - # - # Discriminate on the mere PRESENCE of a ``choices`` attribute, not on - # it being a non-empty list: an adapter may hand back a completed - # response whose ``choices`` is ``None`` or empty (an error / - # content-filter / terminal frame), and every such shape is still a - # whole response — not a token stream — that would crash iteration just - # the same. A genuine provider stream (SDK ``Stream`` object, - # generator) exposes no ``choices`` attribute, so it is left untouched. - if hasattr(stream, "choices"): - logger.info( - "Streaming request returned a final response object instead of " - "an iterator; switching %s/%s to non-streaming for this session.", - agent.provider or "unknown", - agent.model or "unknown", - ) - agent._disable_streaming = True - # An empty/None ``choices`` carries no message to surface; return the - # completed object as-is so the outer loop's normal invalid-response - # validation (conversation_loop.py) handles it via the retry path, - # never ``for chunk in stream``. - choices = stream.choices - first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None - message = getattr(first_choice, "message", None) - if message is not None: - reasoning_text = ( - getattr(message, "reasoning_content", None) - or getattr(message, "reasoning", None) - ) - if isinstance(reasoning_text, str) and reasoning_text: - _fire_first_delta() - agent._fire_reasoning_delta(reasoning_text) - content = getattr(message, "content", None) - if isinstance(content, str) and content: - _fire_first_delta() - agent._fire_stream_delta(content) - return stream - - # Capture rate limit headers from the initial HTTP response. - # The OpenAI SDK Stream object exposes the underlying httpx - # response via .response before any chunks are consumed. - agent._capture_rate_limits(getattr(stream, "response", None)) - agent._capture_credits(getattr(stream, "response", None)) - # Snapshot diagnostic headers (cf-ray, x-openrouter-provider, etc.) - # so they survive even when the stream dies before any chunk - # arrives. Best-effort; never raises. - agent._stream_diag_capture_response(_diag, getattr(stream, "response", None)) - - # Log OpenRouter response cache status when present. - agent._check_openrouter_cache_status(getattr(stream, "response", None)) - content_parts: list = [] tool_calls_acc: dict = {} tool_gen_notified: set = set() @@ -2909,19 +2942,123 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= role = "assistant" reasoning_parts: list = [] usage_obj = None - for chunk in stream: - # Stop the moment a newer attempt has claimed the delta sink - # (#65991): this attempt has been superseded, so it must neither - # fire deltas (incl. the tool-suppressed raw-callback path below) - # nor keep consuming a stream that would interleave into the turn. - if not stream_writer_is_current(agent, _writer_token): + _diag = agent._stream_diag_init() + request_client_holder["diag"] = _diag + _writer_token = {"value": None} + + def _open_stream(next_api_kwargs: dict[str, Any]): + stream_kwargs = { + **next_api_kwargs, + "stream": True, + "timeout": _httpx.Timeout( + connect=_conn_cap, + read=_stream_read_timeout, + write=_base_timeout, + pool=_conn_cap, + ), + } + # Native Gemini rejects OpenAI's usage-streaming extension. + if not is_native_gemini_base_url(agent.base_url): + stream_kwargs["stream_options"] = {"include_usage": True} + request_client = _set_request_client( + agent._create_request_openai_client( + reason="chat_completion_stream_request", + api_kwargs=stream_kwargs, + ) + ) + last_chunk_time["t"] = time.time() + agent._touch_activity("waiting for provider response (streaming)") + return request_client.chat.completions.create(**stream_kwargs) + + def _stream_created(raw_stream: Any) -> None: + response = getattr(raw_stream, "response", None) + agent._capture_rate_limits(response) + agent._capture_credits(response) + agent._stream_diag_capture_response(_diag, response) + agent._check_openrouter_cache_status(response) + _writer_token["value"] = claim_stream_writer(agent) + + def _accept_stream_chunk(_chunk: Any) -> bool: + # A stale-attempt fence can win while Relay is handing an + # already-received tool-call chunk back to Hermes. Preserve only + # the fact that a tool call was in flight so retry policy does not + # misclassify the attempt as a partial text response. The chunk + # itself is still rejected below and never reaches callbacks. + try: + choices = getattr(_chunk, "choices", None) + delta = getattr(choices[0], "delta", None) if choices else None + if getattr(delta, "tool_calls", None): + provider_tool_in_flight["yes"] = True + except Exception: + pass + if not _stream_attempt_is_active(stream_attempt_id): + return False + token = _writer_token["value"] + if token is not None and not stream_writer_is_current(agent, token): logger.warning( "Streaming attempt superseded by a newer stream; stopping " "consumption to preserve the single-writer invariant " "(model=%s).", api_kwargs.get("model", "unknown"), ) - break + return False + # Record provider activity before Relay processes the chunk. This + # prevents the stale watchdog from cancelling a live stream while + # an interceptor or codec is still handling an already-received + # event. + last_chunk_time["t"] = time.time() + return True + + def _relay_final_response() -> dict[str, Any]: + tool_calls = [tool_calls_acc[index] for index in sorted(tool_calls_acc)] + return { + "model": model_name, + "choices": [ + { + "message": { + "role": role, + "content": "".join(content_parts) or None, + "reasoning_content": "".join(reasoning_parts) or None, + "tool_calls": tool_calls or None, + }, + "finish_reason": finish_reason or "stop", + } + ], + "usage": usage_obj, + } + + from agent import relay_llm + + stream = _set_managed_stream( + relay_llm.stream( + api_kwargs, + _open_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=_relay_final_response, + on_stream_created=_stream_created, + accept_chunk=_accept_stream_chunk, + completed_response_predicate=lambda value: hasattr(value, "choices"), + metadata={ + "api_mode": "chat_completions", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + ) + if agent.provider == "moa": + # Hermes interrupts the managed stream; Relay retains sole + # ownership of closing the underlying provider stream. + _set_request_stream_handle(stream) + for chunk in stream: last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") @@ -3075,11 +3212,46 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if hasattr(chunk, "usage") and chunk.usage: usage_obj = chunk.usage + _close_managed_stream() + if _stream_attempt_was_cancelled(stream_attempt_id): raise _httpx.RemoteProtocolError( f"stream attempt {stream_attempt_id} was superseded" ) + # Some OpenAI-compatible adapters accept ``stream=True`` but return a + # completed response. Relay records that attempt while Hermes preserves + # its existing switch-to-non-streaming behavior for later calls. + if stream.final_response is not None: + final_response = stream.final_response + logger.info( + "Streaming request returned a final response object instead of " + "an iterator; switching %s/%s to non-streaming for this session.", + agent.provider or "unknown", + agent.model or "unknown", + ) + agent._disable_streaming = True + choices = final_response.choices + first_choice = ( + choices[0] + if isinstance(choices, (list, tuple)) and choices + else None + ) + message = getattr(first_choice, "message", None) + if message is not None: + reasoning_text = ( + getattr(message, "reasoning_content", None) + or getattr(message, "reasoning", None) + ) + if isinstance(reasoning_text, str) and reasoning_text: + _fire_first_delta() + agent._fire_reasoning_delta(reasoning_text) + content = getattr(message, "content", None) + if isinstance(content, str) and content: + _fire_first_delta() + agent._fire_stream_delta(content) + return final_response + # Build mock response matching non-streaming shape full_content = "".join(content_parts) or None mock_tool_calls = None @@ -3241,72 +3413,95 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # fabricated "successful" empty turn. saw_stream_event = False - # Reset stale-stream timer for this attempt last_chunk_time["t"] = time.time() - # Per-attempt diagnostic dict for the retry block to consume. _diag = agent._stream_diag_init() request_client_holder["diag"] = _diag - # Defensive: strip Responses-only kwargs (instructions, input, ...) - # that can leak in under an api_mode-flip race. The Anthropic SDK - # raises a non-retryable TypeError on them, killing the turn. See - # #31673 / sanitize_anthropic_kwargs(). + _writer_token = {"value": None} + _stream_context = {"manager": None, "stream": None} + base_final_message = None + + from agent import relay_llm from agent.anthropic_adapter import sanitize_anthropic_kwargs - sanitize_anthropic_kwargs( - api_kwargs, log_prefix=getattr(agent, "log_prefix", "") - ) - # Use the Anthropic SDK's streaming context manager - with request_client.messages.stream(**api_kwargs) as stream: + + accumulator = relay_llm.AnthropicStreamAccumulator() + + def _open_anthropic_stream(next_api_kwargs: dict[str, Any]): + final_kwargs = dict(next_api_kwargs) + sanitize_anthropic_kwargs( + final_kwargs, + log_prefix=getattr(agent, "log_prefix", ""), + ) + manager = request_client.messages.stream(**final_kwargs) + _stream_context["manager"] = manager + return manager.__enter__() + + def _anthropic_stream_created(raw_stream: Any) -> None: + _stream_context["stream"] = raw_stream # The Anthropic SDK exposes the raw httpx response on - # ``stream.response``. Snapshot diagnostic headers - # immediately so they survive a stream that dies before the - # first event. + # ``stream.response``. Snapshot diagnostics immediately so they + # survive a stream that dies before the first event. try: agent._stream_diag_capture_response( - _diag, getattr(stream, "response", None) + _diag, + getattr(raw_stream, "response", None), ) except Exception: pass - # Claim the delta sink for THIS attempt (#65991) — parity with the - # chat_completions path so a superseded anthropic stream is fenced. - _writer_token = claim_stream_writer(agent) + _writer_token["value"] = claim_stream_writer(agent) + + def _accept_anthropic_event(_event: Any) -> bool: + token = _writer_token["value"] + if token is None or stream_writer_is_current(agent, token): + return True + logger.warning( + "Anthropic streaming attempt superseded by a newer stream; " + "stopping consumption to preserve the single-writer " + "invariant (model=%s).", + api_kwargs.get("model", "unknown"), + ) + return False + + stream = _set_managed_stream( + relay_llm.stream( + api_kwargs, + _open_anthropic_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "anthropic"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=accumulator.finalize, + on_stream_created=_anthropic_stream_created, + on_chunk=accumulator.observe, + accept_chunk=_accept_anthropic_event, + metadata={ + "api_mode": "anthropic_messages", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + ) + try: for event in stream: - # Bail the instant a newer attempt supersedes this one so a - # stale stream can't interleave tokens into the turn. - if not stream_writer_is_current(agent, _writer_token): - logger.warning( - "Anthropic streaming attempt superseded by a newer " - "stream; stopping consumption to preserve the " - "single-writer invariant (model=%s).", - api_kwargs.get("model", "unknown"), - ) - break saw_stream_event = True - # Update stale-stream timer on every event so the - # outer poll loop knows data is flowing. Without - # this, the detector kills healthy long-running - # Opus streams after 180 s even when events are - # actively arriving (the chat_completions path - # already does this at the top of its chunk loop). last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") - - # Update per-attempt diagnostic counters (best-effort). try: _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 if _diag.get("first_chunk_at") is None: _diag["first_chunk_at"] = last_chunk_time["t"] - try: - _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) - except Exception: - pass + _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) except Exception: pass - if agent._interrupt_requested: break event_type = getattr(event, "type", None) - if event_type == "content_block_start": block = getattr(event, "content_block", None) if block and getattr(block, "type", None) == "tool_use": @@ -3315,7 +3510,6 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if tool_name: _fire_first_delta() agent._fire_tool_gen_started(tool_name) - elif event_type == "content_block_delta": delta = getattr(event, "delta", None) if delta: @@ -3331,48 +3525,49 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if thinking_text: _fire_first_delta() agent._fire_reasoning_delta(thinking_text) - - # Return the native Anthropic Message for downstream processing. - # If the stream was interrupted (the event loop broke out above on - # agent._interrupt_requested), do NOT call get_final_message() — on - # a partially-consumed stream the SDK may hang draining remaining - # events or return a Message with incomplete tool_use blocks (partial - # JSON in `input`). The outer poll loop raises InterruptedError, so - # this return value is discarded anyway. - if agent._interrupt_requested: - return None - # Zero-event guard (parity with the chat_completions zero-chunk - # guard above). Real SDK: an eventless stream has no - # message_start, so get_final_message() raises AssertionError - # (final-message snapshot is None) — normalize that to - # EmptyStreamError so it gets the transient retry budget - # instead of surfacing raw. + if not agent._interrupt_requested: + raw_stream = _stream_context["stream"] + if raw_stream is not None: + try: + base_final_message = raw_stream.get_final_message() + except AssertionError: + if not saw_stream_event: + raise EmptyStreamError( + "Provider returned an empty stream with no events " + "(possible upstream error or malformed event stream)." + ) from None + raise + finally: try: - _final_message = stream.get_final_message() - except AssertionError: - if not saw_stream_event: - raise EmptyStreamError( - "Provider returned an empty stream with no events " - "(possible upstream error or malformed event stream)." - ) from None - raise - # Shim variants of the same failure: an OpenAI-compat adapter - # may fabricate a contentless Message with no stop_reason, or - # return None where the SDK assert would have fired (e.g. - # ``python -O``). A real completed response always carries a - # stop_reason, so this cannot fire on legitimate turns. - if not saw_stream_event and ( - _final_message is None - or ( - not getattr(_final_message, "content", None) - and getattr(_final_message, "stop_reason", None) is None - ) - ): - raise EmptyStreamError( - "Provider returned an empty stream with no stop_reason " - "(possible upstream error or malformed event stream)." - ) - return _final_message + _close_managed_stream() + finally: + manager = _stream_context["manager"] + if manager is not None: + manager.__exit__(None, None, None) + + if agent._interrupt_requested: + return None + if ( + base_final_message is not None + and not getattr(base_final_message, "content", None) + and getattr(base_final_message, "stop_reason", None) is None + ): + raise EmptyStreamError( + "Provider returned an empty stream with no stop_reason " + "(possible upstream error or malformed event stream)." + ) + if base_final_message is not None and not stream.output_modified: + return base_final_message + final_message = accumulator.response(base_final_message) + if ( + not getattr(final_message, "content", None) + and getattr(final_message, "stop_reason", None) is None + ): + raise EmptyStreamError( + "Provider returned an empty stream with no stop_reason " + "(possible upstream error or malformed event stream)." + ) + return final_message def _call(): import httpx as _httpx @@ -3407,6 +3602,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= result["response"] = _call_chat_completions(stream_attempt_id) return # success except Exception as e: + _close_managed_stream() # If the main poll loop force-closed this request because # of an interrupt, the resulting transport error is the # expected consequence of our own close — NOT a transient @@ -3446,7 +3642,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if deltas_were_sent["yes"]: _partial_tool_in_flight = bool( result.get("partial_tool_names") - ) + ) or provider_tool_in_flight["yes"] _is_sse_conn_err_preview = False if not _is_timeout and not _is_conn_err: from openai import APIError as _APIError @@ -3695,6 +3891,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= result["error"] = e return finally: + _close_managed_stream() _close_request_client_once("stream_request_complete") # Provider-configured stale timeout takes priority over env default. @@ -3757,7 +3954,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if _reasoning_floor is not None: _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor) - t = threading.Thread(target=_call, daemon=True) + t = threading.Thread(target=_context_thread_target(_call), daemon=True) t.start() _last_heartbeat = time.time() _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index da3bc4f95695..0c5bb7b70503 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -1236,6 +1236,8 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta """ import httpx as _httpx + from agent import relay_llm + active_client = client or agent._ensure_primary_openai_client(reason="codex_stream_direct") max_stream_retries = 1 # Accumulate streamed text so callers / compat shims can read it. @@ -1260,48 +1262,88 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta if agent._interrupt_requested: raise InterruptedError("Agent interrupted before Codex stream retry") - stream_kwargs = dict(api_kwargs) - stream_kwargs["stream"] = True + intercepted_events = [] + writer_token = {"value": None} + + def _open_codex_stream(next_api_kwargs: dict[str, Any]): + stream_kwargs = dict(next_api_kwargs) + stream_kwargs["stream"] = True + return active_client.responses.create(**stream_kwargs) + + def _codex_stream_created(_raw_stream: Any) -> None: + # Claim the delta sink for THIS physical attempt. A newer attempt + # supersedes this token and fences late deltas out of the turn. + writer_token["value"] = claim_stream_writer(agent) + + def _accept_codex_chunk(_chunk: Any) -> bool: + token = writer_token["value"] + if token is None or stream_writer_is_current(agent, token): + return True + logger.warning( + "Codex streaming attempt superseded by a newer stream; " + "stopping consumption to preserve the single-writer " + "invariant (model=%s).", + api_kwargs.get("model", "unknown"), + ) + return False + + def _finalize_codex_stream() -> Any: + return _consume_codex_event_stream( + list(intercepted_events), + model=api_kwargs.get("model"), + ) try: - event_stream = active_client.responses.create(**stream_kwargs) - except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: + event_stream = relay_llm.stream( + dict(api_kwargs), + _open_codex_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "codex"), + model_name=str(api_kwargs.get("model") or ""), + finalizer=_finalize_codex_stream, + on_stream_created=_codex_stream_created, + on_chunk=intercepted_events.append, + chunk_adapter=lambda chunk: chunk, + accept_chunk=_accept_codex_chunk, + completed_response_predicate=lambda response: bool( + hasattr(response, "output") and not hasattr(response, "__iter__") + ), + metadata={ + "api_mode": "codex_responses", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + "retry_count": attempt, + }, + defer_logical_completion=True, + ) + except ( + _httpx.RemoteProtocolError, + _httpx.ReadTimeout, + _httpx.ConnectError, + ConnectionError, + ) as exc: if attempt < max_stream_retries: logger.debug( - "Codex Responses stream connect failed (attempt %s/%s); retrying. %s error=%s", - attempt + 1, max_stream_retries + 1, - agent._client_log_context(), exc, + "Codex Responses stream connect failed (attempt %s/%s); " + "retrying. %s error=%s", + attempt + 1, + max_stream_retries + 1, + agent._client_log_context(), + exc, ) continue raise - # Claim the delta sink for THIS attempt (#65991) — parity with the - # chat_completions/anthropic/bedrock paths. If a prior attempt's - # stream is somehow still alive, this claim supersedes it so its - # late deltas are fenced out of the turn; conversely, a newer - # attempt supersedes us and the interrupt_check below stops our - # consumption immediately. - _writer_token = claim_stream_writer(agent) - - def _interrupt_or_superseded(_tok=_writer_token) -> bool: - if agent._interrupt_requested: - return True - if not stream_writer_is_current(agent, _tok): - logger.warning( - "Codex streaming attempt superseded by a newer stream; " - "stopping consumption to preserve the single-writer " - "invariant (model=%s).", - api_kwargs.get("model", "unknown"), - ) - return True - return False + def _interrupt_or_superseded() -> bool: + return bool(agent._interrupt_requested) try: - # Compatibility: some mocks/providers return a concrete response - # instead of an iterable. Pass it straight through. - if hasattr(event_stream, "output") and not hasattr(event_stream, "__iter__"): - return event_stream - try: final = _consume_codex_event_stream( event_stream, @@ -1320,6 +1362,12 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta on_event=_on_event, interrupt_check=_interrupt_or_superseded, ) + # The terminal SSE frame is contractually last. Request the + # end-of-stream marker so Relay can run its response finalizer + # and close the physical attempt scope before Hermes returns. + if not agent._interrupt_requested: + for _ignored in event_stream: + pass except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: if attempt < max_stream_retries: logger.debug( @@ -1330,6 +1378,10 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta ) continue raise + except RuntimeError: + if event_stream.final_response is not None: + return event_stream.final_response + raise if final.status in {"incomplete", "failed"}: logger.warning( diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 9a1d91a1a9e2..d03a2cfa3d23 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -494,7 +494,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # session is created (not on continuation). Plugins can use this # to initialise session-scoped state (e.g. warm a memory cache). try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_start", session_id=agent.session_id, @@ -2063,7 +2063,7 @@ def run_conversation( _llm_middleware_trace = [] try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -2104,6 +2104,7 @@ def run_conversation( base_url=agent.base_url, api_mode=agent.api_mode, api_call_count=api_call_count, + retry_count=retry_count, request_messages=list(request_messages) if isinstance(request_messages, list) else [], @@ -2193,7 +2194,28 @@ def run_conversation( return agent._interruptible_streaming_api_call( next_api_kwargs, on_first_delta=_stop_spinner ) - return agent._interruptible_api_call(next_api_kwargs) + from agent import relay_llm + + return relay_llm.execute( + next_api_kwargs, + agent._interruptible_api_call, + session_id=str(agent.session_id or ""), + name=str(agent.provider or "provider"), + model_name=str(agent.model or ""), + metadata={ + "api_mode": agent.api_mode, + "api_request_id": api_request_id, + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + "retry_count": retry_count, + }, + defer_logical_completion=True, + ) from hermes_cli.middleware import run_llm_execution_middleware @@ -3233,6 +3255,12 @@ def run_conversation( clear_nous_rate_limit() except Exception: pass + from agent import relay_llm + + relay_llm.complete_logical_call( + api_request_id, + outcome="success", + ) agent._touch_activity(f"API call #{api_call_count} completed") break # Success, exit retry loop @@ -5360,7 +5388,7 @@ def run_conversation( assistant_message.content = str(raw) try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -6693,7 +6721,8 @@ def run_conversation( _attempt = getattr(agent, "_pre_verify_nudges", 0) try: from agent.verify_hooks import max_verify_nudges - from hermes_cli.plugins import get_pre_verify_continue_message, has_hook + from hermes_cli.lifecycle import has_hook + from hermes_cli.plugins import get_pre_verify_continue_message if _edited and has_hook("pre_verify") and _attempt < max_verify_nudges(): # Posture is fixed for the session — resolve once + cache. diff --git a/agent/relay_llm.py b/agent/relay_llm.py new file mode 100644 index 000000000000..85286c47dae7 --- /dev/null +++ b/agent/relay_llm.py @@ -0,0 +1,1108 @@ +"""Core NeMo Relay adapters for physical Hermes provider attempts.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import logging +from collections.abc import Callable, Iterator +from types import SimpleNamespace +from typing import Any + +from agent import relay_runtime + +logger = logging.getLogger(__name__) + + +_PROVIDER_MESSAGE_EXTENSION_KEYS = frozenset( + {"reasoning_content", "reasoning_details"} +) +_RELAY_INTERNAL_PROVIDER_HEADERS = frozenset( + {"x-dynamo-parent-session-id", "x-dynamo-session-id"} +) + + +def execute( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run one non-streaming physical provider attempt through Relay.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + raw = callback_context.copy().run(callback, final_request) + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = _run_awaitable( + runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if _recover_successful_callback( + raw_response, + relay_error=exc, + callback_error=callback_error, + logical=logical, + defer_logical_completion=defer_logical_completion, + ): + return raw_response["value"] + raise + + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return _namespace(managed) + + +async def execute_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run one asynchronous physical provider attempt through Relay.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return await callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + async def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + async def call_provider() -> Any: + return await callback(final_request) + + task = callback_context.copy().run( + asyncio.create_task, + call_provider(), + ) + raw = await task + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = await runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if _recover_successful_callback( + raw_response, + relay_error=exc, + callback_error=callback_error, + logical=logical, + defer_logical_completion=defer_logical_completion, + ): + return raw_response["value"] + raise + + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return _namespace(managed) + + +def execute_current( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run a provider attempt under the inherited Hermes turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return callback(request) + return execute( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +async def execute_current_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run an async provider attempt under the inherited turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return await callback(request) + return await execute_async( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +def stream_current( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + finalizer: Callable[[], Any], + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run a provider stream under the inherited Hermes turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return stream_factory(request) + return stream( + request, + stream_factory, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +def stream( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + finalizer: Callable[[], Any], + on_stream_created: Callable[[Any], None] | None = None, + on_chunk: Callable[[Any], None] | None = None, + chunk_adapter: Callable[[Any], Any] | None = None, + accept_chunk: Callable[[Any], bool] | None = None, + completed_response_predicate: Callable[[Any], bool] | None = None, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> "ManagedLlmStream": + """Return a synchronous view of one Relay-managed provider stream.""" + return ManagedLlmStream( + request, + stream_factory, + session_id=session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + on_stream_created=on_stream_created, + on_chunk=on_chunk, + chunk_adapter=chunk_adapter, + accept_chunk=accept_chunk, + completed_response_predicate=completed_response_predicate, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +class ManagedLlmStream(Iterator[Any]): + """Drive Relay's async stream from Hermes's provider worker thread.""" + + def __init__( + self, + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + finalizer: Callable[[], Any], + on_stream_created: Callable[[Any], None] | None, + on_chunk: Callable[[Any], None] | None, + chunk_adapter: Callable[[Any], Any] | None, + accept_chunk: Callable[[Any], bool] | None, + completed_response_predicate: Callable[[Any], bool] | None, + metadata: dict[str, Any] | None, + defer_logical_completion: bool, + ) -> None: + self.final_response: Any = None + self._loop: asyncio.AbstractEventLoop | None = None + self._stream: Any = None + self._raw_stream_resource: Any = None + self._closed = False + self._callback_error: BaseException | None = None + self._logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None = None + self._defer_logical_completion = defer_logical_completion + self._on_chunk = on_chunk + self._chunk_adapter = chunk_adapter or _namespace + self._accept_chunk = accept_chunk + self._relay_observes_chunks = False + self._provider_completed = False + self._raw_chunks: list[tuple[Any, Any]] = [] + self.output_modified = False + callback_context = contextvars.copy_context() + + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if ( + runtime is None + or session is None + or not runtime.managed_execution_enabled() + ): + raw_stream = stream_factory(request) + if completed_response_predicate is not None and completed_response_predicate( + raw_stream + ): + self.final_response = raw_stream + self._stream = iter(()) + else: + self._raw_stream_resource = raw_stream + if on_stream_created is not None: + on_stream_created(raw_stream) + self._stream = iter(raw_stream) + return + + self._logical = _logical_parent(runtime, session, parent, metadata) + if self._logical is not None: + parent = self._logical[1] + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + + async def provider_stream(next_request: Any): + raw_stream = None + try: + raw_stream = callback_context.run( + stream_factory, + _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + ) + if ( + completed_response_predicate is not None + and callback_context.run( + completed_response_predicate, + raw_stream, + ) + ): + self.final_response = raw_stream + self._provider_completed = True + return + if on_stream_created is not None: + callback_context.run(on_stream_created, raw_stream) + raw_iterator = callback_context.run(iter, raw_stream) + while True: + try: + chunk = callback_context.run(next, raw_iterator) + except StopIteration: + break + if self._accept_chunk is not None and not callback_context.run( + self._accept_chunk, + chunk, + ): + break + encoded_chunk = _jsonable(chunk) + self._raw_chunks.append((encoded_chunk, chunk)) + yield encoded_chunk + self._provider_completed = True + except BaseException as exc: + self._callback_error = exc + raise + finally: + close = getattr(raw_stream, "close", None) + if callable(close): + callback_context.run(close) + + def observe_chunk(chunk: Any) -> None: + if self._on_chunk is not None: + callback_context.run(self._on_chunk, _jsonable(chunk)) + + def relay_finalizer() -> Any: + try: + if self.final_response is not None: + return _jsonable(self.final_response) + return _jsonable(callback_context.run(finalizer)) + except BaseException as exc: + self._callback_error = exc + raise + + loop = asyncio.new_event_loop() + self._loop = loop + self._relay_observes_chunks = True + try: + self._stream = loop.run_until_complete( + runtime.run_in_session_async( + session, + runtime.relay.llm.stream_execute, + name, + relay_request, + provider_stream, + observe_chunk, + relay_finalizer, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + ) + except BaseException as exc: + if ( + isinstance(exc, Exception) + and self._provider_completed + and self._callback_error is None + ): + logger.warning( + "NeMo Relay stream post-processing failed after provider success; " + "preserving the provider result", + exc_info=True, + ) + self._preserve_pending_provider_chunks() + return + if not self._defer_logical_completion: + _complete_logical( + self._logical, + outcome="cancelled" if _is_cancellation(exc) else "failed", + ) + self._logical = None + loop.close() + self._loop = None + raise + + def __iter__(self) -> "ManagedLlmStream": + return self + + def __next__(self) -> Any: + if self._closed: + raise StopIteration + if self._loop is None: + try: + chunk = next(self._stream) + except StopIteration: + self.close() + raise + if self._accept_chunk is not None and not self._accept_chunk(chunk): + self.close() + raise StopIteration + return chunk + + async def next_chunk() -> Any: + return await anext(self._stream) + + try: + chunk = self._loop.run_until_complete(next_chunk()) + except StopAsyncIteration: + if self._raw_chunks: + self.output_modified = True + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome="success") + self._logical = None + self.close() + raise StopIteration from None + except BaseException as exc: + callback_error = self._callback_error + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + self._close(logical_outcome="failed") + raise callback_error + if ( + isinstance(exc, Exception) + and self._provider_completed + and callback_error is None + ): + logger.warning( + "NeMo Relay stream post-processing failed after provider success; " + "preserving the provider result", + exc_info=True, + ) + self._preserve_pending_provider_chunks() + return next(self) + self._close( + logical_outcome="cancelled" if _is_cancellation(exc) else "failed" + ) + raise + if not self._relay_observes_chunks and self._on_chunk is not None: + self._on_chunk(chunk) + for index, (encoded, raw) in enumerate(self._raw_chunks): + if _json_equal(chunk, encoded): + if index > 0: + self.output_modified = True + del self._raw_chunks[: index + 1] + return raw + self.output_modified = True + return self._chunk_adapter(chunk) + + def close(self) -> None: + """Close an explicitly abandoned stream and cancel its logical call.""" + self._close(logical_outcome="cancelled") + + def _preserve_pending_provider_chunks(self) -> None: + """Switch a failed Relay stream to its undelivered provider chunks.""" + pending = [raw for _encoded, raw in self._raw_chunks] + self._raw_chunks.clear() + loop = self._loop + relay_stream = self._stream + self._loop = None + self._stream = iter(pending) + self._raw_stream_resource = None + self._accept_chunk = None + if loop is not None: + close = getattr(relay_stream, "aclose", None) + if callable(close): + + async def close_stream() -> None: + await close() + + try: + loop.run_until_complete(close_stream()) + except Exception: + logger.debug( + "Relay stream cleanup failed during provider fallback", + exc_info=True, + ) + loop.close() + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome="success") + self._logical = None + + def _close(self, *, logical_outcome: str) -> None: + if self._closed: + return + self._closed = True + loop = self._loop + self._loop = None + if loop is None: + resources = (self._stream, self._raw_stream_resource) + self._stream = None + self._raw_stream_resource = None + closed_ids: set[int] = set() + for resource in resources: + if resource is None or id(resource) in closed_ids: + continue + closed_ids.add(id(resource)) + close = getattr(resource, "close", None) + if callable(close): + try: + close() + except Exception: + logger.debug( + "Provider stream cleanup failed", + exc_info=True, + ) + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome=logical_outcome) + self._logical = None + return + close = getattr(self._stream, "aclose", None) + if callable(close): + + async def close_stream() -> None: + await close() + + try: + loop.run_until_complete(close_stream()) + except Exception: + pass + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome=logical_outcome) + self._logical = None + loop.close() + + def __del__(self) -> None: + self.close() + + +class AnthropicStreamAccumulator: + """Rebuild an Anthropic Message from post-intercept SSE events.""" + + def __init__(self) -> None: + self._message: dict[str, Any] = {} + self._blocks: dict[int, dict[str, Any]] = {} + + def observe(self, event: Any) -> None: + payload = _jsonable(event) + if not isinstance(payload, dict): + return + event_type = payload.get("type") + if event_type == "message_start": + message = payload.get("message") + if isinstance(message, dict): + for key in ("id", "type", "role", "model", "usage"): + if key in message: + self._message[key] = message[key] + return + if event_type == "content_block_start": + index = payload.get("index") + block = payload.get("content_block") + if isinstance(index, int) and isinstance(block, dict): + self._blocks[index] = dict(block) + return + if event_type == "content_block_delta": + index = payload.get("index") + delta = payload.get("delta") + if not isinstance(index, int) or not isinstance(delta, dict): + return + block = self._blocks.setdefault(index, {}) + delta_type = delta.get("type") + if delta_type == "text_delta": + block["text"] = str(block.get("text") or "") + str( + delta.get("text") or "" + ) + elif delta_type == "thinking_delta": + block["thinking"] = str(block.get("thinking") or "") + str( + delta.get("thinking") or "" + ) + elif delta_type == "signature_delta": + block["signature"] = str(block.get("signature") or "") + str( + delta.get("signature") or "" + ) + elif delta_type == "input_json_delta": + partial = str(block.pop("_partial_json", "")) + str( + delta.get("partial_json") or "" + ) + block["_partial_json"] = partial + elif delta_type == "citations_delta" and "citation" in delta: + block.setdefault("citations", []).append(delta["citation"]) + return + if event_type == "message_delta": + delta = payload.get("delta") + if isinstance(delta, dict): + for key in ("stop_reason", "stop_sequence"): + if key in delta: + self._message[key] = delta[key] + if "usage" in payload: + usage = payload["usage"] + current_usage = self._message.get("usage") + if isinstance(current_usage, dict) and isinstance(usage, dict): + self._message["usage"] = {**current_usage, **usage} + else: + self._message["usage"] = usage + + def finalize(self) -> dict[str, Any]: + blocks = [] + for index in sorted(self._blocks): + block = dict(self._blocks[index]) + partial = block.pop("_partial_json", None) + if partial is not None: + try: + block["input"] = json.loads(partial) + except (TypeError, ValueError): + block["input"] = partial + blocks.append(block) + return {**self._message, "content": blocks} + + def response(self, base: Any = None) -> Any: + """Return the attribute-shaped response consumed by Hermes.""" + assembled = self.finalize() + base_payload = _jsonable(base) + if not isinstance(base_payload, dict): + base_payload = {} + content = assembled.pop("content", []) + merged = {**base_payload, **assembled} + if content or "content" not in merged: + merged["content"] = content + return _namespace(merged) + + +def _logical_parent( + runtime: relay_runtime.RelayRuntime, + session: Any, + parent: Any, + metadata: dict[str, Any] | None, +) -> tuple[relay_runtime.RelayTurnContext, Any, str] | None: + turn = relay_runtime.active_turn(session.session_id) + request_id = str((metadata or {}).get("api_request_id") or "") + if turn is None or not request_id or turn.lease.host is not runtime: + return None + with turn.finalize_lock: + if turn.closed: + return None + with turn.logical_llm_lock: + handle = turn.logical_llm_calls.get(request_id) + if handle is None: + handle = runtime.run_in_session( + session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=parent, + input={}, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id, + "hermes.call_role": str( + (metadata or {}).get("call_role") or "primary" + ), + }, + ) + turn.logical_llm_calls[request_id] = handle + return turn, handle, request_id + + +def _complete_logical( + logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, + *, + outcome: str, +) -> None: + if logical is None: + return + turn, handle, request_id = logical + lease = turn.lease + if not isinstance(lease.host, relay_runtime.RelayRuntime): + return + with turn.finalize_lock: + with turn.logical_llm_lock: + if turn.logical_llm_calls.get(request_id) is not handle: + return + if lease.session is None: + return + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + handle, + output={"outcome": outcome}, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + # The provider result is authoritative. Retain the handle so turn + # finalization can retry cleanup without changing that result. + logger.warning( + "Hermes Relay logical LLM finalization failed", + exc_info=True, + ) + return + with turn.logical_llm_lock: + if turn.logical_llm_calls.get(request_id) is handle: + turn.logical_llm_calls.pop(request_id, None) + + +def _recover_successful_callback( + raw_response: dict[str, Any], + *, + relay_error: BaseException, + callback_error: BaseException | None, + logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, + defer_logical_completion: bool, +) -> bool: + if ( + not isinstance(relay_error, Exception) + or callback_error is not None + or "value" not in raw_response + ): + return False + logger.warning( + "NeMo Relay LLM post-processing failed after provider success; " + "returning the provider response", + exc_info=True, + ) + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + return True + + +def _is_cancellation(error: BaseException) -> bool: + return isinstance( + error, + (asyncio.CancelledError, InterruptedError, KeyboardInterrupt), + ) + + +def complete_logical_call(api_request_id: str, *, outcome: str) -> None: + """Complete the active turn's logical LLM call after caller validation.""" + turn = relay_runtime.active_turn() + if turn is None or not api_request_id: + return + with turn.logical_llm_lock: + handle = turn.logical_llm_calls.get(api_request_id) + if handle is not None: + _complete_logical((turn, handle, api_request_id), outcome=outcome) + + +def _provider_request( + original: dict[str, Any], + request: Any, + *, + relay_request_body: dict[str, Any], + codec_baseline_body: dict[str, Any] | None, + metadata: dict[str, Any] | None, +) -> dict[str, Any]: + content = getattr(request, "content", request) + if not isinstance(content, dict): + content = relay_request_body + if codec_baseline_body is None or _json_equal(content, relay_request_body): + final = dict(original) + else: + baseline = codec_baseline_body + intercepted = _provider_request_body(content, metadata) + final = dict(original) + # Typed codecs may not represent provider-specific fields. Overlay only + # values that changed from the codec-facing baseline so unrelated + # intercepts cannot delete or normalize unknown provider arguments. + for key in baseline.keys() | intercepted.keys(): + if key not in intercepted: + final.pop(key, None) + elif key not in baseline or not _json_equal( + intercepted[key], + baseline[key], + ): + final[key] = intercepted[key] + _restore_provider_message_extensions( + original, + final, + baseline=baseline, + intercepted=intercepted, + ) + headers = getattr(request, "headers", None) + if isinstance(headers, dict): + headers = { + key: value + for key, value in headers.items() + if str(key).lower() not in _RELAY_INTERNAL_PROVIDER_HEADERS + } + if headers: + final["extra_headers"] = { + **dict(final.get("extra_headers") or {}), + **headers, + } + return final + + +def _relay_request_body( + request: dict[str, Any], metadata: dict[str, Any] | None +) -> dict[str, Any]: + body = _jsonable(request) + if not isinstance(body, dict): + return {} + # The Responses SDK accepts ``tools=None`` as "no tools", while Relay's + # typed Responses codec correctly expects either an array or an absent + # field. Normalize only the codec-facing copy; the original provider + # request is restored when no interceptor changes it. + if str((metadata or {}).get("api_mode") or "") == "codex_responses": + body = dict(body) + if body.get("tools") is None: + body.pop("tools", None) + elif isinstance(body.get("tools"), list): + body["tools"] = [ + { + "type": "function", + "function": { + key: value + for key, value in tool.items() + if key != "type" + }, + } + if isinstance(tool, dict) + and tool.get("type") == "function" + and "function" not in tool + else tool + for tool in body["tools"] + ] + elif str((metadata or {}).get("api_mode") or "") == "chat_completions": + tools = body.get("tools") + if isinstance(tools, list): + body = dict(body) + body["tools"] = [ + {"type": "function", **tool} + if isinstance(tool, dict) + and "function" in tool + and "type" not in tool + else tool + for tool in tools + ] + return body + + +def _restore_provider_message_extensions( + original: dict[str, Any], + final: dict[str, Any], + *, + baseline: dict[str, Any], + intercepted: dict[str, Any], +) -> None: + """Restore provider wire fields that Relay's typed codec cannot represent.""" + original_messages = original.get("messages") + final_messages = final.get("messages") + baseline_messages = baseline.get("messages") + intercepted_messages = intercepted.get("messages") + if not all( + isinstance(messages, list) + for messages in ( + original_messages, + final_messages, + baseline_messages, + intercepted_messages, + ) + ): + return + if not ( + len(original_messages) + == len(final_messages) + == len(baseline_messages) + == len(intercepted_messages) + ): + return + for original_message, final_message, baseline_message, intercepted_message in zip( + original_messages, + final_messages, + baseline_messages, + intercepted_messages, + strict=True, + ): + if not all( + isinstance(message, dict) + for message in ( + original_message, + final_message, + baseline_message, + intercepted_message, + ) + ): + continue + for key in _PROVIDER_MESSAGE_EXTENSION_KEYS: + if ( + key in original_message + and key not in baseline_message + and key not in intercepted_message + and key not in final_message + ): + final_message[key] = original_message[key] + + +def _codec_round_trip_request_body( + relay: Any, + relay_request: Any, + *, + relay_request_body: dict[str, Any], + metadata: dict[str, Any] | None, +) -> dict[str, Any] | None: + """Return the codec-only request shape used to identify real rewrites.""" + codec = _codec(relay, metadata) + if codec is None: + return _provider_request_body(relay_request_body, metadata) + try: + annotated = codec.decode(relay_request) + encoded = codec.encode(annotated, relay_request) + content = getattr(encoded, "content", encoded) + if isinstance(content, dict): + return _provider_request_body(content, metadata) + except Exception: + logger.warning( + "NeMo Relay request codec baseline failed; ignoring request rewrites", + exc_info=True, + ) + return None + logger.warning( + "NeMo Relay request codec returned an unsupported baseline; " + "ignoring request rewrites" + ) + return None + + +def _provider_request_body( + content: dict[str, Any], metadata: dict[str, Any] | None +) -> dict[str, Any]: + body = dict(content) + if str((metadata or {}).get("api_mode") or "") != "codex_responses": + return body + tools = body.get("tools") + if not isinstance(tools, list): + return body + body["tools"] = [ + { + "type": "function", + **dict(tool["function"]), + } + if isinstance(tool, dict) + and tool.get("type") == "function" + and isinstance(tool.get("function"), dict) + else tool + for tool in tools + ] + return body + + +def _codec(relay: Any, metadata: dict[str, Any] | None) -> Any: + api_mode = str((metadata or {}).get("api_mode") or "") + codecs = getattr(relay, "codecs", None) + if codecs is None: + return None + if api_mode == "chat_completions": + codec = getattr(codecs, "OpenAIChatCodec", None) + elif api_mode == "anthropic_messages": + codec = getattr(codecs, "AnthropicMessagesCodec", None) + elif api_mode == "codex_responses": + codec = getattr(codecs, "OpenAIResponsesCodec", None) + else: + codec = None + return codec() if callable(codec) else None + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + model_dump = getattr(type(value), "model_dump", None) + if callable(model_dump): + try: + return _jsonable(value.model_dump(mode="json")) + except Exception: + pass + try: + attributes = { + str(key): item + for key, item in vars(value).items() + if not str(key).startswith("_") + } + except (TypeError, AttributeError): + return str(value) + return _jsonable(attributes) if attributes else str(value) + + +def _namespace(value: Any) -> Any: + if isinstance(value, dict): + return SimpleNamespace(**{ + str(key): _namespace(item) for key, item in value.items() + }) + if isinstance(value, list): + return [_namespace(item) for item in value] + return value + + +def _json_equal(left: Any, right: Any) -> bool: + try: + return json.dumps( + _jsonable(left), sort_keys=True, separators=(",", ":") + ) == json.dumps(_jsonable(right), sort_keys=True, separators=(",", ":")) + except (TypeError, ValueError): + return False + + +def _run_awaitable(value: Any) -> Any: + if not inspect.isawaitable(value): + return value + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(value) + raise RuntimeError( + "Synchronous Relay LLM execution cannot run on an event-loop thread" + ) diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py new file mode 100644 index 000000000000..4e3654cf7780 --- /dev/null +++ b/agent/relay_runtime.py @@ -0,0 +1,991 @@ +"""Profile-scoped NeMo Relay runtimes owned by the Hermes agent core.""" + +from __future__ import annotations + +import atexit +import asyncio +import contextvars +import importlib +import inspect +import logging +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable + +from hermes_constants import get_hermes_home + +logger = logging.getLogger(__name__) + +SESSION_SCOPE = "hermes.session" +TURN_SCOPE = "hermes.turn" +LOGICAL_LLM_SCOPE = "hermes.logical_llm_call" +RUNTIME_SCHEMA_KEY = "hermes.relay.schema_version" +RUNTIME_SCHEMA_VERSION = "hermes.relay.runtime.v1" +RUNTIME_INSTANCE_KEY = "hermes.relay.runtime_instance" +_PROFILE_KEY_CACHE: dict[str, str] = {} + + +@dataclass +class RelaySession: + """One isolated Relay scope stack owned by a Hermes session.""" + + session_id: str + parent_session_id: str = "" + lock: threading.RLock = field(default_factory=threading.RLock, repr=False) + closing: bool = False + handle: Any = None + context: contextvars.Context | None = None + + +class RelayRuntime: + """Own Relay session scopes independently of any exporter or plugin.""" + + def __init__(self, relay: Any = None, *, profile_key: str | None = None) -> None: + self.relay = relay or _load_nemo_relay() + self.profile_key = profile_key or current_profile_key() + self.runtime_id = uuid.uuid4().hex + self._sessions_lock = threading.RLock() + self._sessions: dict[str, RelaySession] = {} + self._subagent_parents: dict[str, str] = {} + self._subagent_parent_handles: dict[str, Any] = {} + self._execution_consumers_lock = threading.RLock() + self._execution_consumers: set[str] = set() + self._shutdown_registered = True + atexit.register(self.shutdown) + + def retain_managed_execution(self, consumer: str) -> None: + """Keep managed LLM and tool execution active for one consumer.""" + if not consumer: + raise ValueError("Relay managed-execution consumer must not be empty") + with self._execution_consumers_lock: + self._execution_consumers.add(consumer) + + def release_managed_execution(self, consumer: str) -> None: + """Release a consumer's managed-execution requirement.""" + with self._execution_consumers_lock: + self._execution_consumers.discard(consumer) + + def managed_execution_enabled(self) -> bool: + """Return whether a Hermes-managed consumer needs the Relay pipeline.""" + with self._execution_consumers_lock: + return bool(self._execution_consumers) + + def ensure_session( + self, + event: dict[str, Any], + *, + data: Any = None, + metadata: dict[str, Any] | None = None, + ) -> RelaySession | None: + """Return the existing session scope or create it once.""" + session_id = _session_id(event) + if not session_id: + return None + with self._sessions_lock: + session = self._sessions.get(session_id) + if session is None: + parent_session_id = self._subagent_parents.get(session_id, "") + session = RelaySession( + session_id=session_id, + parent_session_id=parent_session_id, + ) + self._sessions[session_id] = session + with session.lock: + if session.closing: + return None + if session.handle is None: + parent_handle = None + scope_metadata = { + **(metadata or {}), + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + } + if session.parent_session_id: + with self._sessions_lock: + parent_handle = self._subagent_parent_handles.get(session_id) + if parent_handle is None: + parent = self.ensure_session({ + "session_id": session.parent_session_id + }) + if parent is not None: + parent_handle = parent.handle + scope_metadata["nemo_relay_scope_role"] = "subagent" + context = contextvars.Context() + try: + session.handle = context.run( + self.relay.scope.push, + SESSION_SCOPE, + self.relay.ScopeType.Agent, + handle=parent_handle, + data=data, + input={}, + metadata=scope_metadata, + ) + except Exception: + session.context = None + raise + session.context = context + return session + + def register_subagent( + self, + event: dict[str, Any], + *, + metadata: dict[str, Any] | None = None, + ) -> RelaySession | None: + """Open a child Agent scope under its spawning turn when available.""" + parent_session_id = str(event.get("parent_session_id") or "") + child_session_id = str(event.get("child_session_id") or "") + if ( + not parent_session_id + or not child_session_id + or parent_session_id == child_session_id + ): + return None + parent = self.ensure_session({"session_id": parent_session_id}) + parent_handle = None if parent is None else parent.handle + turn = active_turn(parent_session_id) + if ( + turn is not None + and not turn.closed + and turn.handle is not None + and turn.lease.host is self + and turn.lease.session is not None + and turn.lease.session.session_id == parent_session_id + ): + parent_handle = turn.handle + with self._sessions_lock: + self._subagent_parents[child_session_id] = parent_session_id + if parent_handle is not None: + self._subagent_parent_handles[child_session_id] = parent_handle + return self.ensure_session( + {"session_id": child_session_id}, + metadata=metadata, + ) + + def unregister_subagent(self, event: dict[str, Any]) -> None: + """Close a delegated session and forget its parent relationship.""" + child_session_id = str(event.get("child_session_id") or "") + if not child_session_id: + return + self.close_session({"session_id": child_session_id}) + with self._sessions_lock: + self._subagent_parents.pop(child_session_id, None) + self._subagent_parent_handles.pop(child_session_id, None) + + def get_session(self, session_id: str) -> RelaySession | None: + """Return an active Hermes Relay session without creating one.""" + with self._sessions_lock: + session = self._sessions.get(str(session_id or "")) + if session is None: + return None + with session.lock: + return None if session.closing else session + + def get_session_handle(self, session_id: str) -> Any: + """Return the Relay parent handle for a Hermes session, if active.""" + session = self.get_session(session_id) + return None if session is None else session.handle + + def run_in_session( + self, + session: RelaySession, + callback: Callable[..., Any], + *args: Any, + allow_closing: bool = False, + **kwargs: Any, + ) -> Any: + """Run a Relay operation against a session's isolated scope stack.""" + with session.lock: + if session.closing and not allow_closing: + raise RuntimeError("Hermes Relay session is closing") + if session.context is None or session.handle is None: + raise RuntimeError("Hermes Relay session context is unavailable") + relay_context = session.context.copy() + + context = contextvars.copy_context() + for variable, value in relay_context.items(): + context.run(variable.set, value) + + def invoke() -> Any: + self.relay.get_scope_stack() + return callback(*args, **kwargs) + + # A copy permits a helper called by an existing Relay callback to + # re-enter the same logical session without re-entering Context. + return context.run(invoke) + + async def run_in_session_async( + self, + session: RelaySession, + callback: Callable[..., Any], + *args: Any, + allow_closing: bool = False, + **kwargs: Any, + ) -> Any: + """Create and await an operation inside the session's saved context.""" + with session.lock: + if session.closing and not allow_closing: + raise RuntimeError("Hermes Relay session is closing") + if session.context is None or session.handle is None: + raise RuntimeError("Hermes Relay session context is unavailable") + relay_context = session.context.copy() + + context = contextvars.copy_context() + for variable, value in relay_context.items(): + context.run(variable.set, value) + + async def invoke() -> Any: + self.relay.get_scope_stack() + result = callback(*args, **kwargs) + if inspect.isawaitable(result): + return await result + return result + + task = context.run(asyncio.create_task, invoke()) + return await task + + def emit_mark( + self, + name: str, + event: dict[str, Any], + *, + data: Any = None, + metadata: Any = None, + ) -> bool: + """Emit a mark parented to the Hermes session identified by ``event``.""" + session = self.ensure_session(event) + if session is None: + return False + self.run_in_session( + session, + self.relay.scope.event, + name, + handle=session.handle, + data=data, + metadata=metadata, + ) + return True + + def apply_tool_request_intercepts( + self, + *, + session_id: str, + tool_name: str, + args: dict[str, Any], + ) -> dict[str, Any]: + """Apply Relay request rewriting before Hermes authorizes a tool call.""" + if not self.managed_execution_enabled(): + return args + request_intercepts = getattr( + getattr(self.relay, "tools", None), + "request_intercepts", + None, + ) + if not callable(request_intercepts): + return args + session = self.ensure_session({"session_id": session_id}) + if session is None: + return args + result = self.run_in_session( + session, + request_intercepts, + tool_name, + args, + ) + return result if isinstance(result, dict) else args + + def close_session(self, event: dict[str, Any]) -> None: + """Close one session scope and remove it from the core registry.""" + session_id = _session_id(event) + with self._sessions_lock: + session = self._sessions.get(session_id) + if session is None: + with self._sessions_lock: + self._subagent_parents.pop(session_id, None) + self._subagent_parent_handles.pop(session_id, None) + return + failures: list[str] = [] + with session.lock: + if session.closing: + return + session.closing = True + if session.handle is not None: + try: + self.run_in_session( + session, + self.relay.scope.pop, + session.handle, + output={}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + }, + allow_closing=True, + ) + except Exception as exc: + failures.append(f"session scope close failed: {exc}") + try: + self.relay.subscribers.flush() + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + with self._sessions_lock: + if self._sessions.get(session_id) is session: + self._sessions.pop(session_id, None) + self._subagent_parents.pop(session_id, None) + self._subagent_parent_handles.pop(session_id, None) + if failures: + logger.warning( + "Hermes Relay session %s closed with errors: %s", + session_id, + "; ".join(failures), + ) + + def shutdown(self) -> None: + """Close all core-owned Relay session scopes.""" + with self._sessions_lock: + session_ids = list(self._sessions) + for session_id in session_ids: + self._safe(self.close_session, {"session_id": session_id}) + if self._shutdown_registered: + try: + atexit.unregister(self.shutdown) + except Exception: + pass + self._shutdown_registered = False + + @staticmethod + def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return callback(*args, **kwargs) + except Exception: + logger.warning("Hermes Relay runtime operation failed", exc_info=True) + return None + + +@dataclass(frozen=True) +class NoopRelayRuntime: + """Explicit reduced-capability host for platforms without Relay wheels.""" + + profile_key: str + reason: str + + @property + def available(self) -> bool: + return False + + def apply_tool_request_intercepts( + self, + *, + session_id: str, + tool_name: str, + args: dict[str, Any], + ) -> dict[str, Any]: + del session_id, tool_name + return args + + @staticmethod + def retain_managed_execution(consumer: str) -> None: + del consumer + + @staticmethod + def release_managed_execution(consumer: str) -> None: + del consumer + + @staticmethod + def managed_execution_enabled() -> bool: + return False + + def shutdown(self) -> None: + """No resources are allocated on unsupported platforms.""" + + +RelayHost = RelayRuntime | NoopRelayRuntime + + +class RelayHostRegistry: + """Own exactly one Relay host for each canonical Hermes profile.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._hosts: dict[str, RelayHost] = {} + + def for_profile( + self, + profile_key: str | None = None, + *, + create: bool = True, + ) -> RelayHost | None: + key = profile_key or current_profile_key() + host = self._hosts.get(key) + if host is not None or not create: + return host + with self._lock: + host = self._hosts.get(key) + if host is not None or not create: + return host + try: + host = RelayRuntime(profile_key=key) + except Exception as exc: + logger.warning( + "Hermes Relay runtime initialization failed", exc_info=True + ) + host = NoopRelayRuntime(profile_key=key, reason=str(exc)) + self._hosts[key] = host + return host + + def shutdown_profile(self, profile_key: str) -> None: + with self._lock: + host = self._hosts.pop(profile_key, None) + if host is not None: + host.shutdown() + + def shutdown_all(self) -> None: + with self._lock: + hosts = list(self._hosts.values()) + self._hosts.clear() + for host in hosts: + host.shutdown() + + +HOST_REGISTRY = RelayHostRegistry() + + +@dataclass +class ConversationLease: + """A resumable reference to one profile-scoped conversation scope.""" + + profile_key: str + session_id: str + platform: str + host: RelayHost + session: RelaySession | None + parent_session_id: str = "" + released: bool = False + + +@dataclass +class RelayTurnContext: + """Runtime-only context for one Hermes turn or top-level task.""" + + lease: ConversationLease + turn_id: str + task_id: str + handle: Any = None + logical_llm_calls: dict[str, Any] = field(default_factory=dict, repr=False) + logical_llm_lock: threading.RLock = field( + default_factory=threading.RLock, + repr=False, + ) + finalize_lock: threading.RLock = field( + default_factory=threading.RLock, + repr=False, + ) + _token: contextvars.Token[RelayTurnContext | None] | None = field( + default=None, + repr=False, + ) + _active_registered: bool = field(default=False, repr=False) + closed: bool = False + + +_CURRENT_TURN: contextvars.ContextVar[RelayTurnContext | None] = contextvars.ContextVar( + "hermes_relay_turn", default=None +) + + +class RelaySessionCoordinator: + """Own semantic conversation and turn lifetimes for Hermes core.""" + + def __init__(self, registry: RelayHostRegistry = HOST_REGISTRY) -> None: + self.registry = registry + self._initializer_lock = threading.RLock() + self._session_initializers: dict[ + str, + Callable[[RelayRuntime, dict[str, Any]], None], + ] = {} + self._active_turns_lock = threading.RLock() + self._active_turns: dict[tuple[str, str], set[int]] = {} + + def register_session_initializer( + self, + name: str, + callback: Callable[[RelayRuntime, dict[str, Any]], None], + ) -> None: + """Register idempotent profile/session preparation before scope creation.""" + with self._initializer_lock: + self._session_initializers[name] = callback + + def unregister_session_initializer(self, name: str) -> None: + """Remove a previously registered session initializer.""" + with self._initializer_lock: + self._session_initializers.pop(name, None) + + def _prepare_session( + self, + host: RelayRuntime, + context: dict[str, Any], + ) -> None: + with self._initializer_lock: + initializers = list(self._session_initializers.items()) + for name, callback in initializers: + try: + callback(host, context) + except Exception: + logger.warning( + "Hermes Relay session initializer failed: %s", + name, + exc_info=True, + ) + + def acquire_conversation( + self, + *, + profile_key: str, + session_id: str, + platform: str, + parent_session_id: str = "", + model: str = "", + ) -> ConversationLease: + host = self.registry.for_profile(profile_key) + if host is None: + host = NoopRelayRuntime(profile_key, "Relay host creation was disabled") + session = None + if isinstance(host, RelayRuntime): + try: + session_context = { + "profile_key": profile_key, + "session_id": session_id, + "platform": platform, + "parent_session_id": parent_session_id, + "model": model, + } + self._prepare_session(host, session_context) + metadata = {"hermes.execution_surface": platform or "unknown"} + if parent_session_id and parent_session_id != session_id: + session = host.register_subagent( + { + "parent_session_id": parent_session_id, + "child_session_id": session_id, + }, + metadata=metadata, + ) + else: + session = host.ensure_session( + {"session_id": session_id}, + metadata=metadata, + ) + except Exception: + logger.warning( + "Hermes Relay conversation initialization failed", + exc_info=True, + ) + return ConversationLease( + profile_key=profile_key, + session_id=session_id, + platform=platform, + host=host, + session=session, + parent_session_id=parent_session_id, + ) + + def begin_turn( + self, + lease: ConversationLease, + *, + turn_id: str, + task_id: str, + ) -> RelayTurnContext: + if lease.released: + raise RuntimeError("Hermes Relay conversation lease is released") + turn = RelayTurnContext(lease=lease, turn_id=turn_id, task_id=task_id) + if isinstance(lease.host, RelayRuntime) and lease.session is not None: + try: + turn.handle = lease.host.run_in_session( + lease.session, + lease.host.relay.scope.push, + TURN_SCOPE, + lease.host.relay.ScopeType.Function, + handle=lease.session.handle, + input={}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + "hermes.execution_surface": lease.platform or "unknown", + }, + ) + except Exception: + logger.warning("Hermes Relay turn initialization failed", exc_info=True) + turn._token = _CURRENT_TURN.set(turn) + key = (lease.profile_key, lease.session_id) + with self._active_turns_lock: + self._active_turns.setdefault(key, set()).add(id(turn)) + turn._active_registered = True + return turn + + def end_turn( + self, + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + with turn.finalize_lock: + if turn.closed: + self._reset_turn_context(turn) + return + turn.closed = True + lease = turn.lease + try: + if isinstance(lease.host, RelayRuntime) and lease.session is not None: + self._finish_logical_calls(turn, outcome=outcome) + if turn.handle is not None: + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + turn.handle, + output={"outcome": outcome}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + logger.warning( + "Hermes Relay turn finalization failed", exc_info=True + ) + finally: + try: + # Delegated agents own one turn. Close their conversation + # while the active-turn guard is still held so a parent + # timeout fallback cannot race this terminal boundary. + if ( + lease.parent_session_id + and isinstance(lease.host, RelayRuntime) + ): + lease.host.unregister_subagent({ + "child_session_id": lease.session_id + }) + except Exception: + logger.warning( + "Hermes Relay child conversation finalization failed", + exc_info=True, + ) + finally: + self._unregister_active_turn(turn) + self._reset_turn_context(turn) + + def has_active_turn(self, *, profile_key: str, session_id: str) -> bool: + """Return whether a turn is still running for one profile/session.""" + key = (profile_key, session_id) + with self._active_turns_lock: + return bool(self._active_turns.get(key)) + + def _unregister_active_turn(self, turn: RelayTurnContext) -> None: + if not turn._active_registered: + return + key = (turn.lease.profile_key, turn.lease.session_id) + with self._active_turns_lock: + active = self._active_turns.get(key) + if active is not None: + active.discard(id(turn)) + if not active: + self._active_turns.pop(key, None) + turn._active_registered = False + + def _reset_active_turns_for_tests(self) -> None: + with self._active_turns_lock: + self._active_turns.clear() + + def finish_logical_calls( + self, + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + """Close logical LLM children before sibling task aggregation scopes.""" + with turn.finalize_lock: + if turn.closed: + return + self._finish_logical_calls(turn, outcome=outcome) + + @staticmethod + def _finish_logical_calls( + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + lease = turn.lease + if not isinstance(lease.host, RelayRuntime) or lease.session is None: + return + with turn.logical_llm_lock: + logical_calls = list(turn.logical_llm_calls.items()) + turn.logical_llm_calls.clear() + for request_id, logical_handle in logical_calls: + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + logical_handle, + output={"outcome": outcome}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + with turn.logical_llm_lock: + turn.logical_llm_calls.setdefault(request_id, logical_handle) + logger.warning( + "Hermes Relay logical LLM finalization failed", + exc_info=True, + ) + + @staticmethod + def _reset_turn_context(turn: RelayTurnContext) -> None: + """Reset the originating ContextVar token when called in that context.""" + if turn._token is None: + return + try: + _CURRENT_TURN.reset(turn._token) + except ValueError: + # A copied async/thread context may own terminal cleanup. Keep the + # token so the originating context can clear its stale reference. + return + turn._token = None + + @staticmethod + def release_conversation(lease: ConversationLease) -> None: + """Release a caller lease without closing a resumable conversation.""" + lease.released = True + + def finalize_conversation( + self, + *, + profile_key: str, + session_id: str, + ) -> None: + host = self.registry.for_profile(profile_key, create=False) + if isinstance(host, RelayRuntime): + host.close_session({"session_id": session_id}) + + def shutdown_profile(self, profile_key: str) -> None: + self.registry.shutdown_profile(profile_key) + + +SESSION_COORDINATOR = RelaySessionCoordinator() + + +def current_turn() -> RelayTurnContext | None: + """Return the turn context inherited by current async and thread work.""" + return _CURRENT_TURN.get() + + +def active_turn(session_id: str | None = None) -> RelayTurnContext | None: + """Return a live turn only when it belongs to the active profile/session.""" + turn = current_turn() + if turn is None or turn.closed or turn.lease.released: + return None + if turn.lease.profile_key != current_profile_key(): + return None + if session_id is not None and turn.lease.session_id != session_id: + return None + if isinstance(turn.lease.host, RelayRuntime): + if turn.lease.session is None: + return None + if turn.lease.host.get_session(turn.lease.session_id) is not turn.lease.session: + return None + return turn + + +def resolve_execution_context( + session_id: str, +) -> tuple[RelayRuntime | None, RelaySession | None, Any]: + """Resolve one active turn/session parent for managed Relay execution.""" + turn = active_turn(session_id) + if ( + turn is not None + and isinstance(turn.lease.host, RelayRuntime) + and turn.lease.session is not None + ): + session = turn.lease.session + return turn.lease.host, session, turn.handle or session.handle + # Managed-execution consumers create and retain the profile host before + # reaching an out-of-turn adapter. Do not initialize Relay for the default + # no-consumer path. + runtime = get_runtime(create=False) + if runtime is None: + return None, None, None + if not runtime.managed_execution_enabled(): + return None, None, None + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + return runtime, session, None if session is None else session.handle + + +def emit_mark( + name: str, + *, + session_id: str, + data: Any = None, + metadata: Any = None, +) -> bool: + """Emit a fail-open Relay mark under a Hermes session.""" + runtime = get_runtime(create=False) + if runtime is None: + return False + try: + return runtime.emit_mark( + name, + {"session_id": session_id}, + data=data, + metadata=metadata, + ) + except Exception: + logger.warning("Hermes Relay mark failed: %s", name, exc_info=True) + return False + + +def apply_tool_request_intercepts( + *, + session_id: str, + tool_name: str, + args: dict[str, Any], +) -> dict[str, Any]: + """Return Relay-rewritten arguments at Hermes's authorization boundary.""" + if not session_id: + return args + runtime = get_runtime(create=False) + if runtime is None: + return args + return runtime.apply_tool_request_intercepts( + session_id=session_id, + tool_name=tool_name, + args=args, + ) + + +def ensure_session(*, session_id: str, **context: Any) -> RelaySession | None: + """Create or return the shared Relay session used by Hermes core.""" + runtime = get_runtime() + if runtime is None: + return None + try: + return runtime.ensure_session({"session_id": session_id, **context}) + except Exception: + logger.warning("Hermes Relay session initialization failed", exc_info=True) + return None + + +def run_in_session( + session_id: str, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> Any: + """Run a scope, LLM, or tool API against a shared Hermes session.""" + runtime = get_runtime() + if runtime is None: + raise RuntimeError("Hermes Relay runtime is unavailable") + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay session is unavailable") + return runtime.run_in_session(session, callback, *args, **kwargs) + + +async def run_in_session_async( + session_id: str, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> Any: + """Await a Relay operation inside a shared Hermes session context.""" + runtime = get_runtime() + if runtime is None: + raise RuntimeError("Hermes Relay runtime is unavailable") + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay session is unavailable") + return await runtime.run_in_session_async(session, callback, *args, **kwargs) + + +def get_session_handle(session_id: str) -> Any: + """Return the shared Relay handle for direct core instrumentation.""" + runtime = get_runtime(create=False) + return None if runtime is None else runtime.get_session_handle(session_id) + + +def _is_relay_wrapped_callback_error( + relay_error: BaseException, + callback_error: BaseException, +) -> bool: + """Match Relay's native callback wrapper without masking policy errors.""" + if relay_error is callback_error: + return True + if not isinstance(relay_error, RuntimeError): + return False + callback_type = callback_error.__class__ + type_names = { + callback_type.__name__, + callback_type.__qualname__, + f"{callback_type.__module__}.{callback_type.__qualname__}", + } + message = str(relay_error) + return any( + message.startswith(f"internal error: {type_name}: {callback_error}") + for type_name in type_names + ) + + +def get_runtime( + *, + create: bool = True, + profile_key: str | None = None, +) -> RelayRuntime | None: + """Return the Relay host for the active Hermes profile.""" + host = HOST_REGISTRY.for_profile(profile_key, create=create) + return host if isinstance(host, RelayRuntime) else None + + +def get_host( + *, + create: bool = True, + profile_key: str | None = None, +) -> RelayHost | None: + """Return the explicit real or reduced-capability host for a profile.""" + return HOST_REGISTRY.for_profile(profile_key, create=create) + + +def current_profile_key() -> str: + """Return the canonical profile identity used for runtime isolation.""" + home = get_hermes_home().expanduser() + if not home.is_absolute(): + return str(home.resolve()) + raw = str(home) + cached = _PROFILE_KEY_CACHE.get(raw) + if cached is not None: + return cached + resolved = str(home.resolve()) + return _PROFILE_KEY_CACHE.setdefault(raw, resolved) + + +def _load_nemo_relay() -> Any: + """Load the binding only when a producer or consumer needs Relay.""" + return importlib.import_module("nemo_relay") + + +def _session_id(event: dict[str, Any]) -> str: + return str(event.get("session_id") or "") + + +def _reset_for_tests() -> None: + """Reset all profile-scoped Relay hosts for isolated tests.""" + SESSION_COORDINATOR._reset_active_turns_for_tests() + HOST_REGISTRY.shutdown_all() + _PROFILE_KEY_CACHE.clear() diff --git a/agent/relay_tools.py b/agent/relay_tools.py new file mode 100644 index 000000000000..5023df1bcf92 --- /dev/null +++ b/agent/relay_tools.py @@ -0,0 +1,123 @@ +"""Core NeMo Relay adapter for Hermes tool execution.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import logging +from collections.abc import Callable +from typing import Any + +from agent import relay_runtime + +logger = logging.getLogger(__name__) + + +def execute( + tool_name: str, + args: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + metadata: dict[str, Any] | None = None, +) -> tuple[Any, dict[str, Any]]: + """Run one tool call through Relay and return its final arguments.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return callback(args), args + + observed_args = args + raw_result: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + def invoke(next_args: Any) -> Any: + nonlocal callback_error, observed_args + observed_args = next_args if isinstance(next_args, dict) else args + try: + result = callback_context.copy().run(callback, observed_args) + except BaseException as exc: + callback_error = exc + raise + raw_result["value"] = result + raw_result["json"] = _jsonable(result) + return raw_result["json"] + + try: + managed = _run_awaitable( + runtime.run_in_session_async( + session, + runtime.relay.tools.execute, + tool_name, + _jsonable(args), + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + ) + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if ( + isinstance(exc, Exception) + and callback_error is None + and "value" in raw_result + ): + logger.warning( + "NeMo Relay tool post-processing failed after dispatch success; " + "returning the Hermes tool result", + exc_info=True, + ) + return raw_result["value"], observed_args + raise + + if "value" in raw_result and _json_equal(managed, raw_result["json"]): + return raw_result["value"], observed_args + if isinstance(managed, str): + return managed, observed_args + return json.dumps(_jsonable(managed), ensure_ascii=False), observed_args + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return _jsonable(model_dump(mode="json")) + except Exception: + pass + try: + return _jsonable(vars(value)) + except (TypeError, AttributeError): + return str(value) + + +def _json_equal(left: Any, right: Any) -> bool: + try: + return json.dumps( + _jsonable(left), sort_keys=True, separators=(",", ":") + ) == json.dumps(_jsonable(right), sort_keys=True, separators=(",", ":")) + except (TypeError, ValueError): + return left == right + + +def _run_awaitable(value: Any) -> Any: + if not inspect.isawaitable(value): + return value + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(value) + raise RuntimeError( + "Synchronous Hermes Relay tool execution cannot run on an active event-loop thread" + ) diff --git a/agent/tool_executor.py b/agent/tool_executor.py index d32fe99c0c55..1dfcf1ebf6ef 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -20,6 +20,7 @@ import os import random import threading import time +from dataclasses import dataclass from typing import Any, Optional from agent.display import ( @@ -292,31 +293,63 @@ def _tool_search_scoped_names(agent) -> frozenset: return names -def _apply_tool_request_middleware_for_agent( - agent, - *, - function_name: str, - function_args: dict, - effective_task_id: str, - tool_call_id: str, -) -> tuple[dict, list[dict[str, Any]]]: - try: - from hermes_cli.middleware import apply_tool_request_middleware +@dataclass +class _ManagedToolResult: + result: Any + args: dict[str, Any] + middleware_trace: list[dict[str, Any]] + blocked: bool - result = apply_tool_request_middleware( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=tool_call_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - ) - payload = result.payload if isinstance(result.payload, dict) else function_args - return payload, list(result.trace) - except Exception as exc: - logger.debug("tool_request middleware error: %s", exc) - return function_args, [] + +class _ConcurrentToolAuthorizationGate: + """Serialize policy prompts and exclude their queue from batch deadlines.""" + + def __init__(self) -> None: + self._serialization_lock = threading.Lock() + self._state_lock = threading.Lock() + self._pending = 0 + self._window_started: float | None = None + self._excluded_seconds = 0.0 + + def run(self, callback): + now = time.monotonic() + with self._state_lock: + if self._pending == 0: + self._window_started = now + self._pending += 1 + try: + with self._serialization_lock: + return callback() + finally: + now = time.monotonic() + with self._state_lock: + self._pending -= 1 + if self._pending == 0: + if self._window_started is not None: + self._excluded_seconds += max( + 0.0, now - self._window_started + ) + self._window_started = None + + def excluded_seconds(self) -> float: + """Return completed plus currently active authorization wait time.""" + now = time.monotonic() + with self._state_lock: + excluded = self._excluded_seconds + if self._window_started is not None: + excluded += max(0.0, now - self._window_started) + return excluded + + +def _managed_values( + outcome: _ManagedToolResult, +) -> tuple[Any, dict[str, Any], list[dict[str, Any]], bool]: + return ( + outcome.result, + outcome.args, + outcome.middleware_trace, + outcome.blocked, + ) def _run_agent_tool_execution_middleware( @@ -327,28 +360,271 @@ def _run_agent_tool_execution_middleware( effective_task_id: str, tool_call_id: str, execute, -) -> tuple[Any, dict]: - observed_args = function_args + scope_block: str | None = None, + display_index: int | None = None, + middleware_trace: list[dict[str, Any]] | None = None, + begin_execution=None, + authorization_gate: _ConcurrentToolAuthorizationGate | None = None, +) -> _ManagedToolResult: + """Run Relay rewrites before Hermes policy and dispatch exactly once.""" + from agent import relay_tools + from hermes_cli.middleware import ( + apply_tool_request_middleware, + run_tool_execution_middleware, + ) - def _execute(next_args: dict) -> Any: - nonlocal observed_args - observed_args = next_args if isinstance(next_args, dict) else function_args - return execute(observed_args) + trace = middleware_trace if middleware_trace is not None else [] + state = { + "args": function_args, + "middleware_trace": trace, + "blocked": False, + "dispatched": False, + } + dispatch_lock = threading.Lock() - from hermes_cli.middleware import run_tool_execution_middleware + def _authorized_dispatch(final_args: dict[str, Any]) -> Any: + with dispatch_lock: + if state["dispatched"]: + raise RuntimeError( + "Hermes tool execution callback invoked more than once" + ) + state["dispatched"] = True + state["blocked"] = False + state["args"] = final_args - result = run_tool_execution_middleware( + def _begin() -> None: + _begin_tool_execution( + agent, + function_name=function_name, + function_args=final_args, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + display_index=display_index, + ) + + def _advance_start_order(callback=None) -> None: + if begin_execution is None: + if callback is not None: + callback() + return + begin_execution(callback) + + block_message = scope_block + block_error_type = "tool_scope_block" + if block_message is None: + block_error_type = "plugin_block" + + def _resolve_pre_tool_block(): + try: + from hermes_cli.plugins import resolve_pre_tool_block + + return resolve_pre_tool_block( + function_name, + final_args, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + middleware_trace=list(state["middleware_trace"]), + ) + except Exception: + return None + + block_message = ( + _resolve_pre_tool_block() + if authorization_gate is None + else authorization_gate.run(_resolve_pre_tool_block) + ) + + guardrail_decision = None + if block_message is None: + guardrail_decision = agent._tool_guardrails.before_call( + function_name, final_args + ) + if guardrail_decision.allows_execution: + guardrail_decision = None + + if block_message is not None or guardrail_decision is not None: + _advance_start_order() + state["blocked"] = True + if block_message is not None: + result = json.dumps({"error": block_message}, ensure_ascii=False) + error_type = block_error_type + error_message = block_message + else: + result = agent._guardrail_block_result(guardrail_decision) + error_type = "guardrail_block" + error_message = ( + getattr(guardrail_decision, "message", None) + or "Tool blocked by guardrail policy" + ) + _emit_terminal_post_tool_call( + agent, + function_name=function_name, + function_args=final_args, + result=result, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + status="blocked", + error_type=error_type, + error_message=error_message, + middleware_trace=list(state["middleware_trace"]), + ) + return result + + if function_name == "memory": + agent._turns_since_memory = 0 + elif function_name == "skill_manage": + agent._iters_since_skill = 0 + + _advance_start_order(_begin) + return execute(final_args) + + def _hermes_pipeline(relay_args: dict[str, Any]) -> Any: + request_result = apply_tool_request_middleware( + function_name, + relay_args, + skip_relay=True, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") or "", + ) + request_args = ( + request_result.payload + if isinstance(request_result.payload, dict) + else relay_args + ) + trace.clear() + trace.extend(request_result.trace) + return run_tool_execution_middleware( + function_name, + request_args, + lambda next_args: _authorized_dispatch( + next_args if isinstance(next_args, dict) else request_args + ), + original_args=function_args, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") or "", + ) + + result, _relay_args = relay_tools.execute( function_name, function_args, - _execute, - original_args=function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=tool_call_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", + _hermes_pipeline, + session_id=str(getattr(agent, "session_id", "") or ""), + metadata={ + "task_id": effective_task_id or "", + "turn_id": getattr(agent, "_current_turn_id", "") or "", + "api_request_id": getattr(agent, "_current_api_request_id", "") or "", + "tool_call_id": tool_call_id or "", + }, ) - return result, observed_args + return _ManagedToolResult( + result=result, + args=state["args"], + middleware_trace=state["middleware_trace"], + blocked=bool(state["blocked"]), + ) + + +def _begin_tool_execution( + agent, + *, + function_name: str, + function_args: dict[str, Any], + effective_task_id: str, + tool_call_id: str, + display_index: int | None, +) -> None: + """Run user-visible and checkpoint preflight on final tool arguments.""" + if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": + display_args = ( + _redact_tool_args_for_display(function_name, function_args) or function_args + ) + args_str = json.dumps(display_args, ensure_ascii=False) + prefix = f"Tool {display_index}" if display_index is not None else "Tool" + if agent.verbose_logging: + print(f" 📞 {prefix}: {function_name}({list(display_args.keys())})") + print( + agent._wrap_verbose( + "Args: ", json.dumps(display_args, indent=2, ensure_ascii=False) + ) + ) + else: + args_preview = ( + args_str[: agent.log_prefix_chars] + "..." + if len(args_str) > agent.log_prefix_chars + else args_str + ) + print( + f" 📞 {prefix}: {function_name}({list(function_args.keys())}) - " + f"{args_preview}" + ) + + agent._current_tool = function_name + agent._touch_activity(f"executing tool: {function_name}") + try: + from tools.environments.base import set_activity_callback + + set_activity_callback(agent._touch_activity) + except Exception: + pass + + if agent.tool_progress_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + preview = _build_tool_preview(function_name, display_args) + agent.tool_progress_callback( + "tool.started", function_name, preview, display_args + ) + except Exception as callback_error: + logging.debug("Tool progress callback error: %s", callback_error) + + if agent.tool_start_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + agent.tool_start_callback( + tool_call_id, function_name, display_args + ) + except Exception as callback_error: + logging.debug("Tool start callback error: %s", callback_error) + + if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: + try: + _ensure_file_checkpoint( + agent, + function_name, + function_args, + effective_task_id, + ) + except Exception: + pass + + if function_name == "terminal" and agent._checkpoint_mgr.enabled: + try: + command = function_args.get("command", "") + if _is_destructive_command(command): + cwd = function_args.get("workdir") or os.getenv( + "TERMINAL_CWD", os.getcwd() + ) + agent._checkpoint_mgr.ensure_checkpoint( + cwd, f"before terminal: {command[:60]}" + ) + except Exception: + pass def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: @@ -386,7 +662,9 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe return # ── Parse args + pre-execution bookkeeping ─────────────────────── - parsed_calls = [] # list of (tool_call, function_name, function_args, middleware_trace, block_result, blocked_by_guardrail) + # (tool call, resolved name, parsed args, middleware trace, parse error, + # tool-search scope block) + parsed_calls = [] for tool_call in tool_calls: function_name = tool_call.function.name @@ -402,17 +680,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe function_args, [], malformed_args_result, - False, + None, ) ) continue - # Reset nudge counters only for a structurally valid invocation. - if function_name == "memory": - agent._turns_since_memory = 0 - elif function_name == "skill_manage": - agent._iters_since_skill = 0 - # ── Tool Search unwrap ──────────────────────────────────────── # When the model invokes the tool_call bridge, peel it open so # every downstream check (checkpointing, guardrails, plugin @@ -446,167 +718,58 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe function_name = _underlying function_args = _underlying_args else: - _ts_scope_block = json.dumps({ - "error": ( - f"'{_underlying}' is not available in this session. " - "Use tool_search to find tools you can call." - ), - }, ensure_ascii=False) + _ts_scope_block = ( + f"'{_underlying}' is not available in this session. " + "Use tool_search to find tools you can call." + ) except Exception: pass - function_args, middleware_trace = _apply_tool_request_middleware_for_agent( - agent, - function_name=function_name, - function_args=function_args, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", + parsed_calls.append( + (tool_call, function_name, function_args, [], None, _ts_scope_block) ) - # ── Block evaluation (BEFORE checkpoint preflight) ─────────── - # We must know whether the tool will execute before touching - # checkpoint state (dedup slot, real snapshots). - block_result = None - blocked_by_guardrail = False - if _ts_scope_block is not None: - # Out-of-scope tool_call: reject before hooks/guardrails/dispatch. - block_result = _ts_scope_block - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="tool_scope_block", - error_message=_ts_scope_block, - middleware_trace=list(middleware_trace), - ) - else: - try: - from hermes_cli.plugins import resolve_pre_tool_block - block_message = resolve_pre_tool_block( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=getattr(tool_call, "id", "") or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - middleware_trace=list(middleware_trace), - ) - except Exception: - block_message = None - - if block_message is not None: - block_result = json.dumps({"error": block_message}, ensure_ascii=False) - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="plugin_block", - error_message=block_message, - middleware_trace=list(middleware_trace), - ) - else: - guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - block_result = agent._guardrail_block_result(guardrail_decision) - blocked_by_guardrail = True - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="guardrail_block", - error_message=getattr(guardrail_decision, "message", None) or "Tool blocked by guardrail policy", - middleware_trace=list(middleware_trace), - ) - - # ── Checkpoint preflight (only for tools that will execute) ── - if block_result is None: - # Checkpoint for file-mutating tools - if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: - try: - _ensure_file_checkpoint( - agent, - function_name, - function_args, - effective_task_id, - ) - except Exception: - pass - - # Checkpoint before destructive terminal commands - if function_name == "terminal" and agent._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - agent._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass - - parsed_calls.append((tool_call, function_name, function_args, middleware_trace, block_result, blocked_by_guardrail)) - # ── Logging / callbacks ────────────────────────────────────────── tool_names_str = ", ".join(name for _, name, _, _, _, _ in parsed_calls) if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}") - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls, 1): - display_args = _redact_tool_args_for_display(name, args) or args - args_str = json.dumps(display_args, ensure_ascii=False) - if agent.verbose_logging: - print(f" 📞 Tool {i}: {name}({list(display_args.keys())})") - print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {name}({list(args.keys())}) - {args_preview}") - - for tc, name, args, middleware_trace, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if agent.tool_progress_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - preview = _build_tool_preview(name, display_args) - agent.tool_progress_callback("tool.started", name, preview, display_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - for tc, name, args, middleware_trace, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if agent.tool_start_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - agent.tool_start_callback(tc.id, name, display_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") # ── Concurrent execution ───────────────────────────────────────── # Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag, middleware_trace) results = [None] * num_tools - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + for i, (tc, name, args, middleware_trace, block_result, _scope_block) in enumerate(parsed_calls): if block_result is not None: results[i] = (name, args, block_result, 0.0, True, True, middleware_trace) + start_condition = threading.Condition() + next_start_order = 0 + authorization_gate = _ConcurrentToolAuthorizationGate() + + def _begin_in_order(order: int, callback=None) -> None: + nonlocal next_start_order + with start_condition: + start_condition.wait_for(lambda: order == next_start_order) + try: + if callback is not None: + callback() + finally: + next_start_order += 1 + start_condition.notify_all() + # Touch activity before launching workers so the gateway knows # we're executing tools (not stuck). agent._current_tool = tool_names_str agent._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}") - def _run_tool(index, tool_call, function_name, function_args, middleware_trace): + def _run_tool( + index, + tool_call, + function_name, + function_args, + middleware_trace, + scope_block, + start_order, + ): """Worker function executed in a thread.""" # Register this worker tid so the agent can fan out an interrupt # to it — see AIAgent.interrupt(). Must happen first thing, and @@ -636,18 +799,50 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe # ContextVars are propagated by propagate_context_to_thread() at the # submit site below (GHSA-qg5c-hvr5-hjgr, #13617). start = time.time() + blocked = False + start_advanced = False + + def _advance_start(callback=None) -> None: + nonlocal start_advanced + if start_advanced: + return + try: + _begin_in_order(start_order, callback) + finally: + start_advanced = True + try: try: - result = agent._invoke_tool( - function_name, - function_args, - effective_task_id, - tool_call.id, - messages=messages, - pre_tool_block_checked=True, - skip_tool_request_middleware=True, - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict[str, Any]) -> Any: + return agent._invoke_tool( + function_name, + next_args, + effective_task_id, + tool_call.id, + messages=messages, + pre_tool_block_checked=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + ) + + managed = _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=scope_block, + display_index=index + 1, + middleware_trace=middleware_trace, + begin_execution=_advance_start, + authorization_gate=authorization_gate, ) + result = managed.result + function_args = managed.args + middleware_trace = managed.middleware_trace + blocked = managed.blocked except KeyboardInterrupt: try: agent.interrupt("keyboard interrupt") @@ -664,7 +859,15 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe ) duration = time.time() - start logger.info("tool %s cancelled (%.2fs)", function_name, duration) - results[index] = (function_name, function_args, result, duration, True, False, middleware_trace) + results[index] = ( + function_name, + function_args, + result, + duration, + True, + False, + middleware_trace, + ) return except Exception as tool_error: result = f"Error executing tool '{function_name}': {tool_error}" @@ -675,8 +878,17 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200]) else: logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result)) - results[index] = (function_name, function_args, result, duration, is_error, False, middleware_trace) + results[index] = ( + function_name, + function_args, + result, + duration, + is_error, + blocked, + middleware_trace, + ) finally: + _advance_start() # Tear down worker-tid tracking. Clear any interrupt bit we may # have set so the next task scheduled onto this recycled tid # starts with a clean slate. This MUST be in a finally block @@ -699,9 +911,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe try: runnable_calls = [ - (i, tc, name, args) - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls) - if block_result is None + (i, tc, name, args, scope_block) + for i, (tc, name, args, _trace, parse_error, scope_block) in enumerate( + parsed_calls + ) + if parse_error is None ] futures = [] future_to_index = {} @@ -719,13 +933,22 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe executor = DaemonThreadPoolExecutor(max_workers=max_workers) abandon_executor = False try: - for submit_index, (i, tc, name, args) in enumerate(runnable_calls): + for submit_index, (i, tc, name, args, scope_block) in enumerate( + runnable_calls + ): # Propagate the agent turn's ContextVars (e.g. # _approval_session_key) AND thread-local approval/sudo # callbacks into the worker thread; clears callbacks on exit. try: f = executor.submit( - propagate_context_to_thread(_run_tool), i, tc, name, args, parsed_calls[i][3] + propagate_context_to_thread(_run_tool), + i, + tc, + name, + args, + parsed_calls[i][3], + scope_block, + submit_index, ) except RuntimeError as submit_error: if not _is_interpreter_shutdown_submit_error(submit_error): @@ -736,7 +959,13 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe "skipping %d unsubmitted tool(s)", len(skipped_calls), ) - for skipped_i, _tc, skipped_name, skipped_args in skipped_calls: + for ( + skipped_i, + _tc, + skipped_name, + skipped_args, + _scope_block, + ) in skipped_calls: if results[skipped_i] is None: middleware_trace = parsed_calls[skipped_i][3] result = ( @@ -766,7 +995,10 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe while True: wait_timeout = 5.0 if deadline is not None: - remaining = deadline - time.monotonic() + effective_deadline = ( + deadline + authorization_gate.excluded_seconds() + ) + remaining = effective_deadline - time.monotonic() if remaining <= 0: done, not_done = set(), { f for f in futures if not f.done() @@ -783,7 +1015,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe if not not_done: break - if deadline is not None and time.monotonic() >= deadline: + if ( + deadline is not None + and time.monotonic() + >= deadline + authorization_gate.excluded_seconds() + ): abandon_executor = True timed_out_indices = { future_to_index[f] @@ -863,7 +1099,9 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total") # ── Post-execution: display per-tool results ───────────────────── - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + for i, (tc, name, args, middleware_trace, _parse_error, _scope_block) in enumerate( + parsed_calls + ): r = results[i] blocked = False is_error = True @@ -923,6 +1161,8 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe tool_duration = 0.0 else: function_name, function_args, function_result, tool_duration, is_error, blocked, middleware_trace = r + name = function_name + args = function_args progress_function_name = function_name if blocked: effect_disposition = "none" @@ -1169,153 +1409,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe except Exception: pass - function_args, middleware_trace = _apply_tool_request_middleware_for_agent( - agent, - function_name=function_name, - function_args=function_args, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - ) - - # Check plugin hooks for a block directive before executing. - _block_msg: Optional[str] = None - _block_error_type = "plugin_block" - if _ts_scope_block is not None: - _block_msg = _ts_scope_block - _block_error_type = "tool_scope_block" - else: - try: - from hermes_cli.plugins import resolve_pre_tool_block - _block_msg = resolve_pre_tool_block( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=getattr(tool_call, "id", "") or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - middleware_trace=list(middleware_trace), - ) - except Exception: - pass - - _guardrail_block_decision: ToolGuardrailDecision | None = None - if _block_msg is None: - guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - _guardrail_block_decision = guardrail_decision - - _execution_blocked = _block_msg is not None or _guardrail_block_decision is not None - - if _execution_blocked: - # Tool blocked by plugin or guardrail policy — skip counters, - # callbacks, checkpointing, activity mutation, and real execution. - pass - # Reset nudge counters when the relevant tool is actually used - elif function_name == "memory": - agent._turns_since_memory = 0 - elif function_name == "skill_manage": - agent._iters_since_skill = 0 - - if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - args_str = json.dumps(display_args, ensure_ascii=False) - if agent.verbose_logging: - print(f" 📞 Tool {i}: {function_name}({list(display_args.keys())})") - print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())}) - {args_preview}") - - if not _execution_blocked: - agent._current_tool = function_name - agent._touch_activity(f"executing tool: {function_name}") - - # Set activity callback for long-running tool execution (terminal - # commands, etc.) so the gateway's inactivity monitor doesn't kill - # the agent while a command is running. - if not _execution_blocked: - try: - from tools.environments.base import set_activity_callback - set_activity_callback(agent._touch_activity) - except Exception: - pass - - if not _execution_blocked and agent.tool_progress_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - preview = _build_tool_preview(function_name, display_args) - agent.tool_progress_callback("tool.started", function_name, preview, display_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - if not _execution_blocked and agent.tool_start_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - agent.tool_start_callback(tool_call.id, function_name, display_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") - - # Checkpoint: snapshot working dir before file-mutating tools - if not _execution_blocked and function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: - try: - _ensure_file_checkpoint( - agent, - function_name, - function_args, - effective_task_id, - ) - except Exception: - pass # never block tool execution - - # Checkpoint before destructive terminal commands - if not _execution_blocked and function_name == "terminal" and agent._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - agent._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass # never block tool execution + middleware_trace: list[dict[str, Any]] = [] + _execution_blocked = False tool_start_time = time.time() - if _block_msg is not None: - # Tool blocked by plugin policy — return error without executing. - function_result = json.dumps({"error": _block_msg}, ensure_ascii=False) - tool_duration = 0.0 - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=function_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type=_block_error_type, - error_message=_block_msg, - middleware_trace=list(middleware_trace), - ) - elif _guardrail_block_decision is not None: - # Tool blocked by tool-loop guardrail — synthesize exactly one - # tool result for the original tool_call_id without executing. - function_result = agent._guardrail_block_result(_guardrail_block_decision) - tool_duration = 0.0 - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=function_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="guardrail_block", - error_message=getattr(_guardrail_block_decision, "message", None) or "Tool blocked by guardrail policy", - middleware_trace=list(middleware_trace), - ) - elif function_name == "todo": + if function_name == "todo": def _execute(next_args: dict) -> Any: from tools.todo_tool import todo_tool as _todo_tool return _todo_tool( @@ -1323,14 +1422,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe merge=next_args.get("merge", False), store=agent._todo_store, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") @@ -1352,14 +1453,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe db=session_db, current_session_id=agent.session_id, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('session_search', function_args, tool_duration, result=function_result)}") @@ -1389,14 +1492,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe ), ) return result - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('memory', function_args, tool_duration, result=function_result)}") @@ -1409,14 +1514,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe multi_select=next_args.get("multi_select", False), callback=agent.clarify_callback, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('clarify', function_args, tool_duration, result=function_result)}") @@ -1428,14 +1535,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe count=next_args.get("count"), callback=getattr(agent, "read_terminal_callback", None), ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}") @@ -1460,14 +1569,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe try: def _execute(next_args: dict) -> Any: return agent._dispatch_delegate_task(next_args) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _delegate_result = function_result finally: agent._delegate_spinner = None @@ -1491,14 +1602,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe try: def _execute(next_args: dict) -> Any: return agent.context_compressor.handle_tool_call(function_name, next_args, messages=messages) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _ce_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Context engine tool '{function_name}' failed: {tool_error}"}) @@ -1525,14 +1638,16 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe try: def _execute(next_args: dict) -> Any: return agent._memory_manager.handle_tool_call(function_name, next_args) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _mem_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Memory tool '{function_name}' failed: {tool_error}"}) @@ -1555,18 +1670,46 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe spinner.start() _spinner_result = None try: - function_result = _ra().handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict) -> Any: + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) + + ( + function_result, + function_args, + middleware_trace, + _execution_blocked, + ) = _managed_values( + _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + middleware_trace=middleware_trace, + ) ) _spinner_result = function_result except KeyboardInterrupt: @@ -1597,18 +1740,46 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe agent._vprint(f" {cute_msg}") else: try: - function_result = _ra().handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict) -> Any: + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) + + ( + function_result, + function_args, + middleware_trace, + _execution_blocked, + ) = _managed_values( + _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + middleware_trace=middleware_trace, + ) ) except KeyboardInterrupt: _emit_cancelled_terminal_post_tool_call( diff --git a/agent/turn_context.py b/agent/turn_context.py index ac9555d012e4..5a0068948879 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -436,7 +436,12 @@ def build_turn_context( # Generate unique task_id if not provided to isolate VMs between tasks. effective_task_id = task_id or str(uuid.uuid4()) agent._current_task_id = effective_task_id - turn_id = f"{agent.session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + turn_id = str(getattr(agent, "_relay_pending_turn_id", "") or "") + if not turn_id: + turn_id = ( + f"{agent.session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + ) + agent._relay_pending_turn_id = None agent._current_turn_id = turn_id agent._current_api_request_id = "" # Tripwire: warn (with both turn ids) when this turn starts before the @@ -1030,7 +1035,7 @@ def build_turn_context( # Plugin hook: pre_llm_call (context injected into user message, not system prompt). plugin_user_context = "" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _pre_results = _invoke_hook( "pre_llm_call", session_id=agent.session_id, @@ -1041,6 +1046,7 @@ def build_turn_context( is_first_turn=(not bool(conversation_history)), model=agent.model, platform=getattr(agent, "platform", None) or "", + parent_session_id=getattr(agent, "_parent_session_id", None) or "", sender_id=getattr(agent, "_user_id", None) or "", ) _ctx_parts: list[str] = [] diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 4e2d318b2e2c..0f636c1dd241 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -488,7 +488,7 @@ def finalize_turn( # First hook to return a string wins; None/empty return leaves text unchanged. if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _transform_results = _invoke_hook( "transform_llm_output", response_text=final_response, @@ -510,7 +510,7 @@ def finalize_turn( # to an external memory system). if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "post_llm_call", session_id=agent.session_id, @@ -669,14 +669,16 @@ def finalize_turn( # Fired at the very end of every run_conversation call. # Plugins can use this for cleanup, flushing buffers, etc. try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=agent.session_id, task_id=effective_task_id, turn_id=turn_id, completed=completed, + failed=failed, interrupted=interrupted, + turn_exit_reason=_turn_exit_reason, model=agent.model, platform=getattr(agent, "platform", None) or "", ) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 932e99929019..437b24c4db8c 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1465,6 +1465,22 @@ display: # hooks_auto_accept: false +# ============================================================================= +# Telemetry +# ============================================================================= +# Shared metrics are disabled by default. When enabled, Hermes writes only +# allowlisted aggregate counters and immutable JSON +# packages under $HERMES_HOME/telemetry/shared_metrics; it does not upload them. +# Packages include a random profile-scoped ID that stays stable until this +# directory is deleted. It is not derived from hardware, account, or host data. +# Successfully exported local history is retained for 30 days; pending deltas +# are retained until they can be exported. +# This profile-owned choice is not overridden by managed-scope configuration. +telemetry: + shared_metrics: + enabled: false + + # ============================================================================= # Update Behavior # ============================================================================= diff --git a/cli.py b/cli.py index d7042309a000..dfa438374664 100644 --- a/cli.py +++ b/cli.py @@ -1267,9 +1267,8 @@ def _notify_session_finalize( reason: str = "shutdown", ) -> None: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=session_id, platform=platform, reason=reason, @@ -1297,7 +1296,7 @@ def _emit_interrupted_session_end(cli, *, reason: str = "keyboard_interrupt") -> pass try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=session_id, @@ -7695,13 +7694,21 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): lifecycle point (shutdown, /new, /reset). """ try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - event_type, - session_id=self.agent.session_id if self.agent else None, - platform=getattr(self, "platform", None) or "cli", - reason="new_session" if event_type == "on_session_reset" else "session_boundary", - ) + from hermes_cli.lifecycle import finalize_session, invoke_hook + + context = { + "session_id": self.agent.session_id if self.agent else None, + "platform": getattr(self, "platform", None) or "cli", + "reason": ( + "new_session" + if event_type == "on_session_reset" + else "session_boundary" + ), + } + if event_type == "on_session_finalize": + finalize_session(**context) + else: + invoke_hook(event_type, **context) except Exception: pass @@ -16596,7 +16603,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # the exit occurred, meaning run_conversation's hook didn't fire. if self.agent and getattr(self, '_agent_running', False): try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=self.agent.session_id, diff --git a/contributors/emails/afournier@nvidia.com b/contributors/emails/afournier@nvidia.com new file mode 100644 index 000000000000..d4289ed44061 --- /dev/null +++ b/contributors/emails/afournier@nvidia.com @@ -0,0 +1 @@ +afourniernv diff --git a/docs/observability/README.md b/docs/observability/README.md index 800c7d73e958..14847f74969f 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -14,6 +14,10 @@ Behavior-changing request or execution wrappers are outside this observer contract. Observer hooks should report what happened; they should not replace provider requests, tool arguments, or execution callbacks. +Hermes also has a first-party NeMo Relay shared-metrics path. It uses these +lifecycle boundaries directly and does not require enabling an observability +plugin. See [Relay shared metrics](relay-shared-metrics.md). + ## Contract Plugins register observer callbacks from `register(ctx)`: diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md new file mode 100644 index 000000000000..8c87da044ff0 --- /dev/null +++ b/docs/observability/relay-shared-metrics.md @@ -0,0 +1,127 @@ +# NeMo Relay Shared Metrics + +Hermes includes NeMo Relay as a normal runtime dependency on platforms for +which Relay publishes a native wheel. The shared-metrics integration is built +into Hermes and does not require `hermes plugins enable +observability/nemo_relay`. Hermes remains importable without Relay on other +native targets. Those targets use an explicit reduced-capability no-op host: +Hermes execution remains available, while Relay scopes, middleware, plugins, +and subscribers are unavailable. The `hermes-agent[nemo-relay]` extra remains +as a no-op compatibility alias for existing installation commands. + +Hermes requires NeMo Relay 0.6.0 or later within the 0.6 release line. That +release establishes the lossless provider-codec contract used for Anthropic +Messages, OpenAI Chat Completions, and OpenAI Responses requests. + +## Runtime Dependency and Data Boundary + +Hermes installs the platform-specific `nemo-relay` native wheel from the +bounded `>=0.6.0,<0.7` dependency range. The published package is built from +the [NVIDIA NeMo Relay repository](https://github.com/NVIDIA/NeMo-Relay). +Unsupported platforms use the explicit no-op runtime described above rather +than downloading a different implementation. + +When Relay managed execution is active, the provider request and response pass +through that native module in the Hermes process so configured interceptors can +operate on the real call. This is separate from the shared-metrics data +contract. Shared-metrics mode installs no network exporter and its subscriber +accepts only the versioned, allowlisted projection described below. Enabling a +separately configured rich-observability or dynamic plugin can create a +different data path and requires its own policy review. + +Collection remains off unless Hermes policy enables it: + +```yaml +telemetry: + shared_metrics: + enabled: true +``` + +This choice is read from the profile's own `config.yaml`. A machine-managed +configuration overlay cannot enable or disable shared metrics on the profile's +behalf. + +The existing `observability/nemo_relay` plugin remains separate. Enable that +plugin only for its opt-in rich observability exporters, adaptive execution, +or dynamic Relay plugins. + +Hermes core owns one Relay host and one isolated Relay session scope per Hermes +session. Core lifecycle producers use +`hermes_cli.observability.relay_runtime` to obtain the shared session handle or +run Relay scope, LLM, tool, and mark APIs in that session context. New product +marks do not require Hermes plugin registration. Shared-metrics marks must +still contain only fields approved by the versioned allowlist; the hard +dependency does not change the collection or privacy policy. + +## Current Slices + +The current vertical slices record logical model calls and top-level task runs: + +```text +Hermes turn, API, and tool hooks + -> Relay session, task, and LLM lifecycle + -> Hermes shared-metrics subscriber + -> SQLite counters + -> immutable JSON delta package +``` + +Hermes sends an empty `LLMRequest` into the metrics-owned lifecycle. This does +not describe the separate managed-execution call through the native runtime +documented above. The terminal metrics event contains only bounded model +family, provider family, locality, call role, and outcome values. Prompts, +responses, exact model IDs, endpoints, errors, session IDs, task IDs, and +request IDs are not included in the metrics event or package. + +Each task run is a Relay `Function` scope named `hermes.task_run`, parented to +the owning Hermes session. The start counter contains only bounded execution +surface and entrypoint values. The terminal counter contains bounded outcome, +end reason, termination status, duration, logical model-call count, terminal +tool-call count, and provider-retry count buckets. Retries are additional +provider attempts for the same Hermes API request ID; they do not inflate the +logical model-call count. Tool calls are deduplicated by their Hermes tool-call +ID after a terminal tool result is observed. The outer `AIAgent` execution +boundary closes the task for normal returns, early returns, exceptions, and +cancellations. Active task ownership follows the task ID if Hermes rotates its +conversation session during context compression. + +Local state is written under: + +```text +$HERMES_HOME/telemetry/shared_metrics/metrics.sqlite3 +$HERMES_HOME/telemetry/shared_metrics/outbox/*.json +``` + +The database keeps transactional aggregate and package-outbox state. Package +files are immutable delta documents that conform to a closed JSON schema and +are written with atomic replacement. Fully packaged aggregate rows and +successfully exported package rows and files are retained locally for 30 days. +Pending package rows and counters with unexported deltas are never pruned. + +Each package contains an `install_id` generated as a random UUID. Despite the +schema field name, its current scope is one `HERMES_HOME`, so it is more +precisely a persistent pseudonymous profile identifier. It is not derived from +hardware, account, host, path, or credential data. It remains stable across +packages from that profile and can therefore link those local packages. +Deleting `$HERMES_HOME/telemetry/shared_metrics` resets the identifier together +with all aggregates and package files. + +This slice has no remote-delivery path. A future remote exporter must not reuse +the persistent local identifier by default. It requires a separate product and +privacy decision covering consent, identity scope, rotation or keyed +pseudonymization, reset behavior, retention, and deletion. + +## Smoke Test + +Run a real Hermes CLI turn against the deterministic local model server: + +```bash +./.venv/bin/python scripts/smoke_nemo_relay_shared_metrics.py +``` + +The script uses the installed `nemo-relay` dependency by default. Pass +`--relay-python ../nemo-relay/python` only when testing a locally built Relay +binding. + +The smoke verifies the model request reached the local server, model and task +counters were stored, one package was exported, and prompt, response, and +exact-model canaries are absent from the package. diff --git a/gateway/run.py b/gateway/run.py index 37dfd9d92cde..500ae8b66ce2 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -6868,9 +6868,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _e: logger.debug("Shutdown transcript flush failed: %s", _e) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=getattr(agent, "session_id", None), platform="gateway", reason="shutdown", @@ -9162,11 +9161,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew for key, entry in _expired_entries: try: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import finalize_session _parts = key.split(":") _platform = _parts[2] if len(_parts) > 2 else "" - _invoke_hook( - "on_session_finalize", + finalize_session( session_id=entry.session_id, platform=_platform, reason="session_expired", @@ -10983,7 +10981,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # (e.g. customer handover ingest) without triggering the pairing flow. if not is_internal: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _hook_results = _invoke_hook( "pre_gateway_dispatch", event=event, diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index cf1d44929cbc..a32e7c6a45bb 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -223,9 +223,8 @@ class GatewaySlashCommandsMixin: # Fire plugin on_session_finalize hook (session boundary) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook - _invoke_hook( - "on_session_finalize", + from hermes_cli.lifecycle import finalize_session + finalize_session( session_id=_old_sid, platform=source.platform.value if source.platform else "", reason="new_session", @@ -302,7 +301,7 @@ class GatewaySlashCommandsMixin: # Fire plugin on_session_reset hook (new session guaranteed to exist) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _new_sid = new_entry.session_id if new_entry else None _invoke_hook( "on_session_reset", diff --git a/hermes_cli/config.py b/hermes_cli/config.py index aab86489c4c6..53f5161262fe 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3418,6 +3418,14 @@ DEFAULT_CONFIG = { "profile_build": "ask", }, + # Privacy-safe aggregate metrics written only to this profile's local + # telemetry directory. Collection is opt-in and no remote sink exists. + "telemetry": { + "shared_metrics": { + "enabled": False, + }, + }, + # ``hermes update`` behaviour. "updates": { # Pre-update safety backup — ONE consolidated mechanism, three modes: diff --git a/hermes_cli/hooks.py b/hermes_cli/hooks.py index 58faf7b41df5..852bfa1c8428 100644 --- a/hermes_cli/hooks.py +++ b/hermes_cli/hooks.py @@ -149,7 +149,17 @@ _DEFAULT_PAYLOADS = { "changed_paths": ["src/app.tsx"], }, "on_session_start": {"session_id": "test-session"}, - "on_session_end": {"session_id": "test-session"}, + "on_session_end": { + "session_id": "test-session", + "task_id": "test-task", + "turn_id": "test-turn", + "completed": True, + "failed": False, + "interrupted": False, + "turn_exit_reason": "text_response(stop)", + "model": "gpt-4", + "platform": "cli", + }, "on_session_finalize": {"session_id": "test-session"}, "on_session_reset": {"session_id": "test-session"}, "pre_api_request": { diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index ca81157decb0..d32f85f81d3c 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -175,7 +175,7 @@ def _fire_kanban_lifecycle_hook(event: str, task_id: str, **fields: Any) -> None it through. """ try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook from hermes_cli.profiles import get_active_profile_name try: profile_name = get_active_profile_name() diff --git a/hermes_cli/lifecycle.py b/hermes_cli/lifecycle.py new file mode 100644 index 000000000000..350e3880aff9 --- /dev/null +++ b/hermes_cli/lifecycle.py @@ -0,0 +1,63 @@ +"""Hermes lifecycle dispatch for first-party observers and plugins.""" + +from __future__ import annotations + +import logging +from typing import Any, List + +logger = logging.getLogger(__name__) + + +def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]: + """Notify first-party observers, then invoke compatibility plugin hooks.""" + try: + from hermes_cli.observability import observe_lifecycle + + observe_lifecycle(hook_name, **kwargs) + except Exception: + logger.warning("Built-in observability hook failed", exc_info=True) + + from hermes_cli import plugins + + return plugins.invoke_hook(hook_name, **kwargs) + + +def has_hook(hook_name: str) -> bool: + """Return whether a first-party observer or plugin consumes a hook.""" + try: + from hermes_cli.observability import handles_hook + + if handles_hook(hook_name): + return True + except Exception: + logger.warning("Unable to inspect built-in observability hooks", exc_info=True) + + from hermes_cli import plugins + + return plugins.has_hook(hook_name) + + +def finalize_session(**kwargs: Any) -> List[Any]: + """Notify observers and hard-close one core-owned Relay conversation.""" + try: + from hermes_cli.observability import observe_lifecycle + + observe_lifecycle("on_session_finalize", **kwargs) + except Exception: + logger.warning("Built-in observability hook failed", exc_info=True) + + session_id = str(kwargs.get("session_id") or "") + if session_id: + try: + from agent import relay_runtime + + relay_runtime.SESSION_COORDINATOR.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=session_id, + ) + except Exception: + logger.warning("Core Relay session finalization failed", exc_info=True) + + from hermes_cli import plugins + + return plugins.invoke_hook("on_session_finalize", **kwargs) diff --git a/hermes_cli/middleware.py b/hermes_cli/middleware.py index 8795952a2b7d..a3b472a45636 100644 --- a/hermes_cli/middleware.py +++ b/hermes_cli/middleware.py @@ -127,18 +127,32 @@ def apply_tool_request_middleware( Middleware may return ``{"args": {...}}`` to replace the effective tool arguments before hooks, guardrails, approvals, and execution see them. """ - if not _has_middleware(TOOL_REQUEST_MIDDLEWARE): - return RequestMiddlewareResult( - payload=args, - original_payload=args, - changed=False, - trace=[], - ) - original_args = _safe_copy(args) current_args = _safe_copy(original_args) trace: List[Dict[str, Any]] = [] + session_id = str(context.get("session_id") or "") + skip_relay = bool(context.pop("skip_relay", False)) + if session_id and not skip_relay: + from agent import relay_runtime + + relay_args = relay_runtime.apply_tool_request_intercepts( + session_id=session_id, + tool_name=tool_name, + args=current_args, + ) + if relay_args != current_args: + current_args = _safe_copy(relay_args) + trace.append({"source": "nemo_relay"}) + + if not _has_middleware(TOOL_REQUEST_MIDDLEWARE): + return RequestMiddlewareResult( + payload=args if not trace else current_args, + original_payload=args, + changed=bool(trace), + trace=trace, + ) + for result in _invoke_middleware( TOOL_REQUEST_MIDDLEWARE, tool_name=tool_name, diff --git a/hermes_cli/observability/__init__.py b/hermes_cli/observability/__init__.py new file mode 100644 index 000000000000..1c4e70300c6f --- /dev/null +++ b/hermes_cli/observability/__init__.py @@ -0,0 +1,31 @@ +"""First-party Hermes observability integrations.""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: + """Dispatch a Hermes lifecycle event to built-in observability features.""" + from . import relay_shared_metrics + + _safe_observe(relay_shared_metrics.observe_lifecycle, hook_name, kwargs) + + +def handles_hook(hook_name: str) -> bool: + """Return whether any built-in observability feature handles a hook.""" + from . import relay_shared_metrics + + return relay_shared_metrics.handles_hook(hook_name) + + +def _safe_observe(callback: Any, hook_name: str, kwargs: dict[str, Any]) -> None: + try: + callback(hook_name, **kwargs) + except Exception: + logger.warning( + "Built-in observability hook failed: %s", hook_name, exc_info=True + ) diff --git a/hermes_cli/observability/relay_runtime.py b/hermes_cli/observability/relay_runtime.py new file mode 100644 index 000000000000..bd6de0b83796 --- /dev/null +++ b/hermes_cli/observability/relay_runtime.py @@ -0,0 +1,14 @@ +"""Compatibility alias for the core Hermes Relay runtime. + +New code should import :mod:`agent.relay_runtime`. This module remains an +alias, rather than a copy, so existing plugins and tests share the same +profile registry and test-reset state during the migration. +""" + +from __future__ import annotations + +import sys + +from agent import relay_runtime as _core_relay_runtime + +sys.modules[__name__] = _core_relay_runtime diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py new file mode 100644 index 000000000000..fb6f885b2f05 --- /dev/null +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -0,0 +1,817 @@ +"""Direct NeMo Relay integration for Hermes shared client metrics.""" + +from __future__ import annotations + +import atexit +import contextvars +import logging +import threading +from dataclasses import dataclass, field +from time import monotonic_ns +from typing import Any, Callable + +from agent import relay_runtime +from hermes_cli import __version__ + +from .shared_metrics import SharedMetricsStore +from .shared_metrics_contract import ( + MODEL_CALL_SCOPE, + SCHEMA_KEY, + SCHEMA_VERSION, + SUBSCRIBER_NAME, + TASK_SCOPE, + model_call_fields, + model_call_outcome, + task_start_fields, + task_terminal_fields, +) +from .shared_metrics_subscriber import SharedMetricsSubscriber + +logger = logging.getLogger(__name__) + +HANDLED_HOOKS = frozenset({ + "on_session_start", + "on_session_end", + "on_session_finalize", + "on_session_reset", + "pre_llm_call", + "pre_api_request", + "post_tool_call", + "post_api_request", + "api_request_error", + "subagent_stop", +}) + +_RUNTIME_FAILED = object() +_RUNTIMES: dict[str, _Runtime | object] = {} +_RUNTIME_LOCK = threading.RLock() + + +def _retry_ordinal(event: dict[str, Any]) -> int | None: + value = event.get("retry_count") + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return None + + +@dataclass +class _ModelCall: + handle: Any + task_id: str + fields: dict[str, str] + retry_ordinal: int | None = None + + +@dataclass +class _TaskRun: + handle: Any + context: contextvars.Context + started_ns: int + start_fields: dict[str, str] + model_call_ids: set[str] = field(default_factory=set) + tool_call_ids: set[str] = field(default_factory=set) + turn_ids: set[str] = field(default_factory=set) + unidentified_tool_calls: int = 0 + retry_count: int = 0 + + +@dataclass +class _MetricsSession: + session_id: str + relay_session: relay_runtime.RelaySession + lock: threading.RLock = field(default_factory=threading.RLock, repr=False) + closing: bool = False + model_calls: dict[str, _ModelCall] = field(default_factory=dict) + tasks: dict[str, _TaskRun] = field(default_factory=dict) + + +class _Runtime: + """Own shared-metrics state layered on the Hermes core Relay host.""" + + def __init__(self, host: relay_runtime.RelayRuntime | None = None) -> None: + resolved_host = host or relay_runtime.get_runtime() + if resolved_host is None: + raise RuntimeError("Hermes core Relay runtime is unavailable") + self.host: relay_runtime.RelayRuntime = resolved_host + self.relay = self.host.relay + self._sessions_lock = threading.RLock() + self._active = True + self._sessions: dict[str, _MetricsSession] = {} + self._task_creation_lock = threading.RLock() + self._task_sessions_lock = threading.RLock() + self._task_sessions: dict[tuple[str, str], _MetricsSession] = {} + self._turn_sessions: dict[tuple[str, str], _MetricsSession] = {} + self._subscriber_name = f"{SUBSCRIBER_NAME}.{self.host.runtime_id}" + self.subscriber = SharedMetricsSubscriber( + SharedMetricsStore(), + __version__, + runtime_id=self.host.runtime_id, + ) + self.relay.subscribers.register(self._subscriber_name, self.subscriber) + self.host.retain_managed_execution(self._subscriber_name) + self._registered = True + atexit.register(self.shutdown) + + def ensure_session(self, event: dict[str, Any]) -> _MetricsSession | None: + session_id = str(event.get("session_id") or "") + if not session_id: + return None + with self._sessions_lock: + if not self._active: + return None + relay_session = self.host.ensure_session(event) + if relay_session is None: + return None + session = self._sessions.get(session_id) + if session is None: + session = _MetricsSession( + session_id=session_id, + relay_session=relay_session, + ) + self._sessions[session_id] = session + with session.lock: + if session.closing: + return None + return session + + def _run_in_session( + self, + session: _MetricsSession, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> Any: + return self.host.run_in_session( + session.relay_session, + callback, + *args, + **kwargs, + ) + + def start_task(self, event: dict[str, Any]) -> _TaskRun | None: + """Open one Relay function scope for a Hermes task run.""" + task_key = self._task_key(event) + if task_key is None: + return None + _, task_id = task_key + with self._task_creation_lock: + owner = self._task_session(event) + if owner is not None: + with owner.lock: + if owner.closing: + return None + task = owner.tasks.get(task_id) + if task is not None: + self._remember_turn(owner, task, event) + return task + + session = self.ensure_session(event) + if session is None: + return None + with session.lock: + if session.closing or session.relay_session.context is None: + return None + task_context = session.relay_session.context.copy() + start_fields = task_start_fields(event) + active_turn = relay_runtime.active_turn(session.session_id) + parent_handle = session.relay_session.handle + if ( + active_turn is not None + and active_turn.lease.session_id == session.session_id + and active_turn.task_id == task_id + and active_turn.handle is not None + ): + parent_handle = active_turn.handle + + def push_task() -> Any: + self.relay.get_scope_stack() + return self.relay.scope.push( + TASK_SCOPE, + self.relay.ScopeType.Function, + handle=parent_handle, + input=start_fields, + metadata=self._event_metadata(), + ) + + handle = task_context.run(push_task) + task = _TaskRun( + handle=handle, + context=task_context, + started_ns=monotonic_ns(), + start_fields=start_fields, + ) + session.tasks[task_id] = task + with self._task_sessions_lock: + self._task_sessions[task_key] = session + self._remember_turn(session, task, event) + return task + + def _run_in_task( + self, + task: _TaskRun, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> Any: + def invoke() -> Any: + self.relay.get_scope_stack() + return callback(*args, **kwargs) + + return task.context.copy().run(invoke) + + def start_model_call(self, event: dict[str, Any]) -> None: + task_id = str(event.get("task_id") or "") + session = self._task_session(event, allow_task_id_fallback=True) + task = session.tasks.get(task_id) if session is not None else None + if task is None: + task = self.start_task(event) + session = self._task_session(event) if task is not None else None + if session is None: + session = self.ensure_session(event) + if session is None: + return + request_id = str(event.get("api_request_id") or "") + if not request_id: + return + fields = model_call_fields(event) + retry_ordinal = _retry_ordinal(event) + model_family = fields["model_family"] + with session.lock: + if session.closing: + return + if task is not None: + self._remember_turn(session, task, event) + existing = session.model_calls.get(request_id) + if existing is not None: + existing.fields = fields + if task is not None: + if retry_ordinal is None or existing.retry_ordinal is None: + task.retry_count += 1 + elif retry_ordinal > existing.retry_ordinal: + task.retry_count += retry_ordinal - existing.retry_ordinal + if retry_ordinal is not None: + existing.retry_ordinal = max( + existing.retry_ordinal or 0, + retry_ordinal, + ) + return + if task is not None: + task.model_call_ids.add(request_id) + if retry_ordinal is not None and retry_ordinal > 0: + # A real Hermes retry can advance api_request_id while + # carrying the retry ordinal. Count that physical attempt. + task.retry_count += 1 + handle = self._run_in_task( + task, + self.relay.llm.call, + MODEL_CALL_SCOPE, + self.relay.LLMRequest({}, {}), + handle=task.handle, + metadata=self._event_metadata(), + model_name=model_family, + ) + else: + handle = self._run_in_session( + session, + self.relay.llm.call, + MODEL_CALL_SCOPE, + self.relay.LLMRequest({}, {}), + handle=session.relay_session.handle, + metadata=self._event_metadata(), + model_name=model_family, + ) + session.model_calls[request_id] = _ModelCall( + handle=handle, + task_id=str(event.get("task_id") or ""), + fields=fields, + retry_ordinal=retry_ordinal, + ) + + def record_tool_call(self, event: dict[str, Any]) -> None: + """Count one unique tool invocation under its owning task.""" + task_id = str(event.get("task_id") or "") + session = self._task_session(event, allow_task_id_fallback=True) + task = session.tasks.get(task_id) if session is not None else None + if task is None: + task = self.start_task(event) + session = self._task_session(event) if task is not None else None + if session is None or task is None: + return + tool_call_id = str(event.get("tool_call_id") or "") + with session.lock: + if session.closing: + return + self._remember_turn(session, task, event) + if tool_call_id: + task.tool_call_ids.add(tool_call_id) + else: + task.unidentified_tool_calls += 1 + + def end_model_call(self, event: dict[str, Any], outcome: str | None = None) -> None: + session = self._task_session(event, allow_task_id_fallback=True) + if session is None: + session = self._session(event) + if session is None: + return + request_id = str(event.get("api_request_id") or "") + with session.lock: + if session.closing: + return + model_call = session.model_calls.get(request_id) + if model_call is None: + return + fields = model_call_fields(event) + model_call.fields = fields + self._finish_model_call( + session, + request_id, + outcome or model_call_outcome(event), + ) + + def end_pending_model_calls(self, event: dict[str, Any]) -> None: + session = self._task_session(event, allow_task_id_fallback=True) + if session is None: + session = self._session(event) + if session is None: + return + with session.lock: + if session.closing: + return + self._end_pending_model_calls(session, event) + + def finish_task(self, event: dict[str, Any]) -> None: + """Close one task scope exactly once with bounded terminal fields.""" + task_id = str(event.get("task_id") or "") + session = self._task_session( + event, + allow_task_id_fallback=True, + ) or self._session(event) + if session is None: + return + with session.lock: + if session.closing: + return + self._finish_task(session, task_id, event) + + def close_session(self, event: dict[str, Any]) -> None: + session = self._session(event) + if session is None: + return + failures: list[str] = [] + with session.lock: + if session.closing: + return + session.closing = True + for task_id in list(session.tasks): + self._finish_task( + session, + task_id, + { + **event, + "task_id": task_id, + "completed": False, + "failed": True, + "interrupted": False, + "turn_exit_reason": "system_aborted", + }, + ) + self._end_pending_model_calls(session, event) + try: + self.relay.subscribers.flush() + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + self._export() + with self._sessions_lock: + if self._sessions.get(session.session_id) is session: + self._sessions.pop(session.session_id, None) + if failures: + logger.warning( + "Hermes shared-metrics session %s closed with errors: %s", + session.session_id, + "; ".join(failures), + ) + + def shutdown(self) -> None: + with self._sessions_lock: + self._active = False + session_ids = list(self._sessions) + for session_id in session_ids: + self._safe(self.close_session, {"session_id": session_id}) + if not self._registered: + return + self._safe(self.relay.subscribers.flush) + self._export() + self._safe(self.relay.subscribers.deregister, self._subscriber_name) + self.host.release_managed_execution(self._subscriber_name) + self._registered = False + try: + atexit.unregister(self.shutdown) + except Exception: + pass + + def deactivate(self) -> None: + """Stop collection without exporting locally aggregated metrics.""" + with self._sessions_lock: + self._active = False + self.subscriber.deactivate() + if self._registered: + self._safe(self.relay.subscribers.deregister, self._subscriber_name) + self.host.release_managed_execution(self._subscriber_name) + self._registered = False + with self._sessions_lock: + sessions = list(self._sessions.values()) + for session in sessions: + with session.lock: + if session.closing: + continue + session.closing = True + for task_id in list(session.tasks): + self._finish_task( + session, + task_id, + { + "session_id": session.session_id, + "task_id": task_id, + "failed": True, + "turn_exit_reason": "system_aborted", + }, + ) + self._end_pending_model_calls(session, {}) + with self._sessions_lock: + self._sessions.clear() + with self._task_sessions_lock: + self._task_sessions.clear() + self._turn_sessions.clear() + try: + atexit.unregister(self.shutdown) + except Exception: + pass + + def _session(self, event: dict[str, Any]) -> _MetricsSession | None: + session_id = str(event.get("session_id") or "") + with self._sessions_lock: + return self._sessions.get(session_id) + + @staticmethod + def _task_key(event: dict[str, Any]) -> tuple[str, str] | None: + session_id = str(event.get("session_id") or "") + task_id = str(event.get("task_id") or "") + if not session_id or not task_id: + return None + return session_id, task_id + + def _task_session( + self, + event: dict[str, Any], + *, + allow_task_id_fallback: bool = False, + ) -> _MetricsSession | None: + task_key = self._task_key(event) + if task_key is None: + return None + turn_key = self._turn_key(event) + with self._task_sessions_lock: + if turn_key is not None: + owner = self._turn_sessions.get(turn_key) + if owner is not None: + return owner + owner = self._task_sessions.get(task_key) + if owner is not None or not allow_task_id_fallback: + return owner + task_id = task_key[1] + candidates: list[_MetricsSession] = [] + for (_, candidate_task_id), session in self._task_sessions.items(): + if candidate_task_id != task_id: + continue + if not any(candidate is session for candidate in candidates): + candidates.append(session) + return candidates[0] if len(candidates) == 1 else None + + @staticmethod + def _turn_key(event: dict[str, Any]) -> tuple[str, str] | None: + session_id = str(event.get("session_id") or "") + turn_id = str(event.get("turn_id") or "") + if not session_id or not turn_id: + return None + return session_id, turn_id + + def _remember_turn( + self, + session: _MetricsSession, + task: _TaskRun, + event: dict[str, Any], + ) -> None: + turn_id = str(event.get("turn_id") or "") + if not turn_id: + return + task.turn_ids.add(turn_id) + with self._task_sessions_lock: + self._turn_sessions[(session.session_id, turn_id)] = session + + def _finish_model_call( + self, + session: _MetricsSession, + request_id: str, + outcome: str, + ) -> None: + model_call = session.model_calls.pop(request_id, None) + if model_call is None: + return + try: + task = session.tasks.get(model_call.task_id) + if task is not None: + self._run_in_task( + task, + self.relay.llm.call_end, + model_call.handle, + {**model_call.fields, "outcome": outcome}, + metadata=self._event_metadata(), + ) + else: + self._run_in_session( + session, + self.relay.llm.call_end, + model_call.handle, + {**model_call.fields, "outcome": outcome}, + metadata=self._event_metadata(), + ) + except Exception: + logger.warning( + "Hermes shared-metrics model call close failed", exc_info=True + ) + + def _end_pending_model_calls( + self, + session: _MetricsSession, + event: dict[str, Any], + ) -> None: + task_id = str(event.get("task_id") or "") + request_ids = [ + request_id + for request_id, model_call in session.model_calls.items() + if not task_id or model_call.task_id == task_id + ] + outcome = "cancelled" if event.get("interrupted") else "failed" + for request_id in request_ids: + self._finish_model_call(session, request_id, outcome) + + def _finish_task( + self, + session: _MetricsSession, + task_id: str, + event: dict[str, Any], + ) -> None: + task = session.tasks.get(task_id) + if task is None: + return + self._end_pending_model_calls(session, {**event, "task_id": task_id}) + fields = task_terminal_fields( + {**task.start_fields, **event}, + duration_ms=max(0, (monotonic_ns() - task.started_ns) // 1_000_000), + model_call_count=len(task.model_call_ids), + tool_call_count=len(task.tool_call_ids) + task.unidentified_tool_calls, + retry_count=task.retry_count, + ) + try: + self._run_in_task( + task, + self.relay.scope.pop, + task.handle, + output=fields, + metadata=self._event_metadata(), + ) + except Exception: + logger.warning("Hermes shared-metrics task close failed", exc_info=True) + finally: + session.tasks.pop(task_id, None) + with self._task_sessions_lock: + task_key = (session.session_id, task_id) + if self._task_sessions.get(task_key) is session: + self._task_sessions.pop(task_key, None) + for turn_id in task.turn_ids: + turn_key = (session.session_id, turn_id) + if self._turn_sessions.get(turn_key) is session: + self._turn_sessions.pop(turn_key, None) + + def _export(self) -> None: + self._safe(self.subscriber.store.create_and_export_package) + + def _event_metadata(self) -> dict[str, str]: + return { + SCHEMA_KEY: SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: self.host.runtime_id, + } + + @staticmethod + def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return callback(*args, **kwargs) + except Exception: + logger.warning("Hermes shared metrics operation failed", exc_info=True) + return None + + +def enabled() -> bool: + """Return the shared-metrics policy for the active Hermes profile.""" + profile_key = relay_runtime.current_profile_key() + try: + from hermes_cli.config import read_raw_config + + # Collection consent is profile-owned. Managed config overlays may + # control runtime policy, but cannot opt a profile into or out of + # shared metrics. + config = read_raw_config() or {} + except Exception: + logger.debug("Unable to read Hermes shared-metrics policy", exc_info=True) + value = False + else: + telemetry = config.get("telemetry") if isinstance(config, dict) else None + shared_metrics = ( + telemetry.get("shared_metrics") if isinstance(telemetry, dict) else None + ) + value = ( + isinstance(shared_metrics, dict) + and shared_metrics.get("enabled") is True + ) + if value: + return True + with _RUNTIME_LOCK: + runtime = _RUNTIMES.pop(profile_key, None) + if isinstance(runtime, _Runtime): + runtime.deactivate() + return False + + +def handles_hook(hook_name: str) -> bool: + return hook_name in HANDLED_HOOKS and enabled() + + +def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: + """Project one Hermes lifecycle event into the core Relay integration.""" + if not handles_hook(hook_name): + return + runtime = _get_runtime() + if runtime is None: + return + try: + if hook_name == "on_session_start": + runtime.ensure_session(kwargs) + elif hook_name == "pre_llm_call": + runtime.start_task(kwargs) + elif hook_name == "pre_api_request": + runtime.start_model_call(kwargs) + elif hook_name == "post_tool_call": + runtime.record_tool_call(kwargs) + elif hook_name == "post_api_request": + runtime.end_model_call(kwargs, "success") + elif hook_name == "api_request_error": + if kwargs.get("retryable") is False: + runtime.end_model_call(kwargs, "failed") + elif hook_name == "on_session_end": + runtime.finish_task(kwargs) + elif hook_name == "subagent_stop": + child_session_id = str(kwargs.get("child_session_id") or "") + if child_session_id: + runtime.close_session({"session_id": child_session_id}) + elif hook_name in {"on_session_finalize", "on_session_reset"}: + runtime.close_session(kwargs) + except Exception: + logger.warning( + "Hermes shared metrics hook failed: %s", hook_name, exc_info=True + ) + + +def prepare_session_start() -> None: + """Register the subscriber before any producer opens the session scope.""" + if enabled(): + _get_runtime(retry_failed=True) + + +def _prepare_core_session( + host: relay_runtime.RelayRuntime, + context: dict[str, Any], +) -> None: + """Prepare the profile subscriber before the coordinator opens a scope.""" + del context + if host.profile_key == relay_runtime.current_profile_key(): + if enabled(): + _get_runtime(retry_failed=True, host=host) + + +def start_task_run( + *, + session_id: str, + task_id: str, + platform: str, + parent_session_id: str = "", +) -> None: + """Start task metrics at the outer Hermes execution boundary.""" + if not enabled(): + return + runtime = _get_runtime(retry_failed=True) + if runtime is None: + return + runtime._safe( + runtime.start_task, + { + "session_id": session_id, + "task_id": task_id, + "platform": platform, + "parent_session_id": parent_session_id, + }, + ) + + +def finish_task_run( + *, + session_id: str, + task_id: str, + platform: str, + result: dict[str, Any] | None = None, + error: BaseException | None = None, +) -> None: + """Finish task metrics for every return or exception path.""" + if not enabled(): + return + runtime = _get_runtime() + if runtime is None: + return + + terminal = result if isinstance(result, dict) else {} + interrupted = terminal.get("interrupted") is True + completed = terminal.get("completed") is True + failed = terminal.get("failed") is True + reason = str( + terminal.get("turn_exit_reason") or terminal.get("failure_reason") or "" + ) + if error is not None: + interrupted = isinstance(error, (KeyboardInterrupt, InterruptedError)) or ( + type(error).__name__ == "CancelledError" + ) + timed_out = isinstance(error, TimeoutError) + completed = False + failed = not interrupted + if interrupted: + reason = "interrupted_by_user" + elif timed_out: + reason = "timed_out" + else: + reason = "system_aborted" + elif not reason: + reason = "failed" if failed else "unknown" + + runtime._safe( + runtime.finish_task, + { + "session_id": session_id, + "task_id": task_id, + "platform": platform, + "completed": completed, + "failed": failed, + "interrupted": interrupted, + "turn_exit_reason": reason, + }, + ) + + +def _get_runtime( + *, + retry_failed: bool = False, + host: relay_runtime.RelayRuntime | None = None, +) -> _Runtime | None: + profile_key = relay_runtime.current_profile_key() + with _RUNTIME_LOCK: + runtime = _RUNTIMES.get(profile_key) + if isinstance(runtime, _Runtime): + if host is None or runtime.host is host: + return runtime + runtime.deactivate() + _RUNTIMES.pop(profile_key, None) + if runtime is _RUNTIME_FAILED and not retry_failed: + return None + if runtime is _RUNTIME_FAILED: + _RUNTIMES.pop(profile_key, None) + try: + runtime = _Runtime(host=host) + except Exception: + logger.warning("Hermes shared metrics initialization failed", exc_info=True) + _RUNTIMES[profile_key] = _RUNTIME_FAILED + return None + _RUNTIMES[profile_key] = runtime + return runtime + + +relay_runtime.SESSION_COORDINATOR.register_session_initializer( + SUBSCRIBER_NAME, + _prepare_core_session, +) + + +def _reset_for_tests() -> None: + """Reset all profile-scoped shared-metrics state for isolated tests.""" + with _RUNTIME_LOCK: + runtimes = list(_RUNTIMES.values()) + _RUNTIMES.clear() + for runtime in runtimes: + if isinstance(runtime, _Runtime): + runtime.shutdown() diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json new file mode 100644 index 000000000000..68496e5ab6f3 --- /dev/null +++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json @@ -0,0 +1,338 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:hermes-agent:schema:shared-metrics:v1", + "title": "Hermes Shared Metrics Package v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "package_id", + "install_id", + "period_start", + "period_end", + "generated_at", + "resource", + "metrics" + ], + "properties": { + "schema_version": { + "const": "hermes.shared_metrics.v1" + }, + "package_id": { + "$ref": "#/$defs/uuid" + }, + "install_id": { + "description": "Random persistent identifier scoped to one HERMES_HOME; local-only in schema v1.", + "$ref": "#/$defs/uuid" + }, + "period_start": { + "type": "string", + "format": "date-time" + }, + "period_end": { + "type": "string", + "format": "date-time" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "resource": { + "type": "object", + "additionalProperties": false, + "required": [ + "hermes_version" + ], + "properties": { + "hermes_version": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + } + }, + "metrics": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "$ref": "#/$defs/model_call_counter" + }, + { + "$ref": "#/$defs/task_started_counter" + }, + { + "$ref": "#/$defs/task_finished_counter" + } + ] + } + } + }, + "$defs": { + "uuid": { + "type": "string", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "model_call_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.model_call.count" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "call_role", + "locality", + "model_family", + "outcome", + "provider_family" + ], + "properties": { + "call_role": { + "const": "primary" + }, + "locality": { + "enum": [ + "local", + "remote", + "unknown" + ] + }, + "model_family": { + "enum": [ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "o1", + "o3", + "o4", + "qwen", + "step", + "trinity", + "unknown" + ] + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success" + ] + }, + "provider_family": { + "enum": [ + "aggregator", + "custom", + "direct", + "local", + "unknown" + ] + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "task_started_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.task_run.started" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "entrypoint", + "execution_surface" + ], + "properties": { + "entrypoint": { + "$ref": "#/$defs/task_entrypoint" + }, + "execution_surface": { + "$ref": "#/$defs/execution_surface" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "task_finished_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.task_run.finished" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "duration_bucket", + "end_reason", + "entrypoint", + "execution_surface", + "model_call_count_bucket", + "outcome", + "retry_count_bucket", + "termination", + "tool_call_count_bucket" + ], + "properties": { + "duration_bucket": { + "$ref": "#/$defs/duration_bucket" + }, + "end_reason": { + "enum": [ + "approval_denied", + "completed", + "failed", + "guardrail_blocked", + "iteration_limit", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled" + ] + }, + "entrypoint": { + "$ref": "#/$defs/task_entrypoint" + }, + "execution_surface": { + "$ref": "#/$defs/execution_surface" + }, + "model_call_count_bucket": { + "$ref": "#/$defs/count_bucket" + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success", + "timed_out", + "unknown" + ] + }, + "retry_count_bucket": { + "$ref": "#/$defs/count_bucket" + }, + "termination": { + "enum": [ + "none", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled" + ] + }, + "tool_call_count_bucket": { + "$ref": "#/$defs/count_bucket" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "execution_surface": { + "enum": [ + "api", + "batch", + "cli", + "desktop", + "gateway", + "other", + "python", + "scheduled_task", + "tui", + "unknown" + ] + }, + "task_entrypoint": { + "enum": [ + "api", + "background", + "batch", + "delegated", + "gateway_message", + "interactive", + "other", + "python", + "scheduled_task", + "unknown" + ] + }, + "duration_bucket": { + "enum": [ + "1s_to_5s", + "2m_to_10m", + "30s_to_2m", + "5s_to_30s", + "gte_10m", + "lt_1s" + ] + }, + "count_bucket": { + "enum": [ + "0", + "1", + "2", + "3_to_5", + "6_to_10", + "gte_11" + ] + } + } +} diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py new file mode 100644 index 000000000000..506fb4ab1314 --- /dev/null +++ b/hermes_cli/observability/shared_metrics.py @@ -0,0 +1,486 @@ +"""Durable aggregation and local export for Hermes shared metrics.""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from hermes_cli.sqlite_util import write_txn +from hermes_constants import get_hermes_home +from utils import atomic_json_write + +from .shared_metrics_contract import ( + COUNTER_METRICS, + MODEL_CALL_METRIC, + counter_dimensions_are_valid, +) + + +_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v1" +_STORE_SCHEMA_VERSION = "1" +_BUSY_TIMEOUT_MS = 250 +_SCHEMA_BUSY_TIMEOUT_MS = 5_000 +_LOCAL_HISTORY_RETENTION_DAYS = 30 + +logger = logging.getLogger(__name__) + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class SharedMetricsStore: + """Persist allowlisted counters and export immutable delta packages.""" + + def __init__( + self, + database_path: Path | None = None, + outbox_directory: Path | None = None, + ) -> None: + root = get_hermes_home() / "telemetry" / "shared_metrics" + self.database_path = database_path or root / "metrics.sqlite3" + self.outbox_directory = outbox_directory or root / "outbox" + self._ensure_private_directory(self.database_path.parent) + self._ensure_private_directory(self.outbox_directory) + self._ensure_private_file(self.database_path) + self._ensure_schema() + + def record_model_call( + self, + dimensions: dict[str, str], + hermes_version: str, + ) -> None: + """Increment the terminal model-call counter for the current UTC day.""" + self.record_counter(MODEL_CALL_METRIC, dimensions, hermes_version) + + def record_counter( + self, + metric_name: str, + dimensions: dict[str, str], + hermes_version: str, + ) -> None: + """Increment one allowlisted counter for the current UTC day.""" + if metric_name not in COUNTER_METRICS: + raise ValueError(f"Unsupported shared metric: {metric_name}") + if not counter_dimensions_are_valid(metric_name, dimensions): + raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}") + dimensions_json = json.dumps( + dimensions, + sort_keys=True, + separators=(",", ":"), + ) + period_start = _utc_now().date().isoformat() + with self._connection() as connection: + connection.execute( + """ + INSERT INTO counter_aggregates( + period_start, + metric_name, + hermes_version, + dimensions_json, + value, + packaged_value + ) VALUES (?, ?, ?, ?, 1, 0) + ON CONFLICT( + period_start, + metric_name, + hermes_version, + dimensions_json + ) + DO UPDATE SET value = value + 1 + """, + ( + period_start, + metric_name, + hermes_version or "unknown", + dimensions_json, + ), + ) + + def create_and_export_package(self) -> list[Path]: + """Commit one pending delta package, then atomically export the outbox.""" + pending_periods = self._pending_period_count() + for _ in range(pending_periods): + if self._create_package() is None: + break + exported = self._export_pending_packages() + try: + self._prune_expired_history() + except Exception: + logger.warning( + "Unable to prune expired shared-metrics history", + exc_info=True, + ) + return exported + + def counter_snapshot(self) -> list[dict[str, Any]]: + """Return cumulative counters for focused tests and local inspection.""" + with self._connection() as connection: + rows = connection.execute( + """ + SELECT + period_start, + metric_name, + hermes_version, + dimensions_json, + value, + packaged_value + FROM counter_aggregates + ORDER BY period_start, hermes_version, metric_name, dimensions_json + """ + ).fetchall() + return [ + { + "period_start": row["period_start"], + "metric_name": row["metric_name"], + "hermes_version": row["hermes_version"], + "dimensions": json.loads(row["dimensions_json"]), + "value": row["value"], + "packaged_value": row["packaged_value"], + } + for row in rows + ] + + @contextmanager + def _connection( + self, + *, + busy_timeout_ms: int = _BUSY_TIMEOUT_MS, + ) -> Iterator[sqlite3.Connection]: + connection = sqlite3.connect( + self.database_path, + timeout=busy_timeout_ms / 1000, + ) + try: + connection.row_factory = sqlite3.Row + connection.execute(f"PRAGMA busy_timeout={busy_timeout_ms}") + with connection: + yield connection + finally: + connection.close() + + @staticmethod + def _ensure_private_directory(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + path.chmod(0o700) + except OSError: + pass + + @staticmethod + def _ensure_private_file(path: Path) -> None: + path.touch(mode=0o600, exist_ok=True) + try: + path.chmod(0o600) + except OSError: + pass + + def _ensure_schema(self) -> None: + with self._connection(busy_timeout_ms=_SCHEMA_BUSY_TIMEOUT_MS) as connection: + # Serialize first-run creation and upgrades across Hermes processes. + with write_txn(connection): + self._ensure_schema_in_transaction(connection) + + @staticmethod + def _ensure_schema_in_transaction(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS telemetry_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + """ + ) + schema_row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'schema_version'" + ).fetchone() + if schema_row is not None and str(schema_row["value"]) != _STORE_SCHEMA_VERSION: + raise RuntimeError( + "Unsupported shared-metrics store schema version: " + f"{schema_row['value']}" + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS counter_aggregates ( + period_start TEXT NOT NULL, + metric_name TEXT NOT NULL, + hermes_version TEXT NOT NULL, + dimensions_json TEXT NOT NULL, + value INTEGER NOT NULL, + packaged_value INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY ( + period_start, + metric_name, + hermes_version, + dimensions_json + ) + ) + """ + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS package_outbox ( + package_id TEXT PRIMARY KEY, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL, + exported_at TEXT + ) + """ + ) + connection.execute( + """ + INSERT OR IGNORE INTO telemetry_state(key, value) + VALUES ('schema_version', ?) + """, + (_STORE_SCHEMA_VERSION,), + ) + + def _install_id(self, connection: sqlite3.Connection) -> str: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'install_id'" + ).fetchone() + if row is not None: + return str(row["value"]) + candidate = str(uuid.uuid4()) + connection.execute( + "INSERT OR IGNORE INTO telemetry_state(key, value) VALUES ('install_id', ?)", + (candidate,), + ) + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'install_id'" + ).fetchone() + if row is None: + raise RuntimeError("Unable to create the shared-metrics install identity") + return str(row["value"]) + + def _pending_period_count(self) -> int: + with self._connection() as connection: + row = connection.execute( + """ + SELECT COUNT(*) AS period_count + FROM ( + SELECT period_start, hermes_version + FROM counter_aggregates + WHERE value > packaged_value + GROUP BY period_start, hermes_version + ) + """ + ).fetchone() + return int(row["period_count"]) if row is not None else 0 + + def _create_package(self) -> dict[str, Any] | None: + now = _utc_now() + with self._connection() as connection: + with write_txn(connection): + return self._create_package_in_transaction(connection, now) + + def _create_package_in_transaction( + self, + connection: sqlite3.Connection, + now: datetime, + ) -> dict[str, Any] | None: + period_row = connection.execute( + """ + SELECT period_start, hermes_version + FROM counter_aggregates + WHERE value > packaged_value + ORDER BY period_start, hermes_version + LIMIT 1 + """ + ).fetchone() + period_value = period_row["period_start"] if period_row is not None else None + if not period_value: + return None + + rows = connection.execute( + """ + SELECT metric_name, dimensions_json, value, packaged_value + FROM counter_aggregates + WHERE period_start = ? + AND hermes_version = ? + AND value > packaged_value + ORDER BY metric_name, dimensions_json + """, + (period_value, period_row["hermes_version"]), + ).fetchall() + period_start = datetime.fromisoformat(str(period_value)).replace( + tzinfo=timezone.utc + ) + period_end = period_start + timedelta(days=1) + package_id = str(uuid.uuid4()) + payload = { + "schema_version": _PACKAGE_SCHEMA_VERSION, + "package_id": package_id, + "install_id": self._install_id(connection), + "period_start": _isoformat(period_start), + "period_end": _isoformat(period_end), + "generated_at": _isoformat(now), + "resource": {"hermes_version": period_row["hermes_version"]}, + "metrics": [self._package_metric(row) for row in rows], + } + payload_json = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ) + connection.execute( + """ + INSERT INTO package_outbox( + package_id, + period_start, + period_end, + payload_json, + created_at + ) VALUES (?, ?, ?, ?, ?) + """, + ( + package_id, + payload["period_start"], + payload["period_end"], + payload_json, + payload["generated_at"], + ), + ) + for row in rows: + connection.execute( + """ + UPDATE counter_aggregates + SET packaged_value = value + WHERE period_start = ? + AND metric_name = ? + AND hermes_version = ? + AND dimensions_json = ? + """, + ( + period_value, + row["metric_name"], + period_row["hermes_version"], + row["dimensions_json"], + ), + ) + return payload + + @staticmethod + def _package_metric(row: sqlite3.Row) -> dict[str, Any]: + metric_name = str(row["metric_name"]) + dimensions = json.loads(row["dimensions_json"]) + if not isinstance(dimensions, dict) or not counter_dimensions_are_valid( + metric_name, dimensions + ): + raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}") + return { + "name": metric_name, + "type": "counter", + "dimensions": dimensions, + "value": row["value"] - row["packaged_value"], + } + + def _export_pending_packages(self) -> list[Path]: + with self._connection() as connection: + rows = connection.execute( + """ + SELECT package_id, payload_json + FROM package_outbox + WHERE exported_at IS NULL + ORDER BY created_at, package_id + """ + ).fetchall() + + exported: list[Path] = [] + for row in rows: + package_id = str(row["package_id"]) + path = self.outbox_directory / f"{package_id}.json" + atomic_json_write( + path, + json.loads(row["payload_json"]), + indent=2, + sort_keys=True, + mode=0o600, + ) + with self._connection() as connection: + connection.execute( + """ + UPDATE package_outbox + SET exported_at = ? + WHERE package_id = ? AND exported_at IS NULL + """, + (_isoformat(_utc_now()), package_id), + ) + exported.append(path) + return exported + + def _prune_expired_history(self, *, now: datetime | None = None) -> None: + """Remove exported local history after the bounded retention window.""" + cutoff = (now or _utc_now()) - timedelta( + days=_LOCAL_HISTORY_RETENTION_DAYS + ) + cutoff_timestamp = _isoformat(cutoff) + cutoff_period = cutoff.date().isoformat() + with self._connection() as connection: + rows = connection.execute( + """ + SELECT package_id + FROM package_outbox + WHERE exported_at IS NOT NULL + AND exported_at < ? + ORDER BY exported_at, package_id + """, + (cutoff_timestamp,), + ).fetchall() + + removable_package_ids: list[str] = [] + for row in rows: + package_id = str(row["package_id"]) + try: + (self.outbox_directory / f"{package_id}.json").unlink( + missing_ok=True + ) + except OSError: + logger.warning( + "Unable to prune expired shared-metrics package %s", + package_id, + exc_info=True, + ) + continue + removable_package_ids.append(package_id) + + with self._connection() as connection: + with write_txn(connection): + for package_id in removable_package_ids: + connection.execute( + """ + DELETE FROM package_outbox + WHERE package_id = ? + AND exported_at IS NOT NULL + AND exported_at < ? + """, + (package_id, cutoff_timestamp), + ) + connection.execute( + """ + DELETE FROM counter_aggregates + WHERE period_start < ? + AND value = packaged_value + AND NOT EXISTS ( + SELECT 1 + FROM package_outbox + WHERE exported_at IS NULL + AND substr(package_outbox.period_start, 1, 10) + = counter_aggregates.period_start + ) + """, + (cutoff_period,), + ) diff --git a/hermes_cli/observability/shared_metrics_contract.py b/hermes_cli/observability/shared_metrics_contract.py new file mode 100644 index 000000000000..a8fefea40c55 --- /dev/null +++ b/hermes_cli/observability/shared_metrics_contract.py @@ -0,0 +1,516 @@ +"""Bounded product contract for the first Hermes shared-metrics slice.""" + +from __future__ import annotations + +import re +from functools import lru_cache +from typing import Any + +from agent.relay_runtime import RUNTIME_INSTANCE_KEY + +SCHEMA_KEY = "hermes.metrics.schema_version" +SCHEMA_VERSION = "hermes.metrics.event.v1" +MODEL_CALL_SCOPE = "hermes.model_call" +TASK_SCOPE = "hermes.task_run" +SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics" +PRIMARY_MODEL_CALL_ROLE = "primary" +MODEL_CALL_METRIC = "hermes.model_call.count" +TASK_STARTED_METRIC = "hermes.task_run.started" +TASK_FINISHED_METRIC = "hermes.task_run.finished" + +EXECUTION_SURFACES: frozenset[str] = frozenset({ + "api", + "batch", + "cli", + "desktop", + "gateway", + "python", + "scheduled_task", + "tui", + "other", + "unknown", +}) +PROVIDER_FAMILIES: frozenset[str] = frozenset({ + "aggregator", + "custom", + "direct", + "local", + "unknown", +}) +MODEL_LOCALITIES: frozenset[str] = frozenset({"local", "remote", "unknown"}) +MODEL_OUTCOMES: frozenset[str] = frozenset({"cancelled", "failed", "success"}) +TASK_OUTCOMES: frozenset[str] = frozenset({ + "cancelled", + "failed", + "success", + "timed_out", + "unknown", +}) +TASK_END_REASONS: frozenset[str] = frozenset({ + "approval_denied", + "completed", + "failed", + "guardrail_blocked", + "iteration_limit", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled", +}) +TASK_TERMINATIONS: frozenset[str] = frozenset({ + "none", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled", +}) +TASK_ENTRYPOINTS: frozenset[str] = frozenset({ + "api", + "background", + "batch", + "delegated", + "gateway_message", + "interactive", + "other", + "python", + "scheduled_task", + "unknown", +}) +DURATION_BUCKETS: frozenset[str] = frozenset({ + "1s_to_5s", + "2m_to_10m", + "30s_to_2m", + "5s_to_30s", + "gte_10m", + "lt_1s", +}) +COUNT_BUCKETS: frozenset[str] = frozenset({ + "0", + "1", + "2", + "3_to_5", + "6_to_10", + "gte_11", +}) + +# Shared metrics use an explicit family allowlist rather than raw model IDs or +# dynamically sourced catalog values. The latter would make the exported schema +# drift independently of this contract. +MODEL_FAMILIES: frozenset[str] = frozenset({ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "qwen", + "step", + "trinity", + "o1", + "o3", + "o4", + "unknown", +}) + +_COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = { + MODEL_CALL_METRIC: { + "call_role": frozenset({PRIMARY_MODEL_CALL_ROLE}), + "locality": MODEL_LOCALITIES, + "model_family": MODEL_FAMILIES, + "outcome": MODEL_OUTCOMES, + "provider_family": PROVIDER_FAMILIES, + }, + TASK_STARTED_METRIC: { + "entrypoint": TASK_ENTRYPOINTS, + "execution_surface": EXECUTION_SURFACES, + }, + TASK_FINISHED_METRIC: { + "duration_bucket": DURATION_BUCKETS, + "end_reason": TASK_END_REASONS, + "entrypoint": TASK_ENTRYPOINTS, + "execution_surface": EXECUTION_SURFACES, + "model_call_count_bucket": COUNT_BUCKETS, + "outcome": TASK_OUTCOMES, + "retry_count_bucket": COUNT_BUCKETS, + "termination": TASK_TERMINATIONS, + "tool_call_count_bucket": COUNT_BUCKETS, + }, +} +COUNTER_METRICS: frozenset[str] = frozenset(_COUNTER_DIMENSION_VALUES) + +_MODEL_FAMILY_PATTERN = re.compile( + r"(?:^|[/_.:-])(" + + "|".join( + re.escape(family) + for family in sorted( + MODEL_FAMILIES - {"unknown"}, + key=lambda value: len(value), + reverse=True, + ) + ) + + r")(?=$|[/_.:-]|\d)" +) + +# These providers route across model families but are not marked as aggregators +# in Hermes's execution metadata because that flag has narrower routing/catalog +# semantics there. +_TELEMETRY_AGGREGATOR_OVERRIDES = frozenset({ + "copilot-acp", + "github-copilot", + "moa", + "nous", +}) + +# Hermes intentionally resolves these local runtimes through the generic custom +# provider path, so canonical provider metadata cannot distinguish them alone. +_LOCAL_CUSTOM_PROVIDER_ALIASES = frozenset({"mlx", "ollama"}) + + +def counter_dimensions_are_valid( + metric_name: str, + dimensions: dict[str, Any], +) -> bool: + """Return whether dimensions match one closed shared-metric contract.""" + contract = _COUNTER_DIMENSION_VALUES.get(metric_name) + if contract is None or set(dimensions) != set(contract): + return False + return all( + isinstance(dimensions[field], str) + and dimensions[field] in allowed_values + for field, allowed_values in contract.items() + ) + + +def model_call_dimensions(event: Any) -> dict[str, str] | None: + """Return package dimensions for one valid primary model-call end event.""" + metadata = getattr(event, "metadata", None) + if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION: + return None + relay_metadata = set(metadata) - {SCHEMA_KEY, RUNTIME_INSTANCE_KEY} + if relay_metadata - {"otel.status_code"} or metadata.get( + "otel.status_code", "OK" + ) not in {"OK", "ERROR"}: + return None + if ( + str(getattr(event, "kind", "") or "") != "scope" + or str(getattr(event, "category", "") or "") != "llm" + or str(getattr(event, "name", "") or "") != MODEL_CALL_SCOPE + or str(getattr(event, "scope_category", "") or "") != "end" + ): + return None + category_profile = getattr(event, "category_profile", None) + if not isinstance(category_profile, dict) or set(category_profile) != { + "model_name" + }: + return None + event_model_family = category_profile.get("model_name") + if event_model_family not in MODEL_FAMILIES: + return None + data = getattr(event, "data", None) + expected_fields = { + "call_role", + "locality", + "model_family", + "outcome", + "provider_family", + } + if not isinstance(data, dict) or set(data) != expected_fields: + return None + dimensions = { + "call_role": data.get("call_role"), + "locality": data.get("locality"), + "model_family": data.get("model_family"), + "outcome": data.get("outcome"), + "provider_family": data.get("provider_family"), + } + if not counter_dimensions_are_valid(MODEL_CALL_METRIC, dimensions): + return None + return dimensions + + +def task_counter(event: Any) -> tuple[str, dict[str, str]] | None: + """Return one validated task counter from a task scope event.""" + metadata = getattr(event, "metadata", None) + if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION: + return None + relay_metadata = set(metadata) - {SCHEMA_KEY, RUNTIME_INSTANCE_KEY} + if relay_metadata - {"otel.status_code"} or metadata.get( + "otel.status_code", "OK" + ) not in {"OK", "ERROR"}: + return None + if ( + str(getattr(event, "kind", "") or "") != "scope" + or str(getattr(event, "category", "") or "") != "function" + or str(getattr(event, "name", "") or "") != TASK_SCOPE + ): + return None + if getattr(event, "category_profile", None) is not None: + return None + + scope_category = str(getattr(event, "scope_category", "") or "") + data = getattr(event, "data", None) + if scope_category == "start": + expected_fields = {"entrypoint", "execution_surface"} + if not isinstance(data, dict) or set(data) != expected_fields: + return None + dimensions = { + "entrypoint": data.get("entrypoint"), + "execution_surface": data.get("execution_surface"), + } + if not counter_dimensions_are_valid(TASK_STARTED_METRIC, dimensions): + return None + return TASK_STARTED_METRIC, dimensions + + expected_fields = { + "duration_bucket", + "end_reason", + "entrypoint", + "execution_surface", + "model_call_count_bucket", + "outcome", + "retry_count_bucket", + "termination", + "tool_call_count_bucket", + } + if ( + scope_category != "end" + or not isinstance(data, dict) + or set(data) != expected_fields + ): + return None + dimensions = {field: data.get(field) for field in sorted(expected_fields)} + if not counter_dimensions_are_valid(TASK_FINISHED_METRIC, dimensions): + return None + return TASK_FINISHED_METRIC, dimensions + + +def execution_surface(kwargs: dict[str, Any]) -> str: + """Normalize the safe session surface carried by the parent Relay scope.""" + value = ( + str(kwargs.get("execution_surface") or kwargs.get("platform") or "unknown") + .strip() + .lower() + ) + if value in EXECUTION_SURFACES: + return value + if value == "api_server": + return "api" + if value in {"cron", "scheduler", "scheduled"}: + return "scheduled_task" + try: + from hermes_cli.platforms import get_all_platforms + + if value in get_all_platforms(): + return "gateway" + except Exception: + pass + if value in {"discord", "email", "slack", "telegram", "teams", "whatsapp"}: + return "gateway" + return "unknown" if value == "unknown" else "other" + + +def task_start_fields(kwargs: dict[str, Any]) -> dict[str, str]: + """Build the bounded fields recorded on a task scope start event.""" + surface = execution_surface(kwargs) + return { + "entrypoint": task_entrypoint(kwargs, surface), + "execution_surface": surface, + } + + +def task_entrypoint(kwargs: dict[str, Any], surface: str | None = None) -> str: + """Normalize the task dispatch owner without exporting source strings.""" + declared = str(kwargs.get("entrypoint") or "").strip().lower() + if declared in TASK_ENTRYPOINTS: + return declared + resolved_surface = surface or execution_surface(kwargs) + if kwargs.get("parent_task_id") or kwargs.get("parent_session_id"): + return "delegated" + return { + "api": "api", + "batch": "batch", + "cli": "interactive", + "desktop": "interactive", + "gateway": "gateway_message", + "python": "python", + "scheduled_task": "scheduled_task", + "tui": "interactive", + "unknown": "unknown", + }.get(resolved_surface, "other") + + +def task_terminal_fields( + kwargs: dict[str, Any], + *, + duration_ms: int, + model_call_count: int, + tool_call_count: int, + retry_count: int, +) -> dict[str, str]: + """Build the bounded terminal payload for one task scope.""" + start_fields = task_start_fields(kwargs) + outcome, end_reason, termination = task_terminal_state(kwargs) + return { + **start_fields, + "duration_bucket": duration_bucket(duration_ms), + "end_reason": end_reason, + "model_call_count_bucket": count_bucket(model_call_count), + "outcome": outcome, + "retry_count_bucket": count_bucket(retry_count), + "termination": termination, + "tool_call_count_bucket": count_bucket(tool_call_count), + } + + +def task_terminal_state(kwargs: dict[str, Any]) -> tuple[str, str, str]: + """Map Hermes terminal state to bounded task outcome dimensions.""" + reason = str(kwargs.get("turn_exit_reason") or "").strip().lower() + if kwargs.get("interrupted") or "interrupt" in reason or "cancel" in reason: + return "cancelled", "user_cancelled", "user_cancelled" + if "timeout" in reason or "timed_out" in reason: + return "timed_out", "timed_out", "timed_out" + if "max_iterations" in reason or "budget_exhausted" in reason: + return "failed", "iteration_limit", "system_aborted" + if "approval" in reason and ("denied" in reason or "rejected" in reason): + return "failed", "approval_denied", "none" + if "guardrail" in reason: + return "failed", "guardrail_blocked", "system_aborted" + if reason == "system_aborted": + return "failed", "system_aborted", "system_aborted" + if kwargs.get("completed") is True: + return "success", "completed", "none" + if kwargs.get("failed") is True or (reason and reason != "unknown"): + return "failed", "failed", "none" + return "unknown", "unknown", "unknown" + + +def duration_bucket(duration_ms: int) -> str: + """Bucket a non-negative task duration into a fixed low-cardinality range.""" + value = max(0, int(duration_ms)) + if value < 1_000: + return "lt_1s" + if value < 5_000: + return "1s_to_5s" + if value < 30_000: + return "5s_to_30s" + if value < 120_000: + return "30s_to_2m" + if value < 600_000: + return "2m_to_10m" + return "gte_10m" + + +def count_bucket(count: int) -> str: + """Bucket a non-negative per-task count into a fixed range.""" + value = max(0, int(count)) + if value <= 2: + return str(value) + if value <= 5: + return "3_to_5" + if value <= 10: + return "6_to_10" + return "gte_11" + + +def provider_family(kwargs: dict[str, Any]) -> str: + """Map a Hermes provider to a bounded product category.""" + raw_provider = str(kwargs.get("provider") or "").strip().lower().replace("_", "-") + if not raw_provider: + return "unknown" + if raw_provider in _LOCAL_CUSTOM_PROVIDER_ALIASES: + return "local" + if raw_provider == "custom" or raw_provider.startswith(("custom-", "custom:")): + return "custom" + provider, is_aggregator, is_known = _provider_metadata(raw_provider) + if provider in {"lmstudio", "local"}: + return "local" + if is_aggregator or provider in _TELEMETRY_AGGREGATOR_OVERRIDES: + return "aggregator" + if provider == "custom": + return "custom" + return "direct" if is_known else "unknown" + + +def _provider_metadata(provider: str) -> tuple[str, bool, bool]: + """Resolve provider identity without refreshing remote provider metadata.""" + try: + from hermes_cli.models import normalize_provider as normalize_model_provider + from hermes_cli.providers import HERMES_OVERLAYS, normalize_provider + + canonical = normalize_provider(normalize_model_provider(provider)) + overlay = HERMES_OVERLAYS.get(canonical) + return ( + canonical, + bool(overlay and overlay.is_aggregator), + canonical in _known_provider_ids(), + ) + except Exception: + return provider, False, False + + +@lru_cache(maxsize=1) +def _known_provider_ids() -> frozenset[str]: + """Cache Hermes's static provider catalog for the process lifetime.""" + try: + from hermes_cli.provider_catalog import provider_catalog_by_slug + + return frozenset(provider_catalog_by_slug()) + except Exception: + return frozenset() + + +def model_locality(kwargs: dict[str, Any]) -> str: + """Classify local endpoints without exporting their URL.""" + return _model_locality(kwargs, provider_family(kwargs)) + + +def _model_locality(kwargs: dict[str, Any], provider_category: str) -> str: + base_url = kwargs.get("base_url") + if isinstance(base_url, str) and base_url: + try: + from agent.model_metadata import is_local_endpoint + + if is_local_endpoint(base_url): + return "local" + except Exception: + pass + if provider_category == "local": + return "local" + if provider_category in {"aggregator", "direct"}: + return "remote" + return "unknown" + + +def model_call_fields(kwargs: dict[str, Any]) -> dict[str, str]: + """Build the bounded producer fields for one logical model call.""" + provider_category = provider_family(kwargs) + return { + "call_role": PRIMARY_MODEL_CALL_ROLE, + "locality": _model_locality(kwargs, provider_category), + "model_family": model_family(kwargs), + "provider_family": provider_category, + } + + +def model_family(kwargs: dict[str, Any]) -> str: + """Map a raw model identifier to an allowlisted family.""" + declared_family = str(kwargs.get("model_family") or "").strip().lower() + if declared_family in MODEL_FAMILIES - {"unknown"}: + return declared_family + model = str(kwargs.get("response_model") or kwargs.get("model") or "").lower() + match = _MODEL_FAMILY_PATTERN.search(model) + return match.group(1) if match is not None else "unknown" + + +def model_call_outcome(kwargs: dict[str, Any]) -> str: + """Fail closed when a terminal model-call outcome is not recognized.""" + value = str(kwargs.get("outcome") or "").lower() + return value if value in MODEL_OUTCOMES else "failed" diff --git a/hermes_cli/observability/shared_metrics_subscriber.py b/hermes_cli/observability/shared_metrics_subscriber.py new file mode 100644 index 000000000000..677308503af8 --- /dev/null +++ b/hermes_cli/observability/shared_metrics_subscriber.py @@ -0,0 +1,67 @@ +"""Relay subscriber for the persisted Hermes shared-metrics slice.""" + +from __future__ import annotations + +import logging +import threading +from typing import Any + +from agent.relay_runtime import RUNTIME_INSTANCE_KEY + +from .shared_metrics import SharedMetricsStore +from .shared_metrics_contract import MODEL_CALL_METRIC, model_call_dimensions, task_counter + +logger = logging.getLogger(__name__) + + +class SharedMetricsSubscriber: + """Persist validated Hermes counters from Relay lifecycle events.""" + + def __init__( + self, + store: SharedMetricsStore, + hermes_version: str, + *, + runtime_id: str | None = None, + ) -> None: + self.store = store + self._hermes_version = hermes_version or "unknown" + self._runtime_id = runtime_id + self._active = True + self._lock = threading.RLock() + + def deactivate(self) -> None: + """Stop accepting events before telemetry is disabled or torn down.""" + with self._lock: + self._active = False + + def __call__(self, event: Any) -> None: + if self._runtime_id is not None: + metadata = getattr(event, "metadata", None) + if ( + not isinstance(metadata, dict) + or metadata.get(RUNTIME_INSTANCE_KEY) != self._runtime_id + ): + return + dimensions = model_call_dimensions(event) + metric_name = MODEL_CALL_METRIC + if dimensions is None: + task_metric = task_counter(event) + if task_metric is None: + return + metric_name, dimensions = task_metric + with self._lock: + if not self._active: + return + try: + self.store.record_counter( + metric_name, + dimensions, + self._hermes_version, + ) + except Exception: + logger.warning( + "Unable to persist the Hermes shared metric: %s", + metric_name, + exc_info=True, + ) diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index d706ca1a468b..8dd02a1bdf5e 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -2066,7 +2066,7 @@ def discover_plugins(force: bool = False) -> None: def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]: - """Invoke a lifecycle hook on all loaded plugins. + """Invoke a lifecycle hook on loaded plugins. Returns a list of non-``None`` return values from plugin callbacks. """ @@ -2091,7 +2091,7 @@ def has_middleware(kind: str) -> bool: def has_hook(hook_name: str) -> bool: - """Return True when a hook has registered callbacks.""" + """Return True when a loaded plugin handles a hook.""" return get_plugin_manager().has_hook(hook_name) diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 4763e35e9359..0f4d413d795b 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2221,6 +2221,37 @@ def setup_tools(config: dict, first_install: bool = False): tools_command(first_install=first_install, config=config) +# ============================================================================= +# Shared Metrics +# ============================================================================= + + +def setup_telemetry(config: dict): + """Configure the local, privacy-safe shared-metrics subscriber.""" + print_header("Shared Metrics") + print_info("Shared metrics contain only bounded counters and histograms.") + print_info("Packages stay under this Hermes profile and are not uploaded.") + + telemetry = config.get("telemetry") + if not isinstance(telemetry, dict): + telemetry = {} + config["telemetry"] = telemetry + shared_metrics = telemetry.get("shared_metrics") + if not isinstance(shared_metrics, dict): + shared_metrics = {} + telemetry["shared_metrics"] = shared_metrics + + current = shared_metrics.get("enabled") is True + shared_metrics["enabled"] = prompt_yes_no( + "Enable local shared metrics?", + default=current, + ) + if shared_metrics["enabled"]: + print_success("Local shared metrics enabled.") + else: + print_info("Local shared metrics disabled.") + + # ============================================================================= # Post-Migration Section Skip Logic # ============================================================================= @@ -2629,6 +2660,7 @@ SETUP_SECTIONS = [ ("terminal", "Terminal Backend", setup_terminal_backend), ("gateway", "Messaging Platforms (Gateway)", setup_gateway), ("tools", "Tools", setup_tools), + ("telemetry", "Shared Metrics", setup_telemetry), ("agent", "Agent Settings", setup_agent_settings), ] @@ -2727,6 +2759,7 @@ def run_setup_wizard(args): hermes setup terminal — just terminal backend hermes setup gateway — just messaging platforms hermes setup tools — just tool configuration + hermes setup telemetry — just local shared metrics hermes setup agent — just agent settings """ from hermes_cli.config import is_managed, managed_error diff --git a/hermes_cli/subcommands/setup.py b/hermes_cli/subcommands/setup.py index 406710a68875..fa0c0dc37c74 100644 --- a/hermes_cli/subcommands/setup.py +++ b/hermes_cli/subcommands/setup.py @@ -18,12 +18,21 @@ def build_setup_parser(subparsers, *, cmd_setup: Callable) -> None: "setup", help="Interactive setup wizard", description="Configure Hermes Agent with an interactive wizard. " - "Run a specific section: hermes setup model|tts|terminal|gateway|tools|agent", + "Run a specific section: " + "hermes setup model|tts|terminal|gateway|tools|telemetry|agent", ) setup_parser.add_argument( "section", nargs="?", - choices=["model", "tts", "terminal", "gateway", "tools", "agent"], + choices=[ + "model", + "tts", + "terminal", + "gateway", + "tools", + "telemetry", + "agent", + ], default=None, help="Run a specific setup section instead of the full wizard", ) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 4c0b105da046..56da56c70ada 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -967,6 +967,9 @@ _CATEGORY_MERGE: Dict[str, str] = { # field — fold it into the agent tab rather than spawning a one-field # orphan category. "computer_use": "agent", + # `telemetry.shared_metrics.enabled` is the only schema-surfaced telemetry + # field — fold it into security alongside the other privacy-posture toggles. + "telemetry": "security", } # Display order for tabs — unlisted categories sort alphabetically after these. diff --git a/model_tools.py b/model_tools.py index 7b5811e8fbfe..47f6ee18e74a 100644 --- a/model_tools.py +++ b/model_tools.py @@ -1056,7 +1056,7 @@ def _emit_post_tool_call_hook( listener will actually consume it). """ try: - from hermes_cli.plugins import has_hook, invoke_hook + from hermes_cli.lifecycle import has_hook, invoke_hook if not has_hook("post_tool_call"): return if status is None: @@ -1093,6 +1093,7 @@ def handle_function_call( enabled_tools: Optional[List[str]] = None, skip_pre_tool_call_hook: bool = False, skip_tool_request_middleware: bool = False, + skip_tool_execution_middleware: bool = False, tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None, enabled_toolsets: Optional[List[str]] = None, disabled_toolsets: Optional[List[str]] = None, @@ -1203,6 +1204,7 @@ def handle_function_call( enabled_tools=enabled_tools, skip_pre_tool_call_hook=skip_pre_tool_call_hook, skip_tool_request_middleware=skip_tool_request_middleware, + skip_tool_execution_middleware=skip_tool_execution_middleware, tool_request_middleware_trace=list(_tool_middleware_trace), enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, @@ -1341,19 +1343,22 @@ def handle_function_call( session_id=session_id, user_task=user_task, ) - from hermes_cli.middleware import run_tool_execution_middleware + if skip_tool_execution_middleware: + result = _dispatch(function_args) + else: + from hermes_cli.middleware import run_tool_execution_middleware - result = run_tool_execution_middleware( - function_name, - function_args, - _dispatch, - original_args=_tool_original_args, - task_id=task_id or "", - session_id=session_id or "", - tool_call_id=tool_call_id or "", - turn_id=turn_id or "", - api_request_id=api_request_id or "", - ) + result = run_tool_execution_middleware( + function_name, + function_args, + _dispatch, + original_args=_tool_original_args, + task_id=task_id or "", + session_id=session_id or "", + tool_call_id=tool_call_id or "", + turn_id=turn_id or "", + api_request_id=api_request_id or "", + ) finally: if _approval_tokens is not None and reset_current_observability_context is not None: try: @@ -1384,7 +1389,7 @@ def handle_function_call( # Gated on has_hook so the no-listener path skips both the result # field derivation and the payload dispatch. try: - from hermes_cli.plugins import has_hook, invoke_hook + from hermes_cli.lifecycle import has_hook, invoke_hook if has_hook("transform_tool_result"): status, error_type, error_message = _tool_result_observer_fields(result) hook_results = invoke_hook( diff --git a/plugins/observability/nemo_relay/README.md b/plugins/observability/nemo_relay/README.md index a1b90c4da4fa..85f7d039031f 100644 --- a/plugins/observability/nemo_relay/README.md +++ b/plugins/observability/nemo_relay/README.md @@ -1,18 +1,19 @@ # NeMo Relay Observability -Optional Hermes observability plugin that maps Hermes observer hooks to -NeMo Relay scopes, LLM spans, tool spans, marks, ATOF, and ATIF. +Optional Hermes observability plugin that configures exporters and maps +Hermes-specific observer hooks to NeMo Relay marks and ATIF state. Hermes core +owns Relay session, turn, LLM, and tool execution scopes. NeMo Relay is NVIDIA's runtime layer for agent execution boundaries. It does not replace Hermes Agent's planner, tools, memory, model provider routing, or -CLI UX. Instead, this plugin lets Hermes emit NeMo Relay lifecycle events for -the work Hermes already owns: sessions, turns, provider/API calls, tool calls, -approval prompts, and delegated subagents. +CLI UX. Hermes core emits NeMo Relay lifecycle events for provider and tool +execution, while this plugin enables rich exporters and observer marks for +sessions, turns, approval prompts, and delegated subagents. With this plugin enabled, Hermes Agent can: -- Preserve Hermes execution as NeMo Relay scopes, LLM spans, tool spans, and - mark events. +- Export the Relay scopes and LLM/tool lifecycles emitted by Hermes core. +- Add Hermes session, turn, approval, and subagent mark events. - Export raw lifecycle events as Agent Trajectory Observability Format (ATOF) JSONL for debugging and offline inspection. - Export Agent Trajectory Interchange Format (ATIF) trajectories for replay, @@ -73,27 +74,24 @@ checkout that contains this plugin. A globally installed older CLI will not see new bundled plugins from your working tree. ```bash -uv sync --extra nemo-relay +uv sync uv run hermes plugins enable observability/nemo_relay uv run hermes chat --query 'Reply exactly ok' --provider custom --model qwen3.6:35b ``` To ship the updated CLI into another environment, build and install a fresh -wheel from this checkout, then install the official NeMo Relay runtime extra: +wheel from this checkout. On platforms for which Relay publishes a native +wheel, Hermes installs its supported NeMo Relay runtime as a normal dependency: ```bash uv build --wheel python -m pip install --force-reinstall dist/hermes_agent-*.whl -python -m pip install "nemo-relay>=0.5,<1.0" hermes plugins enable observability/nemo_relay ``` -The plugin fails open when `nemo-relay` is not installed. Install a supported -NeMo Relay 0.x distribution beginning with 0.5: - -```bash -pip install "nemo-relay>=0.5,<1.0" -``` +The plugin remains opt-in even though the runtime dependency is installed by +default. Enabling this plugin controls rich observability and adaptive +behavior; it does not control Hermes shared client metrics. ## Export Configuration @@ -170,8 +168,9 @@ Relay owns exporter lifecycle through that config. The direct double-export trajectories on teardown. If `plugins.toml` initialization fails, Hermes keeps the direct env-var fallbacks active for that run. -To enable NeMo Relay managed execution intercepts for provider and tool calls, -include an adaptive component in the same `plugins.toml`: +Hermes core routes provider and tool execution through NeMo Relay managed APIs +regardless of whether this plugin is enabled. To install adaptive interceptors +on those boundaries, include an adaptive component in the same `plugins.toml`: ```toml [[components]] @@ -182,18 +181,15 @@ enabled = true mode = "observe_only" ``` -When the adaptive component is enabled and the installed NeMo Relay runtime -exposes `llm.execute(...)` / `tools.execute(...)`, Hermes routes LLM and tool -execution through those middleware boundaries. The observer hooks still emit -session, turn, approval, and subagent marks; the plugin skips its manual -`llm.call` and `tools.call` spans for executions that are already managed by -NeMo Relay. `tool_parallelism.mode = "observe_only"` keeps tool scheduling -observational while still wrapping the real execution boundary. +The observer hooks emit session, turn, approval, and subagent marks. They do not +create a second LLM or tool lifecycle. `tool_parallelism.mode = "observe_only"` +keeps tool scheduling observational while still intercepting the core-managed +execution boundary. ### Dynamic Plugins -Hermes feature-detects the dynamic-plugin activation API available in NeMo Relay -0.6 and later. Configure native or worker plugins with Hermes-owned +Hermes uses the dynamic-plugin activation API available in NeMo Relay 0.6 and +later. Configure native or worker plugins with Hermes-owned `[[dynamic_plugins]]` entries that match the Python binding's activation-spec fields: @@ -242,24 +238,15 @@ During shutdown it closes session exporters, flushes Relay subscribers, and then closes the activation so callbacks are removed before plugin code is unloaded. -NeMo Relay 0.5 does not expose dynamic activation through its Python binding. -When dynamic plugin configuration is present with a binding that lacks the -activation API, Hermes logs an actionable warning and continues with the -ordinary static component configuration, so ATOF and ATIF observability remain -available. No dynamic plugin is loaded in that degraded mode. - For the full generic Hermes middleware contract, see [`docs/middleware/README.md`](../../../docs/middleware/README.md). ## Canonical Local Examples -The observe-only examples in this section use a supported NeMo Relay 0.x -distribution beginning with 0.5 and a local Ollama model served through the -OpenAI-compatible API. +The observe-only examples in this section use the NeMo Relay runtime installed +with Hermes and a local Ollama model served through the OpenAI-compatible API. ```bash -pip install "nemo-relay>=0.5,<1.0" - export HERMES_HOME=/tmp/hermes-nemo-relay-docs/hermes-home mkdir -p "$HERMES_HOME" @@ -443,8 +430,8 @@ Sanitized ATIF excerpt: The plugin keeps NeMo Relay's native event model: - Hermes sessions map to `agent` scopes. -- Hermes API request hooks map to `llm` scope start/end events. -- Hermes tool hooks map to `tool` scope start/end events. +- Hermes core managed provider calls map to `llm` scope start/end events. +- Hermes core managed tool calls map to `tool` scope start/end events. - Turn, approval, subagent, and diagnostic fallback events map to `mark` events. @@ -454,11 +441,13 @@ subagent IDs, role/status fields when present, and derived stream lossless for later ATIF conversion that can compact subagents into separate trajectories. -## Adaptive Middleware Example +## Adaptive Execution Example -The `observability/nemo_relay` plugin uses Hermes execution middleware to hand -LLM and tool calls to NeMo Relay managed execution when an adaptive component is -enabled. +Hermes core owns the LLM and tool boundaries and enters NeMo Relay managed +execution while a Hermes-managed Relay consumer is active. With no shared +metrics subscriber or explicitly configured Relay plugin, Hermes calls the +provider or tool directly. The `observability/nemo_relay` plugin retains the +managed path while its adaptive components are installed on those boundaries. Minimal `plugins.toml`: @@ -479,33 +468,28 @@ Enable it for Hermes: export HERMES_NEMO_RELAY_PLUGINS_TOML=/tmp/hermes-middleware-test/plugins.toml ``` -When the adaptive component is enabled and the installed NeMo Relay runtime -exposes `llm.execute(...)` and `tools.execute(...)`, Hermes routes execution -through these boundaries: +Execution follows these boundaries with or without an adaptive component: ```text Hermes provider call - -> llm_execution middleware - -> nemo_relay.llm.execute(...) - -> Hermes provider adapter next_call(...) + -> nemo_relay.llm.execute(...) + -> Hermes provider adapter callback(...) Hermes tool call - -> tool_execution middleware - -> nemo_relay.tools.execute(...) - -> Hermes tool dispatcher next_call(...) + -> nemo_relay.tools.execute(...) + -> Hermes authorization and dispatch callback(...) ``` -The plugin still emits observer marks for sessions, turns, approvals, and -subagents. When adaptive managed execution is active, it skips manual -`llm.call` and `tools.call` observer spans to avoid duplicate LLM/tool events -for the same execution. +The plugin emits observer marks for sessions, turns, approvals, and subagents. +It does not register provider or tool lifecycle hooks, so each managed call +produces one Relay lifecycle. ### Local Adaptive E2E This example enables both NeMo Relay observability export and adaptive execution middleware for a local Hermes run. This path requires a NeMo Relay runtime that -supports `[components.config.tool_parallelism]`, as provided by the supported -0.x release range beginning with 0.5. +supports `[components.config.tool_parallelism]`, as provided by NeMo Relay 0.6 +and later. ```bash export HERMES_HOME=/tmp/hermes-middleware-test/hermes-home diff --git a/plugins/observability/nemo_relay/__init__.py b/plugins/observability/nemo_relay/__init__.py index 56c6140ca58c..13218ed5e4a6 100644 --- a/plugins/observability/nemo_relay/__init__.py +++ b/plugins/observability/nemo_relay/__init__.py @@ -15,34 +15,30 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional +from agent import relay_runtime + logger = logging.getLogger(__name__) _INIT_FAILED = object() _LOCK = threading.RLock() -_RUNTIME: "_Runtime | object | None" = None -_RELAY_LLM_SURFACE_BY_API_MODE = { - "anthropic_messages": "anthropic.messages", - "chat_completions": "openai.chat_completions", - "codex_responses": "openai.responses", -} +_RUNTIMES: dict[str, "_Runtime | object"] = {} +_SESSION_INITIALIZER_NAME = "hermes.nemo_relay.rich_observability" @dataclass class _SessionState: session_id: str + relay_session: relay_runtime.RelaySession | None = None handle: Any = None atif_exporter: Any = None atif_subscriber_name: str = "" is_embedded_subagent: bool = False parent_session_id: str = "" - llm_spans: dict[str, Any] = field(default_factory=dict) - tool_spans: dict[str, Any] = field(default_factory=dict) @dataclass -class _SubagentParent: +class _SubagentContext: parent_session_id: str - parent_handle: Any metadata: dict[str, Any] @@ -51,8 +47,6 @@ class _Settings: plugins_toml_path: str = "" plugins_config: dict[str, Any] | None = None dynamic_plugins: list[dict[str, Any]] = field(default_factory=list) - adaptive_enabled: bool = False - adaptive_mode: str = "observe_only" atof_enabled: bool = False atof_output_directory: str = "" atof_filename: str = "hermes-atof.jsonl" @@ -66,20 +60,179 @@ class _Settings: atif_model_name: str = "unknown" +class _ProcessPluginConfiguration: + """Own Relay's process-global plugin configuration across profile runtimes.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._key: str | None = None + self._plugin_mod: Any = None + self._activation: Any = None + self._owners: set[int] = set() + + def acquire( + self, + owner: "_Runtime", + plugin_mod: Any, + plugin_config: dict[str, Any], + dynamic_plugins: list[dict[str, Any]], + ) -> tuple[bool, Any]: + owner_id = id(owner) + key = _plugin_configuration_key(plugin_config, dynamic_plugins) + with self._lock: + if owner_id in self._owners: + return True, self._activation + if self._owners: + if self._plugin_mod is plugin_mod and self._key == key: + self._owners.add(owner_id) + return True, self._activation + logger.warning( + "NeMo Relay plugin configuration is already active for another " + "Hermes profile; keeping the existing process-global configuration " + "and using direct observability for this profile." + ) + return False, None + + activation = None + if dynamic_plugins: + initialize_dynamic = getattr( + plugin_mod, + "initialize_with_dynamic_plugins", + None, + ) + if callable(initialize_dynamic): + try: + activation = _resolve_awaitable( + initialize_dynamic(plugin_config, dynamic_plugins) + ) + except Exception as exc: + logger.warning( + "NeMo Relay dynamic plugin activation failed; continuing " + "with static observability only: %s", + exc, + ) + else: + logger.warning( + "NeMo Relay dynamic plugins require a binding that exposes " + "plugin.initialize_with_dynamic_plugins (available in NeMo " + "Relay 0.6+). Continuing with static observability only." + ) + + if activation is None: + initialize = getattr(plugin_mod, "initialize", None) + if not callable(initialize): + return False, None + try: + _resolve_awaitable(initialize(plugin_config)) + except Exception as exc: + logger.debug( + "NeMo Relay plugins.toml init failed: %s", + exc, + exc_info=True, + ) + return False, None + + self._key = key + self._plugin_mod = plugin_mod + self._activation = activation + self._owners.add(owner_id) + return True, activation + + def release(self, owner: "_Runtime", nemo_relay: Any) -> None: + owner_id = id(owner) + with self._lock: + if owner_id not in self._owners: + return + self._owners.remove(owner_id) + if self._owners: + return + + failures: list[str] = [] + activation = self._activation + plugin_mod = self._plugin_mod + try: + if activation is not None: + try: + _flush_relay_subscribers(nemo_relay) + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + close = getattr(activation, "close", None) + if callable(close): + try: + _resolve_awaitable(close()) + except Exception as exc: + failures.append( + f"dynamic plugin activation close failed: {exc}" + ) + else: + failures.append("dynamic plugin activation has no close method") + else: + clear = getattr(plugin_mod, "clear", None) + if callable(clear): + try: + _resolve_awaitable(clear()) + except Exception as exc: + failures.append( + f"static plugin configuration clear failed: {exc}" + ) + finally: + self._key = None + self._plugin_mod = None + self._activation = None + + if failures: + raise RuntimeError("; ".join(failures)) + + def reset_for_tests(self) -> None: + with self._lock: + self._key = None + self._plugin_mod = None + self._activation = None + self._owners.clear() + + +_PLUGIN_CONFIGURATION = _ProcessPluginConfiguration() + + class _Runtime: - def __init__(self, nemo_relay: Any, settings: _Settings) -> None: + def __init__( + self, + nemo_relay: Any, + settings: _Settings, + host: relay_runtime.RelayRuntime, + ) -> None: self.nemo_relay = nemo_relay self.settings = settings + self.host = host + self._sessions_lock = threading.RLock() self.sessions: dict[str, _SessionState] = {} - self.subagent_parents: dict[str, _SubagentParent] = {} + self.subagent_contexts: dict[str, _SubagentContext] = {} self.atof_exporter: Any = None - self._atof_subscriber_name = "hermes.nemo_relay.atof" + self._atof_subscriber_name = f"hermes.nemo_relay.atof.{self.host.runtime_id}" + self._execution_consumer_name = ( + f"hermes.nemo_relay.rich_observability.{self.host.runtime_id}" + ) + self._execution_consumer_retained = False self._plugin_activation: Any = None self._shutdown_registered = False self._plugin_config_initialized = self._configure_plugins_toml() self._plugin_config_needs_reinit = False if not self._plugin_config_initialized: self._activate_direct_fallbacks() + self._sync_managed_execution() + + def _sync_managed_execution(self) -> None: + required = bool( + self._plugin_config_initialized + or self.atof_exporter is not None + or self.settings.atif_enabled + ) + if required and not self._execution_consumer_retained: + self.host.retain_managed_execution(self._execution_consumer_name) + self._execution_consumer_retained = True + elif not required and self._execution_consumer_retained: + self.host.release_managed_execution(self._execution_consumer_name) + self._execution_consumer_retained = False def _configure_plugins_toml(self) -> bool: if not self.settings.plugins_config: @@ -88,38 +241,17 @@ class _Runtime: if plugin_mod is None: return False plugin_config = _static_plugin_config(self.settings.plugins_config) - if self.settings.dynamic_plugins: - activate_dynamic = getattr(plugin_mod, "activate_dynamic_plugins", None) - if callable(activate_dynamic): - try: - self._ensure_plugin_config_output_dirs(plugin_config) - self._plugin_activation = _resolve_awaitable( - activate_dynamic(plugin_config, self.settings.dynamic_plugins) - ) - self._ensure_shutdown_registered() - return True - except Exception as exc: - logger.warning( - "NeMo Relay dynamic plugin activation failed; continuing with static " - "observability only: %s", - exc, - ) - else: - logger.warning( - "NeMo Relay dynamic plugins require a binding that exposes " - "plugin.activate_dynamic_plugins (available in NeMo Relay 0.6+). " - "Continuing with static observability only." - ) - initialize = getattr(plugin_mod, "initialize", None) - if not callable(initialize): - return False - try: - self._ensure_plugin_config_output_dirs(plugin_config) - _resolve_awaitable(initialize(plugin_config)) - return True - except Exception as exc: - logger.debug("NeMo Relay plugins.toml init failed: %s", exc, exc_info=True) - return False + self._ensure_plugin_config_output_dirs(plugin_config) + initialized, activation = _PLUGIN_CONFIGURATION.acquire( + self, + plugin_mod, + plugin_config, + self.settings.dynamic_plugins, + ) + self._plugin_activation = activation + if activation is not None: + self._ensure_shutdown_registered() + return initialized def _ensure_shutdown_registered(self) -> None: if self._shutdown_registered: @@ -130,43 +262,12 @@ class _Runtime: def _clear_plugins_toml(self) -> None: if not self._plugin_config_initialized: return - failures: list[str] = [] - if self._plugin_activation is not None: - activation = self._plugin_activation - try: - _flush_relay_subscribers(self.nemo_relay) - except Exception as exc: - failures.append(f"subscriber flush failed: {exc}") - - close = getattr(activation, "close", None) - if callable(close): - try: - _resolve_awaitable(close()) - except Exception as exc: - failures.append(f"dynamic plugin activation close failed: {exc}") - finally: - # Retain the owned activation through the complete close - # attempt. The binding transitions it to a terminal state - # before its awaitable resolves, including error results. - self._plugin_activation = None - self._plugin_config_initialized = False - self._plugin_config_needs_reinit = bool(self.settings.plugins_config) - else: - failures.append("dynamic plugin activation has no close method") - else: - try: - plugin_mod = getattr(self.nemo_relay, "plugin", None) - clear = getattr(plugin_mod, "clear", None) - if callable(clear): - _resolve_awaitable(clear()) - except Exception as exc: - failures.append(f"static plugin configuration clear failed: {exc}") - finally: - self._plugin_config_initialized = False - self._plugin_config_needs_reinit = bool(self.settings.plugins_config) - - if failures: - raise RuntimeError("; ".join(failures)) + try: + _PLUGIN_CONFIGURATION.release(self, self.nemo_relay) + finally: + self._plugin_activation = None + self._plugin_config_initialized = False + self._plugin_config_needs_reinit = bool(self.settings.plugins_config) def _activate_direct_fallbacks(self) -> None: self._plugin_config_needs_reinit = False @@ -178,9 +279,11 @@ class _Runtime: self._plugin_config_initialized = self._configure_plugins_toml() if not self._plugin_config_initialized: self._activate_direct_fallbacks() + self._sync_managed_execution() return self._clear_atof() self._plugin_config_needs_reinit = False + self._sync_managed_execution() def _plugins_toml_owns_exporter(self, exporter_name: str) -> bool: return self._plugin_config_initialized and _observability_exporter_enabled( @@ -232,45 +335,76 @@ class _Runtime: except Exception: logger.debug("NeMo Relay ATOF deregister failed", exc_info=True) self.atof_exporter = None + self._sync_managed_execution() - def ensure_session(self, kwargs: dict[str, Any]) -> _SessionState: - self._maybe_reinitialize_plugins_toml() + def prepare_session(self, kwargs: dict[str, Any]) -> _SessionState: + """Register per-session subscribers without opening the core scope.""" session_id = _session_id(kwargs) - state = self.sessions.get(session_id) - if state is not None: + with self._sessions_lock: + self._maybe_reinitialize_plugins_toml() + state = self.sessions.get(session_id) + if state is not None: + return state + + state = _SessionState(session_id=session_id) + if self.settings.atif_enabled and not self._plugins_toml_owns_exporter("atif"): + state.atif_exporter = self.nemo_relay.AtifExporter( + session_id, + self.settings.atif_agent_name, + self.settings.atif_agent_version, + model_name=str(kwargs.get("model") or self.settings.atif_model_name), + extra={ + "source": "hermes-agent", + "plugin": "observability/nemo_relay", + }, + ) + state.atif_subscriber_name = ( + f"hermes.nemo_relay.atif.{self.host.runtime_id}.{session_id}" + ) + state.atif_exporter.register(state.atif_subscriber_name) + self.sessions[session_id] = state return state - state = _SessionState(session_id=session_id) - if self.settings.atif_enabled and not self._plugins_toml_owns_exporter("atif"): - state.atif_exporter = self.nemo_relay.AtifExporter( - session_id, - self.settings.atif_agent_name, - self.settings.atif_agent_version, - model_name=str(kwargs.get("model") or self.settings.atif_model_name), - extra={"source": "hermes-agent", "plugin": "observability/nemo_relay"}, - ) - state.atif_subscriber_name = f"hermes.nemo_relay.atif.{session_id}" - state.atif_exporter.register(state.atif_subscriber_name) + def ensure_session(self, kwargs: dict[str, Any]) -> _SessionState: + state = self.prepare_session(kwargs) + if state.relay_session is not None: + return state - subagent_parent = self.subagent_parents.get(session_id) - metadata = _metadata(kwargs) - parent_handle = None - if subagent_parent is not None: - parent_handle = subagent_parent.parent_handle - metadata = {**metadata, **subagent_parent.metadata} - state.is_embedded_subagent = True - state.parent_session_id = subagent_parent.parent_session_id - - state.handle = self.nemo_relay.scope.push( - f"hermes-session-{session_id}", - self.nemo_relay.ScopeType.Agent, - handle=parent_handle, - data={"session_id": session_id}, - metadata=metadata, + rich_metadata = _metadata(kwargs) + with self._sessions_lock: + subagent_context = self.subagent_contexts.get(state.session_id) + if subagent_context is not None: + rich_metadata = {**rich_metadata, **subagent_context.metadata} + relay_session = self.host.ensure_session( + kwargs, + data={"session_id": state.session_id}, + metadata=rich_metadata, ) - self.sessions[session_id] = state + if relay_session is None: + raise RuntimeError("Hermes core Relay session is unavailable") + state.relay_session = relay_session + state.handle = relay_session.handle + if subagent_context is not None: + state.is_embedded_subagent = True + state.parent_session_id = subagent_context.parent_session_id return state + def run_in_session( + self, + state: _SessionState, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, + ) -> Any: + if state.relay_session is None: + raise RuntimeError("Hermes core Relay session is unavailable") + return self.host.run_in_session( + state.relay_session, + callback, + *args, + **kwargs, + ) + def export_atif(self, state: _SessionState) -> None: if not self.settings.atif_enabled or state.atif_exporter is None: return @@ -283,22 +417,24 @@ class _Runtime: filename = self.settings.atif_filename_template.format(session_id=state.session_id) Path(output_dir, filename).write_text(state.atif_exporter.export_json(), encoding="utf-8") - def close_session(self, kwargs: dict[str, Any]) -> None: + def close_session( + self, + kwargs: dict[str, Any], + *, + close_host: bool = True, + ) -> None: session_id = _session_id(kwargs) - self.subagent_parents.pop(session_id, None) - state = self.sessions.pop(session_id, None) + with self._sessions_lock: + self.subagent_contexts.pop(session_id, None) + state = self.sessions.pop(session_id, None) if state is None: return failures: list[str] = [] - if state.handle is not None: + if close_host: try: - self.nemo_relay.scope.pop(state.handle, output=_jsonable(kwargs)) + self.host.close_session(kwargs) except Exception as exc: - failures.append(f"session scope pop failed: {exc}") - try: - _flush_relay_subscribers(self.nemo_relay) - except Exception as exc: - failures.append(f"subscriber flush failed: {exc}") + failures.append(f"core session close failed: {exc}") try: self.export_atif(state) except Exception as exc: @@ -308,21 +444,22 @@ class _Runtime: state.atif_exporter.deregister(state.atif_subscriber_name) except Exception as exc: failures.append(f"ATIF deregister failed: {exc}") - if ( - self._plugin_config_initialized - and self._plugin_activation is None - and not self.sessions - ): - try: - self._clear_plugins_toml() - except Exception as exc: - failures.append(f"plugin configuration clear failed: {exc}") - elif ( - self.settings.plugins_config - and self._plugin_activation is None - and not self.sessions - ): - self._plugin_config_needs_reinit = True + with self._sessions_lock: + if ( + self._plugin_config_initialized + and self._plugin_activation is None + and not self.sessions + ): + try: + self._clear_plugins_toml() + except Exception as exc: + failures.append(f"plugin configuration clear failed: {exc}") + elif ( + self.settings.plugins_config + and self._plugin_activation is None + and not self.sessions + ): + self._plugin_config_needs_reinit = True if failures: logger.warning( "NeMo Relay session %s teardown completed with errors: %s", @@ -333,7 +470,9 @@ class _Runtime: def shutdown(self) -> None: """Close active sessions and the process-lifetime plugin activation.""" failures: list[str] = [] - for session_id in list(self.sessions): + with self._sessions_lock: + session_ids = list(self.sessions) + for session_id in session_ids: try: self.close_session({"session_id": session_id, "reason": "runtime_shutdown"}) except Exception as exc: @@ -344,6 +483,9 @@ class _Runtime: except Exception as exc: failures.append(f"plugin runtime close failed: {exc}") self._clear_atof() + if self._execution_consumer_retained: + self.host.release_managed_execution(self._execution_consumer_name) + self._execution_consumer_retained = False if self._shutdown_registered and self._plugin_activation is None: atexit.unregister(self.shutdown) self._shutdown_registered = False @@ -355,7 +497,9 @@ class _Runtime: def mark(self, name: str, kwargs: dict[str, Any]) -> None: state = self.ensure_session(kwargs) - self.nemo_relay.scope.event( + self.run_in_session( + state, + self.nemo_relay.scope.event, name, handle=state.handle, data=_jsonable(kwargs), @@ -367,12 +511,14 @@ class _Runtime: metadata = _metadata(kwargs) child_session_id = _child_session_id(kwargs) if child_session_id: - self.subagent_parents[child_session_id] = _SubagentParent( - parent_session_id=parent_state.session_id, - parent_handle=parent_state.handle, - metadata=_subagent_child_metadata(kwargs, metadata), - ) - self.nemo_relay.scope.event( + with self._sessions_lock: + self.subagent_contexts[child_session_id] = _SubagentContext( + parent_session_id=parent_state.session_id, + metadata=_subagent_child_metadata(kwargs, metadata), + ) + self.run_in_session( + parent_state, + self.nemo_relay.scope.event, "hermes.subagent.start", handle=parent_state.handle, data=_jsonable(kwargs), @@ -382,148 +528,19 @@ class _Runtime: def mark_subagent_stop(self, kwargs: dict[str, Any]) -> None: child_session_id = _child_session_id(kwargs) if child_session_id: - self.subagent_parents.pop(child_session_id, None) + self.close_session( + {"session_id": child_session_id}, + close_host=False, + ) + with self._sessions_lock: + self.subagent_contexts.pop(child_session_id, None) self.mark("hermes.subagent.stop", kwargs) - def managed_llm_enabled(self) -> bool: - return ( - (self.settings.adaptive_enabled or self._plugin_activation is not None) - and callable(getattr(getattr(self.nemo_relay, "llm", None), "execute", None)) - and callable(getattr(self.nemo_relay, "LLMRequest", None)) - ) - - def managed_tool_enabled(self) -> bool: - return ( - (self.settings.adaptive_enabled or self._plugin_activation is not None) - and callable(getattr(getattr(self.nemo_relay, "tools", None), "execute", None)) - ) - - def _run_managed_with_downstream_preservation( - self, - next_call: Callable[[Any], Any], - normalize_payload: Callable[[Any], Any], - shape_response: Callable[[Any], Any], - make_managed_execute: Callable[[Callable[[Any], Any]], Any], - *, - preserve_raw_response: bool, - ) -> Any: - # NeMo Relay's native managed execution may wrap a failing callback as an - # internal runtime error, hiding the real downstream provider/tool - # exception. Capture the original here and re-raise it after managed - # execution so Hermes retry classification still sees it. The LLM and tool - # paths share this scaffolding; they differ only in payload normalization, - # response shaping, and the Relay call itself. - raw_response: dict[str, Any] = {"set": False, "value": None, "normalized": None} - callback_error: Exception | None = None - downstream_error: BaseException | None = None - - def _impl(next_payload: Any) -> Any: - nonlocal callback_error, downstream_error - try: - raw = next_call(normalize_payload(next_payload)) - except Exception as exc: - callback_error = exc - downstream_error = _original_downstream_error(exc) - raise - raw_response["set"] = True - raw_response["value"] = raw - raw_response["normalized"] = shape_response(raw) - return raw_response["normalized"] - - try: - managed_result = _resolve_awaitable(make_managed_execute(_impl)) - except Exception as exc: - if downstream_error is not None and _is_relay_wrapped_callback_error(exc, callback_error): - raise downstream_error - raise - if ( - preserve_raw_response - and raw_response["set"] - and _json_semantically_equal(managed_result, raw_response["normalized"]) - ): - return raw_response["value"] - return managed_result - - def execute_llm(self, kwargs: dict[str, Any]) -> Any: - state = self.ensure_session(kwargs) - request_body = _jsonable(kwargs.get("request") or {}) - request = self.nemo_relay.LLMRequest({}, request_body) - next_call = kwargs.get("next_call") - if not callable(next_call): - return request_body - - def _normalize(next_request: Any) -> Any: - next_body = getattr(next_request, "content", next_request) - return next_body if isinstance(next_body, dict) else request_body - - def _make_managed(impl: Callable[[Any], Any]) -> Any: - async def _managed_execute() -> Any: - result = self.nemo_relay.llm.execute( - _relay_llm_surface(kwargs), - request, - impl, - handle=state.handle, - data=_jsonable( - { - "turn_id": kwargs.get("turn_id"), - "api_request_id": kwargs.get("api_request_id"), - "api_call_count": kwargs.get("api_call_count"), - "mode": self.settings.adaptive_mode, - } - ), - metadata=_metadata(kwargs), - model_name=str(kwargs.get("model") or ""), - ) - if inspect.isawaitable(result): - return await result - return result - - return _managed_execute() - - return self._run_managed_with_downstream_preservation( - next_call, _normalize, _llm_response_payload, _make_managed, preserve_raw_response=True - ) - - def execute_tool(self, kwargs: dict[str, Any]) -> Any: - state = self.ensure_session(kwargs) - tool_name = str(kwargs.get("tool_name") or "tool") - args = _jsonable(kwargs.get("args") or {}) - next_call = kwargs.get("next_call") - if not callable(next_call): - return args - - def _normalize(next_args: Any) -> Any: - return next_args if isinstance(next_args, dict) else args - - def _make_managed(impl: Callable[[Any], Any]) -> Any: - async def _managed_execute() -> Any: - result = self.nemo_relay.tools.execute( - tool_name, - args, - impl, - handle=state.handle, - data=_jsonable( - { - "turn_id": kwargs.get("turn_id"), - "api_request_id": kwargs.get("api_request_id"), - "tool_call_id": kwargs.get("tool_call_id"), - "mode": self.settings.adaptive_mode, - } - ), - metadata=_metadata(kwargs), - ) - if inspect.isawaitable(result): - return await result - return result - - return _managed_execute() - - return self._run_managed_with_downstream_preservation( - next_call, _normalize, _jsonable, _make_managed, preserve_raw_response=False - ) - - def register(ctx) -> None: + relay_runtime.SESSION_COORDINATOR.register_session_initializer( + _SESSION_INITIALIZER_NAME, + _prepare_core_session, + ) # Activate dynamic plugins before Hermes installs the managed execution # boundaries that invoke their interceptors. if _load_settings().dynamic_plugins: @@ -534,17 +551,10 @@ def register(ctx) -> None: ctx.register_hook("on_session_reset", on_session_reset) ctx.register_hook("pre_llm_call", on_pre_llm_call) ctx.register_hook("post_llm_call", on_post_llm_call) - ctx.register_hook("pre_api_request", on_pre_api_request) - ctx.register_hook("post_api_request", on_post_api_request) - ctx.register_hook("api_request_error", on_api_request_error) - ctx.register_hook("pre_tool_call", on_pre_tool_call) - ctx.register_hook("post_tool_call", on_post_tool_call) ctx.register_hook("pre_approval_request", on_pre_approval_request) ctx.register_hook("post_approval_response", on_post_approval_response) ctx.register_hook("subagent_start", on_subagent_start) ctx.register_hook("subagent_stop", on_subagent_stop) - ctx.register_middleware("llm_execution", on_llm_execution_middleware) - ctx.register_middleware("tool_execution", on_tool_execution_middleware) def on_session_start(**kwargs: Any) -> None: @@ -562,13 +572,13 @@ def on_session_end(**kwargs: Any) -> None: def on_session_finalize(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: - _safe(lambda: runtime.close_session(kwargs)) + _safe(lambda: runtime.close_session(kwargs, close_host=False)) def on_session_reset(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: - _safe(lambda: runtime.close_session(kwargs)) + _safe(lambda: runtime.close_session(kwargs, close_host=False)) def on_pre_llm_call(**kwargs: Any) -> None: @@ -583,122 +593,6 @@ def on_post_llm_call(**kwargs: Any) -> None: _safe(lambda: runtime.mark("hermes.turn.end", kwargs)) -def on_pre_api_request(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - request_payload = kwargs.get("request") - request_body = request_payload.get("body") if isinstance(request_payload, dict) else {} - request = runtime.nemo_relay.LLMRequest({}, _jsonable(request_body)) - span = runtime.nemo_relay.llm.call( - str(kwargs.get("provider") or "llm"), - request, - handle=state.handle, - data=_jsonable({"turn_id": kwargs.get("turn_id"), "api_request_id": kwargs.get("api_request_id")}), - metadata=_metadata(kwargs), - model_name=str(kwargs.get("model") or ""), - ) - state.llm_spans[_api_key(kwargs)] = span - - _safe(_record) - - -def on_post_api_request(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.llm_spans.pop(_api_key(kwargs), None) - if span is None: - runtime.mark("hermes.api.response.unmatched", kwargs) - return - runtime.nemo_relay.llm.call_end( - span, - _jsonable(kwargs.get("response") or {}), - data=_jsonable({"usage": kwargs.get("usage"), "finish_reason": kwargs.get("finish_reason")}), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - -def on_api_request_error(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_llm_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.llm_spans.pop(_api_key(kwargs), None) - if span is None: - runtime.mark("hermes.api.error", kwargs) - return - runtime.nemo_relay.llm.call_end( - span, - {"error": _jsonable(kwargs.get("error") or {})}, - data=_jsonable(kwargs), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - -def on_pre_tool_call(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_tool_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = runtime.nemo_relay.tools.call( - str(kwargs.get("tool_name") or "tool"), - _jsonable(kwargs.get("args") or {}), - handle=state.handle, - data=_jsonable({"turn_id": kwargs.get("turn_id"), "api_request_id": kwargs.get("api_request_id")}), - metadata=_metadata(kwargs), - tool_call_id=str(kwargs.get("tool_call_id") or ""), - ) - state.tool_spans[_tool_key(kwargs)] = span - - _safe(_record) - - -def on_post_tool_call(**kwargs: Any) -> None: - runtime = _get_runtime() - if runtime is None: - return - if runtime.managed_tool_enabled(): - return - - def _record() -> None: - state = runtime.ensure_session(kwargs) - span = state.tool_spans.pop(_tool_key(kwargs), None) - if span is None: - runtime.mark("hermes.tool.response.unmatched", kwargs) - return - runtime.nemo_relay.tools.call_end( - span, - _jsonable(kwargs.get("result")), - data=_jsonable({"status": kwargs.get("status"), "duration_ms": kwargs.get("duration_ms")}), - metadata=_metadata(kwargs), - ) - - _safe(_record) - - def on_pre_approval_request(**kwargs: Any) -> None: runtime = _get_runtime() if runtime is not None: @@ -723,60 +617,58 @@ def on_subagent_stop(**kwargs: Any) -> None: _safe(lambda: runtime.mark_subagent_stop(kwargs)) -def on_llm_execution_middleware(**kwargs: Any) -> Any: - runtime = _get_runtime() - next_call = kwargs.get("next_call") - request = kwargs.get("request") or {} - if runtime is not None and runtime.managed_llm_enabled(): - return runtime.execute_llm(kwargs) - if callable(next_call): - return next_call(request) - return request +def _prepare_core_session( + host: relay_runtime.RelayRuntime, + context: dict[str, Any], +) -> None: + """Register rich subscribers before core creates the conversation scope.""" + runtime = _get_runtime( + profile_key=str(context.get("profile_key") or host.profile_key), + host=host, + ) + if runtime is not None: + runtime.prepare_session(context) -def on_tool_execution_middleware(**kwargs: Any) -> Any: - runtime = _get_runtime() - next_call = kwargs.get("next_call") - args = kwargs.get("args") or {} - if runtime is not None and runtime.managed_tool_enabled(): - return runtime.execute_tool(kwargs) - if callable(next_call): - return next_call(args) - return args - - -def _get_runtime() -> Optional[_Runtime]: - global _RUNTIME +def _get_runtime( + *, + profile_key: str | None = None, + host: relay_runtime.RelayRuntime | None = None, +) -> Optional[_Runtime]: + profile_key = profile_key or relay_runtime.current_profile_key() with _LOCK: - if _RUNTIME is _INIT_FAILED: + runtime = _RUNTIMES.get(profile_key) + if runtime is _INIT_FAILED: return None - if isinstance(_RUNTIME, _Runtime): - return _RUNTIME + if isinstance(runtime, _Runtime): + if host is None or runtime.host is host: + return runtime + runtime.shutdown() + _RUNTIMES.pop(profile_key, None) try: - import nemo_relay as nemo_runtime - except Exception as exc: - logger.debug("NeMo Relay plugin disabled: import failed: %s", exc) - _RUNTIME = _INIT_FAILED - return None - try: - _RUNTIME = _Runtime(nemo_relay=nemo_runtime, settings=_load_settings()) + resolved_host = host or relay_runtime.get_runtime(profile_key=profile_key) + if resolved_host is None: + raise RuntimeError("Hermes core Relay runtime is unavailable") + runtime = _Runtime( + nemo_relay=resolved_host.relay, + settings=_load_settings(), + host=resolved_host, + ) except Exception as exc: logger.debug("NeMo Relay plugin disabled: init failed: %s", exc, exc_info=True) - _RUNTIME = _INIT_FAILED + _RUNTIMES[profile_key] = _INIT_FAILED return None - return _RUNTIME + _RUNTIMES[profile_key] = runtime + return runtime def _load_settings() -> _Settings: plugins_toml_path = _env("HERMES_NEMO_RELAY_PLUGINS_TOML") plugins_config = _load_plugins_config(plugins_toml_path) - adaptive_config = _enabled_component_config(plugins_config, "adaptive") return _Settings( plugins_toml_path=plugins_toml_path, plugins_config=plugins_config, dynamic_plugins=_dynamic_plugin_specs(plugins_config, plugins_toml_path), - adaptive_enabled=adaptive_config is not None, - adaptive_mode=_adaptive_mode(adaptive_config), atof_enabled=_env_bool("HERMES_NEMO_RELAY_ATOF_ENABLED"), atof_output_directory=_env("HERMES_NEMO_RELAY_ATOF_OUTPUT_DIRECTORY"), atof_filename=_env("HERMES_NEMO_RELAY_ATOF_FILENAME") or "hermes-atof.jsonl", @@ -800,6 +692,18 @@ def _static_plugin_config(plugins_config: dict[str, Any]) -> dict[str, Any]: } +def _plugin_configuration_key( + plugin_config: dict[str, Any], + dynamic_plugins: list[dict[str, Any]], +) -> str: + return json.dumps( + {"config": plugin_config, "dynamic_plugins": dynamic_plugins}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + def _dynamic_plugin_specs( plugins_config: dict[str, Any] | None, plugins_toml_path: str = "", @@ -950,20 +854,6 @@ def _enabled_component_config( return None -def _adaptive_mode(config: dict[str, Any] | None) -> str: - if not isinstance(config, dict): - return "observe_only" - tool_parallelism = config.get("tool_parallelism") - if isinstance(tool_parallelism, dict): - mode = tool_parallelism.get("mode") - if isinstance(mode, str) and mode.strip(): - return mode.strip() - mode = config.get("mode") - if isinstance(mode, str) and mode.strip(): - return mode.strip() - return "observe_only" - - def _observability_exporter_enabled( plugins_config: dict[str, Any] | None, exporter_name: str, @@ -998,7 +888,10 @@ def _child_session_id(kwargs: dict[str, Any]) -> str: return str(kwargs.get("child_session_id") or "") -def _subagent_child_metadata(kwargs: dict[str, Any], parent_metadata: dict[str, Any]) -> dict[str, Any]: +def _subagent_child_metadata( + kwargs: dict[str, Any], + parent_metadata: dict[str, Any], +) -> dict[str, Any]: child_session_id = _child_session_id(kwargs) metadata = { "session_id": child_session_id, @@ -1022,25 +915,6 @@ def _subagent_child_metadata(kwargs: dict[str, Any], parent_metadata: dict[str, return metadata -def _api_key(kwargs: dict[str, Any]) -> str: - return str(kwargs.get("api_request_id") or f"{_session_id(kwargs)}:{kwargs.get('api_call_count') or 'api'}") - - -def _tool_key(kwargs: dict[str, Any]) -> str: - return str( - kwargs.get("tool_call_id") - or f"{_session_id(kwargs)}:{kwargs.get('turn_id') or ''}:{kwargs.get('tool_name') or 'tool'}" - ) - - -def _relay_llm_surface(kwargs: dict[str, Any]) -> str: - api_mode = str(kwargs.get("api_mode") or "").strip().lower() - return _RELAY_LLM_SURFACE_BY_API_MODE.get( - api_mode, - str(kwargs.get("provider") or "llm"), - ) - - def _metadata(kwargs: dict[str, Any]) -> dict[str, Any]: keys = ( "telemetry_schema_version", @@ -1100,100 +974,6 @@ def _jsonable(value: Any) -> Any: return str(value) -def _json_semantically_equal(left: Any, right: Any) -> bool: - """Compare JSON-compatible values without conflating booleans and numbers.""" - try: - options = {"ensure_ascii": False, "sort_keys": True, "separators": (",", ":")} - return json.dumps(_jsonable(left), **options) == json.dumps(_jsonable(right), **options) - except (TypeError, ValueError): - return False - - -def _value(obj: Any, key: str, default: Any = None) -> Any: - if isinstance(obj, dict): - return obj.get(key, default) - return getattr(obj, key, default) - - -def _original_downstream_error(exc: Exception) -> BaseException: - # Hermes wraps downstream execution failures in a local/private exception - # class, so detect the wrapper by shape instead of importing it here. - original = getattr(exc, "original", None) - if exc.__class__.__name__ == "_DownstreamExecutionError" and isinstance(original, BaseException): - return original - return exc - - -def _is_relay_wrapped_callback_error(exc: Exception, callback_error: Exception | None) -> bool: - # NeMo Relay re-wraps a failing callback as ``RuntimeError("internal error: - # : ")``. Match by prefix rather than exact equality so a - # trailing traceback/suffix in a future Relay version doesn't silently defeat - # the unwrap; the class-name + message prefix still discriminates the real - # downstream failure from unrelated Relay-internal errors. If Relay drops the - # leading ``internal error:`` shape entirely, this returns False and Hermes - # falls back to surfacing Relay's error (the pre-fix behavior) rather than - # masking it. - if callback_error is None or not isinstance(exc, RuntimeError): - return False - expected = f"internal error: {callback_error.__class__.__name__}: {callback_error}" - return str(exc).startswith(expected) - - -def _llm_response_payload(response: Any) -> Any: - """Return the LLM response shape NeMo Relay's ATIF conversion expects.""" - payload = _jsonable(response) - if isinstance(payload, dict) and "assistant_message" in payload: - return payload - - choices = _value(response, "choices") - if choices is None and isinstance(payload, dict): - choices = payload.get("choices") - first_choice = choices[0] if isinstance(choices, list) and choices else None - message = _value(first_choice, "message") - finish_reason = _value(first_choice, "finish_reason") - - assistant_message: dict[str, Any] = {"role": "assistant", "content": ""} - if message is not None: - assistant_message["role"] = _value(message, "role", "assistant") or "assistant" - content = _value(message, "content") - if content is not None: - assistant_message["content"] = _jsonable(content) - tool_calls = _tool_calls_payload(_value(message, "tool_calls")) - if tool_calls: - assistant_message["tool_calls"] = tool_calls - reasoning = _value(message, "reasoning_content") - if reasoning is not None: - assistant_message["reasoning_content"] = _jsonable(reasoning) - elif isinstance(payload, dict): - assistant_message["content"] = payload.get("content") or payload.get("output_text") or "" - - return { - "model": _value(response, "model", payload.get("model") if isinstance(payload, dict) else None), - "assistant_message": assistant_message, - "finish_reason": finish_reason, - "usage": _jsonable(_value(response, "usage", payload.get("usage") if isinstance(payload, dict) else None)), - } - - -def _tool_calls_payload(tool_calls: Any) -> list[dict[str, Any]]: - if not isinstance(tool_calls, list): - return [] - normalized: list[dict[str, Any]] = [] - for call in tool_calls: - function = _value(call, "function") - normalized.append( - { - "id": _value(call, "id"), - "type": _value(call, "type", "function") or "function", - "function": { - "name": _value(function, "name"), - "arguments": _value(function, "arguments"), - }, - } - ) - return normalized - - def _safe(fn) -> None: try: fn() @@ -1231,8 +1011,13 @@ def _resolve_awaitable(value: Any) -> Any: def reset_for_tests() -> None: - global _RUNTIME + relay_runtime.SESSION_COORDINATOR.unregister_session_initializer( + _SESSION_INITIALIZER_NAME + ) with _LOCK: - if isinstance(_RUNTIME, _Runtime): - _RUNTIME.shutdown() - _RUNTIME = None + runtimes = list(_RUNTIMES.values()) + _RUNTIMES.clear() + for runtime in runtimes: + if isinstance(runtime, _Runtime): + runtime.shutdown() + _PLUGIN_CONFIGURATION.reset_for_tests() diff --git a/plugins/observability/nemo_relay/plugin.yaml b/plugins/observability/nemo_relay/plugin.yaml index b1b00f25d819..046d5d0d8510 100644 --- a/plugins/observability/nemo_relay/plugin.yaml +++ b/plugins/observability/nemo_relay/plugin.yaml @@ -9,11 +9,6 @@ hooks: - on_session_reset - pre_llm_call - post_llm_call - - pre_api_request - - post_api_request - - api_request_error - - pre_tool_call - - post_tool_call - pre_approval_request - post_approval_response - subagent_start diff --git a/pyproject.toml b/pyproject.toml index 02c00a096752..d623df04b9ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,11 @@ dependencies = [ # Hence the ``sys_platform == 'win32'`` marker: the dep (and its portalocker # / pywin32 tree) ships only where it's actually used. "concurrent-log-handler==0.9.29; sys_platform == 'win32'", + # First-party lifecycle and shared-metrics runtime. Relay 0.6 is the minimum + # lossless provider-codec contract. Managed calls pass request/response data + # through this native module in-process; shared metrics installs no network + # exporter and consumes only its bounded projection. + "nemo-relay>=0.6.0,<0.7; (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'linux' and platform_machine == 'aarch64') or (sys_platform == 'win32' and platform_machine == 'AMD64') or (sys_platform == 'win32' and platform_machine == 'ARM64')", ] [project.optional-dependencies] @@ -202,7 +207,9 @@ pty = [] # extra that exposes a Starlette-backed server surface so pip/uv can't resolve # a vulnerable pre-1.0.1 transitive. Bump in lockstep with uv.lock. mcp = ["mcp==1.26.0", "starlette==1.3.1"] # starlette: CVE-2026-48710 -nemo-relay = ["nemo-relay>=0.5,<1.0"] +# Backwards-compatible no-op alias. Relay is a core dependency on supported +# wheel targets and intentionally unavailable on other platforms. +nemo-relay = [] homeassistant = ["aiohttp==3.14.1"] sms = ["aiohttp==3.14.1"] teams = ["microsoft-teams-apps==2.0.13.4", "aiohttp==3.14.1"] # aiohttp 3.14.1: CVE-2026-34993(RCE)/47265 + 34513/34518/34519/34520/34525 @@ -319,6 +326,7 @@ py-modules = ["run_agent", "model_tools", "toolsets", "batch_runner", "trajector include = ["agent", "agent.*", "tools", "tools.*", "hermes_cli", "hermes_cli.*", "gateway", "gateway.*", "tui_gateway", "tui_gateway.*", "cron", "cron.*", "acp_adapter", "plugins", "plugins.*", "providers", "providers.*"] [tool.setuptools.package-data] +hermes_cli = ["observability/schemas/*.json"] # gateway/assets/ ships status_phrases.yaml and the Telegram BotFather # screenshot. Without this, sealed venvs (uv2nix) silently lose both — # status phrases fall back to the tiny hardcoded set and the Telegram diff --git a/run_agent.py b/run_agent.py index f4d501914fe5..d624656facd7 100644 --- a/run_agent.py +++ b/run_agent.py @@ -2685,17 +2685,16 @@ class AIAgent: retryable: Optional[bool] = None, reason: Optional[str] = None, ) -> None: - # Lazy module import (not from-import) so tests that - # ``monkeypatch.setattr("hermes_cli.plugins.has_hook", ...)`` still - # take effect on this call site. After first call the import is a + # Lazy module import (not from-import) so tests can replace lifecycle + # dispatch at this call site. After first call the import is a # ``sys.modules`` dict lookup, so retries don't repay any real cost. try: - from hermes_cli import plugins as _plugins + from hermes_cli import lifecycle as _lifecycle - if not _plugins.has_hook("api_request_error"): + if not _lifecycle.has_hook("api_request_error"): return ended_at = time.time() - _plugins.invoke_hook( + _lifecycle.invoke_hook( "api_request_error", task_id=task_id, turn_id=turn_id, @@ -6749,7 +6748,8 @@ class AIAgent: tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False, skip_tool_request_middleware: bool = False, - tool_request_middleware_trace: Optional[list[dict[str, Any]]] = None) -> str: + tool_request_middleware_trace: Optional[list[dict[str, Any]]] = None, + skip_tool_execution_middleware: bool = False) -> str: """Forwarder — see ``agent.agent_runtime_helpers.invoke_tool``.""" from agent.agent_runtime_helpers import invoke_tool return invoke_tool( @@ -6762,6 +6762,7 @@ class AIAgent: pre_tool_block_checked, skip_tool_request_middleware, tool_request_middleware_trace, + skip_tool_execution_middleware, ) @staticmethod @@ -6849,48 +6850,140 @@ class AIAgent: reset_accounting_context, set_accounting_context, ) + from agent import relay_runtime from agent.conversation_loop import run_conversation from agent.portal_tags import ( reset_conversation_context, set_conversation_context, ) - from agent.subagent_lifecycle import bind_subagent_parent - - # Publish the conversation id for ambient Nous Portal tagging. Every - # LLM call made inside this turn — main loop, compression, vision, - # web_extract, session_search, MoA slots, background-review forks - # (which copy this Context into their thread) — inherits the - # ``conversation=`` tag with zero per-call-site plumbing. - token = set_conversation_context(self._conversation_root_id()) - # Publish the session accounting handles the same way so auxiliary - # calls record their token usage into session_model_usage (task - # dimension) — the fix for aux spend being invisible in analytics - # (issue #23270). - acct_token = set_accounting_context( - getattr(self, "_session_db", None), getattr(self, "session_id", None) + from hermes_cli.observability.relay_shared_metrics import ( + finish_task_run, + start_task_run, ) - from agent.auxiliary_client import scoped_runtime_main + from agent.subagent_lifecycle import bind_subagent_parent + effective_task_id = task_id or str(uuid.uuid4()) + session_id = str(getattr(self, "session_id", None) or "") + task_context = { + "session_id": session_id, + "task_id": effective_task_id, + "platform": getattr(self, "platform", None) or "", + } + relay_turn_id = ( + f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + ) + self._relay_pending_turn_id = relay_turn_id + relay_parent_session_id = ( + str(getattr(self, "_parent_session_id", None) or "") + if task_context["platform"] == "subagent" + else "" + ) + relay_lease = None + relay_turn = None + token = None + acct_token = None + task_started = False + task_finished = False + relay_outcome = "failed" + try: + relay_lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=task_context["session_id"], + platform=task_context["platform"], + parent_session_id=relay_parent_session_id, + model=str(getattr(self, "model", None) or ""), + ) + relay_turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + relay_lease, + turn_id=relay_turn_id, + task_id=effective_task_id, + ) + start_task_run( + **task_context, + parent_session_id=getattr(self, "_parent_session_id", None) or "", + ) + task_started = True + # Publish the conversation id for ambient Nous Portal tagging. Every + # LLM call made inside this turn — main loop, compression, vision, + # web_extract, session_search, MoA slots, background-review forks + # (which copy this Context into their thread) — inherits the + # ``conversation=`` tag with zero per-call-site plumbing. + token = set_conversation_context(self._conversation_root_id()) + # Publish the session accounting handles the same way so auxiliary + # calls record their token usage into session_model_usage (task + # dimension) — the fix for aux spend being invisible in analytics + # (issue #23270). + acct_token = set_accounting_context( + getattr(self, "_session_db", None), + getattr(self, "session_id", None), + ) + from agent.auxiliary_client import scoped_runtime_main - # The outer token restores the caller's Context even though turn setup - # replaces the value with the live runtime after fallback restoration. - # Keep the scope local instead of storing ContextVar tokens on the agent, - # which may be observed from another thread. - with bind_subagent_parent(self), scoped_runtime_main({}): - try: - return run_conversation( + # The outer token restores the caller's Context even though turn setup + # replaces the value with the live runtime after fallback restoration. + # Keep the scope local instead of storing ContextVar tokens on the agent, + # which may be observed from another thread. + with bind_subagent_parent(self), scoped_runtime_main({}): + result = run_conversation( self, user_message, system_message, conversation_history, - task_id, + effective_task_id, stream_callback, persist_user_message, persist_user_timestamp=persist_user_timestamp, moa_config=moa_config, ) + terminal = result if isinstance(result, dict) else {} + if terminal.get("interrupted") is True: + relay_outcome = "cancelled" + elif terminal.get("failed") is True: + relay_outcome = "failed" + else: + relay_outcome = "success" + relay_runtime.SESSION_COORDINATOR.finish_logical_calls( + relay_turn, + outcome=relay_outcome, + ) + task_finished = True + finish_task_run(**task_context, result=result) + return result + except BaseException as exc: + if isinstance(exc, (KeyboardInterrupt, InterruptedError)) or ( + type(exc).__name__ == "CancelledError" + ): + relay_outcome = "cancelled" + elif isinstance(exc, TimeoutError): + relay_outcome = "timed_out" + if relay_turn is not None: + relay_runtime.SESSION_COORDINATOR.finish_logical_calls( + relay_turn, + outcome=relay_outcome, + ) + if task_started and not task_finished: + task_finished = True + finish_task_run(**task_context, error=exc) + raise + finally: + try: + if relay_turn is not None: + relay_runtime.SESSION_COORDINATOR.end_turn( + relay_turn, + outcome=relay_outcome, + ) finally: - reset_accounting_context(acct_token) - reset_conversation_context(token) + try: + if relay_lease is not None: + relay_runtime.SESSION_COORDINATOR.release_conversation( + relay_lease + ) + finally: + if getattr(self, "_relay_pending_turn_id", None) == relay_turn_id: + self._relay_pending_turn_id = None + if acct_token is not None: + reset_accounting_context(acct_token) + if token is not None: + reset_conversation_context(token) def chat(self, message: str, stream_callback: Optional[callable] = None) -> str: """ diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.py new file mode 100644 index 000000000000..38a42641d1b1 --- /dev/null +++ b/scripts/smoke_nemo_relay_shared_metrics.py @@ -0,0 +1,443 @@ +"""Run a real Hermes CLI turn and validate the Relay shared-metrics output.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +PROMPT_CANARY = "relay-smoke-sensitive-prompt" +MODEL_CANARY = "gpt-relay-smoke-sensitive-model" +RESPONSE_CANARY = "relay-smoke-sensitive-response" + + +def _resolve_hermes_executable(hermes_repo: Path) -> Path: + for relative_path in ( + Path(".venv") / "bin" / "hermes", + Path(".venv") / "Scripts" / "hermes.exe", + ): + candidate = hermes_repo / relative_path + if candidate.is_file(): + return candidate + discovered = shutil.which("hermes") + if discovered: + return Path(discovered) + raise SystemExit( + "Hermes executable not found in the repository virtual environment " + "or on PATH" + ) + + +class _ModelHandler(BaseHTTPRequestHandler): + """Minimal OpenAI-compatible model server for one deterministic turn.""" + + protocol_version = "HTTP/1.1" + requests: list[dict[str, Any]] = [] + + def do_GET(self) -> None: # noqa: N802 + if self.path.rstrip("/") != "/v1/models": + self.send_error(404) + return + self._write_json({ + "object": "list", + "data": [ + { + "id": MODEL_CANARY, + "object": "model", + "created": 0, + "owned_by": "smoke-test", + } + ], + }) + + def do_POST(self) -> None: # noqa: N802 + if self.path.rstrip("/") != "/v1/chat/completions": + self.send_error(404) + return + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length) or b"{}") + type(self).requests.append(request) + if request.get("stream"): + self._write_stream() + else: + self._write_json({ + "id": "chatcmpl-relay-smoke", + "object": "chat.completion", + "created": int(time.time()), + "model": MODEL_CANARY, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": RESPONSE_CANARY, + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 1, + "total_tokens": 11, + }, + }) + + def log_message(self, format: str, *args: Any) -> None: + return + + def _write_json(self, payload: dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body) + self.close_connection = True + + def _write_stream(self) -> None: + now = int(time.time()) + chunks = [ + { + "id": "chatcmpl-relay-smoke", + "object": "chat.completion.chunk", + "created": now, + "model": MODEL_CANARY, + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": RESPONSE_CANARY, + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-relay-smoke", + "object": "chat.completion.chunk", + "created": now, + "model": MODEL_CANARY, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + "id": "chatcmpl-relay-smoke", + "object": "chat.completion.chunk", + "created": now, + "model": MODEL_CANARY, + "choices": [], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 1, + "total_tokens": 11, + }, + }, + ] + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + for chunk in chunks: + self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode("utf-8")) + self.wfile.flush() + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + self.close_connection = True + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--hermes-repo", + type=Path, + default=Path.cwd(), + help="Hermes source checkout containing .venv/bin/hermes", + ) + parser.add_argument( + "--relay-python", + type=Path, + default=None, + help="Optional NeMo Relay checkout's python directory", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help="Directory for the isolated HERMES_HOME and captured output", + ) + return parser.parse_args() + + +def _write_config(home: Path, port: int) -> None: + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text( + f"""model: + default: {MODEL_CANARY} + provider: custom + base_url: http://127.0.0.1:{port}/v1 + api_mode: chat_completions + api_key: no-key-required +security: + tirith_enabled: false +telemetry: + shared_metrics: + enabled: true +""", + encoding="utf-8", + ) + + +def _validate_store(database_path: Path) -> list[dict[str, Any]]: + if not database_path.is_file(): + raise AssertionError(f"Metrics database was not created: {database_path}") + with sqlite3.connect(database_path) as connection: + rows = connection.execute( + """ + SELECT metric_name, dimensions_json, value, packaged_value + FROM counter_aggregates + ORDER BY metric_name, dimensions_json + """ + ).fetchall() + counters = [ + { + "name": name, + "dimensions": json.loads(dimensions), + "value": value, + "packaged_value": packaged_value, + } + for name, dimensions, value, packaged_value in rows + ] + by_name = {counter["name"]: counter for counter in counters} + if set(by_name) != { + "hermes.model_call.count", + "hermes.task_run.finished", + "hermes.task_run.started", + }: + raise AssertionError( + f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}" + ) + expected_model = { + "name": "hermes.model_call.count", + "dimensions": { + "call_role": "primary", + "locality": "local", + "model_family": "gpt", + "outcome": "success", + "provider_family": "custom", + }, + "value": 1, + "packaged_value": 1, + } + if by_name["hermes.model_call.count"] != expected_model: + raise AssertionError( + f"Unexpected model counter: {by_name['hermes.model_call.count']}" + ) + expected_start = { + "name": "hermes.task_run.started", + "dimensions": { + "entrypoint": "interactive", + "execution_surface": "cli", + }, + "value": 1, + "packaged_value": 1, + } + if by_name["hermes.task_run.started"] != expected_start: + raise AssertionError( + f"Unexpected task start: {by_name['hermes.task_run.started']}" + ) + terminal = by_name["hermes.task_run.finished"] + expected_terminal_dimensions = { + "duration_bucket": terminal["dimensions"].get("duration_bucket"), + "end_reason": "completed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "success", + "retry_count_bucket": "0", + "termination": "none", + "tool_call_count_bucket": "0", + } + if ( + terminal["dimensions"] != expected_terminal_dimensions + or terminal["value"] != 1 + or terminal["packaged_value"] != 1 + ): + raise AssertionError(f"Unexpected task terminal counter: {terminal}") + return counters + + +def _validate_package(outbox: Path, schema_path: Path) -> tuple[Path, dict[str, Any]]: + packages = sorted(outbox.glob("*.json")) + if len(packages) != 1: + raise AssertionError(f"Expected one package in {outbox}, found {len(packages)}") + package_path = packages[0] + package = json.loads(package_path.read_text(encoding="utf-8")) + try: + import jsonschema + except ImportError as exc: + raise RuntimeError( + "The Hermes development environment requires jsonschema" + ) from exc + schema = json.loads(schema_path.read_text(encoding="utf-8")) + jsonschema.validate(package, schema) + + serialized = json.dumps(package) + for prohibited in (PROMPT_CANARY, MODEL_CANARY, RESPONSE_CANARY): + if prohibited in serialized: + raise AssertionError( + f"Exported package leaked prohibited value: {prohibited!r}" + ) + metrics = {metric["name"]: metric for metric in package.get("metrics", [])} + if set(metrics) != { + "hermes.model_call.count", + "hermes.task_run.finished", + "hermes.task_run.started", + }: + raise AssertionError( + f"Unexpected package metrics:\n{json.dumps(package.get('metrics'), indent=2)}" + ) + if metrics["hermes.model_call.count"]["dimensions"] != { + "call_role": "primary", + "locality": "local", + "model_family": "gpt", + "outcome": "success", + "provider_family": "custom", + }: + raise AssertionError( + f"Unexpected model metric: {metrics['hermes.model_call.count']}" + ) + terminal = metrics["hermes.task_run.finished"] + if terminal["dimensions"] != { + "duration_bucket": terminal["dimensions"].get("duration_bucket"), + "end_reason": "completed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "success", + "retry_count_bucket": "0", + "termination": "none", + "tool_call_count_bucket": "0", + }: + raise AssertionError(f"Unexpected task terminal metric: {terminal}") + return package_path, package + + +def main() -> int: + args = _arguments() + hermes_repo = args.hermes_repo.resolve() + relay_python = args.relay_python.resolve() if args.relay_python else None + hermes = _resolve_hermes_executable(hermes_repo) + if relay_python is not None and not any( + (relay_python / "nemo_relay").glob("_native.*") + ): + raise SystemExit( + "Built NeMo Relay Python binding not found under " + f"{relay_python}; run the Relay Python build first" + ) + + if args.output_dir: + root = args.output_dir.resolve() + if root.exists(): + raise SystemExit(f"Refusing to replace existing output directory: {root}") + root.mkdir(parents=True) + else: + root = Path(tempfile.mkdtemp(prefix="hermes-relay-shared-metrics-")) + home = root / "hermes-home" + workdir = root / "workspace" + workdir.mkdir() + home.mkdir() + (home / ".no-bundled-skills").touch() + + _ModelHandler.requests = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _ModelHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + _write_config(home, server.server_port) + env = os.environ.copy() + env["HERMES_HOME"] = str(home) + if relay_python is not None: + env["PYTHONPATH"] = os.pathsep.join([ + str(relay_python), + env.get("PYTHONPATH", ""), + ]).rstrip(os.pathsep) + result = subprocess.run( + [ + str(hermes), + "chat", + "--query", + PROMPT_CANARY, + "--provider", + "custom", + "--model", + MODEL_CANARY, + "--quiet", + "--ignore-rules", + "--toolsets", + "search", + "--max-turns", + "2", + ], + cwd=workdir, + env=env, + text=True, + capture_output=True, + timeout=120, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + (root / "hermes.stdout.txt").write_text(result.stdout, encoding="utf-8") + (root / "hermes.stderr.txt").write_text(result.stderr, encoding="utf-8") + if result.returncode != 0: + raise AssertionError( + f"Hermes exited with {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if not _ModelHandler.requests: + raise AssertionError("Hermes did not call the local model endpoint") + request = _ModelHandler.requests[0] + if request.get("model") != MODEL_CANARY: + raise AssertionError(f"Unexpected model request: {request.get('model')!r}") + if PROMPT_CANARY not in json.dumps(request.get("messages", [])): + raise AssertionError("Hermes model request did not contain the prompt canary") + if RESPONSE_CANARY not in result.stdout: + raise AssertionError("Hermes did not print the mock model response") + + telemetry = home / "telemetry" / "shared_metrics" + counters = _validate_store(telemetry / "metrics.sqlite3") + package_path, package = _validate_package( + telemetry / "outbox", + hermes_repo + / "hermes_cli" + / "observability" + / "schemas" + / "hermes.shared_metrics.v1.schema.json", + ) + + print("Hermes -> NeMo Relay shared-metrics smoke test passed") + print(f"Artifact directory: {root}") + print(f"Model requests: {len(_ModelHandler.requests)}") + print(f"SQLite counters: {json.dumps(counters, indent=2)}") + print(f"Export package: {package_path}") + print(json.dumps(package, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/agent/test_auxiliary_relay.py b/tests/agent/test_auxiliary_relay.py new file mode 100644 index 000000000000..0902f0fb91c3 --- /dev/null +++ b/tests/agent/test_auxiliary_relay.py @@ -0,0 +1,445 @@ +from types import SimpleNamespace + +import pytest + +pytest.importorskip("nemo_relay") + +from agent import auxiliary_client, relay_llm, relay_runtime + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + try: + yield lease.host.relay, turn + finally: + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_auxiliary_retries_share_logical_relay_identity(monkeypatch): + attempts = [] + logical_completions = [] + responses = iter([ + SimpleNamespace(choices=[]), + SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ), + ]) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: next(responses), + ) + ) + ) + + def execute_current(request, callback, **kwargs): + attempts.append(kwargs) + return callback(request) + + monkeypatch.setattr(relay_llm, "execute_current", execute_current) + monkeypatch.setattr( + relay_llm, + "complete_logical_call", + lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + result = run("compression") + + assert result.choices[0].message.content == "ok" + assert attempts[0]["metadata"]["api_request_id"] == ( + attempts[1]["metadata"]["api_request_id"] + ) + assert [attempt["metadata"]["retry_count"] for attempt in attempts] == [0, 1] + assert attempts[0]["metadata"]["call_role"] == "auxiliary:compression" + assert all(attempt["defer_logical_completion"] is True for attempt in attempts) + assert logical_completions == [ + (attempts[0]["metadata"]["api_request_id"], "success") + ] + + +@pytest.mark.asyncio +async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch): + captured = {} + logical_completions = [] + + async def create(**kwargs): + return SimpleNamespace( + request=kwargs, + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))], + ) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + async def execute_current_async(request, callback, **kwargs): + captured.update(kwargs) + return await callback(request) + + monkeypatch.setattr( + relay_llm, + "execute_current_async", + execute_current_async, + ) + monkeypatch.setattr( + relay_llm, + "complete_logical_call", + lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ) + + @auxiliary_client._relay_auxiliary_call_async + async def run(task): + auxiliary_client._set_relay_auxiliary_route( + "anthropic", + "claude-test", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ), + task, + ) + + result = await run("title_generation") + + assert result.request["model"] == "claude-test" + assert captured["name"] == "anthropic" + assert captured["metadata"]["call_role"] == "auxiliary:title_generation" + assert captured["defer_logical_completion"] is True + assert logical_completions == [ + (captured["metadata"]["api_request_id"], "success") + ] + + +def test_terminal_auxiliary_failure_stays_failed_when_caller_catches_it( + relay_turn, monkeypatch +): + _relay, turn = relay_turn + consumer = "test.terminal-auxiliary-failure" + turn.lease.host.retain_managed_execution(consumer) + outcomes = [] + original_pop = turn.lease.host.relay.scope.pop + + def record_pop(*args, **kwargs): + outcomes.append((kwargs.get("output") or {}).get("outcome")) + return original_pop(*args, **kwargs) + + monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: SimpleNamespace(choices=[]), + ) + ) + ) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + assert len(turn.logical_llm_calls) == 1 + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + try: + with pytest.raises(RuntimeError, match="invalid response"): + run("compression") + + assert outcomes == ["failed"] + assert turn.logical_llm_calls == {} + + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + + assert outcomes == ["failed", "success"] + finally: + turn.lease.host.release_managed_execution(consumer) + + +@pytest.mark.asyncio +async def test_async_terminal_auxiliary_failure_closes_logical_call(relay_turn): + _relay, turn = relay_turn + consumer = "test.async-terminal-auxiliary-failure" + turn.lease.host.retain_managed_execution(consumer) + + async def create(**_kwargs): + return SimpleNamespace(choices=[]) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + + @auxiliary_client._relay_auxiliary_call_async + async def run(task): + auxiliary_client._set_relay_auxiliary_route( + "anthropic", + "claude-test", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ), + task, + ) + assert len(turn.logical_llm_calls) == 1 + return auxiliary_client._validate_llm_response( + await auxiliary_client._relay_async_completion( + client, + {"model": "claude-test", "messages": []}, + ), + task, + ) + + try: + with pytest.raises(RuntimeError, match="invalid response"): + await run("title_generation") + + assert turn.logical_llm_calls == {} + finally: + turn.lease.host.release_managed_execution(consumer) + + +def test_auxiliary_stream_uses_streaming_relay_primitive(monkeypatch): + captured = {} + raw_stream = iter([{"delta": "one"}, {"delta": "two"}]) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace(create=lambda **_kwargs: raw_stream) + ) + ) + + def stream_current(request, stream_factory, **kwargs): + captured.update(kwargs) + return stream_factory(request) + + monkeypatch.setattr(relay_llm, "stream_current", stream_current) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "moa-model", + "chat_completions", + ) + return auxiliary_client._relay_sync_stream( + client, + {"model": "moa-model", "messages": [], "stream": True}, + ) + + assert list(run("moa")) == [{"delta": "one"}, {"delta": "two"}] + assert captured["metadata"]["call_role"] == "auxiliary:moa" + + +def test_partial_auxiliary_stream_failure_closes_before_recovery( + relay_turn, monkeypatch +): + _relay, turn = relay_turn + consumer = "test.partial-auxiliary-stream-failure" + turn.lease.host.retain_managed_execution(consumer) + outcomes = [] + original_pop = turn.lease.host.relay.scope.pop + + def record_pop(*args, **kwargs): + outcomes.append((kwargs.get("output") or {}).get("outcome")) + return original_pop(*args, **kwargs) + + monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop) + + class ProviderError(Exception): + pass + + provider_error = ProviderError("stream failed") + partial_chunk = SimpleNamespace( + model="test-model", + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="partial", tool_calls=None), + finish_reason=None, + ) + ], + usage=None, + ) + + def partial_stream(): + yield partial_chunk + raise provider_error + + stream_client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: partial_stream(), + ) + ) + ) + recovery_client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: SimpleNamespace( + choices=[ + SimpleNamespace(message=SimpleNamespace(content="recovered")) + ] + ), + ) + ) + ) + + @auxiliary_client._relay_auxiliary_call + def start_stream(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._relay_sync_stream( + stream_client, + {"model": "test-model", "messages": [], "stream": True}, + ) + + @auxiliary_client._relay_auxiliary_call + def recover(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + recovery_client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + try: + stream = start_stream("moa") + assert next(stream) is partial_chunk + + with pytest.raises(ProviderError) as caught: + next(stream) + + assert caught.value is provider_error + assert outcomes == ["failed"] + assert turn.logical_llm_calls == {} + + result = recover("moa") + + assert result.choices[0].message.content == "recovered" + assert outcomes == ["failed", "success"] + assert turn.logical_llm_calls == {} + finally: + turn.lease.host.release_managed_execution(consumer) + + +def test_auxiliary_attempt_uses_real_relay_request_intercepts(relay_turn): + relay, turn = relay_turn + consumer = "test.auxiliary-request-intercept" + turn.lease.host.retain_managed_execution(consumer) + captured_requests = [] + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **kwargs: captured_requests.append(kwargs) + or SimpleNamespace( + choices=[ + SimpleNamespace(message=SimpleNamespace(content="ok")) + ] + ), + ) + ) + ) + + def rewrite_request(_name, request, annotated): + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + relay.intercepts.register_llm_request( + "hermes-auxiliary-request", + 1, + False, + rewrite_request, + ) + try: + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "openrouter", + "test-model", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "test-model", "messages": []}, + ), + task, + ) + + result = run("compression") + finally: + relay.intercepts.deregister_llm_request("hermes-auxiliary-request") + turn.lease.host.release_managed_execution(consumer) + + assert result.choices[0].message.content == "ok" + assert captured_requests[0]["temperature"] == 0.25 + assert turn.logical_llm_calls == {} diff --git a/tests/agent/test_bedrock_interrupt_post_worker.py b/tests/agent/test_bedrock_interrupt_post_worker.py index 0ee5e4fce33d..52c6877d1f48 100644 --- a/tests/agent/test_bedrock_interrupt_post_worker.py +++ b/tests/agent/test_bedrock_interrupt_post_worker.py @@ -86,7 +86,21 @@ def test_bedrock_stream_returns_normally_when_not_interrupted(): agent._interrupt_requested = False resp = SimpleNamespace(choices=[], usage=None, stop_reason="end_turn") - fake_client = SimpleNamespace(converse_stream=lambda **kw: {"stream": []}) + + class ProviderStream: + def __init__(self): + self.closed = False + + def __iter__(self): + return iter(()) + + def close(self): + self.closed = True + + provider_stream = ProviderStream() + fake_client = SimpleNamespace( + converse_stream=lambda **kw: {"stream": provider_stream} + ) with patch("agent.bedrock_adapter._get_bedrock_runtime_client", return_value=fake_client), \ patch("agent.bedrock_adapter.stream_converse_with_callbacks", return_value=resp), \ @@ -97,3 +111,4 @@ def test_bedrock_stream_returns_normally_when_not_interrupted(): api_kwargs = {"__bedrock_region__": "us-east-1", "__bedrock_converse__": True} out = cch.interruptible_streaming_api_call(agent, api_kwargs) assert out is resp + assert provider_stream.closed is True diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py new file mode 100644 index 000000000000..0266d21bd2f2 --- /dev/null +++ b/tests/agent/test_relay_llm.py @@ -0,0 +1,1498 @@ +"""Tests for the core Relay-managed physical LLM attempt adapter.""" + +from __future__ import annotations + +import asyncio +import contextvars +import json +from types import SimpleNamespace + +import pytest + +pytest.importorskip("nemo_relay") + +from agent import relay_llm, relay_runtime + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + lease.host.retain_managed_execution("test.relay_llm") + try: + yield lease.host.relay, turn + finally: + lease.host.release_managed_execution("test.relay_llm") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn): + relay, turn = relay_turn + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + content = {**request.content, "temperature": 0.25} + return relay.LLMRequestInterceptOutcome( + relay.LLMRequest(request.headers, content), + annotated, + ) + + def rewrite_stream(request, next_call): + async def generate(): + upstream = await next_call(request) + async for chunk in upstream: + updated = dict(chunk) + choices = [dict(choice) for choice in updated.get("choices", [])] + if choices: + delta = dict(choices[0].get("delta") or {}) + if delta.get("content"): + delta["content"] = delta["content"].upper() + choices[0]["delta"] = delta + updated["choices"] = choices + yield updated + + return generate() + + def raw_stream(request): + captured_requests.append(request) + return iter([ + SimpleNamespace( + model="test-model", + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="hello", tool_calls=None), + finish_reason=None, + ) + ], + usage=None, + ), + SimpleNamespace( + model="test-model", + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content=None, tool_calls=None), + finish_reason="stop", + ) + ], + usage=None, + ), + ]) + + relay.intercepts.register_llm_request( + "hermes-test-request", + 1, + False, + rewrite_request, + ) + relay.intercepts.register_llm_stream_execution( + "hermes-test-stream", + 1, + rewrite_stream, + ) + try: + stream = relay_llm.stream( + { + "model": "test-model", + "messages": [], + "extra_headers": {"authorization": "Bearer provider-token"}, + }, + raw_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: { + "model": "test-model", + "choices": [ + { + "message": {"role": "assistant", "content": "HELLO"}, + "finish_reason": "stop", + } + ], + }, + metadata={ + "api_mode": "custom", + "api_request_id": "request-1", + "call_role": "primary", + }, + ) + chunks = list(stream) + finally: + relay.intercepts.deregister_llm_stream_execution("hermes-test-stream") + relay.intercepts.deregister_llm_request("hermes-test-request") + + assert captured_requests[0]["temperature"] == 0.25 + assert captured_requests[0]["extra_headers"] == { + "authorization": "Bearer provider-token" + } + assert chunks[0].choices[0].delta.content == "HELLO" + assert stream.output_modified is True + assert turn.logical_llm_calls == {} + + +def test_deferred_stream_preserves_provider_error_and_logical_scope_for_retry( + relay_turn, +): + _relay, turn = relay_turn + + class ProviderError(Exception): + pass + + provider_error = ProviderError("provider failed") + + def failing_stream(_request): + def generate(): + raise provider_error + yield # pragma: no cover + + return generate() + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + failing_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-2", + }, + defer_logical_completion=True, + ) + + with pytest.raises(ProviderError) as caught: + list(stream) + + assert caught.value is provider_error + assert "request-2" in turn.logical_llm_calls + + +def test_non_deferred_partial_stream_close_cancels_logical_call( + relay_turn, + monkeypatch, +): + relay, turn = relay_turn + original_pop = relay.scope.pop + terminal_outputs = [] + + def record_pop(handle, *args, **kwargs): + terminal_outputs.append(kwargs.get("output")) + return original_pop(handle, *args, **kwargs) + + monkeypatch.setattr(relay.scope, "pop", record_pop) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "partial"}, {"delta": "unused"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "partial"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-partial-close", + }, + ) + + assert next(stream) == {"delta": "partial"} + assert "request-partial-close" in turn.logical_llm_calls + + stream.close() + + assert "request-partial-close" not in turn.logical_llm_calls + assert {"outcome": "cancelled"} in terminal_outputs + + +def test_direct_stream_close_reaches_original_provider_resource(monkeypatch): + class ProviderStream: + def __init__(self): + self.closed = False + + def __iter__(self): + return iter([{"delta": "partial"}, {"delta": "unused"}]) + + def close(self): + self.closed = True + + provider_stream = ProviderStream() + monkeypatch.setattr( + relay_runtime, + "resolve_execution_context", + lambda _session_id: (None, None, None), + ) + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: provider_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + ) + + assert next(stream) == {"delta": "partial"} + stream.close() + + assert provider_stream.closed is True + + +def test_anthropic_stream_accumulator_merges_terminal_usage(): + accumulator = relay_llm.AnthropicStreamAccumulator() + accumulator.observe({ + "type": "message_start", + "message": { + "id": "message-1", + "type": "message", + "role": "assistant", + "model": "claude-test", + "usage": { + "input_tokens": 100, + "output_tokens": 1, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 30, + }, + }, + }) + accumulator.observe({ + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 12}, + }) + + response = accumulator.finalize() + + assert response["usage"] == { + "input_tokens": 100, + "output_tokens": 12, + "cache_creation_input_tokens": 20, + "cache_read_input_tokens": 30, + } + + +def test_anthropic_stream_accumulator_merges_plain_provider_object(): + accumulator = relay_llm.AnthropicStreamAccumulator() + accumulator.observe({ + "type": "message_start", + "message": { + "id": "message-1", + "type": "message", + "role": "assistant", + "model": "claude-test", + "usage": {"input_tokens": 10}, + }, + }) + accumulator.observe({ + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": "hello"}, + }) + + response = accumulator.response( + SimpleNamespace( + id="message-1", + type="message", + role="assistant", + model="claude-test", + content=[], + stop_reason=None, + usage={"input_tokens": 10}, + ) + ) + + assert response.id == "message-1" + assert response.content[0].text == "hello" + assert response.usage.input_tokens == 10 + + +def test_jsonable_does_not_probe_dynamic_attributes(): + class DynamicProviderObject: + def __getattr__(self, name): + raise AssertionError(f"unexpected dynamic attribute lookup: {name}") + + def __str__(self): + return "opaque-provider-object" + + assert relay_llm._jsonable(DynamicProviderObject()) == "opaque-provider-object" + + +def test_non_stream_preserves_raw_provider_response_identity(relay_turn): + _relay, _turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-raw"}, + ) + + assert result is raw_response + + +def test_non_stream_provider_callback_preserves_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar("llm_caller_value", default="default") + caller_value.set("caller") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"caller_value": caller_value.get()}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-context"}, + ) + + assert result == {"caller_value": "caller"} + + +@pytest.mark.asyncio +async def test_async_provider_callback_preserves_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar( + "async_llm_caller_value", + default="default", + ) + caller_value.set("caller") + + async def provider(_request): + await asyncio.sleep(0) + return {"caller_value": caller_value.get()} + + result = await relay_llm.execute_async( + {"model": "test-model", "messages": []}, + provider, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-async-context", + }, + ) + + assert result == {"caller_value": "caller"} + + +def test_stream_provider_callbacks_preserve_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar( + "stream_llm_caller_value", + default="default", + ) + caller_value.set("caller") + observed = [] + + def stream_factory(_request): + observed.append(("factory", caller_value.get())) + + def generate(): + observed.append(("next", caller_value.get())) + yield {"delta": "hello"} + + return generate() + + def on_chunk(_chunk): + observed.append(("chunk", caller_value.get())) + + def finalizer(): + observed.append(("finalizer", caller_value.get())) + return {"content": "hello"} + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + stream_factory, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=finalizer, + on_chunk=on_chunk, + metadata={ + "api_mode": "custom", + "api_request_id": "request-stream-context", + }, + ) + + assert list(stream) == [{"delta": "hello"}] + assert observed == [ + ("factory", "caller"), + ("next", "caller"), + ("chunk", "caller"), + ("finalizer", "caller"), + ] + + +def test_non_stream_does_not_forward_relay_session_headers(relay_turn): + _relay, _turn = relay_turn + captured_requests = [] + + relay_llm.execute( + { + "model": "test-model", + "messages": [], + "extra_headers": {"x-provider-header": "provider-value"}, + }, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-headers"}, + ) + + assert captured_requests[0]["extra_headers"] == { + "x-provider-header": "provider-value" + } + + +def test_non_stream_defers_logical_success_and_reuses_scope_for_retry(relay_turn): + _relay, turn = relay_turn + metadata = {"api_mode": "custom", "api_request_id": "request-retry"} + + first = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "invalid"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata=metadata, + defer_logical_completion=True, + ) + first_handle = turn.logical_llm_calls["request-retry"] + + second = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "valid"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata=metadata, + defer_logical_completion=True, + ) + + assert first == {"content": "invalid"} + assert second == {"content": "valid"} + assert turn.logical_llm_calls == {"request-retry": first_handle} + + relay_llm.complete_logical_call("request-retry", outcome="success") + + assert turn.logical_llm_calls == {} + + +def test_non_stream_result_survives_logical_scope_close_failure( + relay_turn, monkeypatch +): + relay, turn = relay_turn + original_pop = relay.scope.pop + pop_calls = 0 + + def fail_first_pop(*args, **kwargs): + nonlocal pop_calls + pop_calls += 1 + if pop_calls == 1: + raise RuntimeError("simulated logical scope close failure") + return original_pop(*args, **kwargs) + + monkeypatch.setattr(relay.scope, "pop", fail_first_pop) + raw_response = SimpleNamespace(model="test-model", content="raw") + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-close"}, + ) + + assert result is raw_response + assert "request-close" in turn.logical_llm_calls + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + assert turn.logical_llm_calls == {} + + +def test_non_stream_returns_provider_response_after_relay_post_processing_failure( + relay_turn, monkeypatch, caplog +): + relay, turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def fail_after_callback(_name, request, callback, **_kwargs): + callback(request) + raise RuntimeError("simulated Relay post-processing failure") + + monkeypatch.setattr(relay.llm, "execute", fail_after_callback) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: raw_response, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-post-failure", + }, + ) + + assert result is raw_response + assert turn.logical_llm_calls == {} + assert "returning the provider response" in caplog.text + + +def test_non_stream_does_not_swallow_interrupt_after_provider_success( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + async def interrupt_after_callback(_name, request, callback, **_kwargs): + callback(request) + raise KeyboardInterrupt + + monkeypatch.setattr(relay.llm, "execute", interrupt_after_callback) + + with pytest.raises(KeyboardInterrupt): + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "already returned"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-post-interrupt", + }, + ) + + assert "request-post-interrupt" in turn.logical_llm_calls + relay_llm.complete_logical_call("request-post-interrupt", outcome="cancelled") + + +@pytest.mark.asyncio +async def test_async_non_stream_preserves_raw_provider_response_identity(relay_turn): + _relay, _turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def provider(_request): + return raw_response + + result = await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async"}, + ) + + assert result is raw_response + + +@pytest.mark.asyncio +async def test_async_non_stream_returns_provider_response_after_relay_failure( + relay_turn, monkeypatch, caplog +): + relay, turn = relay_turn + raw_response = SimpleNamespace(model="test-model", content="raw") + + async def provider(_request): + return raw_response + + async def fail_after_callback(_name, request, callback, **_kwargs): + await callback(request) + raise RuntimeError("simulated Relay post-processing failure") + + monkeypatch.setattr(relay.llm, "execute", fail_after_callback) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + result = await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-async-post-failure", + }, + ) + + assert result is raw_response + assert turn.logical_llm_calls == {} + assert "returning the provider response" in caplog.text + + +@pytest.mark.asyncio +async def test_async_non_stream_does_not_swallow_cancellation_after_provider_success( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + async def provider(_request): + return {"content": "already returned"} + + async def cancel_after_callback(_name, request, callback, **_kwargs): + await callback(request) + raise asyncio.CancelledError + + monkeypatch.setattr(relay.llm, "execute", cancel_after_callback) + + with pytest.raises(asyncio.CancelledError): + await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "custom", + "api_request_id": "request-async-post-cancel", + }, + ) + + assert "request-async-post-cancel" in turn.logical_llm_calls + relay_llm.complete_logical_call( + "request-async-post-cancel", + outcome="cancelled", + ) + + +@pytest.mark.asyncio +async def test_async_non_stream_defers_logical_success_for_validation(relay_turn): + _relay, turn = relay_turn + + async def provider(_request): + return {"content": "pending-validation"} + + await relay_llm.execute_current_async( + {"model": "test-model", "messages": []}, + provider, + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async-defer"}, + defer_logical_completion=True, + ) + + assert "request-async-defer" in turn.logical_llm_calls + + relay_llm.complete_logical_call("request-async-defer", outcome="success") + + assert turn.logical_llm_calls == {} + + +def test_stream_finishes_after_relay_post_processing_failure( + relay_turn, monkeypatch, caplog +): + relay, turn = relay_turn + + async def fail_after_stream( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + async for chunk in upstream: + observe_chunk(chunk) + yield chunk + finalizer() + raise RuntimeError("simulated Relay post-processing failure") + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", fail_after_stream) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "complete"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-stream-post-failure", + }, + ) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + chunks = list(stream) + + assert chunks == [{"delta": "complete"}] + assert turn.logical_llm_calls == {} + assert "preserving the provider result" in caplog.text + + +def test_stream_flushes_buffered_provider_chunks_after_relay_failure( + relay_turn, monkeypatch +): + relay, turn = relay_turn + raw_chunks = [{"delta": "first"}, {"delta": "second"}] + + async def fail_with_buffered_chunk( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + first = await anext(upstream) + observe_chunk(first) + yield first + second = await anext(upstream) + observe_chunk(second) + with pytest.raises(StopAsyncIteration): + await anext(upstream) + finalizer() + raise RuntimeError("simulated buffered Relay failure") + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", fail_with_buffered_chunk) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter(raw_chunks), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-buffered-failure", + }, + ) + + assert list(stream) == raw_chunks + assert turn.logical_llm_calls == {} + + +def test_stream_constructor_flushes_provider_chunks_after_relay_failure( + relay_turn, monkeypatch +): + relay, turn = relay_turn + raw_chunks = [{"delta": "first"}, {"delta": "second"}] + + async def fail_during_stream_setup( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + upstream = callback(request) + async for chunk in upstream: + observe_chunk(chunk) + finalizer() + raise RuntimeError("simulated Relay setup failure") + + monkeypatch.setattr(relay.llm, "stream_execute", fail_during_stream_setup) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter(raw_chunks), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-setup-failure", + }, + ) + + assert list(stream) == raw_chunks + assert turn.logical_llm_calls == {} + + +def test_stream_does_not_swallow_interrupt_after_provider_success( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + async def interrupt_after_stream( + _name, + request, + callback, + observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + async for chunk in upstream: + observe_chunk(chunk) + yield chunk + finalizer() + raise KeyboardInterrupt + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", interrupt_after_stream) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "complete"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "complete"}, + metadata={ + "api_mode": "custom", + "api_request_id": "request-stream-post-interrupt", + }, + ) + + assert next(stream) == {"delta": "complete"} + with pytest.raises(KeyboardInterrupt): + next(stream) + assert turn.logical_llm_calls == {} + + +def test_stream_does_not_swallow_hermes_finalizer_failure(relay_turn, monkeypatch): + relay, _turn = relay_turn + finalizer_error = RuntimeError("Hermes finalizer failed") + + def fail_finalizer(): + raise finalizer_error + + async def execute_stream( + _name, + request, + callback, + _observe_chunk, + finalizer, + **_kwargs, + ): + async def generate(): + upstream = callback(request) + async for chunk in upstream: + yield chunk + finalizer() + + return generate() + + monkeypatch.setattr(relay.llm, "stream_execute", execute_stream) + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "complete"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=fail_finalizer, + metadata={ + "api_mode": "custom", + "api_request_id": "request-finalizer-failure", + }, + ) + + with pytest.raises(Exception) as caught: + list(stream) + + assert caught.value is finalizer_error + + +def test_stream_defers_logical_success_for_response_validation(relay_turn): + _relay, turn = relay_turn + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + lambda _request: iter([{"delta": "pending-validation"}]), + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=lambda: {"content": "pending-validation"}, + metadata={"api_mode": "custom", "api_request_id": "request-stream-defer"}, + defer_logical_completion=True, + ) + + assert list(stream) == [{"delta": "pending-validation"}] + assert stream.output_modified is False + assert "request-stream-defer" in turn.logical_llm_calls + + relay_llm.complete_logical_call("request-stream-defer", outcome="success") + + assert turn.logical_llm_calls == {} + + +def test_current_attempt_bypasses_relay_without_an_active_turn(monkeypatch): + monkeypatch.setattr(relay_runtime, "current_turn", lambda: None) + request = {"model": "test-model", "messages": []} + + result = relay_llm.execute_current( + request, + lambda value: value, + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_non_stream_bypasses_relay_without_an_active_consumer(relay_turn, monkeypatch): + relay, turn = relay_turn + turn.lease.host.release_managed_execution("test.relay_llm") + request = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]} + + monkeypatch.setattr( + relay.llm, + "execute", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not manage the provider call") + ), + ) + + result = relay_llm.execute( + request, + lambda value: value, + session_id="session-1", + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_stream_bypasses_relay_without_an_active_consumer(relay_turn, monkeypatch): + relay, turn = relay_turn + turn.lease.host.release_managed_execution("test.relay_llm") + request = {"model": "test-model", "messages": [{"role": "user", "content": "hi"}]} + observed = [] + + monkeypatch.setattr( + relay.llm, + "stream_execute", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not manage the provider stream") + ), + ) + + stream = relay_llm.stream( + request, + lambda value: (observed.append(value), iter([{"delta": "ok"}]))[1], + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + ) + + assert list(stream) == [{"delta": "ok"}] + assert observed == [request] + + +def test_bypassed_stream_still_honors_chunk_acceptance(relay_turn): + _relay, turn = relay_turn + turn.lease.host.release_managed_execution("test.relay_llm") + provider_closed = [] + + def provider_stream(_request): + try: + yield {"delta": "accepted"} + yield {"delta": "rejected"} + yield {"delta": "unreachable"} + finally: + provider_closed.append(True) + + stream = relay_llm.stream( + {"model": "test-model", "messages": []}, + provider_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=dict, + accept_chunk=lambda chunk: chunk["delta"] != "rejected", + ) + + assert list(stream) == [{"delta": "accepted"}] + assert provider_closed == [True] + + +def test_anthropic_codec_preserves_tool_history_and_cached_system_blocks(relay_turn): + _relay, _turn = relay_turn + request = { + "model": "claude-sonnet-4-5", + "max_tokens": 512, + "system": [ + { + "type": "text", + "text": "You are Hermes.", + "cache_control": {"type": "ephemeral"}, + } + ], + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Run pwd"}]}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "terminal", + "input": {"command": "pwd"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": "/tmp/worktree"}], + } + ], + }, + ], + } + original_wire = json.dumps(request, ensure_ascii=False, separators=(",", ":")) + observed_body_wire = "" + + def provider(final_request): + nonlocal observed_body_wire + provider_body = { + key: value for key, value in final_request.items() if key != "extra_headers" + } + observed_body_wire = json.dumps( + provider_body, + ensure_ascii=False, + separators=(",", ":"), + ) + return { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + + relay_llm.execute( + request, + provider, + session_id="session-1", + name="anthropic", + model_name="claude-sonnet-4-5", + metadata={ + "api_mode": "anthropic_messages", + "api_request_id": "request-anthropic", + }, + ) + + assert observed_body_wire == original_wire + + +def test_current_attempt_bypasses_a_closed_turn_from_a_copied_context( + relay_turn, + monkeypatch, +): + _relay, turn = relay_turn + stale_context = contextvars.copy_context() + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + request = {"model": "test-model", "messages": []} + + def fail_execute(*_args, **_kwargs): + raise AssertionError("a closed turn must not manage later provider work") + + monkeypatch.setattr(relay_llm, "execute", fail_execute) + + result = stale_context.run( + relay_llm.execute_current, + request, + lambda value: value, + name="test-provider", + model_name="test-model", + ) + + assert result is request + + +def test_non_stream_returns_post_execution_interceptor_result(relay_turn, monkeypatch): + relay, _turn = relay_turn + + async def post_execute(_name, request, callback, **_kwargs): + response = callback(request) + return {**response, "post_interceptor": True} + + monkeypatch.setattr(relay.llm, "execute", post_execute) + + result = relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: {"content": "raw"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-post"}, + ) + + assert result.content == "raw" + assert result.post_interceptor is True + + +@pytest.mark.asyncio +async def test_async_non_stream_returns_namespaced_interceptor_result( + relay_turn, + monkeypatch, +): + relay, _turn = relay_turn + + async def post_execute(_name, request, callback, **_kwargs): + response = await callback(request) + return { + **response, + "post_interceptor": True, + "usage": {"input_tokens": 10}, + } + + monkeypatch.setattr(relay.llm, "execute", post_execute) + + async def provider(_request): + return {"content": "raw"} + + result = await relay_llm.execute_async( + {"model": "test-model", "messages": []}, + provider, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-async-post"}, + ) + + assert result.content == "raw" + assert result.post_interceptor is True + assert result.usage.input_tokens == 10 + + +def test_non_stream_preserves_provider_error_from_relay_wrapper_suffix( + relay_turn, monkeypatch +): + relay, turn = relay_turn + + class ProviderError(Exception): + pass + + provider_error = ProviderError("provider failed") + + async def wrapping_execute(_name, request, callback, **_kwargs): + try: + return callback(request) + except Exception as exc: + raise RuntimeError( + f"internal error: {type(exc).__name__}: {exc} (retried 3x)" + ) from None + + monkeypatch.setattr(relay.llm, "execute", wrapping_execute) + + with pytest.raises(ProviderError) as caught: + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: (_ for _ in ()).throw(provider_error), + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-error"}, + ) + + assert caught.value is provider_error + assert "request-error" in turn.logical_llm_calls + + +def test_non_stream_does_not_mask_relay_error_after_callback_failure( + relay_turn, monkeypatch +): + relay, _turn = relay_turn + provider_error = RuntimeError("provider failed") + relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") + + async def translating_execute(_name, request, callback, **_kwargs): + try: + callback(request) + except Exception: + raise relay_error + + monkeypatch.setattr(relay.llm, "execute", translating_execute) + + with pytest.raises(RuntimeError) as caught: + relay_llm.execute( + {"model": "test-model", "messages": []}, + lambda _request: (_ for _ in ()).throw(provider_error), + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={"api_mode": "custom", "api_request_id": "request-policy"}, + ) + + assert caught.value is relay_error + + +def test_chat_codec_preserves_provider_message_extensions_after_rewrite(relay_turn): + relay, _turn = relay_turn + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + def provider(request): + captured_requests.append(request) + return { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + } + + relay.intercepts.register_llm_request( + "hermes-provider-extension-request", + 1, + False, + rewrite_request, + ) + try: + relay_llm.execute( + { + "model": "test-model", + "messages": [ + { + "role": "assistant", + "content": "", + "reasoning_content": "provider scratchpad", + } + ], + }, + provider, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-3", + }, + ) + finally: + relay.intercepts.deregister_llm_request( + "hermes-provider-extension-request" + ) + + assert captured_requests[0]["temperature"] == 0.25 + assert captured_requests[0]["messages"][0]["reasoning_content"] == ( + "provider scratchpad" + ) + + +def test_request_rewrite_preserves_unmodified_provider_objects(relay_turn): + relay, _turn = relay_turn + timeout = object() + captured_requests = [] + + def rewrite_request(name, request, annotated): + del name + annotated.params = {**(annotated.params or {}), "temperature": 0.25} + return relay.LLMRequestInterceptOutcome(request, annotated) + + relay.intercepts.register_llm_request( + "hermes-provider-object-request", + 1, + False, + rewrite_request, + ) + try: + relay_llm.execute( + {"model": "test-model", "messages": [], "timeout": timeout}, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-provider-object", + }, + ) + finally: + relay.intercepts.deregister_llm_request( + "hermes-provider-object-request" + ) + + assert captured_requests[0]["timeout"] is timeout + assert captured_requests[0]["temperature"] == 0.25 + + +def test_request_rewrite_preserves_fields_dropped_by_codec(relay_turn, monkeypatch): + relay, _turn = relay_turn + captured_requests = [] + vendor_body = { + "routing": {"provider": "nim", "region": "us-west-2"}, + "trace_vendor_request": False, + } + + async def lossy_execute(_name, request, callback, **_kwargs): + content = { + key: value + for key, value in request.content.items() + if key != "extra_body" + } + content["temperature"] = 0.25 + return callback(relay.LLMRequest(request.headers, content)) + + monkeypatch.setattr(relay.llm, "execute", lossy_execute) + monkeypatch.setattr( + relay_llm, + "_codec_round_trip_request_body", + lambda *_args, relay_request_body, **_kwargs: { + key: value + for key, value in relay_request_body.items() + if key != "extra_body" + }, + ) + + relay_llm.execute( + { + "model": "test-model", + "messages": [], + "temperature": 0.0, + "extra_body": vendor_body, + }, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-lossy-codec", + }, + ) + + assert captured_requests == [ + { + "model": "test-model", + "messages": [], + "temperature": 0.25, + "extra_body": vendor_body, + } + ] + + +def test_request_rewrite_is_ignored_when_codec_baseline_fails( + relay_turn, monkeypatch +): + relay, _turn = relay_turn + captured_requests = [] + original = { + "model": "test-model", + "messages": [], + "temperature": 0.0, + "extra_body": {"routing": {"provider": "nim"}}, + } + + async def lossy_execute(_name, request, callback, **_kwargs): + rewritten = { + key: value + for key, value in request.content.items() + if key != "extra_body" + } + rewritten["temperature"] = 0.25 + return callback(relay.LLMRequest(request.headers, rewritten)) + + monkeypatch.setattr(relay.llm, "execute", lossy_execute) + monkeypatch.setattr( + relay_llm, + "_codec_round_trip_request_body", + lambda *_args, **_kwargs: None, + ) + + relay_llm.execute( + original, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-codec-failure", + }, + ) + + assert captured_requests == [original] + + +def test_codec_baseline_failure_is_explicit(relay_turn, monkeypatch, caplog): + relay, _turn = relay_turn + request_body = {"model": "test-model", "messages": []} + request = relay.LLMRequest({}, request_body) + + class FailingCodec: + def decode(self, _request): + raise RuntimeError("simulated codec failure") + + monkeypatch.setattr(relay_llm, "_codec", lambda *_args, **_kwargs: FailingCodec()) + + with caplog.at_level("WARNING", logger="agent.relay_llm"): + baseline = relay_llm._codec_round_trip_request_body( + relay, + request, + relay_request_body=request_body, + metadata={"api_mode": "chat_completions"}, + ) + + assert baseline is None + assert "ignoring request rewrites" in caplog.text + + +def test_request_rewrite_can_remove_codec_represented_field(relay_turn, monkeypatch): + relay, _turn = relay_turn + captured_requests = [] + + async def remove_temperature(_name, request, callback, **_kwargs): + content = dict(request.content) + content.pop("temperature") + return callback(relay.LLMRequest(request.headers, content)) + + monkeypatch.setattr(relay.llm, "execute", remove_temperature) + + relay_llm.execute( + { + "model": "test-model", + "messages": [], + "temperature": 0.25, + "extra_body": {"routing": {"provider": "nim"}}, + }, + lambda request: captured_requests.append(request) or {"content": "ok"}, + session_id="session-1", + name="test-provider", + model_name="test-model", + metadata={ + "api_mode": "chat_completions", + "api_request_id": "request-remove-field", + }, + ) + + assert captured_requests == [ + { + "model": "test-model", + "messages": [], + "extra_body": {"routing": {"provider": "nim"}}, + } + ] diff --git a/tests/agent/test_relay_tools.py b/tests/agent/test_relay_tools.py new file mode 100644 index 000000000000..8f605dfd4d9d --- /dev/null +++ b/tests/agent/test_relay_tools.py @@ -0,0 +1,262 @@ +"""Tests for the core Relay-managed Hermes tool adapter.""" + +from __future__ import annotations + +import contextvars +import json + +import pytest + +pytest.importorskip("nemo_relay") + +from agent import relay_runtime, relay_tools + + +@pytest.fixture() +def relay_turn(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="session-1", + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + lease.host.retain_managed_execution("test.relay_tools") + try: + yield lease.host.relay + finally: + lease.host.release_managed_execution("test.relay_tools") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + +def test_tool_adapter_bypasses_relay_without_an_active_consumer( + relay_turn, monkeypatch +): + relay = relay_turn + runtime = relay_runtime.get_runtime() + assert runtime is not None + runtime.release_managed_execution("test.relay_tools") + args = {"command": "pwd"} + + monkeypatch.setattr( + relay.tools, + "execute", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not manage the tool call") + ), + ) + + result, final_args = relay_tools.execute( + "terminal", + args, + lambda value: value, + session_id="session-1", + ) + + assert result is args + assert final_args is args + + +def test_tool_request_intercepts_bypass_relay_without_an_active_consumer( + relay_turn, monkeypatch +): + relay = relay_turn + runtime = relay_runtime.get_runtime() + assert runtime is not None + runtime.release_managed_execution("test.relay_tools") + args = {"command": "pwd"} + + monkeypatch.setattr( + relay.tools, + "request_intercepts", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("inactive Relay must not run tool request intercepts") + ), + ) + + final_args = runtime.apply_tool_request_intercepts( + session_id="session-1", + tool_name="terminal", + args=args, + ) + + assert final_args is args + + +def test_request_rewrite_reaches_authorized_callback_once(relay_turn): + relay = relay_turn + callback_args = [] + + def rewrite_request(_name, args): + return {**args, "path": "/approved/path"} + + async def wrap_execution(_name, args, next_call): + result = await next_call(args) + return relay.ToolExecutionInterceptOutcome({**result, "wrapped": True}) + + relay.intercepts.register_tool_request( + "hermes-test-tool-request", 1, False, rewrite_request + ) + relay.intercepts.register_tool_execution( + "hermes-test-tool-execution", 1, wrap_execution + ) + try: + result, observed_args = relay_tools.execute( + "write_file", + {"path": "/original/path"}, + lambda args: callback_args.append(args) or {"ok": True}, + session_id="session-1", + metadata={"tool_call_id": "call-1"}, + ) + finally: + relay.intercepts.deregister_tool_execution("hermes-test-tool-execution") + relay.intercepts.deregister_tool_request("hermes-test-tool-request") + + assert callback_args == [{"path": "/approved/path"}] + assert observed_args == {"path": "/approved/path"} + assert isinstance(result, str) + assert json.loads(result) == {"ok": True, "wrapped": True} + + +def test_authorized_tool_callback_preserves_caller_context(relay_turn): + del relay_turn + caller_value = contextvars.ContextVar("tool_caller_value", default="default") + caller_value.set("caller") + + result, _observed_args = relay_tools.execute( + "terminal", + {"command": "true"}, + lambda _args: caller_value.get(), + session_id="session-1", + ) + + assert result == "caller" + + +def test_provider_error_identity_is_preserved(relay_turn): + del relay_turn + + class ToolError(Exception): + pass + + tool_error = ToolError("dispatch failed") + + def fail(_args): + raise tool_error + + with pytest.raises(ToolError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + fail, + session_id="session-1", + ) + + assert caught.value is tool_error + + +def test_tool_error_is_preserved_from_relay_wrapper_suffix(relay_turn, monkeypatch): + relay = relay_turn + + class ToolError(Exception): + pass + + tool_error = ToolError("dispatch failed") + + async def wrapping_execute(_name, args, callback, **_kwargs): + try: + return callback(args) + except Exception as exc: + raise RuntimeError( + f"internal error: {type(exc).__name__}: {exc} (worker trace)" + ) from None + + monkeypatch.setattr(relay.tools, "execute", wrapping_execute) + + with pytest.raises(ToolError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + lambda _args: (_ for _ in ()).throw(tool_error), + session_id="session-1", + ) + + assert caught.value is tool_error + + +def test_tool_adapter_does_not_mask_relay_error_after_callback_failure( + relay_turn, monkeypatch +): + relay = relay_turn + tool_error = RuntimeError("dispatch failed") + relay_error = RuntimeError("internal error: RelayPolicyError: policy blocked") + + async def translating_execute(_name, args, callback, **_kwargs): + try: + callback(args) + except Exception: + raise relay_error + + monkeypatch.setattr(relay.tools, "execute", translating_execute) + + with pytest.raises(RuntimeError) as caught: + relay_tools.execute( + "terminal", + {"command": "false"}, + lambda _args: (_ for _ in ()).throw(tool_error), + session_id="session-1", + ) + + assert caught.value is relay_error + + +def test_tool_adapter_returns_dispatch_result_after_relay_post_processing_failure( + relay_turn, monkeypatch, caplog +): + relay = relay_turn + raw_result = '{"ok":true}' + + async def fail_after_callback(_name, args, callback, **_kwargs): + callback(args) + raise RuntimeError("simulated Relay post-processing failure") + + monkeypatch.setattr(relay.tools, "execute", fail_after_callback) + + with caplog.at_level("WARNING", logger="agent.relay_tools"): + result, observed_args = relay_tools.execute( + "terminal", + {"command": "pwd"}, + lambda _args: raw_result, + session_id="session-1", + ) + + assert result is raw_result + assert observed_args == {"command": "pwd"} + assert "returning the Hermes tool result" in caplog.text + + +def test_tool_adapter_does_not_swallow_interrupt_after_dispatch_success( + relay_turn, monkeypatch +): + relay = relay_turn + + async def interrupt_after_callback(_name, args, callback, **_kwargs): + callback(args) + raise KeyboardInterrupt + + monkeypatch.setattr(relay.tools, "execute", interrupt_after_callback) + + with pytest.raises(KeyboardInterrupt): + relay_tools.execute( + "terminal", + {"command": "pwd"}, + lambda _args: '{"ok":true}', + session_id="session-1", + ) diff --git a/tests/cli/test_session_boundary_hooks.py b/tests/cli/test_session_boundary_hooks.py index 52c64c01c2d8..0a78ad1f88c5 100644 --- a/tests/cli/test_session_boundary_hooks.py +++ b/tests/cli/test_session_boundary_hooks.py @@ -10,8 +10,12 @@ def test_session_hooks_in_valid_hooks(): assert "on_session_reset" in VALID_HOOKS -@patch("hermes_cli.plugins.invoke_hook") -def test_session_finalize_on_reset(mock_invoke_hook): +# These tests pin CLI ownership of the finalize request. The end-to-end +# built-in/core/plugin dispatch order is exercised by +# tests/hermes_cli/test_lifecycle.py::test_finalize_session_closes_core_before_plugin_export. +@patch("hermes_cli.lifecycle.invoke_hook") +@patch("hermes_cli.lifecycle.finalize_session") +def test_session_finalize_on_reset(mock_finalize_session, mock_invoke_hook): """Verify on_session_finalize fires when /new or /reset is used.""" cli = HermesCLI() cli.agent = MagicMock() @@ -22,10 +26,10 @@ def test_session_finalize_on_reset(mock_invoke_hook): # Check if on_session_finalize was called for the old session assert any( - c.args == ("on_session_finalize",) + not c.args and c.kwargs["session_id"] == "test-session-id" and c.kwargs["platform"] == "cli" - for c in mock_invoke_hook.call_args_list + for c in mock_finalize_session.call_args_list ) # Check if on_session_reset was called for the new session assert any( @@ -36,8 +40,8 @@ def test_session_finalize_on_reset(mock_invoke_hook): ) -@patch("hermes_cli.plugins.invoke_hook") -def test_session_finalize_on_cleanup(mock_invoke_hook): +@patch("hermes_cli.lifecycle.finalize_session") +def test_session_finalize_on_cleanup(mock_finalize_session): """Verify on_session_finalize fires during CLI exit cleanup.""" import cli as cli_mod @@ -49,15 +53,15 @@ def test_session_finalize_on_cleanup(mock_invoke_hook): cli_mod._run_cleanup() assert any( - c.args == ("on_session_finalize",) + not c.args and c.kwargs["session_id"] == "cleanup-session-id" and c.kwargs["platform"] == "cli" and c.kwargs["reason"] == "shutdown" - for c in mock_invoke_hook.call_args_list + for c in mock_finalize_session.call_args_list ) -@patch("hermes_cli.plugins.invoke_hook") +@patch("hermes_cli.lifecycle.invoke_hook") def test_interrupted_session_end_helper_emits_observer_shape(mock_invoke_hook): """Verify quiet single-query interruption emits a correlated session end.""" import cli as cli_mod diff --git a/tests/hermes_cli/test_lifecycle.py b/tests/hermes_cli/test_lifecycle.py new file mode 100644 index 000000000000..999d6057b3bf --- /dev/null +++ b/tests/hermes_cli/test_lifecycle.py @@ -0,0 +1,60 @@ +from types import SimpleNamespace + +from agent import relay_runtime +from hermes_cli import lifecycle, observability, plugins + + +def test_invoke_hook_notifies_builtin_observers_before_plugins(monkeypatch): + calls = [] + manager = SimpleNamespace( + invoke_hook=lambda name, **kwargs: calls.append(("plugin", name, kwargs)) or ["ok"] + ) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda name, **kwargs: calls.append(("builtin", name, kwargs)), + ) + monkeypatch.setattr(plugins, "invoke_hook", manager.invoke_hook) + + result = lifecycle.invoke_hook("on_session_start", session_id="session-1") + + assert result == ["ok"] + assert [call[0] for call in calls] == ["builtin", "plugin"] + + +def test_finalize_session_closes_core_before_plugin_export(monkeypatch): + calls = [] + manager = SimpleNamespace( + invoke_hook=lambda name, **kwargs: calls.append(("plugin", name, kwargs)) or [] + ) + coordinator = SimpleNamespace( + finalize_conversation=lambda **kwargs: calls.append(("core", kwargs)) + ) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda name, **kwargs: calls.append(("builtin", name, kwargs)), + ) + monkeypatch.setattr(plugins, "invoke_hook", manager.invoke_hook) + monkeypatch.setattr(relay_runtime, "SESSION_COORDINATOR", coordinator) + monkeypatch.setattr(relay_runtime, "current_profile_key", lambda: "profile-1") + + lifecycle.finalize_session(session_id="session-1", platform="cli") + + assert [call[0] for call in calls] == ["builtin", "core", "plugin"] + assert calls[1][1] == { + "profile_key": "profile-1", + "session_id": "session-1", + } + + +def test_plugin_only_dispatch_does_not_reenter_builtin_observers(monkeypatch): + manager = SimpleNamespace(invoke_hook=lambda name, **kwargs: [name, kwargs]) + monkeypatch.setattr(plugins, "get_plugin_manager", lambda: manager) + monkeypatch.setattr( + observability, + "observe_lifecycle", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unexpected")), + ) + + assert plugins.invoke_hook("custom", value=1) == ["custom", {"value": 1}] diff --git a/tests/hermes_cli/test_plugins.py b/tests/hermes_cli/test_plugins.py index ee9746a34de9..2a6de58ac702 100644 --- a/tests/hermes_cli/test_plugins.py +++ b/tests/hermes_cli/test_plugins.py @@ -189,6 +189,32 @@ class TestPluginDiscovery: assert result.changed is True assert result.trace == [{"source": "same-payload"}] + def test_tool_request_middleware_hides_internal_skip_relay_flag( + self, + monkeypatch, + ): + observed = [] + + def middleware(**kwargs): + observed.append(kwargs) + return {"args": kwargs["args"]} + + manager = types.SimpleNamespace( + _middleware={"tool_request": [middleware]}, + invoke_middleware=lambda kind, **kwargs: [middleware(**kwargs)], + ) + monkeypatch.setattr("hermes_cli.plugins.get_plugin_manager", lambda: manager) + + apply_tool_request_middleware( + "read_file", + {"path": "README.md"}, + session_id="", + skip_relay=True, + ) + + assert len(observed) == 1 + assert "skip_relay" not in observed[0] + def test_execution_middleware_post_next_call_error_does_not_retry(self, monkeypatch): calls = [] diff --git a/tests/hermes_cli/test_relay_shared_metrics.py b/tests/hermes_cli/test_relay_shared_metrics.py new file mode 100644 index 000000000000..058e07db7bc7 --- /dev/null +++ b/tests/hermes_cli/test_relay_shared_metrics.py @@ -0,0 +1,839 @@ +"""Focused tests for the Hermes shared-metrics durable store.""" + +from __future__ import annotations + +import json +import multiprocessing as mp +import os +import sqlite3 +import stat +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from hermes_cli.observability.shared_metrics import SharedMetricsStore +from hermes_cli.observability.shared_metrics_contract import ( + COUNT_BUCKETS, + DURATION_BUCKETS, + EXECUTION_SURFACES, + MODEL_FAMILIES, + MODEL_LOCALITIES, + MODEL_OUTCOMES, + PRIMARY_MODEL_CALL_ROLE, + PROVIDER_FAMILIES, + TASK_END_REASONS, + TASK_ENTRYPOINTS, + TASK_OUTCOMES, + TASK_TERMINATIONS, + count_bucket, + duration_bucket, + execution_surface, + model_call_outcome, + model_call_dimensions, + model_family, + model_locality, + provider_family, + task_counter, + task_start_fields, + task_terminal_fields, + task_terminal_state, +) + + +SCHEMA_PATH = ( + Path(__file__).resolve().parents[2] + / "hermes_cli" + / "observability" + / "schemas" + / "hermes.shared_metrics.v1.schema.json" +) + + +def _schema_validator(): + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(schema) + return jsonschema.Draft202012Validator( + schema, + format_checker=jsonschema.FormatChecker(), + ) + + +def _package_dimension_schema() -> dict[str, object]: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + return schema["$defs"]["model_call_counter"]["properties"]["dimensions"] + + +def _task_dimension_schema(kind: str) -> dict[str, object]: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + return schema["$defs"][kind]["properties"]["dimensions"] + + +def _dimensions() -> dict[str, str]: + return { + "call_role": PRIMARY_MODEL_CALL_ROLE, + "locality": "remote", + "model_family": "claude", + "outcome": "success", + "provider_family": "direct", + } + + +def _record_model_calls_in_process( + database_path: str, + outbox_directory: str, + count: int, + start_barrier: Any | None = None, +) -> None: + if start_barrier is not None: + start_barrier.wait() + store = SharedMetricsStore(Path(database_path), Path(outbox_directory)) + for _ in range(count): + store.record_model_call(_dimensions(), "test-version") + + +def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + store.record_model_call(_dimensions(), "test-version") + + first_paths = store.create_and_export_package() + + assert len(first_paths) == 1 + first_package = json.loads(first_paths[0].read_text(encoding="utf-8")) + _schema_validator().validate(first_package) + uuid.UUID(first_package["package_id"]) + uuid.UUID(first_package["install_id"]) + assert first_package["schema_version"] == "hermes.shared_metrics.v1" + assert first_package["resource"] == {"hermes_version": "test-version"} + assert first_package["metrics"] == [ + { + "name": "hermes.model_call.count", + "type": "counter", + "dimensions": _dimensions(), + "value": 2, + } + ] + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 2 + assert restarted.counter_snapshot()[0]["packaged_value"] == 2 + assert restarted.create_and_export_package() == [] + assert len(list(outbox_directory.glob("*.json"))) == 1 + + restarted.record_model_call(_dimensions(), "test-version") + second_paths = restarted.create_and_export_package() + + assert len(second_paths) == 1 + second_package = json.loads(second_paths[0].read_text(encoding="utf-8")) + assert second_package["package_id"] != first_package["package_id"] + assert second_package["install_id"] == first_package["install_id"] + assert second_package["metrics"][0]["value"] == 1 + assert restarted.counter_snapshot()[0]["value"] == 3 + assert restarted.counter_snapshot()[0]["packaged_value"] == 3 + + +def test_package_schema_matches_the_model_call_contract(): + properties = _package_dimension_schema()["properties"] + + assert properties["call_role"] == {"const": PRIMARY_MODEL_CALL_ROLE} + assert set(properties["locality"]["enum"]) == MODEL_LOCALITIES + assert set(properties["model_family"]["enum"]) == MODEL_FAMILIES + assert set(properties["outcome"]["enum"]) == MODEL_OUTCOMES + assert set(properties["provider_family"]["enum"]) == PROVIDER_FAMILIES + + +def test_package_schema_matches_the_task_contract(): + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + start = _task_dimension_schema("task_started_counter")["properties"] + terminal = _task_dimension_schema("task_finished_counter")["properties"] + + assert set(schema["$defs"]["execution_surface"]["enum"]) == EXECUTION_SURFACES + assert set(schema["$defs"]["task_entrypoint"]["enum"]) == TASK_ENTRYPOINTS + assert set(schema["$defs"]["duration_bucket"]["enum"]) == DURATION_BUCKETS + assert set(schema["$defs"]["count_bucket"]["enum"]) == COUNT_BUCKETS + assert start["entrypoint"] == {"$ref": "#/$defs/task_entrypoint"} + assert set(terminal["end_reason"]["enum"]) == TASK_END_REASONS + assert set(terminal["outcome"]["enum"]) == TASK_OUTCOMES + assert set(terminal["termination"]["enum"]) == TASK_TERMINATIONS + + +@pytest.mark.parametrize( + ("provider", "expected"), + [ + ("", "unknown"), + ("not-a-hermes-provider", "unknown"), + ("custom", "custom"), + ("custom-local", "custom"), + ("custom:private-endpoint", "custom"), + ("lmstudio", "local"), + ("lm_studio", "local"), + ("ollama", "local"), + ("nous", "aggregator"), + ("openrouter", "aggregator"), + ("kilo", "aggregator"), + ("copilot-acp", "aggregator"), + ("huggingface", "aggregator"), + ("novita", "aggregator"), + ("anthropic", "direct"), + ("google", "direct"), + ("openai-api", "direct"), + ], +) +def test_provider_family_uses_bounded_product_categories(provider, expected): + assert provider_family({"provider": provider}) == expected + + +def test_provider_family_does_not_resolve_live_provider_metadata(monkeypatch): + def fail_live_lookup(_provider): + raise AssertionError("telemetry must not refresh provider metadata") + + monkeypatch.setattr("hermes_cli.providers.get_provider", fail_live_lookup) + assert provider_family({"provider": "anthropic"}) == "direct" + + +def test_locality_uses_the_endpoint_only_for_local_classification(): + kwargs = { + "provider": "custom", + "base_url": "http://127.0.0.1:11434/v1", + } + + assert provider_family(kwargs) == "custom" + assert model_locality(kwargs) == "local" + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("google/gemma-3", "gemma"), + ("x-ai/grok-4", "grok"), + ("minimax/minimax-m2.5", "minimax"), + ("xiaomi/mimo-v2", "mimo"), + ("amazon/nova-pro", "nova"), + ("stepfun/step-3.5", "step"), + ("arcee-ai/trinity-large", "trinity"), + ], +) +def test_model_family_covers_families_evidenced_by_the_hermes_catalog(model, expected): + assert model_family({"model": model}) == expected + + +@pytest.mark.parametrize( + "model", + [ + "private-gptish-model", + "innovation-private", + "mimosa-private", + "stepstone-private", + "supernova-private", + ], +) +def test_model_family_requires_identifier_boundaries(model): + assert model_family({"model": model}) == "unknown" + + +def test_model_family_accepts_only_allowlisted_declared_metadata(): + assert model_family({"model": "private", "model_family": "qwen"}) == "qwen" + assert model_family({"model": "private", "model_family": "private"}) == "unknown" + + +def test_model_family_prefers_the_provider_reported_terminal_model(): + assert ( + model_family({"model": "gpt-5", "response_model": "claude-sonnet"}) == "claude" + ) + + +@pytest.mark.parametrize( + ("platform", "expected"), + [ + ("", "unknown"), + ("cli", "cli"), + ("api_server", "api"), + ("cron", "scheduled_task"), + ("whatsapp_cloud", "gateway"), + ("private-surface", "other"), + ], +) +def test_execution_surface_uses_the_hermes_platform_registry(platform, expected): + assert execution_surface({"platform": platform}) == expected + + +@pytest.mark.parametrize( + ("platform", "expected"), + [ + ("cli", "interactive"), + ("tui", "interactive"), + ("whatsapp_cloud", "gateway_message"), + ("cron", "scheduled_task"), + ("api_server", "api"), + ("private-surface", "other"), + ], +) +def test_task_start_fields_use_bounded_surface_and_entrypoint(platform, expected): + fields = task_start_fields({"platform": platform}) + + assert fields["entrypoint"] == expected + assert fields["execution_surface"] in EXECUTION_SURFACES + + +def test_task_start_fields_identify_delegated_work_without_exporting_parent_id(): + fields = task_start_fields({ + "platform": "cli", + "parent_session_id": "private-parent-session", + }) + + assert fields == { + "entrypoint": "delegated", + "execution_surface": "cli", + } + assert "private-parent-session" not in json.dumps(fields) + + +@pytest.mark.parametrize( + ("duration_ms", "expected"), + [ + (0, "lt_1s"), + (999, "lt_1s"), + (1_000, "1s_to_5s"), + (5_000, "5s_to_30s"), + (30_000, "30s_to_2m"), + (120_000, "2m_to_10m"), + (600_000, "gte_10m"), + ], +) +def test_duration_bucket_boundaries(duration_ms, expected): + assert duration_bucket(duration_ms) == expected + + +@pytest.mark.parametrize( + ("count", "expected"), + [ + (0, "0"), + (1, "1"), + (2, "2"), + (3, "3_to_5"), + (6, "6_to_10"), + (11, "gte_11"), + ], +) +def test_count_bucket_boundaries(count, expected): + assert count_bucket(count) == expected + + +@pytest.mark.parametrize( + ("event", "expected"), + [ + ( + {"completed": True, "turn_exit_reason": "text_response(stop)"}, + ("success", "completed", "none"), + ), + ( + {"failed": True, "turn_exit_reason": "all_retries_exhausted_no_response"}, + ("failed", "failed", "none"), + ), + ( + {"interrupted": True, "turn_exit_reason": "interrupted_by_user"}, + ("cancelled", "user_cancelled", "user_cancelled"), + ), + ( + {"turn_exit_reason": "budget_exhausted"}, + ("failed", "iteration_limit", "system_aborted"), + ), + ( + {"turn_exit_reason": "guardrail_halt"}, + ("failed", "guardrail_blocked", "system_aborted"), + ), + ( + {"failed": True, "turn_exit_reason": "provider_timeout"}, + ("timed_out", "timed_out", "timed_out"), + ), + ( + {"failed": True, "turn_exit_reason": "approval_denied"}, + ("failed", "approval_denied", "none"), + ), + ], +) +def test_task_terminal_state_is_bounded(event, expected): + assert task_terminal_state(event) == expected + + +def test_model_outcome_fails_closed_to_a_bounded_value(): + assert model_call_outcome({"outcome": "private"}) == "failed" + + +def test_unlisted_model_collapses_to_a_bounded_value(): + assert model_family({"model": "private-model-name"}) == "unknown" + + +def test_subscriber_contract_rejects_unknown_fields_and_dimension_values(): + event = SimpleNamespace( + kind="scope", + category="llm", + category_profile={"model_name": "gpt"}, + name="hermes.model_call", + scope_category="end", + metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"}, + data={ + "call_role": "primary", + "locality": "remote", + "model_family": "gpt", + "outcome": "success", + "provider_family": "direct", + }, + ) + + assert model_call_dimensions(event) == { + "call_role": "primary", + "locality": "remote", + "model_family": "gpt", + "outcome": "success", + "provider_family": "direct", + } + event.category_profile["model_name"] = "private-model-name" + assert model_call_dimensions(event) is None + event.category_profile["model_name"] = "gpt" + event.data["prompt"] = "must-not-pass" + assert model_call_dimensions(event) is None + event.data.pop("prompt") + event.metadata["prompt"] = "must-not-pass" + assert model_call_dimensions(event) is None + event.metadata.pop("prompt") + event.category_profile["private"] = "must-not-pass" + assert model_call_dimensions(event) is None + event.category_profile.pop("private") + event.category = "function" + assert model_call_dimensions(event) is None + + +def test_task_subscriber_contract_accepts_only_bounded_scope_events(): + start = SimpleNamespace( + kind="scope", + category="function", + category_profile=None, + name="hermes.task_run", + scope_category="start", + metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"}, + data={"entrypoint": "interactive", "execution_surface": "cli"}, + ) + assert task_counter(start) == ( + "hermes.task_run.started", + {"entrypoint": "interactive", "execution_surface": "cli"}, + ) + + terminal_fields = task_terminal_fields( + { + "platform": "cli", + "completed": True, + "turn_exit_reason": "text_response(stop)", + }, + duration_ms=6_000, + model_call_count=2, + tool_call_count=3, + retry_count=1, + ) + end = SimpleNamespace(**{ + **start.__dict__, + "scope_category": "end", + "data": terminal_fields, + }) + assert task_counter(end) == ( + "hermes.task_run.finished", + terminal_fields, + ) + + end.data["task_id"] = "must-not-pass" + assert task_counter(end) is None + end.data.pop("task_id") + end.data["outcome"] = "private" + assert task_counter(end) is None + end.data["outcome"] = "success" + end.metadata["prompt"] = "must-not-pass" + assert task_counter(end) is None + + +def test_store_rejects_an_unsupported_schema_version(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + connection.execute( + "INSERT INTO telemetry_state(key, value) VALUES ('schema_version', '999')" + ) + + with pytest.raises(RuntimeError, match="Unsupported shared-metrics store schema"): + SharedMetricsStore(database_path, tmp_path / "outbox") + + with sqlite3.connect(database_path) as connection: + [schema_version] = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'schema_version'" + ).fetchone() + assert schema_version == "999" + + +def test_pending_metrics_keep_the_version_recorded_at_event_time(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_model_call(_dimensions(), "version-a") + store.record_model_call(_dimensions(), "version-b") + + packages = [ + json.loads(path.read_text(encoding="utf-8")) + for path in store.create_and_export_package() + ] + + assert {package["resource"]["hermes_version"] for package in packages} == { + "version-a", + "version-b", + } + assert all(package["metrics"][0]["value"] == 1 for package in packages) + + +def test_store_exports_task_started_and_terminal_counters(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_counter( + "hermes.task_run.started", + {"entrypoint": "interactive", "execution_surface": "cli"}, + "test-version", + ) + terminal = task_terminal_fields( + { + "platform": "cli", + "completed": True, + "turn_exit_reason": "text_response(stop)", + }, + duration_ms=2_000, + model_call_count=1, + tool_call_count=2, + retry_count=0, + ) + store.record_counter("hermes.task_run.finished", terminal, "test-version") + + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + _schema_validator().validate(package) + + assert {metric["name"] for metric in package["metrics"]} == { + "hermes.task_run.finished", + "hermes.task_run.started", + } + + +def test_package_schema_rejects_unknown_fields(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + store.record_model_call(_dimensions(), "test-version") + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + invalid_package = deepcopy(package) + invalid_package["prompt"] = "must-not-be-accepted" + + jsonschema = pytest.importorskip("jsonschema") + with pytest.raises(jsonschema.ValidationError): + _schema_validator().validate(invalid_package) + + +def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + + with pytest.raises(ValueError, match="Unsupported dimensions"): + store.record_counter( + "hermes.model_call.count", + {"prompt": "must-not-be-persisted"}, + "test-version", + ) + + assert store.counter_snapshot() == [] + + +def test_package_builder_rejects_tampered_dimensions(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + with sqlite3.connect(database_path) as connection: + connection.execute( + "UPDATE counter_aggregates SET dimensions_json = ?", + (json.dumps({"prompt": "must-not-be-exported"}),), + ) + + with pytest.raises(ValueError, match="Unsupported dimensions"): + store.create_and_export_package() + + assert list(outbox_directory.glob("*.json")) == [] + + +def test_pending_package_retry_reuses_the_same_package_and_file(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + [package_path] = store.create_and_export_package() + original_payload = package_path.read_bytes() + + with sqlite3.connect(database_path) as connection: + connection.execute("UPDATE package_outbox SET exported_at = NULL") + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.create_and_export_package() == [package_path] + assert package_path.read_bytes() == original_payload + assert list(outbox_directory.glob("*.json")) == [package_path] + + +def test_retention_prunes_only_expired_exported_history(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + + store.record_model_call(_dimensions(), "expired-version") + [expired_path] = store.create_and_export_package() + store.record_model_call(_dimensions(), "current-version") + [current_path] = store.create_and_export_package() + store.record_model_call(_dimensions(), "pending-version") + pending_package = store._create_package() + assert pending_package is not None + + with sqlite3.connect(database_path) as connection: + connection.execute( + """ + UPDATE counter_aggregates + SET period_start = '2026-05-01' + WHERE hermes_version = 'expired-version' + """ + ) + connection.execute( + """ + UPDATE package_outbox + SET period_start = '2026-05-01T00:00:00Z', + period_end = '2026-05-02T00:00:00Z', + exported_at = '2026-05-02T00:00:00Z' + WHERE package_id = ? + """, + (expired_path.stem,), + ) + + store._prune_expired_history( + now=datetime(2026, 7, 23, tzinfo=timezone.utc) + ) + + assert not expired_path.exists() + assert current_path.exists() + assert not (outbox_directory / f"{pending_package['package_id']}.json").exists() + with sqlite3.connect(database_path) as connection: + outbox_rows = connection.execute( + "SELECT package_id, exported_at FROM package_outbox ORDER BY package_id" + ).fetchall() + aggregate_versions = { + row[0] + for row in connection.execute( + "SELECT hermes_version FROM counter_aggregates" + ).fetchall() + } + assert {row[0] for row in outbox_rows} == { + current_path.stem, + pending_package["package_id"], + } + assert next( + row[1] for row in outbox_rows if row[0] == pending_package["package_id"] + ) is None + assert aggregate_versions == {"current-version", "pending-version"} + + +def test_retention_failure_does_not_fail_a_committed_export(tmp_path, monkeypatch): + store = SharedMetricsStore( + tmp_path / "metrics.sqlite3", + tmp_path / "outbox", + ) + store.record_model_call(_dimensions(), "test-version") + + def fail_pruning(): + raise OSError("retention unavailable") + + monkeypatch.setattr(store, "_prune_expired_history", fail_pruning) + + [package_path] = store.create_and_export_package() + + assert package_path.exists() + assert store.counter_snapshot()[0]["packaged_value"] == 1 + + +def test_file_export_failure_retries_committed_outbox_without_duplicate_delta( + tmp_path, monkeypatch +): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + + def fail_write(*_args, **_kwargs): + raise OSError("simulated atomic export failure") + + module_globals = SharedMetricsStore._export_pending_packages.__globals__ + original_write = module_globals["atomic_json_write"] + monkeypatch.setitem(module_globals, "atomic_json_write", fail_write) + with pytest.raises(OSError, match="simulated atomic export failure"): + store.create_and_export_package() + + with sqlite3.connect(database_path) as connection: + package_id, exported_at = connection.execute( + "SELECT package_id, exported_at FROM package_outbox" + ).fetchone() + assert exported_at is None + assert store.counter_snapshot()[0]["packaged_value"] == 1 + assert list(outbox_directory.glob("*.json")) == [] + + monkeypatch.setitem(module_globals, "atomic_json_write", original_write) + assert store.create_and_export_package() == [ + outbox_directory / f"{package_id}.json" + ] + assert len(list(outbox_directory.glob("*.json"))) == 1 + assert store.create_and_export_package() == [] + + +def test_package_export_does_not_chase_concurrent_updates(tmp_path, monkeypatch): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + original_create = store._create_package + create_calls = 0 + + def create_and_record_another(): + nonlocal create_calls + create_calls += 1 + package = original_create() + if create_calls == 1: + store.record_model_call(_dimensions(), "test-version") + return package + + monkeypatch.setattr(store, "_create_package", create_and_record_another) + first_paths = store.create_and_export_package() + + assert create_calls == 1 + assert len(first_paths) == 1 + [counter] = store.counter_snapshot() + assert counter["metric_name"] == "hermes.model_call.count" + assert counter["dimensions"] == _dimensions() + assert counter["value"] == 2 + assert counter["packaged_value"] == 1 + + second_paths = store.create_and_export_package() + assert len(second_paths) == 1 + assert store.counter_snapshot()[0]["packaged_value"] == 2 + + +def test_concurrent_package_builders_commit_one_delta(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + ready = threading.Barrier(2) + + def export() -> list[Path]: + worker_store = SharedMetricsStore(database_path, outbox_directory) + ready.wait(timeout=5) + return worker_store.create_and_export_package() + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(export) for _ in range(2)] + for future in futures: + future.result() + + with sqlite3.connect(database_path) as connection: + [outbox_count] = connection.execute( + "SELECT COUNT(*) FROM package_outbox" + ).fetchone() + [package_path] = list(outbox_directory.glob("*.json")) + package = json.loads(package_path.read_text(encoding="utf-8")) + + assert outbox_count == 1 + assert package["metrics"][0]["value"] == 1 + assert store.counter_snapshot()[0]["packaged_value"] == 1 + + +def test_concurrent_model_call_updates_are_transactional(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + SharedMetricsStore(database_path, outbox_directory) + + def record_calls(count: int) -> None: + store = SharedMetricsStore(database_path, outbox_directory) + for _ in range(count): + store.record_model_call(_dimensions(), "test-version") + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(record_calls, 10) for _ in range(2)] + for future in futures: + future.result() + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 20 + + +def test_cross_process_model_call_updates_are_transactional(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + context = mp.get_context("spawn") + start_barrier = context.Barrier(2) + processes = [ + context.Process( + target=_record_model_calls_in_process, + args=(str(database_path), str(outbox_directory), 10, start_barrier), + ) + for _ in range(2) + ] + + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + assert not process.is_alive() + assert process.exitcode == 0 + + restarted = SharedMetricsStore(database_path, outbox_directory) + assert restarted.counter_snapshot()[0]["value"] == 20 + + +def test_schema_initialization_waits_for_an_existing_writer(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + database_path.touch() + blocker = sqlite3.connect(database_path) + blocker.execute("BEGIN IMMEDIATE") + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + SharedMetricsStore, + database_path, + outbox_directory, + ) + try: + time.sleep(0.4) + assert not future.done() + finally: + blocker.rollback() + blocker.close() + store = future.result(timeout=2) + + assert store.counter_snapshot() == [] + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX permission modes are unavailable") +def test_store_and_export_are_owner_only(tmp_path): + database_path = tmp_path / "private-store" / "metrics.sqlite3" + outbox_directory = tmp_path / "private-outbox" + store = SharedMetricsStore(database_path, outbox_directory) + store.record_model_call(_dimensions(), "test-version") + [package_path] = store.create_and_export_package() + + assert stat.S_IMODE(database_path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(outbox_directory.stat().st_mode) == 0o700 + assert stat.S_IMODE(database_path.stat().st_mode) == 0o600 + assert stat.S_IMODE(package_path.stat().st_mode) == 0o600 diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py new file mode 100644 index 000000000000..23258def3385 --- /dev/null +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -0,0 +1,2403 @@ +"""Tests for the direct Hermes-to-Relay shared-metrics runtime.""" + +from __future__ import annotations + +import contextvars +import asyncio +import json +import threading +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from hermes_cli import lifecycle, plugins +from hermes_cli.observability import relay_runtime, relay_shared_metrics +from hermes_cli.plugins import PluginManager + + +class _Request: + def __init__(self, headers: dict[str, Any], content: dict[str, Any]) -> None: + self.headers = headers + self.content = content + + +class _Relay: + def __init__(self) -> None: + self.events: list[tuple[Any, ...]] = [] + self._callbacks: dict[str, Any] = {} + self._starts: dict[Any, dict[str, Any]] = {} + self._scope_starts: dict[Any, dict[str, Any]] = {} + self._scope = contextvars.ContextVar("relay_scope", default=None) + self._scope_serial = 0 + self.ScopeType = SimpleNamespace(Agent="agent", Function="function") + self.LLMRequest = _Request + self.scope = SimpleNamespace( + push=self._scope_push, + pop=self._scope_pop, + event=self._scope_event, + ) + self.llm = SimpleNamespace(call=self._llm_call, call_end=self._llm_call_end) + self.subscribers = SimpleNamespace( + register=self._register, + deregister=self._deregister, + flush=self._flush, + ) + self.get_scope_stack = self._get_scope_stack + + def _scope_push(self, name: str, scope_type: Any, **kwargs: Any) -> Any: + self._scope_serial += 1 + handle = ("scope", name, self._scope_serial) + self._scope.set(handle) + self.events.append(("scope.push", name, scope_type, kwargs)) + if scope_type == self.ScopeType.Function: + self._scope_starts[handle] = kwargs + event = SimpleNamespace( + kind="scope", + category="function", + name=name, + scope_category="start", + category_profile=None, + metadata=kwargs.get("metadata"), + data=kwargs.get("input"), + ) + for callback in list(self._callbacks.values()): + callback(event) + return handle + + def _scope_pop(self, handle: Any, **kwargs: Any) -> None: + self.events.append(("scope.pop", handle, kwargs)) + start = self._scope_starts.pop(handle, None) + if start is not None: + event = SimpleNamespace( + kind="scope", + category="function", + name=handle[1], + scope_category="end", + category_profile=None, + metadata={ + **(start.get("metadata") or {}), + **(kwargs.get("metadata") or {}), + }, + data=kwargs.get("output"), + ) + for callback in list(self._callbacks.values()): + callback(event) + + def _scope_event(self, name: str, **kwargs: Any) -> None: + self.events.append(("scope.event", name, kwargs)) + + def _get_scope_stack(self) -> Any: + current = self._scope.get() + self.events.append(("scope.sync", current)) + return current + + def _llm_call( + self, + name: str, + request: _Request, + **kwargs: Any, + ) -> Any: + handle = ("llm", name, len(self._starts)) + self._starts[handle] = kwargs + self.events.append(("llm.call", name, request.content, kwargs)) + return handle + + def _llm_call_end( + self, + handle: Any, + response: dict[str, Any], + **kwargs: Any, + ) -> None: + start = self._starts.pop(handle) + self.events.append(("llm.call_end", handle, response, kwargs)) + event = SimpleNamespace( + kind="scope", + category="llm", + name=handle[1], + scope_category="end", + category_profile={"model_name": start["model_name"]}, + metadata={ + **start["metadata"], + **kwargs["metadata"], + "otel.status_code": "OK", + }, + data=response, + ) + for callback in list(self._callbacks.values()): + callback(event) + + def _register(self, name: str, callback: Any) -> None: + self._callbacks[name] = callback + self.events.append(("subscribers.register", name)) + + def _deregister(self, name: str) -> None: + self._callbacks.pop(name, None) + self.events.append(("subscribers.deregister", name)) + + def _flush(self) -> None: + self.events.append(("subscribers.flush",)) + + +@pytest.fixture +def direct_runtime(tmp_path, monkeypatch): + fake = _Relay() + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + yield fake + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +@pytest.fixture +def real_binding_runtime(tmp_path, monkeypatch): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + yield relay + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_path): + base = { + "session_id": "sensitive-session", + "task_id": "task-1", + "api_request_id": "request-1", + "platform": "cli", + "provider": "custom", + "model": "gpt-sensitive-model-id", + "base_url": "http://127.0.0.1:11434/v1", + } + + assert lifecycle.has_hook("pre_api_request") + lifecycle.invoke_hook("on_session_start", **base) + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook( + "pre_api_request", + **base, + request={"body": {"messages": ["sensitive-prompt"]}}, + ) + lifecycle.invoke_hook( + "post_tool_call", + **base, + tool_call_id="sensitive-tool-call", + tool_name="terminal", + args={"command": "sensitive-command"}, + result={"output": "sensitive-tool-result"}, + status="ok", + ) + lifecycle.invoke_hook( + "api_request_error", + **base, + retryable=True, + error={"message": "sensitive-error"}, + ) + lifecycle.invoke_hook( + "pre_api_request", + **{ + **base, + "provider": "anthropic", + "model": "claude-sonnet", + "base_url": "https://api.anthropic.com", + }, + request={"body": {"messages": ["sensitive-prompt"]}}, + ) + lifecycle.invoke_hook( + "post_api_request", + **{ + **base, + "provider": "anthropic", + "model": "claude-sonnet", + "base_url": "https://api.anthropic.com", + }, + response={"content": "sensitive-response"}, + ) + lifecycle.invoke_hook( + "on_session_end", + **base, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id=base["session_id"]) + + starts = [event for event in direct_runtime.events if event[0] == "llm.call"] + ends = [event for event in direct_runtime.events if event[0] == "llm.call_end"] + scope_starts = [ + event for event in direct_runtime.events if event[0] == "scope.push" + ] + assert len(scope_starts) == 2 + assert scope_starts[0][2] == direct_runtime.ScopeType.Agent + assert scope_starts[1][1] == "hermes.task_run" + assert scope_starts[1][2] == direct_runtime.ScopeType.Function + assert scope_starts[1][3]["handle"][1] == relay_runtime.SESSION_SCOPE + assert scope_starts[1][3]["input"] == { + "entrypoint": "interactive", + "execution_surface": "cli", + } + assert len(starts) == 1 + assert len(ends) == 1 + assert starts[0][2] == {} + assert starts[0][3]["model_name"] == "gpt" + assert ends[0][2] == { + "call_role": "primary", + "locality": "remote", + "model_family": "claude", + "outcome": "success", + "provider_family": "direct", + } + serialized_events = json.dumps(direct_runtime.events) + assert "sensitive-prompt" not in serialized_events + assert "sensitive-response" not in serialized_events + assert "sensitive-error" not in serialized_events + assert "sensitive-command" not in serialized_events + assert "sensitive-tool-result" not in serialized_events + assert "sensitive-tool-call" not in serialized_events + assert "gpt-sensitive-model-id" not in serialized_events + assert plugins.get_plugin_manager().list_plugins() == [] + + root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" + packages = list((root / "outbox").glob("*.json")) + assert len(packages) == 1 + package = json.loads(packages[0].read_text(encoding="utf-8")) + metrics = {metric["name"]: metric for metric in package["metrics"]} + assert set(metrics) == { + "hermes.model_call.count", + "hermes.task_run.finished", + "hermes.task_run.started", + } + assert metrics["hermes.model_call.count"]["dimensions"]["model_family"] == "claude" + assert metrics["hermes.model_call.count"]["value"] == 1 + assert metrics["hermes.task_run.started"] == { + "name": "hermes.task_run.started", + "type": "counter", + "dimensions": { + "entrypoint": "interactive", + "execution_surface": "cli", + }, + "value": 1, + } + terminal = metrics["hermes.task_run.finished"]["dimensions"] + assert terminal["duration_bucket"] in { + "lt_1s", + "1s_to_5s", + "5s_to_30s", + "30s_to_2m", + "2m_to_10m", + "gte_10m", + } + assert { + key: value for key, value in terminal.items() if key != "duration_bucket" + } == { + "end_reason": "completed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "success", + "retry_count_bucket": "1", + "termination": "none", + "tool_call_count_bucket": "1", + } + + +def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot( + real_binding_runtime, + tmp_path, +): + assert real_binding_runtime._native is not None + prompt_canary = "real-relay-sensitive-prompt" + response_canary = "real-relay-sensitive-response" + model_canary = "gpt-real-relay-sensitive-model" + tool_canary = "real-relay-sensitive-tool-result" + + def base(index: int) -> dict[str, Any]: + return { + "session_id": f"sensitive-session-{index}", + "task_id": f"sensitive-task-{index}", + "turn_id": f"sensitive-turn-{index}", + "api_request_id": f"sensitive-request-{index}", + "platform": "cli", + "provider": "custom", + "model": model_canary, + "base_url": "http://127.0.0.1:11434/v1", + } + + success = base(1) + lifecycle.invoke_hook("on_session_start", **success) + lifecycle.invoke_hook("pre_llm_call", **success, messages=[prompt_canary]) + lifecycle.invoke_hook("pre_api_request", **success, retry_count=0) + lifecycle.invoke_hook( + "api_request_error", + **success, + retry_count=0, + retryable=True, + error={"message": prompt_canary}, + ) + lifecycle.invoke_hook("pre_api_request", **success, retry_count=1) + lifecycle.invoke_hook( + "post_tool_call", + **success, + tool_call_id="sensitive-tool-call", + tool_name="terminal", + args={"command": prompt_canary}, + result={"output": tool_canary}, + status="ok", + ) + lifecycle.invoke_hook( + "post_api_request", + **success, + retry_count=1, + response={"content": response_canary}, + ) + lifecycle.invoke_hook( + "on_session_end", + **success, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id=success["session_id"]) + + failed = base(2) + lifecycle.invoke_hook("on_session_start", **failed) + lifecycle.invoke_hook("pre_llm_call", **failed, messages=[prompt_canary]) + lifecycle.invoke_hook("pre_api_request", **failed, retry_count=0) + lifecycle.invoke_hook( + "api_request_error", + **failed, + retry_count=0, + retryable=False, + error={"message": response_canary}, + ) + lifecycle.invoke_hook( + "on_session_end", + **failed, + completed=False, + failed=True, + interrupted=False, + turn_exit_reason="system_aborted", + ) + lifecycle.finalize_session(session_id=failed["session_id"]) + + cancelled = base(3) + lifecycle.invoke_hook("on_session_start", **cancelled) + lifecycle.invoke_hook("pre_llm_call", **cancelled, messages=[prompt_canary]) + lifecycle.invoke_hook("pre_api_request", **cancelled, retry_count=0) + lifecycle.invoke_hook( + "on_session_end", + **cancelled, + completed=False, + failed=False, + interrupted=True, + turn_exit_reason="interrupted_by_user", + ) + lifecycle.finalize_session(session_id=cancelled["session_id"]) + + from hermes_cli.observability.shared_metrics import SharedMetricsStore + + root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" + store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") + snapshot = store.counter_snapshot() + by_metric: dict[str, list[dict[str, Any]]] = {} + for counter in snapshot: + by_metric.setdefault(counter["metric_name"], []).append(counter) + + assert len(by_metric["hermes.task_run.started"]) == 1 + assert by_metric["hermes.task_run.started"][0]["value"] == 3 + assert { + counter["dimensions"]["outcome"] + for counter in by_metric["hermes.model_call.count"] + } == {"success", "failed", "cancelled"} + terminal_by_outcome = { + counter["dimensions"]["outcome"]: counter + for counter in by_metric["hermes.task_run.finished"] + } + assert set(terminal_by_outcome) == {"success", "failed", "cancelled"} + assert terminal_by_outcome["success"]["dimensions"]["retry_count_bucket"] == "1" + assert terminal_by_outcome["success"]["dimensions"]["tool_call_count_bucket"] == "1" + assert terminal_by_outcome["failed"]["dimensions"]["end_reason"] == ( + "system_aborted" + ) + assert terminal_by_outcome["cancelled"]["dimensions"]["termination"] == ( + "user_cancelled" + ) + assert all(counter["packaged_value"] == counter["value"] for counter in snapshot) + + snapshot_values = { + ( + counter["metric_name"], + tuple(sorted(counter["dimensions"].items())), + ): counter["value"] + for counter in snapshot + } + package_values: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {} + packages = sorted((root / "outbox").glob("*.json")) + assert len(packages) == 3 + package_payloads = [ + json.loads(package.read_text(encoding="utf-8")) for package in packages + ] + for package in package_payloads: + assert package["schema_version"] == "hermes.shared_metrics.v1" + for metric in package["metrics"]: + key = (metric["name"], tuple(sorted(metric["dimensions"].items()))) + package_values[key] = package_values.get(key, 0) + metric["value"] + assert package_values == snapshot_values + + serialized_analytics = json.dumps({ + "snapshot": snapshot, + "packages": package_payloads, + }) + for canary in ( + prompt_canary, + response_canary, + model_canary, + tool_canary, + "sensitive-session", + "sensitive-task", + "sensitive-request", + "sensitive-tool-call", + ): + assert canary not in serialized_analytics + + +def test_direct_runtime_is_disabled_by_default(tmp_path, monkeypatch): + fake = _Relay() + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {}) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + + assert not plugins.has_hook("pre_api_request") + lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") + lifecycle.finalize_session(session_id="s1") + + assert fake.events == [] + assert not (tmp_path / "hermes-home" / "telemetry").exists() + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_tool_intercept_bypass_does_not_create_relay_host(monkeypatch): + relay_runtime._reset_for_tests() + imports = [] + + def load_relay(): + imports.append("nemo_relay") + raise AssertionError("disabled helper created Relay host") + + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) + args = {"command": "true"} + + assert ( + relay_runtime.apply_tool_request_intercepts( + session_id="s1", + tool_name="terminal", + args=args, + ) + is args + ) + assert relay_runtime.get_host(create=False) is None + assert imports == [] + + +def test_execution_adapters_do_not_create_relay_host_without_a_consumer( + monkeypatch, +): + from agent import relay_llm, relay_tools + + relay_runtime._reset_for_tests() + imports = [] + + def load_relay(): + imports.append("nemo_relay") + raise AssertionError("disabled execution adapter created Relay host") + + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) + request = {"model": "test-model", "messages": []} + response = object() + tool_args = {"command": "true"} + tool_result = object() + + assert ( + relay_llm.execute( + request, + lambda observed: response if observed is request else None, + session_id="llm-session", + name="test-provider", + model_name="test-model", + ) + is response + ) + result, observed_args = relay_tools.execute( + "terminal", + tool_args, + lambda observed: tool_result if observed is tool_args else None, + session_id="tool-session", + ) + + assert result is tool_result + assert observed_args is tool_args + assert relay_runtime.get_host(create=False) is None + assert imports == [] + + +def test_profile_key_caches_absolute_path_resolution(monkeypatch): + relay_runtime._reset_for_tests() + + class Home: + def __init__(self): + self.resolve_calls = 0 + + def expanduser(self): + return self + + def is_absolute(self): + return True + + def resolve(self): + self.resolve_calls += 1 + return self + + def __str__(self): + return "/profiles/cached" + + home = Home() + monkeypatch.setattr(relay_runtime, "get_hermes_home", lambda: home) + + assert relay_runtime.current_profile_key() == "/profiles/cached" + assert relay_runtime.current_profile_key() == "/profiles/cached" + assert home.resolve_calls == 1 + + +def test_host_registry_reads_existing_host_without_lock(): + registry = relay_runtime.RelayHostRegistry() + host = relay_runtime.NoopRelayRuntime("profile", "test") + registry._hosts["profile"] = host + + class UnexpectedLock: + def __enter__(self): + raise AssertionError("registry read acquired the write lock") + + def __exit__(self, *_args): + return False + + registry._lock = UnexpectedLock() + + assert registry.for_profile("profile", create=False) is host + assert registry.for_profile("missing", create=False) is None + + +def test_core_runtime_is_fail_open_without_a_published_binding(monkeypatch, caplog): + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + def missing_relay(name: str): + assert name == "nemo_relay" + raise ModuleNotFoundError(name) + + monkeypatch.setattr(relay_runtime.importlib, "import_module", missing_relay) + + assert relay_runtime.get_runtime() is None + host = relay_runtime.get_host() + assert isinstance(host, relay_runtime.NoopRelayRuntime) + assert host.profile_key == relay_runtime.current_profile_key() + assert "nemo_relay" in host.reason + assert host.apply_tool_request_intercepts( + session_id="s1", + tool_name="terminal", + args={"command": "true"}, + ) == {"command": "true"} + assert not relay_runtime.emit_mark("hermes.probe", session_id="s1") + assert "Hermes Relay runtime initialization failed" in caplog.text + relay_runtime._reset_for_tests() + + +def test_core_mark_uses_the_shared_session_handle_without_a_plugin(direct_runtime): + lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") + + handle = relay_runtime.get_session_handle("s1") + assert handle is not None + assert relay_runtime.emit_mark( + "hermes.skill.created", + session_id="s1", + data={"provenance": "agent_created"}, + metadata={"data_schema": "hermes.skill.lifecycle.v1"}, + ) + + [mark] = [event for event in direct_runtime.events if event[0] == "scope.event"] + assert mark[1] == "hermes.skill.created" + assert mark[2]["handle"] == handle + assert plugins.get_plugin_manager().list_plugins() == [] + + +def test_core_task_instrumentation_preserves_prompt_history_and_tool_schema( + direct_runtime, + monkeypatch, +): + from run_agent import AIAgent + + agent = object.__new__(AIAgent) + agent.session_id = "cache-stable-session" + agent.platform = "cli" + agent._parent_session_id = None + agent._session_db = None + agent._cached_system_prompt = "byte-stable-system-prompt\nwith exact spacing" + agent.tools = [ + { + "type": "function", + "function": { + "name": "probe", + "parameters": { + "type": "object", + "properties": {"value": {"type": "string"}}, + }, + }, + } + ] + history = [{"role": "user", "content": "sensitive-history-canary"}] + prompt_before = agent._cached_system_prompt.encode("utf-8") + history_before = json.dumps(history, ensure_ascii=False, sort_keys=True) + tools_before = json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) + + def fake_run_conversation( + active_agent, + user_message, + system_message, + conversation_history, + task_id, + stream_callback, + persist_user_message, + **kwargs, + ): + del ( + user_message, + system_message, + task_id, + stream_callback, + persist_user_message, + kwargs, + ) + assert active_agent is agent + assert conversation_history is history + return {"final_response": "ok", "completed": True} + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + fake_run_conversation, + ) + + for task_id in ("cache-task-1", "cache-task-2"): + result = AIAgent.run_conversation( + agent, + "hello", + conversation_history=history, + task_id=task_id, + ) + assert result["final_response"] == "ok" + + assert agent._cached_system_prompt.encode("utf-8") == prompt_before + assert json.dumps(history, ensure_ascii=False, sort_keys=True) == history_before + assert json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) == tools_before + + +def test_session_coordinator_separates_turn_release_from_hard_finalize( + direct_runtime, +): + profile_key = relay_runtime.current_profile_key() + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="coordinated-session", + platform="cli", + ) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + + assert relay_runtime.current_turn() is turn + assert turn.handle is not None + session_handle = lease.session.handle + turn_push = next( + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == relay_runtime.TURN_SCOPE + ) + assert turn_push[3]["handle"] == session_handle + + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + + assert relay_runtime.current_turn() is None + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + assert runtime.get_session("coordinated-session") is not None + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="coordinated-session", + ) + assert runtime.get_session("coordinated-session") is None + + +def test_turn_cleanup_can_be_retried_from_its_originating_context(direct_runtime): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="cross-context-session", + platform="cli", + ) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + + contextvars.Context().run(coordinator.end_turn, turn, outcome="success") + + assert turn.closed + assert relay_runtime.active_turn("cross-context-session") is None + assert relay_runtime.current_turn() is turn + + coordinator.end_turn(turn, outcome="success") + assert relay_runtime.current_turn() is None + coordinator.release_conversation(lease) + + +def test_session_coordinator_prepares_subscribers_before_opening_scope( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + initializer_name = "test.pre_scope_subscriber" + + def prepare(host, context): + assert host.profile_key == relay_runtime.current_profile_key() + direct_runtime.events.append(("session.prepare", dict(context))) + + coordinator.register_session_initializer(initializer_name, prepare) + try: + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="prepared-session", + platform="cli", + model="demo-model", + ) + finally: + coordinator.unregister_session_initializer(initializer_name) + + prepared = next( + index + for index, event in enumerate(direct_runtime.events) + if event[0] == "session.prepare" + ) + opened = next( + index + for index, event in enumerate(direct_runtime.events) + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ) + assert prepared < opened + assert direct_runtime.events[prepared][1] == { + "profile_key": relay_runtime.current_profile_key(), + "session_id": "prepared-session", + "platform": "cli", + "parent_session_id": "", + "model": "demo-model", + } + coordinator.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=lease.session_id, + ) + + +def test_session_initializer_failure_does_not_block_conversation( + direct_runtime, + caplog, +): + coordinator = relay_runtime.SESSION_COORDINATOR + initializer_name = "test.failing_subscriber" + + def fail(_host, _context): + raise RuntimeError("subscriber setup failed") + + coordinator.register_session_initializer(initializer_name, fail) + try: + with caplog.at_level("WARNING"): + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="initializer-failure", + platform="cli", + ) + finally: + coordinator.unregister_session_initializer(initializer_name) + + assert lease.session is not None + assert "Hermes Relay session initializer failed" in caplog.text + coordinator.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=lease.session_id, + ) + + +def test_profile_host_recreation_rebinds_shared_metrics_subscriber( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + first_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="before-restart", + platform="cli", + ) + first_metrics = relay_shared_metrics._get_runtime() + assert first_metrics is not None + first_host = first_lease.host + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=first_lease.session_id, + ) + coordinator.shutdown_profile(profile_key) + + second_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="after-restart", + platform="cli", + ) + second_metrics = relay_shared_metrics._get_runtime() + + assert second_metrics is not None + assert second_lease.host is not first_host + assert second_metrics is not first_metrics + assert second_metrics.host is second_lease.host + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=second_lease.session_id, + ) + + +def test_core_mark_does_not_start_relay_without_metrics_or_a_plugin( + tmp_path, + monkeypatch, +): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + imports = [] + + def load_relay(): + imports.append("nemo_relay") + return _Relay() + + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) + monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {}) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) + + assert not relay_runtime.emit_mark( + "hermes.skill.created", + session_id="s1", + data={"provenance": "agent_created"}, + ) + + assert imports == [] + assert relay_runtime.get_host(create=False) is None + assert not (tmp_path / "hermes-home" / "telemetry").exists() + relay_runtime._reset_for_tests() + + +def test_core_runtime_creates_one_session_under_concurrent_access(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + ready = threading.Barrier(8) + sessions: list[Any] = [] + + def ensure() -> None: + ready.wait(timeout=5) + sessions.append(runtime.ensure_session({"session_id": "shared"})) + + threads = [threading.Thread(target=ensure) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + + assert all(not thread.is_alive() for thread in threads) + assert len({id(session) for session in sessions}) == 1 + assert ( + len([event for event in direct_runtime.events if event[0] == "scope.push"]) == 1 + ) + + +def test_core_runtime_isolates_same_session_id_by_profile(direct_runtime, tmp_path): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + + token = set_hermes_home_override(profile_a) + try: + runtime_a = relay_runtime.get_runtime() + session_a = runtime_a.ensure_session({"session_id": "shared"}) + finally: + reset_hermes_home_override(token) + + token = set_hermes_home_override(profile_b) + try: + runtime_b = relay_runtime.get_runtime() + session_b = runtime_b.ensure_session({"session_id": "shared"}) + finally: + reset_hermes_home_override(token) + + assert runtime_a is not None + assert runtime_b is not None + assert runtime_a is not runtime_b + assert runtime_a.profile_key == str(profile_a.resolve()) + assert runtime_b.profile_key == str(profile_b.resolve()) + assert session_a is not session_b + assert session_a.handle != session_b.handle + + +@pytest.mark.parametrize( + ("profile_enabled", "managed_enabled"), + ((None, True), (False, True), (True, False)), +) +def test_managed_config_cannot_override_shared_metrics_consent( + tmp_path, + monkeypatch, + profile_enabled, + managed_enabled, +): + from hermes_cli import config, managed_scope + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + profile = tmp_path / "profile" + managed = tmp_path / "managed" + profile.mkdir() + managed.mkdir() + profile_config = "{}\n" + if profile_enabled is not None: + profile_config = ( + "telemetry:\n" + " shared_metrics:\n" + f" enabled: {str(profile_enabled).lower()}\n" + ) + (profile / "config.yaml").write_text(profile_config, encoding="utf-8") + (managed / "config.yaml").write_text( + "telemetry:\n" + " shared_metrics:\n" + f" enabled: {str(managed_enabled).lower()}\n", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed)) + config._LOAD_CONFIG_CACHE.clear() + config._RAW_CONFIG_CACHE.clear() + managed_scope.invalidate_managed_cache() + + token = set_hermes_home_override(profile) + try: + assert ( + config.load_config_readonly()["telemetry"]["shared_metrics"]["enabled"] + is managed_enabled + ) + assert relay_shared_metrics.enabled() is (profile_enabled is True) + finally: + reset_hermes_home_override(token) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + managed_scope.invalidate_managed_cache() + + +def test_shared_metrics_policy_and_store_are_profile_scoped(tmp_path, monkeypatch): + from hermes_constants import ( + get_hermes_home, + reset_hermes_home_override, + set_hermes_home_override, + ) + + fake = _Relay() + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: { + "telemetry": { + "shared_metrics": {"enabled": get_hermes_home() == profile_a} + } + }, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + token = set_hermes_home_override(profile_a) + try: + assert relay_shared_metrics.enabled() + relay_shared_metrics.start_task_run( + session_id="shared", + task_id="task-a", + platform="cli", + ) + relay_shared_metrics.finish_task_run( + session_id="shared", + task_id="task-a", + platform="cli", + result={"completed": True}, + ) + relay_shared_metrics._get_runtime().close_session({"session_id": "shared"}) + finally: + reset_hermes_home_override(token) + + token = set_hermes_home_override(profile_b) + try: + assert not relay_shared_metrics.enabled() + relay_shared_metrics.start_task_run( + session_id="shared", + task_id="task-b", + platform="cli", + ) + finally: + reset_hermes_home_override(token) + + assert list((profile_a / "telemetry" / "shared_metrics" / "outbox").glob("*.json")) + assert not (profile_b / "telemetry").exists() + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_shared_metrics_subscribers_isolate_two_enabled_profiles(tmp_path, monkeypatch): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + fake = _Relay() + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + for profile, task_id in ((profile_a, "task-a"), (profile_b, "task-b")): + token = set_hermes_home_override(profile) + try: + relay_shared_metrics.start_task_run( + session_id="shared", + task_id=task_id, + platform="cli", + ) + relay_shared_metrics.finish_task_run( + session_id="shared", + task_id=task_id, + platform="cli", + result={"completed": True}, + ) + relay_shared_metrics._get_runtime().close_session( + {"session_id": "shared"} + ) + finally: + reset_hermes_home_override(token) + + for profile in (profile_a, profile_b): + packages = list( + (profile / "telemetry" / "shared_metrics" / "outbox").glob("*.json") + ) + assert len(packages) == 1 + package = json.loads(packages[0].read_text(encoding="utf-8")) + metrics = {metric["name"]: metric for metric in package["metrics"]} + assert metrics["hermes.task_run.started"]["value"] == 1 + assert metrics["hermes.task_run.finished"]["value"] == 1 + + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + +def test_shared_metrics_isolates_same_task_id_across_sessions(direct_runtime): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + + task_a = runtime.start_task({ + "session_id": "session-a", + "task_id": "shared-task", + "platform": "cli", + }) + task_b = runtime.start_task({ + "session_id": "session-b", + "task_id": "shared-task", + "platform": "gateway", + }) + + assert task_a is not None + assert task_b is not None + assert task_a is not task_b + assert task_a.handle != task_b.handle + + runtime.finish_task({ + "session_id": "session-a", + "task_id": "shared-task", + "platform": "cli", + "completed": True, + }) + runtime.finish_task({ + "session_id": "session-b", + "task_id": "shared-task", + "platform": "gateway", + "completed": True, + }) + + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + task_ends = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert len(task_starts) == 2 + assert len(task_ends) == 2 + + +def test_shared_metrics_keys_turn_ownership_by_session(direct_runtime): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + event_a = { + "session_id": "session-a", + "task_id": "task-a", + "turn_id": "shared-turn", + "platform": "cli", + } + event_b = { + "session_id": "session-b", + "task_id": "task-b", + "turn_id": "shared-turn", + "platform": "gateway", + } + task_a = runtime.start_task(event_a) + task_b = runtime.start_task(event_b) + + runtime.start_model_call({ + **event_a, + "api_request_id": "request-a", + "provider": "anthropic", + "model": "claude-sonnet", + }) + + session_a = runtime._session(event_a) + session_b = runtime._session(event_b) + assert task_a is not None + assert task_b is not None + assert session_a is not None + assert session_b is not None + assert "request-a" in session_a.model_calls + assert "request-a" not in session_b.model_calls + [model_start] = [ + event for event in direct_runtime.events if event[0] == "llm.call" + ] + assert model_start[3]["handle"] == task_a.handle + + +def test_disabling_shared_metrics_stops_collection_and_shutdown_export( + tmp_path, monkeypatch +): + from hermes_cli.observability.shared_metrics import SharedMetricsStore + + fake = _Relay() + profile = tmp_path / "profile" + policy = {"enabled": True} + monkeypatch.setenv("HERMES_HOME", str(profile)) + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": dict(policy)}}, + ) + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + + relay_shared_metrics.start_task_run( + session_id="session", + task_id="task", + platform="cli", + ) + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + policy["enabled"] = False + + assert not relay_shared_metrics.enabled() + counters_before_stale_event = runtime.subscriber.store.counter_snapshot() + runtime.subscriber(SimpleNamespace( + kind="scope", + category="function", + category_profile=None, + name="hermes.task_run", + scope_category="start", + metadata={ + "hermes.metrics.schema_version": "hermes.metrics.event.v1", + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.host.runtime_id, + }, + data={"entrypoint": "interactive", "execution_surface": "cli"}, + )) + assert runtime.subscriber.store.counter_snapshot() == counters_before_stale_event + assert runtime.start_task({ + "session_id": "session", + "task_id": "stale-runtime-task", + "platform": "cli", + }) is None + relay_shared_metrics.finish_task_run( + session_id="session", + task_id="task", + platform="cli", + result={"completed": True}, + ) + relay_shared_metrics._reset_for_tests() + + root = profile / "telemetry" / "shared_metrics" + store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") + assert [row["metric_name"] for row in store.counter_snapshot()] == [ + "hermes.task_run.started" + ] + assert list((root / "outbox").glob("*.json")) == [] + relay_runtime._reset_for_tests() + + +def test_shared_metrics_retries_transient_initialization_failure( + direct_runtime, monkeypatch +): + real_store = relay_shared_metrics.SharedMetricsStore + attempts = 0 + + def flaky_store(): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError("transient store failure") + return real_store() + + monkeypatch.setattr(relay_shared_metrics, "SharedMetricsStore", flaky_store) + + relay_shared_metrics.start_task_run( + session_id="session", + task_id="first", + platform="cli", + ) + relay_shared_metrics.start_task_run( + session_id="session", + task_id="second", + platform="cli", + ) + + assert attempts == 2 + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + assert len(task_starts) == 1 + + +def test_async_session_runner_awaits_inside_saved_relay_context(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "async-session"}) + assert session is not None + + async def probe() -> Any: + await asyncio.sleep(0) + return direct_runtime._scope.get() + + result = asyncio.run(runtime.run_in_session_async(session, probe)) + + assert result == session.handle + + +def test_session_runners_preserve_caller_context_and_profile_override( + direct_runtime, + tmp_path, +): + from hermes_constants import ( + get_hermes_home_override, + reset_hermes_home_override, + set_hermes_home_override, + ) + + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "caller-context-session"}) + assert session is not None + caller_value = contextvars.ContextVar("caller_value", default="default") + caller_value.set("caller") + profile = tmp_path / "caller-profile" + profile_token = set_hermes_home_override(profile) + + def sync_probe() -> tuple[Any, str, str | None]: + return ( + direct_runtime._scope.get(), + caller_value.get(), + get_hermes_home_override(), + ) + + async def async_probe() -> tuple[Any, str, str | None]: + await asyncio.sleep(0) + return sync_probe() + + try: + sync_result = runtime.run_in_session(session, sync_probe) + async_result = asyncio.run(runtime.run_in_session_async(session, async_probe)) + finally: + reset_hermes_home_override(profile_token) + + expected = (session.handle, "caller", str(profile)) + assert sync_result == expected + assert async_result == expected + + +@pytest.mark.asyncio +async def test_async_session_runner_isolates_concurrent_caller_contexts( + direct_runtime, +): + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "concurrent-context-session"}) + assert session is not None + caller_value = contextvars.ContextVar("concurrent_caller", default="default") + ready = asyncio.Event() + entered = 0 + + async def run(value: str) -> tuple[Any, str]: + nonlocal entered + caller_value.set(value) + + async def probe() -> tuple[Any, str]: + nonlocal entered + entered += 1 + if entered == 2: + ready.set() + await ready.wait() + return direct_runtime._scope.get(), caller_value.get() + + return await runtime.run_in_session_async(session, probe) + + results = await asyncio.gather(run("first"), run("second")) + + assert results == [ + (session.handle, "first"), + (session.handle, "second"), + ] + + +def test_sync_session_runner_releases_lock_before_callback(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + session = runtime.ensure_session({"session_id": "sync-session"}) + assert session is not None + acquired = threading.Event() + contender = None + + def probe() -> Any: + nonlocal contender + + def acquire_session_lock() -> None: + with session.lock: + acquired.set() + + contender = threading.Thread(target=acquire_session_lock) + contender.start() + assert acquired.wait(timeout=1) + return direct_runtime._scope.get() + + result = runtime.run_in_session(session, probe) + assert contender is not None + contender.join(timeout=1) + + assert result == session.handle + assert contender.is_alive() is False + + +def test_active_turn_requires_matching_session_and_profile( + direct_runtime, + tmp_path, + monkeypatch, +): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="session-1", + platform="cli", + ) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + + assert relay_runtime.active_turn("session-1") is turn + assert relay_runtime.active_turn("session-2") is None + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "other-profile")) + assert relay_runtime.active_turn("session-1") is None + + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + assert relay_runtime.active_turn("session-1") is None + + +def test_turn_cleanup_drains_logical_calls_after_turn_scope_start_failure( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="session-1", + platform="cli", + ) + assert lease.session is not None + original_push = direct_runtime.scope.push + + def fail_turn_push(name, *args, **kwargs): + if name == relay_runtime.TURN_SCOPE: + raise RuntimeError("simulated turn scope failure") + return original_push(name, *args, **kwargs) + + direct_runtime.scope.push = fail_turn_push + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="task-1", + ) + direct_runtime.scope.push = original_push + assert turn.handle is None + + runtime = lease.host + logical_handle = runtime.run_in_session( + lease.session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=lease.session.handle, + input={}, + ) + turn.logical_llm_calls["request-1"] = logical_handle + + coordinator.end_turn(turn, outcome="failed") + coordinator.release_conversation(lease) + + logical_closes = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1] == logical_handle + ] + assert len(logical_closes) == 1 + assert turn.logical_llm_calls == {} + + +def test_shared_metrics_creates_one_task_under_concurrent_access(direct_runtime): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + ready = threading.Barrier(8) + tasks: list[Any] = [] + + def start() -> None: + ready.wait(timeout=5) + tasks.append( + runtime.start_task({"session_id": "s1", "task_id": "t1", "platform": "cli"}) + ) + + threads = [threading.Thread(target=start) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + + assert all(not thread.is_alive() for thread in threads) + assert len({id(task) for task in tasks}) == 1 + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + assert len(task_starts) == 1 + + +def test_core_runtime_parents_subagent_session_without_exposing_ids( + direct_runtime, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + parent_turn = coordinator.begin_turn( + parent_lease, + turn_id="parent-turn", + task_id="parent-task", + ) + child_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="sensitive-child", + platform="subagent", + parent_session_id="parent", + ) + + runtime = relay_runtime.get_runtime() + assert runtime is not None + child = runtime.get_session("sensitive-child") + assert child is not None + assert child.parent_session_id == "parent" + session_pushes = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ] + assert len(session_pushes) == 2 + child_kwargs = session_pushes[1][3] + assert child_kwargs["handle"] == parent_turn.handle + assert child_kwargs["metadata"] == { + "hermes.execution_surface": "subagent", + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id, + "nemo_relay_scope_role": "subagent", + } + assert "sensitive-child" not in json.dumps(session_pushes) + + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="sensitive-child", + ) + coordinator.release_conversation(child_lease) + coordinator.end_turn(parent_turn, outcome="success") + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_subagent_stop_hook_does_not_own_child_session_lifetime(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + child = runtime.register_subagent( + { + "parent_session_id": "parent", + "child_session_id": "child", + } + ) + assert child is not None + + lifecycle.invoke_hook( + "subagent_stop", + parent_session_id="parent", + child_session_id="child", + child_status="completed", + ) + + assert runtime.get_session("child") is child + + runtime.unregister_subagent({"child_session_id": "child"}) + + assert runtime.get_session("child") is None + child_closes = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1] == child.handle + ] + assert len(child_closes) == 1 + + +@pytest.mark.parametrize( + "terminal", + ["return", "exception", "cancelled", "timeout"], +) +def test_subagent_agent_boundary_closes_its_own_scope( + direct_runtime, + monkeypatch, + terminal, +): + from run_agent import AIAgent + + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + parent_turn = coordinator.begin_turn( + parent_lease, + turn_id="parent-turn", + task_id="parent-task", + ) + child_agent = SimpleNamespace( + session_id="child", + platform="subagent", + _parent_session_id="parent", + _session_db=None, + _conversation_root_id=lambda: "parent", + ) + + if terminal == "return": + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + lambda *_args, **_kwargs: { + "final_response": "done", + "completed": True, + "interrupted": False, + }, + ) + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + elif terminal == "exception": + def fail(*_args, **_kwargs): + raise RuntimeError("child failed") + + monkeypatch.setattr("agent.conversation_loop.run_conversation", fail) + with pytest.raises(RuntimeError, match="child failed"): + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + elif terminal == "cancelled": + def cancel(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr("agent.conversation_loop.run_conversation", cancel) + with pytest.raises(KeyboardInterrupt): + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + else: + def time_out(*_args, **_kwargs): + raise TimeoutError("child timed out") + + monkeypatch.setattr("agent.conversation_loop.run_conversation", time_out) + with pytest.raises(TimeoutError, match="child timed out"): + AIAgent.run_conversation(child_agent, "private", task_id="child-task") + + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + assert runtime.get_session("child") is None + child_push = next( + event + for event in direct_runtime.events + if event[0] == "scope.push" + and event[1] == relay_runtime.SESSION_SCOPE + and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent" + ) + assert child_push[3]["handle"] == parent_turn.handle + child_closes = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == relay_runtime.SESSION_SCOPE + ] + assert len(child_closes) == 1 + assert relay_runtime.current_turn() is parent_turn + + coordinator.end_turn(parent_turn, outcome="success") + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_concurrent_subagents_inherit_parent_turn_and_close_independently( + direct_runtime, +): + from concurrent.futures import ThreadPoolExecutor + + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + parent_turn = coordinator.begin_turn( + parent_lease, + turn_id="parent-turn", + task_id="parent-task", + ) + + def run_child(child_id): + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id=child_id, + platform="subagent", + parent_session_id="parent", + ) + turn = coordinator.begin_turn( + lease, + turn_id=f"{child_id}-turn", + task_id=f"{child_id}-task", + ) + coordinator.end_turn(turn, outcome="success") + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=child_id, + ) + coordinator.release_conversation(lease) + + contexts = [contextvars.copy_context(), contextvars.copy_context()] + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(context.run, run_child, f"child-{index}") + for index, context in enumerate(contexts) + ] + for future in futures: + future.result() + + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + assert runtime.get_session("child-0") is None + assert runtime.get_session("child-1") is None + child_pushes = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" + and event[1] == relay_runtime.SESSION_SCOPE + and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent" + ] + assert len(child_pushes) == 2 + assert all(event[3]["handle"] == parent_turn.handle for event in child_pushes) + + coordinator.end_turn(parent_turn, outcome="success") + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_coordinator_tracks_active_turns_across_threads(direct_runtime): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="cross-thread-child", + platform="subagent", + ) + + assert not coordinator.has_active_turn( + profile_key=profile_key, + session_id="cross-thread-child", + ) + + turn = coordinator.begin_turn( + lease, + turn_id="child-turn", + task_id="child-task", + ) + + observed = [] + thread = threading.Thread( + target=lambda: observed.append( + coordinator.has_active_turn( + profile_key=profile_key, + session_id="cross-thread-child", + ) + ) + ) + thread.start() + thread.join(timeout=5) + + assert observed == [True] + + coordinator.end_turn(turn, outcome="success") + + assert not coordinator.has_active_turn( + profile_key=profile_key, + session_id="cross-thread-child", + ) + coordinator.release_conversation(lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="cross-thread-child", + ) + + +def test_child_session_closes_before_active_turn_guard_is_released( + direct_runtime, + monkeypatch, +): + del direct_runtime + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + parent_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="parent", + platform="cli", + ) + child_lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="child", + platform="subagent", + parent_session_id="parent", + ) + child_turn = coordinator.begin_turn( + child_lease, + turn_id="child-turn", + task_id="child-task", + ) + runtime = relay_runtime.get_runtime(create=False) + assert runtime is not None + close_observations = [] + original_close = runtime.close_session + + def observe_close(event): + close_observations.append( + coordinator.has_active_turn( + profile_key=profile_key, + session_id="child", + ) + ) + original_close(event) + + monkeypatch.setattr(runtime, "close_session", observe_close) + + coordinator.end_turn(child_turn, outcome="success") + + assert close_observations == [True] + assert runtime.get_session("child") is None + assert not coordinator.has_active_turn( + profile_key=profile_key, + session_id="child", + ) + coordinator.release_conversation(child_lease) + coordinator.release_conversation(parent_lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id="parent", + ) + + +def test_core_runtime_ignores_self_parenting_subagent_event(direct_runtime): + runtime = relay_runtime.get_runtime() + assert runtime is not None + + runtime.register_subagent({"parent_session_id": "same", "child_session_id": "same"}) + session = runtime.ensure_session({"session_id": "same"}) + + assert session is not None + assert session.parent_session_id == "" + + +def test_terminal_model_error_is_counted_as_failed(direct_runtime): + base = { + "session_id": "s1", + "task_id": "t1", + "api_request_id": "r1", + "provider": "anthropic", + "model": "claude-sonnet", + } + + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=False) + lifecycle.finalize_session(session_id="s1") + + [end] = [event for event in direct_runtime.events if event[0] == "llm.call_end"] + assert end[2]["outcome"] == "failed" + + +def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runtime): + base = { + "session_id": "s1", + "task_id": "t1", + "api_request_id": "r1", + "platform": "cli", + "provider": "nvidia", + "model": "nvidia/nemotron-3-super-120b-a12b", + } + + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=True) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=True) + lifecycle.invoke_hook("pre_api_request", **base) + lifecycle.invoke_hook("api_request_error", **base, retryable=False) + for tool_call_id in ("tool-1", "tool-1", "tool-2"): + lifecycle.invoke_hook( + "post_tool_call", + **base, + tool_call_id=tool_call_id, + tool_name="terminal", + result={"output": "private"}, + status="ok", + ) + lifecycle.invoke_hook( + "on_session_end", + **base, + completed=False, + failed=True, + interrupted=False, + turn_exit_reason="all_retries_exhausted_no_response", + ) + lifecycle.finalize_session(session_id="s1") + + model_starts = [event for event in direct_runtime.events if event[0] == "llm.call"] + model_ends = [ + event for event in direct_runtime.events if event[0] == "llm.call_end" + ] + assert len(model_starts) == 1 + assert len(model_ends) == 1 + assert model_ends[0][2]["outcome"] == "failed" + [task_end] = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end[2]["output"] == { + "duration_bucket": task_end[2]["output"]["duration_bucket"], + "end_reason": "failed", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "1", + "outcome": "failed", + "retry_count_bucket": "2", + "termination": "none", + "tool_call_count_bucket": "2", + } + + +def test_task_terminal_counts_explicit_retry_with_new_request_id(direct_runtime): + base = { + "session_id": "s1", + "task_id": "t1", + "platform": "cli", + "provider": "nvidia", + "model": "nvidia/nemotron-3-super-120b-a12b", + } + + lifecycle.invoke_hook("pre_llm_call", **base) + lifecycle.invoke_hook( + "pre_api_request", + **base, + api_request_id="r1", + retry_count=0, + ) + lifecycle.invoke_hook( + "api_request_error", + **base, + api_request_id="r1", + retryable=True, + ) + lifecycle.invoke_hook( + "pre_api_request", + **base, + api_request_id="r2", + retry_count=1, + ) + lifecycle.invoke_hook( + "post_api_request", + **base, + api_request_id="r2", + ) + lifecycle.invoke_hook( + "on_session_end", + **base, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id="s1") + + [task_end] = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end[2]["output"]["model_call_count_bucket"] == "2" + assert task_end[2]["output"]["retry_count_bucket"] == "1" + + +def test_outer_agent_boundary_closes_early_returns_and_exceptions( + direct_runtime, + monkeypatch, +): + from run_agent import AIAgent + + agent = SimpleNamespace( + session_id="s1", + platform="cli", + _parent_session_id=None, + _session_db=None, + _conversation_root_id=lambda: "s1", + ) + + def early_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return { + "final_response": "private failure detail", + "completed": False, + "failed": True, + "interrupted": False, + } + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + early_failure, + ) + result = AIAgent.run_conversation(agent, "private prompt", task_id="early") + assert result["failed"] is True + + def raise_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise RuntimeError("private exception detail") + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + raise_failure, + ) + with pytest.raises(RuntimeError, match="private exception detail"): + AIAgent.run_conversation(agent, "private prompt", task_id="exception") + + def raise_interrupt(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise KeyboardInterrupt + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + raise_interrupt, + ) + with pytest.raises(KeyboardInterrupt): + AIAgent.run_conversation(agent, "private prompt", task_id="cancelled") + + def raise_timeout(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + raise TimeoutError("private timeout detail") + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + raise_timeout, + ) + with pytest.raises(TimeoutError, match="private timeout detail"): + AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") + + lifecycle.finalize_session(session_id="s1") + + task_ends = [ + event[2]["output"] + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert len(task_ends) == 4 + assert task_ends[0]["outcome"] == "failed" + assert task_ends[0]["end_reason"] == "failed" + assert task_ends[0]["termination"] == "none" + assert task_ends[1]["outcome"] == "failed" + assert task_ends[1]["end_reason"] == "system_aborted" + assert task_ends[1]["termination"] == "system_aborted" + assert task_ends[2]["outcome"] == "cancelled" + assert task_ends[2]["end_reason"] == "user_cancelled" + assert task_ends[2]["termination"] == "user_cancelled" + assert task_ends[3]["outcome"] == "timed_out" + assert task_ends[3]["end_reason"] == "timed_out" + assert task_ends[3]["termination"] == "timed_out" + serialized = json.dumps(direct_runtime.events) + assert "private prompt" not in serialized + assert "private failure detail" not in serialized + assert "private exception detail" not in serialized + assert "private timeout detail" not in serialized + + +def test_outer_agent_boundary_preserves_a_returned_timeout_reason( + direct_runtime, + monkeypatch, +): + from run_agent import AIAgent + + agent = SimpleNamespace( + session_id="s1", + platform="cli", + _parent_session_id=None, + _session_db=None, + _conversation_root_id=lambda: "s1", + ) + + monkeypatch.setattr( + "agent.conversation_loop.run_conversation", + lambda *_args, **_kwargs: { + "final_response": "private timeout response", + "completed": False, + "failed": True, + "failure_reason": "timeout", + }, + ) + AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") + + [task_end] = [ + event[2]["output"] + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end["outcome"] == "timed_out" + assert task_end["end_reason"] == "timed_out" + assert task_end["termination"] == "timed_out" + serialized = json.dumps(direct_runtime.events) + assert "private prompt" not in serialized + assert "private timeout response" not in serialized + + +def test_session_finalize_closes_a_pending_task_as_system_aborted(direct_runtime): + lifecycle.invoke_hook( + "pre_llm_call", + session_id="s1", + task_id="t1", + platform="cli", + ) + + lifecycle.finalize_session(session_id="s1") + + [task_end] = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + assert task_end[2]["output"] == { + "duration_bucket": task_end[2]["output"]["duration_bucket"], + "end_reason": "system_aborted", + "entrypoint": "interactive", + "execution_surface": "cli", + "model_call_count_bucket": "0", + "outcome": "failed", + "retry_count_bucket": "0", + "termination": "system_aborted", + "tool_call_count_bucket": "0", + } + + +def test_sequential_tasks_in_one_session_aggregate_once_each(direct_runtime, tmp_path): + for task_id in ("t1", "t2"): + lifecycle.invoke_hook( + "pre_llm_call", + session_id="s1", + task_id=task_id, + platform="cli", + ) + lifecycle.invoke_hook( + "on_session_end", + session_id="s1", + task_id=task_id, + platform="cli", + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id="s1") + + outbox = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" / "outbox" + [package_path] = list(outbox.glob("*.json")) + package = json.loads(package_path.read_text(encoding="utf-8")) + metrics = {metric["name"]: metric for metric in package["metrics"]} + assert metrics["hermes.task_run.started"]["value"] == 2 + assert metrics["hermes.task_run.finished"]["value"] == 2 + + +def test_task_ownership_survives_session_id_rotation(direct_runtime): + lifecycle.invoke_hook( + "pre_llm_call", + session_id="before-compression", + task_id="t1", + platform="cli", + ) + lifecycle.invoke_hook( + "pre_api_request", + session_id="after-compression", + task_id="t1", + api_request_id="r1", + platform="cli", + provider="nvidia", + model="nvidia/nemotron-3-super-120b-a12b", + ) + lifecycle.invoke_hook( + "post_api_request", + session_id="after-compression", + task_id="t1", + api_request_id="r1", + platform="cli", + provider="nvidia", + model="nvidia/nemotron-3-super-120b-a12b", + ) + lifecycle.invoke_hook( + "on_session_end", + session_id="after-compression", + task_id="t1", + platform="cli", + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + lifecycle.finalize_session(session_id="before-compression") + + task_starts = [ + event + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + task_ends = [ + event + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" + ] + model_ends = [ + event for event in direct_runtime.events if event[0] == "llm.call_end" + ] + assert len(task_starts) == 1 + assert len(task_ends) == 1 + assert len(model_ends) == 1 + assert model_ends[0][2]["outcome"] == "success" + assert task_ends[0][2]["output"]["model_call_count_bucket"] == "1" + assert task_ends[0][2]["output"]["outcome"] == "success" + + +def test_gateway_and_delegated_entrypoints_flow_through_relay(direct_runtime): + tasks = [ + { + "session_id": "gateway-session", + "task_id": "gateway-task", + "platform": "whatsapp_cloud", + }, + { + "session_id": "child-session", + "task_id": "delegated-task", + "platform": "cli", + "parent_session_id": "private-parent-session", + }, + ] + for task in tasks: + lifecycle.invoke_hook("pre_llm_call", **task) + lifecycle.invoke_hook( + "on_session_end", + **task, + completed=True, + failed=False, + interrupted=False, + turn_exit_reason="text_response(stop)", + ) + + starts = [ + event[3]["input"] + for event in direct_runtime.events + if event[0] == "scope.push" and event[1] == "hermes.task_run" + ] + assert starts == [ + {"entrypoint": "gateway_message", "execution_surface": "gateway"}, + {"entrypoint": "delegated", "execution_surface": "cli"}, + ] + assert "private-parent-session" not in json.dumps(direct_runtime.events) + + +def test_persistence_failure_does_not_escape_the_hook( + direct_runtime, + monkeypatch, + caplog, +): + runtime = relay_shared_metrics._get_runtime() + assert runtime is not None + + def fail_record(*_args: Any, **_kwargs: Any) -> None: + raise OSError("store unavailable") + + monkeypatch.setattr(runtime.subscriber.store, "record_counter", fail_record) + lifecycle.invoke_hook( + "pre_api_request", + session_id="s1", + task_id="t1", + api_request_id="r1", + provider="openai", + model="gpt-5", + ) + lifecycle.invoke_hook( + "post_api_request", + session_id="s1", + task_id="t1", + api_request_id="r1", + provider="openai", + model="gpt-5", + ) + + assert "Unable to persist the Hermes shared metric" in caplog.text + + +def test_close_does_not_reopen_a_session_after_scope_start_failure( + direct_runtime, + monkeypatch, +): + runtime = relay_runtime.get_runtime() + assert runtime is not None + original_push = direct_runtime.scope.push + push_attempts = 0 + + def fail_first_push(*args: Any, **kwargs: Any) -> Any: + nonlocal push_attempts + push_attempts += 1 + if push_attempts == 1: + raise RuntimeError("simulated scope failure") + return original_push(*args, **kwargs) + + direct_runtime.scope.push = fail_first_push + with pytest.raises(RuntimeError, match="simulated scope failure"): + runtime.ensure_session({"session_id": "s1"}) + + close_started = threading.Event() + allow_close = threading.Event() + original_flush = direct_runtime.subscribers.flush + + def block_flush(): + session = runtime._sessions["s1"] + assert session.closing is True + close_started.set() + assert allow_close.wait(timeout=5) + original_flush() + + direct_runtime.subscribers.flush = block_flush + close_thread = threading.Thread( + target=runtime.close_session, + args=({"session_id": "s1"},), + ) + close_thread.start() + assert close_started.wait(timeout=5) + + ensure_thread = threading.Thread( + target=runtime.ensure_session, + args=({"session_id": "s1"},), + ) + ensure_thread.start() + allow_close.set() + close_thread.join(timeout=5) + ensure_thread.join(timeout=5) + + assert not close_thread.is_alive() + assert not ensure_thread.is_alive() + assert push_attempts == 1 + assert "s1" not in runtime._sessions diff --git a/tests/hermes_cli/test_setup_telemetry.py b/tests/hermes_cli/test_setup_telemetry.py new file mode 100644 index 000000000000..e6ebcb428c45 --- /dev/null +++ b/tests/hermes_cli/test_setup_telemetry.py @@ -0,0 +1,37 @@ +"""Tests for shared-metrics configuration discovery and setup.""" + +from __future__ import annotations + +import argparse + +from hermes_cli.config import DEFAULT_CONFIG +from hermes_cli.setup import setup_telemetry +from hermes_cli.subcommands.setup import build_setup_parser + + +def test_shared_metrics_are_registered_disabled_by_default(): + assert DEFAULT_CONFIG["telemetry"]["shared_metrics"]["enabled"] is False + + +def test_setup_telemetry_enables_shared_metrics(monkeypatch): + config = {} + monkeypatch.setattr( + "hermes_cli.setup.prompt_yes_no", + lambda _question, default: not default, + ) + + setup_telemetry(config) + + assert config["telemetry"]["shared_metrics"]["enabled"] is True + + +def test_setup_parser_accepts_telemetry_section(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + handler = object() + build_setup_parser(subparsers, cmd_setup=handler) + + args = parser.parse_args(["setup", "telemetry"]) + + assert args.section == "telemetry" + assert args.func is handler diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 61958194e585..6081f0660f73 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio -import builtins +import contextvars import gc import importlib import json @@ -15,6 +15,8 @@ from types import SimpleNamespace import pytest import yaml +from hermes_cli import lifecycle, plugins as plugin_api +from hermes_cli.observability import relay_runtime, relay_shared_metrics from hermes_cli.plugins import PluginManager @@ -25,7 +27,13 @@ PLUGIN_DIR = REPO_ROOT / "plugins" / "observability" / "nemo_relay" class _FakeNemoRelay: def __init__(self): self.events = [] - self.ScopeType = SimpleNamespace(Agent="agent") + self._callbacks = {} + self._llm_starts = {} + self._scope_serial = 0 + self._scope_context = contextvars.ContextVar( + "fake_nemo_relay_scope", default=None + ) + self.ScopeType = SimpleNamespace(Agent="agent", Function="function") self.scope = SimpleNamespace( push=self._scope_push, pop=self._scope_pop, @@ -40,21 +48,29 @@ class _FakeNemoRelay: call=self._tool_call, call_end=self._tool_call_end, execute=self._tool_execute, + request_intercepts=self._tool_request_intercepts, ) self.plugin = SimpleNamespace( initialize=self._plugin_initialize, clear=self._plugin_clear, - activate_dynamic_plugins=self._plugin_activate_dynamic, + initialize_with_dynamic_plugins=self._plugin_initialize_with_dynamic, + ) + self.subscribers = SimpleNamespace( + register=self._register_subscriber, + deregister=self._deregister_subscriber, + flush=self._flush_subscribers, ) - self.subscribers = SimpleNamespace(flush=self._flush_subscribers) self.LLMRequest = _FakeLLMRequest self.AtofExporterConfig = _FakeAtofExporterConfig self.AtofExporterMode = SimpleNamespace(Append="append", Overwrite="overwrite") self.AtofExporter = self._make_atof_exporter self.AtifExporter = self._make_atif_exporter + self.get_scope_stack = self._get_scope_stack def _scope_push(self, name, scope_type, **kwargs): - handle = ("scope", name) + self._scope_serial += 1 + handle = ("scope", name, self._scope_serial) + self._scope_context.set(handle) self.events.append(("scope.push", name, scope_type, kwargs)) return handle @@ -64,17 +80,45 @@ class _FakeNemoRelay: def _scope_event(self, name, **kwargs): self.events.append(("scope.event", name, kwargs)) + def _get_scope_stack(self): + current = self._scope_context.get() + self.events.append(("scope.sync", current)) + return current + def _llm_call(self, name, request, **kwargs): handle = ("llm", name) + self._llm_starts[handle] = kwargs self.events.append(("llm.call", name, request.content, kwargs)) return handle def _llm_call_end(self, handle, response, **kwargs): self.events.append(("llm.call_end", handle, response, kwargs)) + start = self._llm_starts.pop(handle, {}) + event = SimpleNamespace( + kind="scope", + category="llm", + name=handle[1], + scope_category="end", + category_profile={"model_name": start.get("model_name")}, + metadata={ + **(start.get("metadata") or {}), + **(kwargs.get("metadata") or {}), + "otel.status_code": "OK", + }, + data=response, + ) + for callback in list(self._callbacks.values()): + callback(event) def _llm_execute(self, name, request, func, **kwargs): self.events.append(("llm.execute.start", name, request.content, kwargs)) + handle = self._llm_call(name, request, **kwargs) result = func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) + self._llm_call_end( + handle, + result, + **{key: value for key, value in kwargs.items() if key != "handle"}, + ) self.events.append(("llm.execute.end", name, result, kwargs)) return result @@ -88,10 +132,20 @@ class _FakeNemoRelay: def _tool_execute(self, name, args, func, **kwargs): self.events.append(("tool.execute.start", name, args, kwargs)) - result = func({"intercepted": True, **args}) + handle = self._tool_call(name, args, **kwargs) + result = func(args) + self._tool_call_end( + handle, + result, + **{key: value for key, value in kwargs.items() if key != "handle"}, + ) self.events.append(("tool.execute.end", name, result, kwargs)) return result + def _tool_request_intercepts(self, name, args): + self.events.append(("tool.request_intercepts", name, args)) + return {"intercepted": True, **args} + def _make_atof_exporter(self, config): return _FakeAtofExporter(self.events, config) @@ -105,10 +159,18 @@ class _FakeNemoRelay: async def _plugin_clear(self): self.events.append(("plugin.clear",)) - async def _plugin_activate_dynamic(self, config, dynamic_plugins): + async def _plugin_initialize_with_dynamic(self, config, dynamic_plugins): self.events.append(("plugin.activate_dynamic", config, dynamic_plugins)) return _FakePluginActivation(self.events) + def _register_subscriber(self, name, callback): + self._callbacks[name] = callback + self.events.append(("subscribers.register", name)) + + def _deregister_subscriber(self, name): + self._callbacks.pop(name, None) + self.events.append(("subscribers.deregister", name)) + def _flush_subscribers(self): self.events.append(("subscribers.flush",)) @@ -169,6 +231,12 @@ class _FakeAtifExporter: def _fresh_plugin(monkeypatch, fake): + existing = sys.modules.get("plugins.observability.nemo_relay") + if existing is not None: + existing.reset_for_tests() + relay_shared_metrics._reset_for_tests() + relay_runtime._reset_for_tests() + monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setitem(sys.modules, "nemo_relay", fake) sys.modules.pop("plugins.observability.nemo_relay", None) plugin = importlib.import_module("plugins.observability.nemo_relay") @@ -176,33 +244,6 @@ def _fresh_plugin(monkeypatch, fake): return plugin -def _wrapped_downstream_error(original): - class _DownstreamExecutionError(Exception): - def __init__(self, original): - super().__init__(str(original)) - self.original = original - - return _DownstreamExecutionError(original) - - -def _enable_adaptive_plugin(tmp_path, monkeypatch) -> None: - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - def _enable_dynamic_plugin(tmp_path, monkeypatch) -> Path: plugins_toml = tmp_path / "plugins.toml" plugins_toml.write_text( @@ -233,11 +274,6 @@ def test_manifest_fields(): "on_session_reset", "pre_llm_call", "post_llm_call", - "pre_api_request", - "post_api_request", - "api_request_error", - "pre_tool_call", - "post_tool_call", "pre_approval_request", "post_approval_response", "subagent_start", @@ -266,7 +302,12 @@ def test_nemo_relay_plugin_uses_nemo_relay_runtime(monkeypatch): assert any(event[0] == "scope.push" for event in fake_relay.events) -def test_nemo_relay_plugin_emits_llm_tool_and_exports_atif(tmp_path, monkeypatch): +def test_nemo_relay_plugin_exports_core_managed_llm_and_tool_events( + tmp_path, + monkeypatch, +): + from agent import relay_llm, relay_tools + fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) monkeypatch.setenv("HERMES_NEMO_RELAY_ATOF_ENABLED", "1") @@ -281,64 +322,153 @@ def test_nemo_relay_plugin_emits_llm_tool_and_exports_atif(tmp_path, monkeypatch "telemetry_schema_version": "hermes.observer.v1", } plugin.on_session_start(**base, model="demo-model", platform="cli") - plugin.on_pre_api_request( - **base, - api_request_id="api-1", - provider="openai", - model="demo-model", - request={"method": "POST", "body": {"messages": [{"role": "user", "content": "hi"}]}}, + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + platform="cli", ) - plugin.on_post_api_request( - **base, - api_request_id="api-1", - response={"assistant_message": {"role": "assistant", "content": "hello"}}, + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="t1", ) - plugin.on_pre_tool_call(**base, tool_name="read_file", tool_call_id="tool-1", args={"path": "x"}) - plugin.on_post_tool_call(**base, tool_name="read_file", tool_call_id="tool-1", result='{"ok": true}', status="ok") + relay_llm.execute( + {"messages": [{"role": "user", "content": "hi"}]}, + lambda request: { + "assistant_message": {"role": "assistant", "content": "hello"}, + "request": request, + }, + session_id="s1", + name="openai", + model_name="demo-model", + metadata={"api_request_id": "api-1", "api_mode": "custom"}, + ) + relay_tools.execute( + "read_file", + {"path": "x"}, + lambda _args: {"ok": True}, + session_id="s1", + metadata={"tool_call_id": "tool-1"}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) plugin.on_session_end(**base, completed=True, interrupted=False) plugin.on_session_finalize(**base, reason="shutdown") event_names = [event[0] for event in fake.events] assert "atof.register" in event_names assert "atif.register" in event_names - assert "llm.call" in event_names - assert "llm.call_end" in event_names - assert "tool.call" in event_names - assert "tool.call_end" in event_names + assert event_names.count("llm.call") == 1 + assert event_names.count("llm.call_end") == 1 + assert event_names.count("tool.call") == 1 + assert event_names.count("tool.call_end") == 1 assert "scope.pop" in event_names assert (tmp_path / "atif" / "hermes-atif-s1.json").exists() -def test_nemo_relay_plugin_closes_api_span_on_error(monkeypatch): +def test_shared_metrics_and_rich_plugin_share_one_core_session( + tmp_path, + monkeypatch, +): + from agent import relay_llm + fake = _FakeNemoRelay() + hermes_home = tmp_path / "hermes-home" + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1") + monkeypatch.setenv( + "HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "atif") + ) + monkeypatch.setattr( + "hermes_cli.config.read_raw_config", + lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, + ) plugin = _fresh_plugin(monkeypatch, fake) - base = { + manager = PluginManager() + + class _Context: + def register_hook(self, name, callback): + manager._hooks.setdefault(name, []).append(callback) + + plugin.register(_Context()) + monkeypatch.setattr(plugin_api, "_plugin_manager", manager) + + event = { "session_id": "s1", "task_id": "t1", - "turn_id": "turn-1", - "telemetry_schema_version": "hermes.observer.v1", + "api_request_id": "api-1", + "provider": "anthropic", + "model": "claude-sonnet", + "platform": "cli", } - - plugin.on_pre_api_request( - **base, - api_request_id="api-err", - provider="openai", - model="demo-model", + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + platform="cli", + model=event["model"], + ) + lifecycle.invoke_hook("on_session_start", **event) + turn = coordinator.begin_turn( + lease, + turn_id="turn-1", + task_id="t1", + ) + lifecycle.invoke_hook( + "pre_api_request", + **event, request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, ) - plugin.on_api_request_error( - **base, - api_request_id="api-err", - error={"type": "RateLimitError", "message": "rate limited"}, - retryable=True, - reason="rate_limit", + relay_llm.execute( + {"messages": [{"role": "user", "content": "hi"}]}, + lambda _request: { + "assistant_message": {"role": "assistant", "content": "hello"} + }, + session_id="s1", + name="anthropic", + model_name="claude-sonnet", + metadata={"api_request_id": "api-1", "api_mode": "custom"}, ) + lifecycle.invoke_hook( + "post_api_request", + **event, + response={"assistant_message": {"role": "assistant", "content": "hello"}}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) + lifecycle.finalize_session(session_id="s1") - call_end = next(event for event in fake.events if event[0] == "llm.call_end") - assert call_end[1] == ("llm", "openai") - assert call_end[2] == {"error": {"type": "RateLimitError", "message": "rate limited"}} - assert call_end[3]["data"]["reason"] == "rate_limit" - assert not plugin._get_runtime().sessions["s1"].llm_spans + session_pushes = [ + item + for item in fake.events + if item[0] == "scope.push" and item[1] == relay_runtime.SESSION_SCOPE + ] + assert len(session_pushes) == 1 + register_metrics = next( + index + for index, item in enumerate(fake.events) + if item[0] == "subscribers.register" + and item[1].startswith("hermes.nemo_relay.shared_metrics.") + ) + register_atif = next( + index for index, item in enumerate(fake.events) if item[0] == "atif.register" + ) + open_session = fake.events.index(session_pushes[0]) + assert register_metrics < register_atif < open_session + + plugin_runtime = plugin._get_runtime() + assert plugin_runtime is not None + assert not plugin_runtime.sessions + assert relay_runtime.get_session_handle("s1") is None + packages = list( + (hermes_home / "telemetry" / "shared_metrics" / "outbox").glob("*.json") + ) + assert len(packages) == 1 + package = json.loads(packages[0].read_text(encoding="utf-8")) + assert package["metrics"][0]["name"] == "hermes.model_call.count" + assert package["metrics"][0]["value"] == 1 + assert (tmp_path / "atif" / "hermes-atif-s1.json").exists() def test_nemo_relay_plugin_emits_approval_marks(monkeypatch): @@ -353,24 +483,6 @@ def test_nemo_relay_plugin_emits_approval_marks(monkeypatch): assert "hermes.approval.response" in mark_names -def test_nemo_relay_plugin_emits_unmatched_fallback_marks(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - plugin.on_post_api_request(session_id="s1", api_request_id="missing-api", response={"ok": True}) - plugin.on_api_request_error( - session_id="s1", - api_request_id="missing-api", - error={"type": "TimeoutError", "message": "timed out"}, - ) - plugin.on_post_tool_call(session_id="s1", tool_call_id="missing-tool", result={"ok": True}) - - mark_names = [event[1] for event in fake.events if event[0] == "scope.event"] - assert "hermes.api.response.unmatched" in mark_names - assert "hermes.api.error" in mark_names - assert "hermes.tool.response.unmatched" in mark_names - - def test_nemo_relay_plugin_metadata_promotes_trajectory_and_subagent_ids(monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -417,7 +529,7 @@ def test_nemo_relay_plugin_metadata_promotes_trajectory_and_subagent_ids(monkeyp assert stop_mark[2]["metadata"]["child_status"] == "completed" -def test_nemo_relay_plugin_reparents_child_session_scope_for_embedded_atif(monkeypatch): +def test_nemo_relay_plugin_reuses_core_parented_child_scope_for_embedded_atif(monkeypatch): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) @@ -430,20 +542,52 @@ def test_nemo_relay_plugin_reparents_child_session_scope_for_embedded_atif(monke child_role="leaf", telemetry_schema_version="hermes.observer.v1", ) + runtime = plugin._get_runtime() + assert runtime is not None + assert runtime.host.get_session("child-session") is None + runtime.host.register_subagent( + { + "parent_session_id": "parent-session", + "child_session_id": "child-session", + } + ) plugin.on_session_start(session_id="child-session") - child_push = next( + session_pushes = [ event for event in fake.events - if event[0] == "scope.push" and event[1] == "hermes-session-child-session" - ) + if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE + ] + assert len(session_pushes) == 2 + child_push = session_pushes[1] child_kwargs = child_push[3] - assert child_kwargs["handle"] == ("scope", "hermes-session-parent-session") - assert child_kwargs["metadata"]["session_id"] == "child-session" - assert child_kwargs["metadata"]["trajectory_id"] == "child-session" + assert child_kwargs["handle"] == runtime.sessions["parent-session"].handle assert child_kwargs["metadata"]["nemo_relay_scope_role"] == "subagent" - assert child_kwargs["metadata"]["subagent_id"] == "child-sa" - assert child_kwargs["metadata"]["parent_session_id"] == "parent-session" + assert "session_id" not in child_kwargs["metadata"] + assert "subagent_id" not in child_kwargs["metadata"] + assert runtime.sessions["child-session"].parent_session_id == "parent-session" + + runtime.host.unregister_subagent({"child_session_id": "child-session"}) + assert runtime.host.get_session("child-session") is None + child_close_index = max( + index + for index, event in enumerate(fake.events) + if event[0] == "scope.pop" and event[1] == runtime.sessions["child-session"].handle + ) + plugin.on_subagent_stop( + parent_session_id="parent-session", + child_session_id="child-session", + child_status="completed", + ) + + assert "child-session" not in runtime.sessions + assert runtime.host.get_session("child-session") is None + stop_mark_index = next( + index + for index, event in enumerate(fake.events) + if event[0] == "scope.event" and event[1] == "hermes.subagent.stop" + ) + assert child_close_index < stop_mark_index def test_nemo_relay_plugin_skips_embedded_child_atif_file_by_default(tmp_path, monkeypatch): @@ -551,6 +695,8 @@ enabled = true def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypatch): + from agent import relay_llm, relay_tools + fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -561,21 +707,42 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa runtime = plugin._get_runtime() assert runtime is not None assert runtime._plugin_activation is not None - llm_result = plugin.on_llm_execution_middleware( + coordinator = relay_runtime.SESSION_COORDINATOR + lease = coordinator.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), session_id="s1", - provider="openai", - model="fixture", - request={"messages": []}, - next_call=lambda request: {"request": request}, + platform="cli", ) - tool_result = plugin.on_tool_execution_middleware( + turn = coordinator.begin_turn(lease, turn_id="turn-1", task_id="task-1") + llm_result = relay_llm.execute( + {"messages": []}, + lambda request: {"request": request}, + session_id="s1", + name="openai", + model_name="fixture", + metadata={"api_mode": "custom", "api_request_id": "api-1"}, + ) + tool_args = relay_runtime.apply_tool_request_intercepts( session_id="s1", tool_name="fixture-tool", args={"value": 1}, - next_call=lambda args: {"args": args}, ) + tool_result, final_args = relay_tools.execute( + "fixture-tool", + tool_args, + lambda args: {"args": args}, + session_id="s1", + metadata={"tool_call_id": "tool-1"}, + ) + coordinator.end_turn(turn, outcome="success") + coordinator.release_conversation(lease) assert llm_result["request"]["intercepted"] is True assert tool_result["args"]["intercepted"] is True + assert final_args["intercepted"] is True + relay_runtime.SESSION_COORDINATOR.finalize_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id="s1", + ) plugin.on_session_finalize(session_id="s1", reason="shutdown") assert runtime._plugin_activation is not None assert not any(event[0] == "plugin.activation.close" for event in fake.events) @@ -603,6 +770,190 @@ def test_nemo_relay_plugin_activates_and_owns_dynamic_plugins(tmp_path, monkeypa assert event_names.index("atif.deregister") < event_names.index("plugin.activation.close") +def test_nemo_relay_plugin_uses_real_0_6_dynamic_activation_api( + tmp_path, + monkeypatch, +): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + plugin = _fresh_plugin(monkeypatch, relay) + _enable_dynamic_plugin(tmp_path, monkeypatch) + calls = [] + + class _NativeActivation: + def __init__(self): + self.report = {"diagnostics": []} + self.is_active = True + + async def close(self): + self.is_active = False + + async def _initialize_with_dynamic_plugins(config, dynamic_plugins): + calls.append((config, dynamic_plugins)) + return _NativeActivation() + + monkeypatch.setattr( + relay.plugin, + "_initialize_with_dynamic_plugins", + _initialize_with_dynamic_plugins, + ) + + plugin.on_session_start(session_id="s1") + runtime = plugin._get_runtime() + + assert runtime is not None + assert isinstance(runtime._plugin_activation, relay.plugin.PluginHostActivation) + assert calls == [ + ( + {"version": 1}, + [ + { + "plugin_id": "fixture", + "kind": "rust_dynamic", + "manifest_ref": str(tmp_path / "fixture" / "relay-plugin.toml"), + "config": {"mode": "test"}, + } + ], + ) + ] + + activation = runtime._plugin_activation + runtime.shutdown() + assert activation.is_active is False + + +def test_real_binding_shares_plugin_configuration_across_two_profiles( + tmp_path, + monkeypatch, +): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + plugin = _fresh_plugin(monkeypatch, relay) + original_initialize = relay.plugin.initialize + original_clear = relay.plugin.clear + original_clear() + initialize_calls = [] + clear_calls = 0 + + async def _initialize(config): + initialize_calls.append(config) + return await original_initialize(config) + + def _clear(): + nonlocal clear_calls + clear_calls += 1 + return original_clear() + + monkeypatch.setattr(relay.plugin, "initialize", _initialize) + monkeypatch.setattr(relay.plugin, "clear", _clear) + monkeypatch.setattr( + plugin, + "_load_settings", + lambda: plugin._Settings(plugins_config={"version": 1}), + ) + profile_a = str(tmp_path / "profile-a") + profile_b = str(tmp_path / "profile-b") + host_a = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_a) + host_b = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_b) + + try: + runtime_a = plugin._get_runtime(profile_key=profile_a, host=host_a) + runtime_b = plugin._get_runtime(profile_key=profile_b, host=host_b) + assert runtime_a is not None + assert runtime_b is not None + runtime_a.ensure_session({"session_id": "session-a"}) + runtime_b.ensure_session({"session_id": "session-b"}) + + assert initialize_calls == [{"version": 1}] + assert relay.plugin.report() is not None + + runtime_a.close_session({"session_id": "session-a"}) + + assert clear_calls == 0 + assert relay.plugin.report() is not None + assert runtime_b.host.get_session("session-b") is not None + + runtime_b.close_session({"session_id": "session-b"}) + + assert clear_calls == 1 + assert relay.plugin.report() is None + finally: + plugin.reset_for_tests() + host_a.shutdown() + host_b.shutdown() + original_clear() + + +def test_real_binding_does_not_replace_another_profiles_plugin_configuration( + tmp_path, + monkeypatch, +): + relay = pytest.importorskip("nemo_relay") + if getattr(relay, "_native", None) is None: + pytest.skip("NeMo Relay native binding is unavailable on this platform") + plugin = _fresh_plugin(monkeypatch, relay) + original_initialize = relay.plugin.initialize + original_clear = relay.plugin.clear + original_clear() + initialize_calls = [] + clear_calls = 0 + + async def _initialize(config): + initialize_calls.append(config) + return await original_initialize(config) + + def _clear(): + nonlocal clear_calls + clear_calls += 1 + return original_clear() + + settings = iter(( + plugin._Settings( + plugins_config={"version": 1, "policy": {"unsupported": "warn"}} + ), + plugin._Settings( + plugins_config={"version": 1, "policy": {"unsupported": "ignore"}} + ), + )) + monkeypatch.setattr(relay.plugin, "initialize", _initialize) + monkeypatch.setattr(relay.plugin, "clear", _clear) + monkeypatch.setattr(plugin, "_load_settings", lambda: next(settings)) + profile_a = str(tmp_path / "profile-a") + profile_b = str(tmp_path / "profile-b") + host_a = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_a) + host_b = relay_runtime.RelayRuntime(relay=relay, profile_key=profile_b) + + try: + runtime_a = plugin._get_runtime(profile_key=profile_a, host=host_a) + runtime_b = plugin._get_runtime(profile_key=profile_b, host=host_b) + assert runtime_a is not None + assert runtime_b is not None + + assert initialize_calls == [ + {"version": 1, "policy": {"unsupported": "warn"}} + ] + assert runtime_a._plugin_config_initialized is True + assert runtime_b._plugin_config_initialized is False + assert relay.plugin.report() is not None + + runtime_b.shutdown() + + assert clear_calls == 0 + assert relay.plugin.report() is not None + + runtime_a.shutdown() + + assert clear_calls == 1 + assert relay.plugin.report() is None + finally: + plugin.reset_for_tests() + host_a.shutdown() + host_b.shutdown() + original_clear() + + def test_nemo_relay_rejects_gateway_dynamic_config_with_actionable_diagnostic( tmp_path, monkeypatch, caplog ): @@ -670,136 +1021,40 @@ environment_ref = "../environments/worker-fixture" runtime.shutdown() -@pytest.mark.parametrize( - ("provider", "api_mode", "expected_surface", "should_rewrite"), - [ - ("custom", "chat_completions", "openai.chat_completions", True), - ("openai-codex", "codex_responses", "openai.responses", True), - ("anthropic", "anthropic_messages", "anthropic.messages", True), - ("custom", "anthropic_messages", "anthropic.messages", True), - ("bedrock", "bedrock_converse", "bedrock", False), - ], -) -def test_nemo_relay_managed_llm_uses_wire_protocol_for_interceptor_dispatch( +def test_relay_tool_request_rewrite_precedes_hermes_authorization_boundary( tmp_path, monkeypatch, - provider, - api_mode, - expected_surface, - should_rewrite, ): - fake = _FakeNemoRelay() - supported_surfaces = { - "anthropic.messages", - "openai.chat_completions", - "openai.responses", - } - - def execute(name, request, func, **kwargs): - fake.events.append(("llm.execute.start", name, request.content, kwargs)) - content = dict(request.content) - if name in supported_surfaces: - content["rewritten_for"] = name - result = func(_FakeLLMRequest(request.headers, content)) - fake.events.append(("llm.execute.end", name, result, kwargs)) - return result - - fake.llm.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_llm_execution_middleware( - session_id="s1", - provider=provider, - api_mode=api_mode, - model="fixture", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: request, - ) - - execute_start = next( - event for event in fake.events if event[0] == "llm.execute.start" - ) - assert execute_start[1] == expected_surface - assert execute_start[3]["metadata"]["provider"] == provider - assert execute_start[3]["metadata"]["api_mode"] == api_mode - if should_rewrite: - assert result["rewritten_for"] == expected_surface - else: - assert "rewritten_for" not in result - - -def test_nemo_relay_managed_llm_returns_post_next_interceptor_result(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - raw_response = SimpleNamespace( - model="fixture", - choices=[ - SimpleNamespace( - message=SimpleNamespace(role="assistant", content="raw", tool_calls=[]), - finish_reason="stop", - ) - ], - usage=None, - ) - - def execute(name, request, func, **kwargs): - del name, kwargs - normalized = func(_FakeLLMRequest(request.headers, request.content)) - return {**normalized, "post_next_interceptor": True} - - fake.llm.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_llm_execution_middleware( - session_id="s1", - provider="openai", - api_mode="chat_completions", - model="fixture", - request={"messages": []}, - next_call=lambda request: raw_response, - ) - - assert result["post_next_interceptor"] is True - assert result["assistant_message"]["content"] == "raw" - assert result is not raw_response - - -def test_nemo_relay_managed_tool_returns_post_interceptor_result(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def execute(name, args, func, **kwargs): - fake.events.append(("tool.execute.start", name, args, kwargs)) - raw = func({"intercepted": True, **args}) - result = {"compressed": True, "raw": raw} - fake.events.append(("tool.execute.end", name, result, kwargs)) - return result - - fake.tools.execute = execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_dynamic_plugin(tmp_path, monkeypatch) - - result = plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="fixture-tool", - args={"value": 1}, - next_call=lambda args: {"tool_output": args}, - ) - - assert result == { - "compressed": True, - "raw": {"tool_output": {"intercepted": True, "value": 1}}, - } - - -def test_nemo_relay_plugin_activates_before_registering_managed_middleware(tmp_path, monkeypatch): + from hermes_cli.middleware import apply_tool_request_middleware + + fake = _FakeNemoRelay() + plugin = _fresh_plugin(monkeypatch, fake) + _enable_dynamic_plugin(tmp_path, monkeypatch) + plugin.on_session_start(session_id="s1") + + result = apply_tool_request_middleware( + "fixture-tool", + {"value": 1}, + session_id="s1", + tool_call_id="tool-1", + ) + + assert result.payload == {"intercepted": True, "value": 1} + assert result.trace[0] == {"source": "nemo_relay"} + + +def test_nemo_relay_plugin_activates_without_duplicate_execution_hooks( + tmp_path, monkeypatch +): fake = _FakeNemoRelay() plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) + registered_hooks = [] class _Context: def register_hook(self, name, callback): - del name, callback + del callback + registered_hooks.append(name) def register_middleware(self, name, callback): del callback @@ -808,9 +1063,15 @@ def test_nemo_relay_plugin_activates_before_registering_managed_middleware(tmp_p plugin.register(_Context()) event_names = [event[0] for event in fake.events] - assert event_names.index("plugin.activate_dynamic") < event_names.index( - "hermes.register_middleware" - ) + assert "plugin.activate_dynamic" in event_names + assert "hermes.register_middleware" not in event_names + assert not { + "pre_api_request", + "post_api_request", + "api_request_error", + "pre_tool_call", + "post_tool_call", + }.intersection(registered_hooks) runtime = plugin._get_runtime() assert runtime is not None runtime.shutdown() @@ -820,7 +1081,7 @@ def test_nemo_relay_plugin_degrades_to_static_config_on_relay_0_5( tmp_path, monkeypatch, caplog ): fake = _FakeNemoRelay() - delattr(fake.plugin, "activate_dynamic_plugins") + delattr(fake.plugin, "initialize_with_dynamic_plugins") plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -899,7 +1160,7 @@ def test_nemo_relay_plugin_registers_shutdown_after_dynamic_retry(tmp_path, monk raise RuntimeError("temporary activation failure") return _FakePluginActivation(fake.events) - fake.plugin.activate_dynamic_plugins = _flaky_activate + fake.plugin.initialize_with_dynamic_plugins = _flaky_activate plugin = _fresh_plugin(monkeypatch, fake) _enable_dynamic_plugin(tmp_path, monkeypatch) @@ -1034,7 +1295,7 @@ enabled = true assert runtime is not None assert runtime._plugin_config_initialized is True scope_push_names = [event[1] for event in fake.events if event[0] == "scope.push"] - assert "hermes-session-s2" in scope_push_names + assert relay_runtime.SESSION_SCOPE in scope_push_names def test_nemo_relay_plugin_retries_plugins_toml_after_clear_failure(tmp_path, monkeypatch): @@ -1075,7 +1336,7 @@ enabled = true assert event_names.count("plugin.initialize.attempt") == 2 assert event_names.count("plugin.clear.failed") == 1 scope_push_names = [event[1] for event in fake.events if event[0] == "scope.push"] - assert "hermes-session-s2" in scope_push_names + assert relay_runtime.SESSION_SCOPE in scope_push_names def test_nemo_relay_plugin_disables_direct_atif_when_plugins_toml_owns_atif(tmp_path, monkeypatch): @@ -1195,652 +1456,3 @@ output_directory = "{(tmp_path / "managed-atof").as_posix()}" assert event_names.count("plugin.initialize.attempt") == 2 assert event_names.count("atof.register") == 1 assert event_names.count("atof.deregister") == 1 - - -def test_nemo_relay_adaptive_llm_execution_middleware_preserves_raw_response(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - seen_request = {} - raw_choice = SimpleNamespace( - message=SimpleNamespace( - role="assistant", - content=None, - tool_calls=[ - SimpleNamespace( - id="tool-1", - type="function", - function=SimpleNamespace(name="terminal", arguments='{"command":"pwd"}'), - ) - ], - reasoning_content="need a tool", - ), - finish_reason="tool_calls", - ) - raw_response = SimpleNamespace( - id="resp-1", - model="demo-model", - choices=[raw_choice], - usage=SimpleNamespace(prompt_tokens=3, completion_tokens=5, total_tokens=8), - ) - - def next_call(request): - seen_request.update(request) - return raw_response - - response = plugin.on_llm_execution_middleware( - session_id="s1", - task_id="t1", - turn_id="turn-1", - api_request_id="api-1", - provider="anthropic", - model="demo-model", - api_call_count=1, - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert response is raw_response - assert response.model == "demo-model" - assert response.choices == [raw_choice] - assert seen_request["intercepted"] is True - execute_start = next(event for event in fake.events if event[0] == "llm.execute.start") - assert execute_start[3]["data"]["mode"] == "observe_only" - execute_end = next(event for event in fake.events if event[0] == "llm.execute.end") - assert execute_end[2] == { - "model": "demo-model", - "assistant_message": { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "tool-1", - "type": "function", - "function": {"name": "terminal", "arguments": '{"command":"pwd"}'}, - } - ], - "reasoning_content": "need a tool", - }, - "finish_reason": "tool_calls", - "usage": {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}, - } - - -def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def native_like_execute(name, request, func, **kwargs): - fake.events.append(("llm.execute.start", name, request.content, kwargs)) - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None - - fake.llm.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ProviderAuthError(Exception): - status_code = 403 - - provider_error = ProviderAuthError("provider auth failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(ProviderAuthError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error_with_relay_suffix( - tmp_path, monkeypatch -): - # Guards the startswith (vs exact ==) match in _is_relay_wrapped_callback_error: - # Relay re-wraps the callback failure with its canonical prefix but APPENDS a - # trailing suffix. Exact equality would miss this and surface Relay's wrapper; - # prefix matching must still recover the original downstream error. - fake = _FakeNemoRelay() - - def native_like_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc} (retried 3x)") from None - - fake.llm.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ProviderAuthError(Exception): - status_code = 403 - - provider_error = ProviderAuthError("provider auth failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(ProviderAuthError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_llm_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - relay_error = RuntimeError("internal error: relay setup failed") - - def internal_error_execute(name, request, func, **kwargs): - raise relay_error - - fake.llm.execute = internal_error_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - with pytest.raises(RuntimeError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_llm_execution_keeps_wrapped_relay_error_after_downstream_failure( - tmp_path, monkeypatch -): - fake = _FakeNemoRelay() - relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream") - - def translated_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception: - raise relay_error - - fake.llm.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - def next_call(request): - raise _wrapped_downstream_error(RuntimeError("provider failed")) - - with pytest.raises(RuntimeError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_llm_execution_keeps_relay_translated_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - class RelayPolicyError(Exception): - pass - - relay_error = RelayPolicyError("relay policy blocked") - - def translated_execute(name, request, func, **kwargs): - try: - return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content})) - except Exception: - raise relay_error - - fake.llm.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - provider_error = RuntimeError("provider failed") - - def next_call(request): - raise _wrapped_downstream_error(provider_error) - - with pytest.raises(RelayPolicyError) as caught: - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_downstream_unwrap_matches_real_middleware_wrapper_shape(monkeypatch): - # Regression guard against core/plugin drift. The synthetic tests above model - # the downstream-error wrapper with a local class, so they keep passing even - # if core middleware renames its private ``_DownstreamExecutionError`` or drops - # ``.original`` -- the exact shape the plugin matches by name at - # ``_original_downstream_error``. Capture the wrapper the REAL - # ``hermes_cli.middleware._run_execution_chain`` hands to a middleware - # callback's ``next_call`` and assert the plugin's detector unwraps it to the - # original exception. If core middleware changes the wrapper shape, this fails - # here instead of silently defeating the unwrap in production. - from hermes_cli import middleware - - from plugins.observability.nemo_relay import _original_downstream_error - - class ProviderError(Exception): - status_code = 403 - - provider_error = ProviderError("provider auth failed") - captured: dict[str, Exception] = {} - - def terminal_call(payload): - raise provider_error - - def capturing_callback(**kwargs): - next_call = kwargs["next_call"] - try: - return next_call(kwargs.get("request")) - except Exception as exc: - captured["wrapper"] = exc - # Surface the original so the chain unwinds without re-wrapping noise. - raise _original_downstream_error(exc) from None - - with pytest.raises(ProviderError) as caught: - middleware._run_execution_chain( - "llm", - [capturing_callback], - terminal_call, - request={"messages": []}, - ) - - wrapper = captured["wrapper"] - # The wrapper the plugin sees must match what _original_downstream_error keys on. - assert wrapper.__class__.__name__ == "_DownstreamExecutionError" - assert isinstance(getattr(wrapper, "original", None), BaseException) - assert _original_downstream_error(wrapper) is provider_error - assert caught.value is provider_error - assert caught.value.status_code == 403 - - -def _adaptive_llm_execute_mode(tmp_path, monkeypatch, plugins_toml_text: str) -> str: - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text(plugins_toml_text, encoding="utf-8") - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - - execute_start = next(event for event in fake.events if event[0] == "llm.execute.start") - return execute_start[3]["data"]["mode"] - - -def test_nemo_relay_adaptive_llm_execution_middleware_defaults_to_observe_only_when_mode_is_unset( - tmp_path, monkeypatch -): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -version = 1 -""", - ) - assert mode == "observe_only" - - -def test_nemo_relay_adaptive_llm_execution_middleware_accepts_legacy_top_level_mode(tmp_path, monkeypatch): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -mode = "route" -""", - ) - assert mode == "route" - - -def test_nemo_relay_adaptive_llm_execution_middleware_prefers_tool_parallelism_mode(tmp_path, monkeypatch): - mode = _adaptive_llm_execute_mode( - tmp_path, - monkeypatch, - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config] -mode = "route" - -[components.config.tool_parallelism] -mode = "schedule" -""", - ) - assert mode == "schedule" - - -def test_nemo_relay_llm_execution_middleware_calls_through_without_adaptive(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - response = plugin.on_llm_execution_middleware( - session_id="s1", - provider="anthropic", - model="demo-model", - request={"messages": []}, - next_call=lambda request: {"raw": request}, - ) - - assert response == {"raw": {"messages": []}} - assert not any(event[0] == "llm.execute.start" for event in fake.events) - - -def test_nemo_relay_adaptive_tool_execution_middleware_preserves_raw_response(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - seen_args = {} - - def next_call(args): - seen_args.update(args) - return {"raw": True, "args": args} - - response = plugin.on_tool_execution_middleware( - session_id="s1", - task_id="t1", - turn_id="turn-1", - api_request_id="api-1", - tool_name="terminal", - tool_call_id="tool-1", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert response == {"raw": True, "args": {"command": "pwd", "intercepted": True}} - assert seen_args["intercepted"] is True - execute_start = next(event for event in fake.events if event[0] == "tool.execute.start") - assert execute_start[3]["data"]["mode"] == "observe_only" - assert execute_start[3]["data"]["tool_call_id"] == "tool-1" - - -def test_nemo_relay_adaptive_tool_execution_preserves_downstream_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - def native_like_execute(name, args, func, **kwargs): - fake.events.append(("tool.execute.start", name, args, kwargs)) - try: - return func({"intercepted": True, **args}) - except Exception as exc: - raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None - - fake.tools.execute = native_like_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - class ToolAuthError(Exception): - status_code = 403 - - tool_error = ToolAuthError("tool auth failed") - - def next_call(args): - raise _wrapped_downstream_error(tool_error) - - with pytest.raises(ToolAuthError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is tool_error - assert caught.value.status_code == 403 - - -def test_nemo_relay_adaptive_tool_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - relay_error = RuntimeError("internal error: relay setup failed") - - def internal_error_execute(name, args, func, **kwargs): - raise relay_error - - fake.tools.execute = internal_error_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - with pytest.raises(RuntimeError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_tool_execution_keeps_wrapped_relay_error_after_downstream_failure( - tmp_path, monkeypatch -): - fake = _FakeNemoRelay() - relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream") - - def translated_execute(name, args, func, **kwargs): - try: - return func({"intercepted": True, **args}) - except Exception: - raise relay_error - - fake.tools.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - def next_call(args): - raise _wrapped_downstream_error(RuntimeError("tool failed")) - - with pytest.raises(RuntimeError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_adaptive_tool_execution_keeps_relay_translated_error(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - - class RelayPolicyError(Exception): - pass - - relay_error = RelayPolicyError("relay policy blocked") - - def translated_execute(name, args, func, **kwargs): - try: - return func({"intercepted": True, **args}) - except Exception: - raise relay_error - - fake.tools.execute = translated_execute - plugin = _fresh_plugin(monkeypatch, fake) - _enable_adaptive_plugin(tmp_path, monkeypatch) - - tool_error = RuntimeError("tool failed") - - def next_call(args): - raise _wrapped_downstream_error(tool_error) - - with pytest.raises(RelayPolicyError) as caught: - plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=next_call, - ) - - assert caught.value is relay_error - - -def test_nemo_relay_tool_execution_middleware_calls_through_without_adaptive(monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - - response = plugin.on_tool_execution_middleware( - session_id="s1", - tool_name="terminal", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - assert response == {"raw": {"command": "pwd"}} - assert not any(event[0] == "tool.execute.start" for event in fake.events) - - -def test_nemo_relay_adaptive_execution_skips_duplicate_observer_spans(tmp_path, monkeypatch): - fake = _FakeNemoRelay() - plugin = _fresh_plugin(monkeypatch, fake) - plugins_toml = tmp_path / "plugins.toml" - plugins_toml.write_text( - """ -version = 1 - -[[components]] -kind = "adaptive" -enabled = true - -[components.config.tool_parallelism] -mode = "observe_only" -""", - encoding="utf-8", - ) - monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml)) - - base = { - "session_id": "s1", - "task_id": "t1", - "turn_id": "turn-1", - "api_request_id": "api-1", - } - plugin.on_pre_api_request( - **base, - provider="anthropic", - model="demo-model", - request={"body": {"messages": [{"role": "user", "content": "hi"}]}}, - ) - plugin.on_post_api_request(**base, response={"ok": True}) - plugin.on_pre_tool_call(**base, tool_name="terminal", tool_call_id="tool-1", args={"command": "pwd"}) - plugin.on_post_tool_call(**base, tool_name="terminal", tool_call_id="tool-1", result={"ok": True}) - - plugin.on_llm_execution_middleware( - **base, - provider="anthropic", - model="demo-model", - request={"messages": [{"role": "user", "content": "hi"}]}, - next_call=lambda request: {"raw": request}, - ) - plugin.on_tool_execution_middleware( - **base, - tool_name="terminal", - tool_call_id="tool-1", - args={"command": "pwd"}, - next_call=lambda args: {"raw": args}, - ) - - event_names = [event[0] for event in fake.events] - assert "llm.call" not in event_names - assert "llm.call_end" not in event_names - assert "tool.call" not in event_names - assert "tool.call_end" not in event_names - assert "llm.execute.start" in event_names - assert "tool.execute.start" in event_names - - -def test_nemo_relay_plugin_noops_without_dependency(monkeypatch): - monkeypatch.delitem(sys.modules, "nemo_relay", raising=False) - sys.modules.pop("plugins.observability.nemo_relay", None) - plugin = importlib.import_module("plugins.observability.nemo_relay") - plugin.reset_for_tests() - - real_import = builtins.__import__ - - def blocked_import(name, *args, **kwargs): - if name == "nemo_relay": - raise ModuleNotFoundError(f"No module named {name!r}") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", blocked_import) - - plugin.on_pre_api_request(session_id="s1", api_request_id="api-1") - plugin.on_post_api_request(session_id="s1", api_request_id="api-1") diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index d6c5887d2706..c2b6db704eac 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2445,8 +2445,8 @@ class TestExecuteToolCalls: hook_calls.append((hook_name, kwargs)) return [] - monkeypatch.setattr("hermes_cli.plugins.invoke_hook", _capture_hook) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook", _capture_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with ( patch("run_agent.handle_function_call", side_effect=KeyboardInterrupt), @@ -3040,6 +3040,69 @@ class TestConcurrentToolExecution: assert "real-a" in messages[0]["content"] assert "real-b" in messages[1]["content"] + def test_concurrent_serializes_post_rewrite_authorization(self, agent, monkeypatch): + tc1 = _mock_tool_call( + name="web_search", arguments='{"q": "a"}', call_id="c1" + ) + tc2 = _mock_tool_call( + name="web_search", arguments='{"q": "b"}', call_id="c2" + ) + mock_msg = _mock_assistant_msg(content="", tool_calls=[tc1, tc2]) + messages = [] + state_lock = threading.Lock() + active = 0 + max_active = 0 + + def authorize(*_args, **_kwargs): + nonlocal active, max_active + with state_lock: + active += 1 + max_active = max(max_active, active) + try: + time.sleep(0.05) + return None + finally: + with state_lock: + active -= 1 + + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + authorize, + ) + + with patch( + "run_agent.handle_function_call", + side_effect=lambda _name, args, _task_id, **_kwargs: f"result-{args['q']}", + ): + agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") + + assert max_active == 1 + assert [message["tool_call_id"] for message in messages] == ["c1", "c2"] + + def test_concurrent_timeout_excludes_authorization_wait(self, agent, monkeypatch): + monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "0.05") + tool_call = _mock_tool_call( + name="web_search", arguments='{"q": "approved"}', call_id="c1" + ) + mock_msg = _mock_assistant_msg(content="", tool_calls=[tool_call]) + messages = [] + + def authorize(*_args, **_kwargs): + time.sleep(0.15) + return None + + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + authorize, + ) + + with patch("run_agent.handle_function_call", return_value="approved-result"): + agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") + + assert len(messages) == 1 + assert "approved-result" in messages[0]["content"] + assert "timed out after" not in messages[0]["content"] + def test_concurrent_interrupt_before_start(self, agent): """If interrupt is requested before concurrent execution, all tools are skipped.""" tc1 = _mock_tool_call(name="web_search", arguments='{}', call_id="c1") @@ -3107,6 +3170,55 @@ class TestConcurrentToolExecution: assert starts == [("c1", "web_search", {"query": "hello"})] assert completes == [("c1", "web_search", {"query": "hello"}, '{"success": true}')] + @pytest.mark.parametrize("quiet_mode", [True, False]) + def test_sequential_registry_tool_forwards_request_middleware_trace( + self, + agent, + monkeypatch, + quiet_mode, + ): + from hermes_cli.middleware import RequestMiddlewareResult + + trace = [{"source": "test-middleware"}] + observed = [] + agent.quiet_mode = quiet_mode + tool_call = _mock_tool_call( + name="web_search", + arguments='{"query":"hello"}', + call_id="c1", + ) + mock_msg = _mock_assistant_msg(content="", tool_calls=[tool_call]) + monkeypatch.setattr( + "hermes_cli.middleware.apply_tool_request_middleware", + lambda _name, args, **_kwargs: RequestMiddlewareResult( + payload=args, + original_payload=args, + changed=True, + trace=trace, + ), + ) + monkeypatch.setattr( + "hermes_cli.middleware.run_tool_execution_middleware", + lambda _name, args, callback, **_kwargs: callback(args), + ) + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + "agent.tool_executor._begin_tool_execution", + lambda *_args, **_kwargs: None, + ) + + def handle_function_call(*_args, **kwargs): + observed.append(kwargs) + return '{"success": true}' + + with patch("run_agent.handle_function_call", side_effect=handle_function_call): + agent._execute_tool_calls_sequential(mock_msg, [], "task-1") + + assert observed[0]["tool_request_middleware_trace"] == trace + def test_sequential_browser_type_callbacks_redact_api_key(self, agent): secret = "sk-proj-ABCD1234567890EFGH" tool_call = _mock_tool_call( @@ -3191,10 +3303,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: result = agent._invoke_tool("todo", {"todos": []}, "task-1", tool_call_id="todo-1") @@ -3274,10 +3386,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: "Blocked by policy", ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("run_agent.handle_function_call", side_effect=AssertionError("should not run")): agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3301,10 +3413,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3348,10 +3460,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: agent._execute_tool_calls_sequential(mock_msg, messages, "task-1") @@ -3386,10 +3498,10 @@ class TestConcurrentToolExecution: lambda *args, **kwargs: None, ) monkeypatch.setattr( - "hermes_cli.plugins.invoke_hook", + "hermes_cli.lifecycle.invoke_hook", lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}'): agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1") @@ -3606,134 +3718,212 @@ class TestConcurrentToolExecution: # Second (allowed) write must checkpoint even though first was blocked. cp_mock.assert_called_once() + def test_managed_tool_pipeline_rejects_second_dispatch(self, agent, monkeypatch): + from agent import relay_tools, tool_executor + + dispatched = [] + duplicate_errors = [] + monkeypatch.setattr( + "hermes_cli.middleware.apply_tool_request_middleware", + lambda _name, args, **_kwargs: SimpleNamespace( + payload=args, + trace=[], + ), + ) + monkeypatch.setattr( + "hermes_cli.middleware.run_tool_execution_middleware", + lambda _name, args, callback, **_kwargs: callback(args), + ) + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr(tool_executor, "_begin_tool_execution", lambda *_a, **_k: None) + + def invoke_twice(name, args, callback, **kwargs): + del name, kwargs + result = callback(args) + try: + callback(args) + except RuntimeError as exc: + duplicate_errors.append(str(exc)) + return result, args + + monkeypatch.setattr(relay_tools, "execute", invoke_twice) + + outcome = tool_executor._run_agent_tool_execution_middleware( + agent, + function_name="terminal", + function_args={"command": "true"}, + effective_task_id="task-1", + tool_call_id="call-1", + execute=lambda args: dispatched.append(args) or "ok", + ) + + assert outcome.result == "ok" + assert dispatched == [{"command": "true"}] + assert duplicate_errors == [ + "Hermes tool execution callback invoked more than once" + ] + assert outcome.blocked is False + + def test_managed_tool_pipeline_allows_one_concurrent_dispatch( + self, + agent, + monkeypatch, + ): + from agent import relay_tools, tool_executor + + dispatched = [] + results = [] + errors = [] + barrier = threading.Barrier(2) + monkeypatch.setattr( + "hermes_cli.middleware.apply_tool_request_middleware", + lambda _name, args, **_kwargs: SimpleNamespace( + payload=args, + trace=[], + ), + ) + monkeypatch.setattr( + "hermes_cli.middleware.run_tool_execution_middleware", + lambda _name, args, callback, **_kwargs: callback(args), + ) + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr(tool_executor, "_begin_tool_execution", lambda *_a, **_k: None) + + def invoke_concurrently(name, args, callback, **kwargs): + del name, kwargs + + def invoke(): + barrier.wait(timeout=2) + try: + results.append(callback(args)) + except RuntimeError as exc: + errors.append(str(exc)) + + threads = [threading.Thread(target=invoke) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=2) + return results[0], args + + monkeypatch.setattr(relay_tools, "execute", invoke_concurrently) + + outcome = tool_executor._run_agent_tool_execution_middleware( + agent, + function_name="terminal", + function_args={"command": "true"}, + effective_task_id="task-1", + tool_call_id="call-1", + execute=lambda args: dispatched.append(args) or "ok", + ) + + assert outcome.result == "ok" + assert dispatched == [{"command": "true"}] + assert errors == ["Hermes tool execution callback invoked more than once"] + assert outcome.blocked is False + class TestAgentRuntimePostHookOwnershipSync: - """Pin the inline-dispatch tool list against the post-hook ownership set. + """Exercise post-hook ownership through both agent-runtime tool paths.""" - The post_tool_call hook fires from two places: the inline dispatcher in - agent/tool_executor.py:execute_tool_calls_sequential (for agent-runtime - tools that never reach handle_function_call) and - model_tools.handle_function_call itself (for registry-dispatched tools). - To prevent the executor from silently dropping or double-emitting, - AGENT_RUNTIME_POST_HOOK_TOOL_NAMES has to match exactly the static - `function_name == "..."` branches in the inline dispatch chain. + _CASES = ( + ("todo", {"todos": []}), + ("session_search", {"query": "needle"}), + ("memory", {"action": "view", "target": "memory"}), + ("clarify", {"question": "Continue?"}), + ("read_terminal", {}), + ("delegate_task", {"goal": "Check the child path"}), + ) - The chain is the if/elif tower anchored on `_block_msg is not None`. - Pre-dispatch `function_name == "..."` checks (counter resets, checkpoint - triggers) live outside the dispatch chain and are explicitly skipped. - """ - - _DISPATCH_ANCHOR_LEFT = "_block_msg" - - @classmethod - def _is_dispatch_anchor(cls, test_node) -> bool: - # Looking for `_block_msg is not None`. - if not isinstance(test_node, ast.Compare): - return False - if not (isinstance(test_node.left, ast.Name) and test_node.left.id == cls._DISPATCH_ANCHOR_LEFT): - return False - if not (len(test_node.ops) == 1 and isinstance(test_node.ops[0], ast.IsNot)): - return False - comparator = test_node.comparators[0] - return isinstance(comparator, ast.Constant) and comparator.value is None - - @staticmethod - def _function_name_literal(test_node) -> str | None: - """Return the string literal X for `function_name == "X"`, else None.""" - if not isinstance(test_node, ast.Compare): - return None - if not (isinstance(test_node.left, ast.Name) and test_node.left.id == "function_name"): - return None - if not (len(test_node.ops) == 1 and isinstance(test_node.ops[0], ast.Eq)): - return None - comparator = test_node.comparators[0] - if isinstance(comparator, ast.Constant) and isinstance(comparator.value, str): - return comparator.value - return None - - @classmethod - def _extract_dispatch_chain_names(cls, func) -> set[str]: - """Find the if/elif chain anchored on `_block_msg is not None`, return its - `function_name == "..."` literals.""" - source = inspect.cleandoc("\n" + inspect.getsource(func)) - tree = ast.parse(source) - names: set[str] = set() - for node in ast.walk(tree): - if not isinstance(node, ast.If): - continue - if not cls._is_dispatch_anchor(node.test): - continue - current = node - while current is not None: - literal = cls._function_name_literal(current.test) - if literal is not None: - names.add(literal) - if current.orelse and len(current.orelse) == 1 and isinstance(current.orelse[0], ast.If): - current = current.orelse[0] - else: - current = None - break - return names - - @classmethod - def _extract_invoke_tool_names(cls, func) -> set[str]: - """invoke_tool uses a flat if/elif on function_name directly; walk every - Compare in the function body (no other static `function_name == "..."` - checks live there).""" - source = inspect.cleandoc("\n" + inspect.getsource(func)) - tree = ast.parse(source) - names: set[str] = set() - for node in ast.walk(tree): - literal = cls._function_name_literal(node) - if literal is not None: - names.add(literal) - return names - - def test_frozenset_matches_inline_dispatch_chain(self): - from agent import tool_executor + @pytest.mark.parametrize(("tool_name", "tool_args"), _CASES) + def test_agent_runtime_tools_emit_once_per_executor_path( + self, + agent, + monkeypatch, + tool_name, + tool_args, + ): from agent.agent_runtime_helpers import AGENT_RUNTIME_POST_HOOK_TOOL_NAMES - inline_names = self._extract_dispatch_chain_names( - tool_executor.execute_tool_calls_sequential + hook_calls = [] + monkeypatch.setattr( + "hermes_cli.plugins.resolve_pre_tool_block", + lambda *args, **kwargs: None, ) - assert inline_names, ( - "Could not find the dispatch chain (anchored on " - "`_block_msg is not None`) in execute_tool_calls_sequential. " - "If the dispatcher was refactored, update _DISPATCH_ANCHOR_LEFT " - "and the walker in this test." + monkeypatch.setattr( + "hermes_cli.lifecycle.invoke_hook", + lambda hook_name, **kwargs: hook_calls.append((hook_name, kwargs)) or [], ) - assert inline_names == set(AGENT_RUNTIME_POST_HOOK_TOOL_NAMES), ( - "Inline dispatch chain in " - "agent/tool_executor.py:execute_tool_calls_sequential has drifted " - "from AGENT_RUNTIME_POST_HOOK_TOOL_NAMES in " - "agent/agent_runtime_helpers.py.\n" - f" Inline branches: {sorted(inline_names)}\n" - f" Ownership frozenset: {sorted(AGENT_RUNTIME_POST_HOOK_TOOL_NAMES)}\n" - "Update both together so post_tool_call fires exactly once per " - "tool execution." + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: True) + monkeypatch.setattr( + "tools.todo_tool.todo_tool", + lambda **kwargs: '{"ok":true}', ) + monkeypatch.setattr( + "tools.memory_tool.memory_tool", + lambda **kwargs: '{"ok":true}', + ) + monkeypatch.setattr( + "tools.clarify_tool.clarify_tool", + lambda **kwargs: '{"ok":true}', + ) + monkeypatch.setattr( + "tools.read_terminal_tool.read_terminal_tool", + lambda **kwargs: '{"ok":true}', + ) + monkeypatch.setattr(agent, "_get_session_db_for_recall", lambda: None) + monkeypatch.setattr( + agent, + "_dispatch_delegate_task", + lambda args: '{"ok":true}', + ) + agent._memory_manager = None - def test_invoke_tool_dispatch_matches_inline_dispatch_chain(self): - """invoke_tool (concurrent path) and the inline dispatcher (sequential - path) must cover the same set of agent-runtime tools — otherwise - post_tool_call fires inconsistently depending on which executor ran - the tool.""" - from agent import agent_runtime_helpers, tool_executor + assert tool_name in AGENT_RUNTIME_POST_HOOK_TOOL_NAMES + with patch( + "run_agent.handle_function_call", + side_effect=AssertionError("agent-runtime tools must stay inline"), + ): + agent._invoke_tool( + tool_name, + dict(tool_args), + "task-concurrent", + tool_call_id=f"{tool_name}-concurrent", + ) + tool_call = _mock_tool_call( + name=tool_name, + arguments=json.dumps(tool_args), + call_id=f"{tool_name}-sequential", + ) + agent._execute_tool_calls_sequential( + _mock_assistant_msg(content="", tool_calls=[tool_call]), + [], + "task-sequential", + ) - invoke_tool_names = self._extract_invoke_tool_names( - agent_runtime_helpers.invoke_tool - ) - inline_names = self._extract_dispatch_chain_names( - tool_executor.execute_tool_calls_sequential - ) - assert invoke_tool_names == inline_names, ( - "Static `function_name == \"...\"` branches diverged between " - "agent/agent_runtime_helpers.py:invoke_tool (concurrent path) " - "and agent/tool_executor.py:execute_tool_calls_sequential " - "(sequential path).\n" - f" invoke_tool: {sorted(invoke_tool_names)}\n" - f" execute_tool_calls_sequential: {sorted(inline_names)}" - ) + post_calls = [ + kwargs + for hook_name, kwargs in hook_calls + if hook_name == "post_tool_call" + ] + assert [call["tool_call_id"] for call in post_calls] == [ + f"{tool_name}-concurrent", + f"{tool_name}-sequential", + ] + assert all(call["tool_name"] == tool_name for call in post_calls) + + def test_post_hook_ownership_contract_lists_exercised_tools(self): + from agent.agent_runtime_helpers import AGENT_RUNTIME_POST_HOOK_TOOL_NAMES + + assert AGENT_RUNTIME_POST_HOOK_TOOL_NAMES == { + tool_name for tool_name, _ in self._CASES + } class TestPathsOverlap: @@ -3876,14 +4066,50 @@ class TestHandleMaxIterations: assert len(result) > 0 assert "summary" in result.lower() + def test_summary_retries_share_relay_identity(self, agent): + agent.client.chat.completions.create.side_effect = [ + _mock_response(content=""), + _mock_response(content="Summary"), + ] + agent._cached_system_prompt = "You are helpful." + relay_calls = [] + + def execute_current(request, callback, **kwargs): + relay_calls.append(kwargs) + return callback(request) + + with ( + patch("agent.relay_llm.execute_current", side_effect=execute_current), + patch("agent.relay_llm.complete_logical_call") as complete_logical, + ): + result = agent._handle_max_iterations( + [{"role": "user", "content": "do stuff"}], + 60, + ) + + assert result == "Summary" + assert [call["metadata"]["retry_count"] for call in relay_calls] == [0, 1] + assert relay_calls[0]["metadata"]["api_request_id"] == ( + relay_calls[1]["metadata"]["api_request_id"] + ) + assert relay_calls[0]["metadata"]["call_role"] == "iteration_summary" + assert all(call["defer_logical_completion"] is True for call in relay_calls) + complete_logical.assert_called_once_with( + relay_calls[0]["metadata"]["api_request_id"], + outcome="success", + ) + def test_api_failure_returns_error(self, agent): agent.client.chat.completions.create.side_effect = Exception("API down") agent._cached_system_prompt = "You are helpful." messages = [{"role": "user", "content": "do stuff"}] - result = agent._handle_max_iterations(messages, 60) + with patch("agent.relay_llm.complete_logical_call") as complete_logical: + result = agent._handle_max_iterations(messages, 60) assert isinstance(result, str) assert "error" in result.lower() assert "API down" in result + complete_logical.assert_called_once() + assert complete_logical.call_args.kwargs == {"outcome": "failed"} def test_summary_skips_reasoning_for_unsupported_openrouter_model(self, agent): agent.base_url = "https://openrouter.ai/api/v1" @@ -4197,6 +4423,50 @@ class TestRunConversation: agent.compression_enabled = False agent.save_trajectories = False + def test_task_start_failure_closes_relay_turn_and_lease(self, agent): + relay_lease = SimpleNamespace( + parent_session_id="", + profile_key="/profile", + session_id=agent.session_id or "", + ) + relay_turn = object() + coordinator = MagicMock() + coordinator.acquire_conversation.return_value = relay_lease + coordinator.begin_turn.return_value = relay_turn + start_error = RuntimeError("task metrics start failed") + + with ( + patch("agent.relay_runtime.SESSION_COORDINATOR", coordinator), + patch( + "agent.relay_runtime.current_profile_key", + return_value="/profile", + ), + patch( + "hermes_cli.observability.relay_shared_metrics.start_task_run", + side_effect=start_error, + ), + patch( + "hermes_cli.observability.relay_shared_metrics.finish_task_run" + ) as finish_task_run, + patch("agent.conversation_loop.run_conversation") as run_conversation, + ): + with pytest.raises(RuntimeError) as caught: + agent.run_conversation("hello", task_id="task-1") + + assert caught.value is start_error + run_conversation.assert_not_called() + finish_task_run.assert_not_called() + coordinator.finish_logical_calls.assert_called_once_with( + relay_turn, + outcome="failed", + ) + coordinator.end_turn.assert_called_once_with( + relay_turn, + outcome="failed", + ) + coordinator.release_conversation.assert_called_once_with(relay_lease) + assert agent._relay_pending_turn_id is None + def test_stop_finish_reason_returns_response(self, agent): self._setup_agent(agent) resp = _mock_response(content="Final answer", finish_reason="stop") @@ -4280,6 +4550,7 @@ class TestRunConversation: usage=None, ) hook_events = [] + logical_completions = [] def _fake_activate(reason=None): agent._fallback_index = len(agent._fallback_chain) @@ -4291,6 +4562,12 @@ class TestRunConversation: patch.object(agent, "_run_codex_stream", side_effect=[content_filter_response, fallback_response]) as mock_run_codex_stream, patch.object(agent, "_try_activate_fallback", side_effect=_fake_activate) as mock_try_activate_fallback, patch.object(agent, "_invoke_api_request_error_hook", side_effect=lambda **kw: hook_events.append(kw)), + patch( + "agent.relay_llm.complete_logical_call", + side_effect=lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -4304,6 +4581,9 @@ class TestRunConversation: assert hook_events[0]["error_type"] == "ContentPolicyBlocked" assert hook_events[0]["retryable"] is False assert hook_events[0]["reason"] == FailoverReason.content_policy_blocked.value + assert logical_completions == [ + (hook_events[0]["api_request_id"], "success") + ] def test_ollama_small_runtime_context_fails_before_api_call(self, agent, caplog): self._setup_agent(agent) @@ -4383,10 +4663,10 @@ class TestRunConversation: with ( patch("run_agent.handle_function_call", return_value="search result"), patch( - "hermes_cli.plugins.has_hook", + "hermes_cli.lifecycle.has_hook", side_effect=lambda name: name in {"pre_api_request", "post_api_request"}, ), - patch("hermes_cli.plugins.invoke_hook", side_effect=_record_hook), + patch("hermes_cli.lifecycle.invoke_hook", side_effect=_record_hook), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -4399,6 +4679,7 @@ class TestRunConversation: assert len(pre_request_calls) == 2 assert len(post_request_calls) == 2 assert [call["api_call_count"] for call in pre_request_calls] == [1, 2] + assert [call["retry_count"] for call in pre_request_calls] == [0, 0] assert [call["api_call_count"] for call in post_request_calls] == [1, 2] assert all(call["session_id"] == agent.session_id for call in pre_request_calls) assert all(call["turn_id"] == pre_request_calls[0]["turn_id"] for call in pre_request_calls + post_request_calls) @@ -4411,6 +4692,41 @@ class TestRunConversation: assert all("usage" in c and "response" in c for c in post_request_calls) assert all("assistant_message" in c["response"] for c in post_request_calls) + def test_terminal_task_closes_logical_calls_before_metrics_scope(self, agent): + from agent import relay_runtime + + order = [] + failed_result = { + "final_response": "provider failed", + "messages": [], + "completed": False, + "failed": True, + "interrupted": False, + } + + with ( + patch( + "agent.conversation_loop.run_conversation", + return_value=failed_result, + ), + patch( + "hermes_cli.observability.relay_shared_metrics.start_task_run", + ), + patch( + "hermes_cli.observability.relay_shared_metrics.finish_task_run", + side_effect=lambda **_kwargs: order.append("metrics"), + ), + patch.object( + relay_runtime.SESSION_COORDINATOR, + "finish_logical_calls", + side_effect=lambda *_args, **_kwargs: order.append("logical"), + ), + ): + result = agent.run_conversation("private prompt") + + assert result is failed_result + assert order == ["logical", "metrics"] + def test_api_request_error_hook_skips_payload_work_without_listener(self, agent, monkeypatch): payload_built = False hook_called = False @@ -4425,8 +4741,8 @@ class TestRunConversation: hook_called = True return [] - monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: False) - monkeypatch.setattr("hermes_cli.plugins.invoke_hook", _invoke_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", lambda name: False) + monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook", _invoke_hook) monkeypatch.setattr(agent, "_api_request_payload_for_hook", _payload_for_hook) agent._invoke_api_request_error_hook( @@ -4465,12 +4781,12 @@ class TestRunConversation: payload_counts["response"] += 1 return {} - monkeypatch.setattr("hermes_cli.plugins.has_hook", _has_hook) + monkeypatch.setattr("hermes_cli.lifecycle.has_hook", _has_hook) monkeypatch.setattr(agent, "_api_request_payload_for_hook", _request_payload) monkeypatch.setattr(agent, "_api_response_payload_for_hook", _response_payload) with ( - patch("hermes_cli.plugins.invoke_hook", return_value=[]), + patch("hermes_cli.lifecycle.invoke_hook", return_value=[]), patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), @@ -6436,6 +6752,50 @@ class TestRetryExhaustion: assert "Invalid API response" in result["error"] assert result.get("final_response") == result["error"] + def test_invalid_response_retry_completes_one_logical_call(self, agent): + self._setup_agent(agent) + agent.client.chat.completions.create.side_effect = [ + SimpleNamespace(choices=[], model="test/model", usage=None), + _mock_response(content="recovered"), + ] + relay_attempts = [] + logical_completions = [] + + def execute(request, callback, **kwargs): + relay_attempts.append(kwargs) + return callback(request) + + from agent import conversation_loop as _conv_loop + + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + patch("run_agent.time", self._make_fast_time_mock()), + patch.object(_conv_loop, "time", self._make_fast_time_mock()), + patch.object(_conv_loop, "jittered_backoff", lambda *a, **k: 0.0), + patch("agent.relay_llm.execute", side_effect=execute), + patch( + "agent.relay_llm.complete_logical_call", + side_effect=lambda request_id, *, outcome: logical_completions.append( + (request_id, outcome) + ), + ), + ): + result = agent.run_conversation("hello") + + assert result["completed"] is True + assert len(relay_attempts) == 2 + assert all( + attempt["defer_logical_completion"] is True + for attempt in relay_attempts + ) + request_ids = { + attempt["metadata"]["api_request_id"] for attempt in relay_attempts + } + assert len(request_ids) == 1 + assert logical_completions == [(request_ids.pop(), "success")] + def test_content_filter_refusal_surfaced_not_retried(self, agent): """A model refusal must be surfaced immediately, NOT laundered into the empty-response retry loop and reported as "rate limited" / "no diff --git a/tests/run_agent/test_stream_single_writer_65991.py b/tests/run_agent/test_stream_single_writer_65991.py index b1972f969f7e..4fbf74ae2514 100644 --- a/tests/run_agent/test_stream_single_writer_65991.py +++ b/tests/run_agent/test_stream_single_writer_65991.py @@ -137,6 +137,21 @@ class TestSingleWriterLoop: assert "".join(delivered) == "first" assert "-stale-tail" not in "".join(delivered) + def test_chat_parser_failure_closes_managed_stream(self): + agent = _make_agent() + managed_stream = MagicMock() + managed_stream.__iter__.return_value = iter([object()]) + managed_stream.final_response = None + + with patch( + "agent.relay_llm.stream", + return_value=managed_stream, + ): + with pytest.raises(AttributeError): + agent._interruptible_streaming_api_call({}) + + managed_stream.close.assert_called_once() + class TestCodexSingleWriter: """The codex_responses path claims the sink and stops when superseded, @@ -208,6 +223,55 @@ class TestCodexSingleWriter: assert "".join(delivered) == "hello world" + def test_codex_interrupt_closes_stream_without_draining_provider(self): + from agent.codex_runtime import run_codex_stream + + agent = _make_agent() + agent.api_mode = "codex_responses" + produced = [] + stream_closed = threading.Event() + + def interrupt_after_first_delta(_text): + agent._interrupt_requested = True + + agent.stream_delta_callback = interrupt_after_first_delta + agent._stream_callback = None + + def event_gen(): + try: + produced.append("first") + yield self._codex_event( + "response.output_text.delta", + delta="first", + item_id="i1", + ) + produced.append("lookahead") + yield self._codex_event( + "response.output_text.delta", + delta="-unused", + item_id="i1", + ) + produced.append("terminal") + yield self._codex_event( + "response.completed", + response=SimpleNamespace( + id="r1", + status="completed", + output=[], + usage=None, + ), + ) + finally: + stream_closed.set() + + mock_client = MagicMock() + mock_client.responses.create.return_value = event_gen() + + run_codex_stream(agent, {"model": "gpt-5.3-codex"}, client=mock_client) + + assert produced == ["first", "lookahead"] + assert stream_closed.is_set() + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/run_agent/test_streaming.py b/tests/run_agent/test_streaming.py index b613220df202..0f2fc27a5158 100644 --- a/tests/run_agent/test_streaming.py +++ b/tests/run_agent/test_streaming.py @@ -96,6 +96,51 @@ class TestStreamingAccumulator: assert response.usage is not None assert response.usage.completion_tokens == 3 + @patch("run_agent.AIAgent._create_request_openai_client") + @patch("run_agent.AIAgent._close_request_openai_client") + def test_chat_stream_closes_original_provider_resource( + self, + mock_close, + mock_create, + ): + from run_agent import AIAgent + + class ProviderStream: + def __init__(self): + self.closed = False + + def __iter__(self): + return iter([ + _make_stream_chunk( + content="Hello", + finish_reason="stop", + model="test-model", + ) + ]) + + def close(self): + self.closed = True + + provider_stream = ProviderStream() + mock_client = MagicMock() + mock_client.chat.completions.create.return_value = provider_stream + mock_create.return_value = mock_client + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "chat_completions" + agent._interrupt_requested = False + + response = agent._interruptible_streaming_api_call({}) + + assert response.choices[0].message.content == "Hello" + assert provider_stream.closed is True + @patch("run_agent.AIAgent._create_request_openai_client") @patch("run_agent.AIAgent._close_request_openai_client") def test_native_gemini_endpoint_omits_stream_options(self, mock_close, mock_create): @@ -1266,6 +1311,7 @@ class TestAnthropicStreamCallbacks: agent._interruptible_streaming_api_call({}) assert touch_calls.count("receiving stream response") == len(events) + mock_stream.close.assert_called_once() @patch("run_agent.AIAgent._rebuild_anthropic_client") @patch("run_agent.AIAgent._replace_primary_openai_client") diff --git a/tests/run_agent/test_tool_call_guardrail_runtime.py b/tests/run_agent/test_tool_call_guardrail_runtime.py index 36ced43c7956..3bc8a83ab592 100644 --- a/tests/run_agent/test_tool_call_guardrail_runtime.py +++ b/tests/run_agent/test_tool_call_guardrail_runtime.py @@ -220,6 +220,109 @@ def test_config_enabled_hard_stop_concurrent_path_does_not_submit_blocked_calls_ assert completed_events[0][1] == "web_search" +def test_relay_rewrite_precedes_sequential_policy_approval_checkpoint_and_dispatch(): + agent = _make_agent("write_file") + original_args = {"path": "/original/path", "content": "old"} + final_args = {"path": "/approved/path", "content": "new"} + tc = _mock_tool_call("write_file", json.dumps(original_args), "c-rewrite") + msg = SimpleNamespace(content="", tool_calls=[tc]) + messages = [] + observed = { + "plugin": [], + "guardrail": [], + "approval": [], + "checkpoint": [], + "start": [], + "dispatch": [], + } + + original_before_call = agent._tool_guardrails.before_call + + def observe_guardrail(name, args): + observed["guardrail"].append((name, dict(args))) + return original_before_call(name, args) + + def relay_execute(name, args, callback, **kwargs): + del name, args, kwargs + return callback(dict(final_args)), dict(final_args) + + def observe_plugin(name, args, **kwargs): + del kwargs + observed["plugin"].append((name, dict(args))) + return None + + def observe_approval(name, args): + observed["approval"].append((name, dict(args))) + return None + + def dispatch(name, args, task_id, **kwargs): + del task_id, kwargs + observed["dispatch"].append((name, dict(args))) + return json.dumps({"ok": True}) + + agent._checkpoint_mgr = SimpleNamespace( + enabled=True, + get_working_dir_for_path=lambda path: path, + ensure_checkpoint=lambda path, reason: observed["checkpoint"].append( + (path, reason) + ), + ) + agent.tool_start_callback = lambda _call_id, name, args: observed["start"].append( + (name, dict(args)) + ) + + with ( + patch("agent.relay_tools.execute", side_effect=relay_execute), + patch( + "hermes_cli.plugins.resolve_pre_tool_block", + side_effect=observe_plugin, + ), + patch.object(agent._tool_guardrails, "before_call", side_effect=observe_guardrail), + patch( + "acp_adapter.edit_approval.maybe_require_edit_approval", + side_effect=observe_approval, + ), + patch("model_tools.registry.dispatch", side_effect=dispatch), + ): + agent._execute_tool_calls_sequential(msg, messages, "task-1") + + expected = [("write_file", final_args)] + assert observed["plugin"] == expected + assert observed["guardrail"] == expected + assert observed["approval"] == expected + assert observed["start"] == expected + assert observed["dispatch"] == expected + assert observed["checkpoint"] == [ + ("/approved/path", "before write_file") + ] + + +def test_relay_rewrite_is_guarded_before_dispatch_in_concurrent_path(): + agent = _make_agent("web_search", config=_hard_stop_config()) + original_args = {"query": "original"} + blocked_args = {"query": "blocked"} + _seed_exact_failures(agent, "web_search", blocked_args) + tc = _mock_tool_call("web_search", json.dumps(original_args), "c-rewrite-block") + msg = SimpleNamespace(content="", tool_calls=[tc]) + messages = [] + starts = [] + + def relay_execute(name, args, callback, **kwargs): + del name, args, kwargs + return callback(dict(blocked_args)), dict(blocked_args) + + agent.tool_start_callback = lambda *args: starts.append(args) + with ( + patch("agent.relay_tools.execute", side_effect=relay_execute), + patch("run_agent.handle_function_call", return_value="SHOULD_NOT_RUN") as dispatch, + ): + agent._execute_tool_calls_concurrent(msg, messages, "task-1") + + dispatch.assert_not_called() + assert starts == [] + assert "repeated_exact_failure_block" in messages[0]["content"] + + def test_plugin_pre_tool_block_wins_without_counting_as_toolguard_block(): agent = _make_agent("web_search") args = {"query": "same"} diff --git a/tests/scripts/test_smoke_nemo_relay_shared_metrics.py b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py new file mode 100644 index 000000000000..d31a90bb4395 --- /dev/null +++ b/tests/scripts/test_smoke_nemo_relay_shared_metrics.py @@ -0,0 +1,41 @@ +"""Tests for the shared-metrics smoke artifact.""" + +from pathlib import Path + +import pytest + +from scripts import smoke_nemo_relay_shared_metrics as smoke + + +@pytest.mark.parametrize( + "relative_path", + [ + Path(".venv") / "bin" / "hermes", + Path(".venv") / "Scripts" / "hermes.exe", + ], +) +def test_resolve_hermes_executable_from_repository_venv( + tmp_path, + monkeypatch, + relative_path, +): + executable = tmp_path / relative_path + executable.parent.mkdir(parents=True) + executable.touch() + monkeypatch.setattr(smoke.shutil, "which", lambda _name: None) + + assert smoke._resolve_hermes_executable(tmp_path) == executable + + +def test_resolve_hermes_executable_falls_back_to_path(tmp_path, monkeypatch): + executable = tmp_path / "bin" / "hermes" + monkeypatch.setattr(smoke.shutil, "which", lambda _name: str(executable)) + + assert smoke._resolve_hermes_executable(tmp_path / "repo") == executable + + +def test_resolve_hermes_executable_reports_missing_binary(tmp_path, monkeypatch): + monkeypatch.setattr(smoke.shutil, "which", lambda _name: None) + + with pytest.raises(SystemExit, match="or on PATH"): + smoke._resolve_hermes_executable(tmp_path) diff --git a/tests/test_model_tools.py b/tests/test_model_tools.py index 469b8a6921e9..e6b0bfd51f33 100644 --- a/tests/test_model_tools.py +++ b/tests/test_model_tools.py @@ -326,6 +326,40 @@ class TestPreToolCallBlocking: assert "post_tool_call" in hook_calls assert "transform_tool_result" in hook_calls + def test_relay_rewrite_is_visible_to_pre_tool_authorization(self, monkeypatch): + observed = {} + + def rewrite(**kwargs): + assert kwargs["tool_name"] == "read_file" + return {**kwargs["args"], "path": "approved.txt"} + + def fake_invoke_hook(hook_name, **kwargs): + if hook_name == "pre_tool_call": + observed["pre_tool_args"] = kwargs["args"] + return [] + + def dispatch(_name, args, **_kwargs): + observed["dispatch_args"] = args + return json.dumps({"ok": True}) + + monkeypatch.setattr( + "hermes_cli.observability.relay_runtime.apply_tool_request_intercepts", + rewrite, + ) + monkeypatch.setattr("hermes_cli.plugins.invoke_hook", fake_invoke_hook) + monkeypatch.setattr("hermes_cli.plugins.has_hook", lambda name: True) + monkeypatch.setattr("model_tools.registry.dispatch", dispatch) + + handle_function_call( + "read_file", + {"path": "original.txt"}, + task_id="t1", + session_id="s1", + ) + + assert observed["pre_tool_args"]["path"] == "approved.txt" + assert observed["dispatch_args"]["path"] == "approved.txt" + def test_run_agent_pattern_fires_pre_tool_call_exactly_once(self, monkeypatch): """End-to-end regression for the double-fire bug. diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index 0b2257426299..4d685011ccf2 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -3,7 +3,6 @@ from pathlib import Path import tomllib - def _load_optional_dependencies(): pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" with pyproject_path.open("rb") as handle: @@ -11,6 +10,13 @@ def _load_optional_dependencies(): return project["optional-dependencies"] +def _load_package_data(): + pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" + with pyproject_path.open("rb") as handle: + tool = tomllib.load(handle)["tool"] + return tool["setuptools"]["package-data"] + + def test_matrix_extra_not_in_all(): """The [matrix] extra pulls `mautrix[encryption]` -> `python-olm`, which has Linux-only wheels and no native build path on Windows or @@ -215,14 +221,10 @@ def test_feishu_extra_includes_qrcode_for_qr_login(): assert any(dep.startswith("qrcode") for dep in feishu_extra) -def test_nemo_relay_extra_uses_supported_official_distribution_range(): - optional_dependencies = _load_optional_dependencies() +def test_shared_metrics_schema_is_packaged(): + package_data = _load_package_data() - assert optional_dependencies["nemo-relay"] == ["nemo-relay>=0.5,<1.0"] - assert not any( - spec == "hermes-agent[nemo-relay]" - for spec in optional_dependencies["all"] - ) + assert "observability/schemas/*.json" in package_data["hermes_cli"] def _uv_lock_version(package: str) -> str: diff --git a/tests/tools/test_zombie_process_cleanup.py b/tests/tools/test_zombie_process_cleanup.py index a8b745f541a9..b4679ffbe3ac 100644 --- a/tests/tools/test_zombie_process_cleanup.py +++ b/tests/tools/test_zombie_process_cleanup.py @@ -317,9 +317,15 @@ class TestGatewayCleanupWiring: class TestDelegationCleanup: """Verify subagent delegation cleans up child agents.""" - def test_run_single_child_calls_close(self): + def test_run_single_child_calls_close(self, monkeypatch, tmp_path): """_run_single_child finally block should call close() on child.""" from unittest.mock import MagicMock + from hermes_constants import ( + get_hermes_home, + reset_hermes_home_override, + set_hermes_home_override, + ) + from hermes_cli.observability import relay_runtime from tools.delegate_tool import _run_single_child parent = MagicMock() @@ -327,18 +333,155 @@ class TestDelegationCleanup: parent._active_children_lock = threading.Lock() child = MagicMock() + child.session_id = "child-session" child._delegate_saved_tool_names = ["tool1"] - child.run_conversation.side_effect = RuntimeError("test abort") + observed = {} + + def run_conversation(**_kwargs): + observed["hermes_home"] = get_hermes_home() + raise RuntimeError("test abort") + + child.run_conversation.side_effect = run_conversation + relay_host = MagicMock() + monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host) parent._active_children.append(child) + profile_home = tmp_path / "profile-a" + token = set_hermes_home_override(profile_home) + try: + result = _run_single_child( + task_index=0, + goal="test goal", + child=child, + parent_agent=parent, + ) + finally: + reset_hermes_home_override(token) + + child.close.assert_called_once() + assert observed["hermes_home"] == profile_home + relay_host.unregister_subagent.assert_called_once_with( + {"child_session_id": "child-session"} + ) + assert child not in parent._active_children + assert result["status"] == "error" + + def test_active_child_turn_owns_relay_scope_cleanup(self, monkeypatch): + from unittest.mock import MagicMock + + from hermes_cli.observability import relay_runtime + from tools.delegate_tool import _run_single_child + + parent = MagicMock() + parent._active_children = [] + parent._active_children_lock = threading.Lock() + child = MagicMock() + child.session_id = "active-child-session" + child._delegate_saved_tool_names = ["tool1"] + child.run_conversation.side_effect = RuntimeError("test abort") + parent._active_children.append(child) + relay_host = MagicMock() + monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host) + monkeypatch.setattr( + relay_runtime.SESSION_COORDINATOR, + "has_active_turn", + lambda **_kwargs: True, + ) + result = _run_single_child( task_index=0, - goal="test goal", + goal="test active turn cleanup", child=child, parent_agent=parent, ) - child.close.assert_called_once() - assert child not in parent._active_children assert result["status"] == "error" + relay_host.unregister_subagent.assert_not_called() + + def test_timed_out_child_keeps_relay_session_until_its_turn_exits( + self, monkeypatch, tmp_path + ): + from unittest.mock import MagicMock + + from agent import relay_runtime + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + from tools.delegate_tool import _run_single_child + + relay_runtime._reset_for_tests() + profile_home = tmp_path / "profile-timeout" + profile_token = set_hermes_home_override(profile_home) + child_started = threading.Event() + release_child = threading.Event() + child_finished = threading.Event() + parent = MagicMock() + parent._active_children = [] + parent._active_children_lock = threading.Lock() + child = MagicMock() + child.session_id = "timed-out-child" + child._delegate_saved_tool_names = ["tool1"] + child.get_activity_summary.return_value = {"api_call_count": 1} + parent._active_children.append(child) + relay_host = MagicMock() + monkeypatch.setattr(relay_runtime, "get_runtime", lambda **_kwargs: relay_host) + monkeypatch.setattr("tools.delegate_tool._get_child_timeout", lambda: 0.1) + + def run_conversation(**kwargs): + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=child.session_id, + platform="subagent", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="timed-out-child-turn", + task_id=kwargs["task_id"], + ) + child_started.set() + try: + release_child.wait(timeout=5) + return { + "final_response": "late result", + "completed": True, + "interrupted": False, + "api_calls": 1, + "messages": [], + } + finally: + relay_runtime.SESSION_COORDINATOR.end_turn( + turn, + outcome="cancelled", + ) + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + child_finished.set() + + child.run_conversation.side_effect = run_conversation + try: + result = _run_single_child( + task_index=0, + goal="test timed-out turn cleanup", + child=child, + parent_agent=parent, + ) + + assert child_started.is_set() + assert result["status"] == "timeout" + assert relay_runtime.SESSION_COORDINATOR.has_active_turn( + profile_key=str(profile_home), + session_id=child.session_id, + ) + relay_host.unregister_subagent.assert_not_called() + + release_child.set() + assert child_finished.wait(timeout=5) + assert not relay_runtime.SESSION_COORDINATOR.has_active_turn( + profile_key=str(profile_home), + session_id=child.session_id, + ) + finally: + release_child.set() + reset_hermes_home_override(profile_token) + relay_runtime._reset_for_tests() diff --git a/tools/approval.py b/tools/approval.py index 24c08f76916a..57300f0cb630 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -104,7 +104,7 @@ def _fire_approval_hook(hook_name: str, **kwargs) -> None: pre_approval_request, post_approval_response. """ try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook except Exception: # Plugin system not available in this execution context # (e.g. bare tool-only imports, minimal test environments). diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 7894a6af9bfc..b1d32e33d146 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -18,6 +18,7 @@ never the child's intermediate tool calls or reasoning. """ import enum +import contextvars import json import logging @@ -1592,7 +1593,7 @@ def _build_child_agent( logger.debug("spawn_requested relay failed: %s", exc) try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "subagent_start", parent_session_id=getattr(parent_agent, "session_id", None), @@ -2183,7 +2184,11 @@ def _run_single_child( stream_callback=_relay_child_text, ) - _child_future = _timeout_executor.submit(_run_with_thread_capture) + _child_context = contextvars.copy_context() + _child_future = _timeout_executor.submit( + _child_context.run, + _run_with_thread_capture, + ) try: result = _child_future.result(timeout=child_timeout) except Exception as _timeout_exc: @@ -2586,6 +2591,23 @@ def _run_single_child( except Exception: logger.debug("Failed to close child agent after delegation") + # The AIAgent turn boundary normally closes the child scope itself. This + # fallback covers failures before that boundary starts, but must not pop + # a scope while a timed-out child worker is still unwinding. + try: + from agent import relay_runtime + + runtime = relay_runtime.get_runtime(create=False) + child_session_id = str(getattr(child, "session_id", "") or "") + child_turn_is_active = relay_runtime.SESSION_COORDINATOR.has_active_turn( + profile_key=relay_runtime.current_profile_key(), + session_id=child_session_id, + ) + if runtime is not None and child_session_id and not child_turn_is_active: + runtime.unregister_subagent({"child_session_id": child_session_id}) + except Exception: + logger.debug("Failed to close child Relay session after delegation") + _PARENT_FINALIZATION_LOCK_GUARD = threading.Lock() _PARENT_FINALIZATION_FALLBACK_LOCK = threading.RLock() @@ -2984,7 +3006,9 @@ def delegate_task( with DaemonThreadPoolExecutor(max_workers=max_children) as executor: futures = {} for i, t, child in children: + child_context = contextvars.copy_context() future = executor.submit( + child_context.run, _run_single_child, task_index=i, goal=t["goal"], diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 859d30a76497..476582d03e1f 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -2798,7 +2798,7 @@ def terminal_tool( # still subject to the final output limit below. # The hook is fail-open, and the first valid string return wins. try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook hook_results = invoke_hook( "transform_terminal_output", command=command, diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 1e15d788b0a3..e7b73107438f 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -473,13 +473,19 @@ def _notify_session_boundary( ) -> None: """Fire session lifecycle hooks with CLI parity.""" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import finalize_session, invoke_hook - _invoke_hook( - event_type, - session_id=session_id, - platform=_resolve_agent_platform(platform), - ) + if event_type == "on_session_finalize": + finalize_session( + session_id=session_id, + platform=_resolve_agent_platform(platform), + ) + else: + invoke_hook( + event_type, + session_id=session_id, + platform=_resolve_agent_platform(platform), + ) except Exception: pass @@ -658,7 +664,7 @@ def _finalize_session(session: dict | None, end_reason: str = "tui_close") -> No # the user Ctrl‑C's mid‑turn. if agent is not None: try: - from hermes_cli.plugins import invoke_hook + from hermes_cli.lifecycle import invoke_hook invoke_hook( "on_session_end", diff --git a/uv.lock b/uv.lock index f0b075a08ee3..dee539ca975d 100644 --- a/uv.lock +++ b/uv.lock @@ -1530,6 +1530,7 @@ dependencies = [ { name = "httpx", extra = ["socks"] }, { name = "jinja2" }, { name = "markdown" }, + { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')" }, { name = "openai" }, { name = "packaging" }, { name = "pathspec" }, @@ -1665,9 +1666,6 @@ mistral = [ modal = [ { name = "modal" }, ] -nemo-relay = [ - { name = "nemo-relay" }, -] parallel-web = [ { name = "parallel-web" }, ] @@ -1807,7 +1805,7 @@ requires-dist = [ { name = "microsoft-teams-apps", marker = "extra == 'teams'", specifier = "==2.0.13.4" }, { name = "mistralai", marker = "extra == 'mistral'", specifier = "==2.4.8" }, { name = "modal", marker = "extra == 'modal'", specifier = "==1.3.4" }, - { name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5,<1.0" }, + { name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')", specifier = ">=0.6.0,<0.7" }, { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "openai", specifier = "==2.24.0" }, { name = "packaging", specifier = "==26.0" }, @@ -2603,14 +2601,14 @@ wheels = [ [[package]] name = "nemo-relay" -version = "0.5.0" +version = "0.6.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/fc/6892859ffbfb60f8cc09a4181c8d407b574d49490e35cd563e7f0a61bf54/nemo_relay-0.5.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:203409e0f392fc4f40f52277347ed75ba9969b2af28ddb3b6ae53f789185a0ed", size = 7769721, upload-time = "2026-07-08T18:43:57.745Z" }, - { url = "https://files.pythonhosted.org/packages/dd/13/25c4fcf6fb979af07c9562238819183588a910c2ae09767603500d6948c2/nemo_relay-0.5.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de61ad0644c5e8c300de814f6884c5de7ae51db14694f5057f938b4da54bfaaf", size = 7037083, upload-time = "2026-07-08T18:44:00.135Z" }, - { url = "https://files.pythonhosted.org/packages/49/97/bd4c77e975d50f7d09f8bfcb74d7e704e05a8b2205f555bf557f84edc5fe/nemo_relay-0.5.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb323569f104506f4f1a0e2104b22d74c9a0bbf3280ec4d58ad3d768de76fb15", size = 7430817, upload-time = "2026-07-08T18:44:02.165Z" }, - { url = "https://files.pythonhosted.org/packages/da/8b/5681f3430e6d25bf7419dbe576da14599e1cfadc083edb74e691a0c3a980/nemo_relay-0.5.0-cp311-abi3-win_amd64.whl", hash = "sha256:da1bd1e1dec79b7bda3984dd8c59b32ba97abf96c37e85cd1f38d208a29f42b9", size = 7359048, upload-time = "2026-07-08T18:44:04.169Z" }, - { url = "https://files.pythonhosted.org/packages/4e/f2/65c3ef0435e671c829063d4872897f9054f369f0aeacab12c5aefe486705/nemo_relay-0.5.0-cp311-abi3-win_arm64.whl", hash = "sha256:87462585f17b40ce82c54347acef5ace96ee11de3015f234e0b902f23b3793fb", size = 6934235, upload-time = "2026-07-08T18:44:06.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/65/d320016505457cc30971f575e8dadffb923b7cfc780ab8bb25a4ce9d305c/nemo_relay-0.6.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:ad5dae6febf6532d7b113abc2a404679c8feffc499df3034b93d9a078185d2bb", size = 9917779, upload-time = "2026-07-22T20:07:48.961Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c0/f33250e71c4206da1b339072893f9a1e39295fe1aceb9a2fef4b8620a0f2/nemo_relay-0.6.0-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0cd9570f64c6956fe3bfb82af1cdb3ee70cb50b51098cdb0de831c3f9b4e904", size = 8888375, upload-time = "2026-07-22T20:07:51.049Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f4/d1dfaed022da0f6f14765a122867f976a69cc520fe1faaf99757f5719d1f/nemo_relay-0.6.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:849daa9e45158ac581e54506e0fcc7a24f557d1ed06dbdc074f5de7a00393cbc", size = 9336372, upload-time = "2026-07-22T20:07:53.224Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b0/908d77f75b9054e1e403da78f7d9430249a45939070d4298abbb41a04b5b/nemo_relay-0.6.0-cp311-abi3-win_amd64.whl", hash = "sha256:bfbbedfd130fa95c9b8c04643c30910df850e0ed3500beeab82b00ca2d94e7ea", size = 9613425, upload-time = "2026-07-22T20:07:55.175Z" }, + { url = "https://files.pythonhosted.org/packages/cd/71/c438b9d746303ff7f270d99f13b250bf947cdac3e52de2a83fd132cbca0b/nemo_relay-0.6.0-cp311-abi3-win_arm64.whl", hash = "sha256:82fe132943399d89e6ec34dc28df0be7bbe41b84f6698c545928b8816b6010f6", size = 9034810, upload-time = "2026-07-22T20:07:57.467Z" }, ] [[package]] From eda54775e274b530656f27db7f52fb187645a0cb Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Mon, 27 Jul 2026 21:11:04 -0700 Subject: [PATCH 002/156] fix(relay): isolate streaming callback contexts Signed-off-by: Alex Fournier --- agent/relay_llm.py | 23 +++++--- tests/agent/test_relay_llm.py | 92 +++++++++++++++++++++++++++++++ tests/run_agent/test_streaming.py | 89 ++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 9 deletions(-) diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 85286c47dae7..3e3f54d4dfd7 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -344,6 +344,11 @@ class ManagedLlmStream(Iterator[Any]): self.output_modified = False callback_context = contextvars.copy_context() + def run_callback(callback: Callable[..., Any], *args: Any) -> Any: + # Relay can invoke stream surfaces while another callback still + # owns the captured Context. A fresh copy is safe to enter. + return callback_context.copy().run(callback, *args) + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) if ( runtime is None @@ -378,7 +383,7 @@ class ManagedLlmStream(Iterator[Any]): async def provider_stream(next_request: Any): raw_stream = None try: - raw_stream = callback_context.run( + raw_stream = run_callback( stream_factory, _provider_request( request, @@ -390,7 +395,7 @@ class ManagedLlmStream(Iterator[Any]): ) if ( completed_response_predicate is not None - and callback_context.run( + and run_callback( completed_response_predicate, raw_stream, ) @@ -399,14 +404,14 @@ class ManagedLlmStream(Iterator[Any]): self._provider_completed = True return if on_stream_created is not None: - callback_context.run(on_stream_created, raw_stream) - raw_iterator = callback_context.run(iter, raw_stream) + run_callback(on_stream_created, raw_stream) + raw_iterator = run_callback(iter, raw_stream) while True: try: - chunk = callback_context.run(next, raw_iterator) + chunk = run_callback(next, raw_iterator) except StopIteration: break - if self._accept_chunk is not None and not callback_context.run( + if self._accept_chunk is not None and not run_callback( self._accept_chunk, chunk, ): @@ -421,17 +426,17 @@ class ManagedLlmStream(Iterator[Any]): finally: close = getattr(raw_stream, "close", None) if callable(close): - callback_context.run(close) + run_callback(close) def observe_chunk(chunk: Any) -> None: if self._on_chunk is not None: - callback_context.run(self._on_chunk, _jsonable(chunk)) + run_callback(self._on_chunk, _jsonable(chunk)) def relay_finalizer() -> Any: try: if self.final_response is not None: return _jsonable(self.final_response) - return _jsonable(callback_context.run(finalizer)) + return _jsonable(run_callback(finalizer)) except BaseException as exc: self._callback_error = exc raise diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py index 0266d21bd2f2..bca9b6d1648d 100644 --- a/tests/agent/test_relay_llm.py +++ b/tests/agent/test_relay_llm.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio import contextvars import json +import threading from types import SimpleNamespace import pytest @@ -436,6 +437,97 @@ def test_stream_provider_callbacks_preserve_caller_context(relay_turn): ] +def test_anthropic_stream_callbacks_do_not_reenter_captured_context( + relay_turn, + monkeypatch, +): + del relay_turn + caller_value = contextvars.ContextVar( + "anthropic_stream_caller_value", + default="default", + ) + caller_value.set("caller") + callback_context = contextvars.copy_context() + real_copy_context = contextvars.copy_context + copy_count = 0 + + def capture_callback_context(): + nonlocal copy_count + copy_count += 1 + if copy_count == 1: + return callback_context + return real_copy_context() + + monkeypatch.setattr( + relay_llm.contextvars, + "copy_context", + capture_callback_context, + ) + observed = [] + accumulator = relay_llm.AnthropicStreamAccumulator() + + def observe_chunk(chunk): + observed.append(caller_value.get()) + accumulator.observe(chunk) + + chunks = [ + { + "type": "message_start", + "message": { + "id": "message-1", + "type": "message", + "role": "assistant", + "model": "claude-test", + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + ] + stream = relay_llm.stream( + { + "model": "claude-test", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + }, + lambda _request: iter(chunks), + session_id="session-1", + name="anthropic", + model_name="claude-test", + finalizer=accumulator.finalize, + on_chunk=observe_chunk, + metadata={ + "api_mode": "anthropic_messages", + "api_request_id": "request-anthropic-context-reentry", + }, + ) + + entered = threading.Event() + release = threading.Event() + + def hold_callback_context() -> None: + def wait() -> None: + entered.set() + assert release.wait(timeout=5) + + callback_context.run(wait) + + holder = threading.Thread(target=hold_callback_context) + holder.start() + assert entered.wait(timeout=1) + try: + assert list(stream) == chunks + finally: + release.set() + holder.join(timeout=1) + + assert holder.is_alive() is False + assert observed == ["caller", "caller"] + + def test_non_stream_does_not_forward_relay_session_headers(relay_turn): _relay, _turn = relay_turn captured_requests = [] diff --git a/tests/run_agent/test_streaming.py b/tests/run_agent/test_streaming.py index 0f2fc27a5158..199d1ae355e1 100644 --- a/tests/run_agent/test_streaming.py +++ b/tests/run_agent/test_streaming.py @@ -1313,6 +1313,95 @@ class TestAnthropicStreamCallbacks: assert touch_calls.count("receiving stream response") == len(events) mock_stream.close.assert_called_once() + @pytest.mark.filterwarnings("ignore:Pydantic serializer warnings:UserWarning") + def test_anthropic_sdk_stream_runs_through_relay_managed_execution( + self, + tmp_path, + monkeypatch, + ): + anthropic = pytest.importorskip("anthropic") + httpx = pytest.importorskip("httpx") + pytest.importorskip("nemo_relay") + from agent import relay_runtime + from run_agent import AIAgent + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + response_body = b"""event: message_start +data: {"type":"message_start","message":{"id":"msg_test","type":"message","role":"assistant","content":[],"model":"claude-test","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":1}} + +event: message_stop +data: {"type":"message_stop"} + +""" + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=response_body, + request=request, + ) + + client = anthropic.Anthropic( + api_key="test-key", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + relay_runtime._reset_for_tests() + agent = AIAgent( + api_key="test-key", + base_url="https://api.anthropic.com", + provider="anthropic", + model="claude-test", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "anthropic_messages" + agent.session_id = "anthropic-relay-session" + agent._interrupt_requested = False + agent._create_request_anthropic_client = lambda *args, **kwargs: client + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=agent.session_id, + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="anthropic-relay-turn", + task_id="anthropic-relay-task", + ) + lease.host.retain_managed_execution("test.anthropic_relay") + + try: + result = agent._interruptible_streaming_api_call( + { + "model": "claude-test", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + } + ) + finally: + lease.host.release_managed_execution("test.anthropic_relay") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + client.close() + + assert result.content[0].text == "hello" + assert result.stop_reason == "end_turn" + @patch("run_agent.AIAgent._rebuild_anthropic_client") @patch("run_agent.AIAgent._replace_primary_openai_client") def test_anthropic_stream_parser_valueerror_retries_before_delivery( From 6623ee9bb2ed40c9591d25b67128b38a1011faad Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 16 Jul 2026 08:32:27 +0000 Subject: [PATCH 003/156] =?UTF-8?q?perf(state):=20read-path=20split=20?= =?UTF-8?q?=E2=80=94=20per-thread=20read-only=20connections=20for=20recall?= =?UTF-8?q?=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway shares ONE SessionDB across every agent, so every recall/browse read (session_search discover/scroll/browse, memory prefetch, title resolve) queued behind every writer flush on self._lock — one Python lock in front of a WAL database that natively supports concurrent readers. Measured convoy: a 0.23s FTS query stretched to 112s and a browse flush to 137s while 6-8 concurrent turns flushed hundreds of tool results. Fix: under WAL, read-only methods (get_session, resolve_session_by_title, list_sessions_rich, get_messages, get_messages_around, get_anchored_view, search_messages) run on a per-thread mode=ro connection via _read_ctx(), taking no lock at all. Fresh read transactions begin per statement, so read-your-committed-writes holds for flush-then-search patterns. Non-WAL (NFS DELETE fallback) or read-conn open failure keeps the legacy locked single-connection path, remembered per thread to avoid per-query retries. --- hermes_state.py | 146 ++++++++++++++++----- tests/test_session_db_read_path_split.py | 156 +++++++++++++++++++++++ 2 files changed, 268 insertions(+), 34 deletions(-) create mode 100644 tests/test_session_db_read_path_split.py diff --git a/hermes_state.py b/hermes_state.py index 5b12134cb2e6..80cce2c5ef6f 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -27,6 +27,7 @@ import sys import threading import time from collections import deque +from contextlib import contextmanager from pathlib import Path from agent.memory_manager import sanitize_context @@ -2000,6 +2001,11 @@ class SessionDB: self.read_only = read_only self._lock = threading.Lock() + # Read-path split (WAL only): recall/browse queries run on per-thread + # read-only connections so they never queue behind writer flushes on + # self._lock. See _read_ctx(). + self._read_local = threading.local() + self._wal_active = False self._write_count = 0 # One-shot guard for the runtime FTS rebuild recovery on the write # path. A corrupt FTS shadow table makes EVERY message write raise @@ -2098,7 +2104,9 @@ class SessionDB: isolation_level=None, ) self._conn.row_factory = sqlite3.Row - apply_wal_with_fallback(self._conn, db_label="state.db") + self._wal_active = ( + apply_wal_with_fallback(self._conn, db_label="state.db") == "wal" + ) self._conn.execute("PRAGMA foreign_keys=ON") self._fts_cjk_loaded = load_fts5_cjk_extension(self._conn) self._init_schema() @@ -2152,6 +2160,62 @@ class SessionDB: _set_last_init_error(f"{type(exc).__name__}: {exc}") raise + # ── Read-path split ── + + def _get_read_conn(self) -> Optional[sqlite3.Connection]: + """Per-thread read-only connection, or None when unavailable. + + Only used under WAL: WAL readers see a consistent snapshot and never + block on (or get blocked by) the writer, so recall/browse queries can + skip self._lock entirely. Under DELETE journal mode (NFS fallback) a + reader can hit SQLITE_BUSY storms during writes, so we keep the + legacy locked single-connection path there. + + Fresh read transactions begin per statement (autocommit), so each + query observes everything committed so far — read-your-writes holds + for the flush-then-search patterns in a turn. + """ + if not self._wal_active or self.read_only: + return None + conn = getattr(self._read_local, "conn", None) + if conn is not None: + return conn + if getattr(self._read_local, "failed", False): + return None + try: + conn = sqlite3.connect( + f"file:{self.db_path}?mode=ro", + uri=True, + timeout=5.0, + isolation_level=None, + ) + conn.row_factory = sqlite3.Row + except sqlite3.Error: + # Mark this thread failed so we don't retry the open on every + # query; the locked writer connection still serves reads. + self._read_local.failed = True + logger.debug("read-only connection open failed for %s", self.db_path, exc_info=True) + return None + self._read_local.conn = conn + return conn + + @contextmanager + def _read_ctx(self): + """Yield a connection for read-only statements. + + WAL: a per-thread read-only connection with NO lock — recall queries + never convoy behind writer flushes (the gateway shares one SessionDB + across every agent, so this lock was a global choke point). + Non-WAL or read-conn failure: the shared writer connection under + self._lock, byte-for-byte the legacy behavior. + """ + conn = self._get_read_conn() + if conn is not None: + yield conn + return + with self._lock: + yield self._conn + # ── Core write helper ── @staticmethod @@ -2681,6 +2745,20 @@ class SessionDB: # (instance, function), so this removes exactly our registration; # no-op when the writer never started. atexit.unregister(self._drain_token_queue_at_exit) + # Close all read-only connections across all threads. Per-thread + # connections live in threading.local() and would otherwise be GC'd + # without calling close(), leaking tracked fds in _live_connections. + # The strong set holds references so short-lived reader threads' + # connections survive until close() drains them. + with self._read_conns_lock: + read_conns = list(self._read_conns) + self._read_conns.clear() + for conn in read_conns: + try: + conn.close() + except Exception: + pass + self._read_local.conn = None with self._lock: if self._conn: try: @@ -5773,8 +5851,8 @@ class SessionDB: # row through here; drain queued token deltas so they see exact # totals. No-op attribute check when nothing is queued. self.flush_token_counts() - with self._lock: - cursor = self._conn.execute( + with self._read_ctx() as conn: + cursor = conn.execute( "SELECT * FROM sessions WHERE id = ?", (session_id,) ) row = cursor.fetchone() @@ -6086,8 +6164,8 @@ class SessionDB: def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]: """Look up a session by exact title. Returns session dict or None.""" - with self._lock: - cursor = self._conn.execute( + with self._read_ctx() as conn: + cursor = conn.execute( "SELECT * FROM sessions WHERE title = ?", (title,) ) row = cursor.fetchone() @@ -6107,8 +6185,8 @@ class SessionDB: # Also search for numbered variants: "title #2", "title #3", etc. # Escape SQL LIKE wildcards (%, _) in the title to prevent false matches escaped = title.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - with self._lock: - cursor = self._conn.execute( + with self._read_ctx() as conn: + cursor = conn.execute( "SELECT id, title, started_at FROM sessions " "WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC", (f"{escaped} #%",), @@ -6510,8 +6588,8 @@ class SessionDB: LIMIT ? OFFSET ? """ params.extend([limit, offset]) - with self._lock: - cursor = self._conn.execute(query, params) + with self._read_ctx() as conn: + cursor = conn.execute(query, params) rows = cursor.fetchall() sessions = [] for row in rows: @@ -7289,8 +7367,8 @@ class SessionDB: # SQLite's OFFSET requires LIMIT; -1 means "no limit". sql += " LIMIT ? OFFSET ?" params.extend([-1 if limit is None else limit, offset]) - with self._lock: - cursor = self._conn.execute(sql, params) + with self._read_ctx() as conn: + cursor = conn.execute(sql, params) rows = cursor.fetchall() result = [] for row in rows: @@ -7335,9 +7413,9 @@ class SessionDB: """ if window < 0: window = 0 - with self._lock: + with self._read_ctx() as conn: # Confirm the anchor exists in this session. - anchor_exists = self._conn.execute( + anchor_exists = conn.execute( "SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1", (around_message_id, session_id), ).fetchone() @@ -7346,13 +7424,13 @@ class SessionDB: # Two queries: anchor + before (DESC, take window+1), and after # (ASC, take window). Final order is id ASC. - before_rows = self._conn.execute( + before_rows = conn.execute( "SELECT * FROM messages " "WHERE session_id = ? AND id <= ? " "ORDER BY id DESC LIMIT ?", (session_id, around_message_id, window + 1), ).fetchall() - after_rows = self._conn.execute( + after_rows = conn.execute( "SELECT * FROM messages " "WHERE session_id = ? AND id > ? " "ORDER BY id ASC LIMIT ?", @@ -7460,7 +7538,7 @@ class SessionDB: bookend_start_rows: List[Any] = [] bookend_end_rows: List[Any] = [] if bookend > 0: - with self._lock: + with self._read_ctx() as conn: role_clause = "" role_params: list = [] if keep_roles is not None: @@ -7468,7 +7546,7 @@ class SessionDB: role_clause = f" AND role IN ({role_placeholders})" role_params = list(keep_roles) - bookend_start_rows = self._conn.execute( + bookend_start_rows = conn.execute( f"SELECT * FROM messages " f"WHERE session_id = ? AND id < ?{role_clause} " f"AND length(content) > 0 " @@ -7476,7 +7554,7 @@ class SessionDB: (session_id, window_min_id, *role_params, bookend), ).fetchall() - bookend_end_rows = self._conn.execute( + bookend_end_rows = conn.execute( f"SELECT * FROM messages " f"WHERE session_id = ? AND id > ?{role_clause} " f"AND length(content) > 0 " @@ -8697,8 +8775,8 @@ class SessionDB: """ tri_params.extend([limit, offset]) try: - with self._lock: - tri_cursor = self._conn.execute(tri_sql, tri_params) + with self._read_ctx() as conn: + tri_cursor = conn.execute(tri_sql, tri_params) matches = [dict(row) for row in tri_cursor.fetchall()] _trigram_succeeded = True except sqlite3.OperationalError: @@ -8718,8 +8796,8 @@ class SessionDB: # messages table, so CJK search stays available. if self._try_runtime_fts_rebuild(exc): try: - with self._lock: - tri_cursor = self._conn.execute( + with self._read_ctx() as conn: + tri_cursor = conn.execute( tri_sql, tri_params ) matches = [ @@ -8786,13 +8864,13 @@ class SessionDB: like_params.extend([limit, offset]) # instr() for snippet uses first search token like_params = [non_op_tokens[0]] + like_params - with self._lock: - like_cursor = self._conn.execute(like_sql, like_params) + with self._read_ctx() as conn: + like_cursor = conn.execute(like_sql, like_params) matches = [dict(row) for row in like_cursor.fetchall()] else: try: - with self._lock: - cursor = self._conn.execute(sql, params) + with self._read_ctx() as conn: + cursor = conn.execute(sql, params) matches = [dict(row) for row in cursor.fetchall()] except sqlite3.OperationalError: # FTS5 query syntax error despite sanitization — return empty @@ -8802,14 +8880,14 @@ class SessionDB: # structure record" class on the MATCH read, the same class the # write path self-heals (#66296). OperationalError (query # syntax) is a subclass caught above; this arm is the corruption - # parent. Rebuild the index in place once — the lock is released - # here, so rebuild_fts() can re-acquire it — and retry, so - # search self-heals for read-only sessions (cron/CLI history - # search) that never trigger a write to repair it first. + # parent. Rebuild the index in place once — the read context + # holds no writer lock, so rebuild_fts() can acquire it — and + # retry, so search self-heals for read-only sessions (cron/CLI + # history search) that never trigger a write to repair it first. if not self._try_runtime_fts_rebuild(exc): raise - with self._lock: - cursor = self._conn.execute(sql, params) + with self._read_ctx() as conn: + cursor = conn.execute(sql, params) matches = [dict(row) for row in cursor.fetchall()] # Deferred-rebuild supplement (schema v23): while the background @@ -8894,8 +8972,8 @@ class SessionDB: # Done outside the lock so we don't hold it across N sequential queries. for match in matches: try: - with self._lock: - ctx_cursor = self._conn.execute( + with self._read_ctx() as conn: + ctx_cursor = conn.execute( """WITH target AS ( SELECT session_id, timestamp, id FROM messages diff --git a/tests/test_session_db_read_path_split.py b/tests/test_session_db_read_path_split.py new file mode 100644 index 000000000000..160637b156fd --- /dev/null +++ b/tests/test_session_db_read_path_split.py @@ -0,0 +1,156 @@ +"""Tests for the SessionDB read-path split (per-thread read-only connections). + +The gateway shares ONE SessionDB across every agent, so recall/browse reads +used to queue behind writer flushes on self._lock — a measured production +convoy (a 0.2s FTS query stretched to 112s while 6-8 concurrent turns +flushed tool results). These tests pin the new contract: reads run on a +per-thread read-only connection under WAL, never touch self._lock, and fall +back to the legacy locked path when WAL or the read connection is missing. +""" + +import threading +import time + +import pytest + +from hermes_state import SessionDB + + +@pytest.fixture() +def db(tmp_path): + d = SessionDB(db_path=tmp_path / "state.db") + d.create_session(session_id="s1", source="cli", model="m") + d.append_message("s1", role="user", content="hello graphiti world") + d.append_message("s1", role="assistant", content="the neo4j daemon is healthy") + yield d + d.close() + + +def test_read_conn_is_per_thread(db): + conns = {} + + def grab(key): + conns[key] = db._get_read_conn() + + t1 = threading.Thread(target=grab, args=(1,)) + t2 = threading.Thread(target=grab, args=(2,)) + t1.start(); t2.start(); t1.join(); t2.join() + assert conns[1] is not None and conns[2] is not None + assert conns[1] is not conns[2] + + +def test_read_conn_reused_within_thread(db): + assert db._get_read_conn() is db._get_read_conn() + + +def test_reads_do_not_take_writer_lock(db): + """Reads must complete while another thread holds self._lock.""" + acquired = db._lock.acquire() + assert acquired + try: + done = {} + + def reader(): + done["session"] = db.get_session("s1") + done["search"] = db.search_messages("graphiti", limit=10) + done["messages"] = db.get_messages("s1") + + t = threading.Thread(target=reader) + t.start() + t.join(timeout=5.0) + assert not t.is_alive(), "read path blocked on writer lock" + assert done["session"]["id"] == "s1" + assert any("graphiti" in (m.get("snippet") or "") for m in done["search"]) + assert len(done["messages"]) == 2 + finally: + db._lock.release() + + +def test_title_resolution_does_not_take_writer_lock(db): + """Exact-title and numbered-variant resolution must not block on self._lock.""" + db.create_session(session_id="t1", source="cli", model="m") + db.set_session_title("t1", "ops sync") + db.create_session(session_id="t2", source="cli", model="m") + db.set_session_title("t2", "ops sync #2") + acquired = db._lock.acquire() + assert acquired + try: + done = {} + + def reader(): + done["exact"] = db.get_session_by_title("ops sync") + done["resolved"] = db.resolve_session_by_title("ops sync") + + t = threading.Thread(target=reader) + t.start() + t.join(timeout=5.0) + assert not t.is_alive(), "title resolution blocked on writer lock" + assert done["exact"]["id"] == "t1" + # Lineage rule: the latest numbered variant wins over the exact match. + assert done["resolved"] == "t2" + finally: + db._lock.release() + + +def test_read_your_writes(db): + """A fresh committed write must be visible to the read connection.""" + db.append_message("s1", role="user", content="zanzibar checkpoint") + rows = db.search_messages("zanzibar", limit=5) + assert rows, "committed write invisible to read connection" + + +def test_fallback_when_read_conn_unavailable(db, monkeypatch): + monkeypatch.setattr(db, "_get_read_conn", lambda: None) + assert db.get_session("s1")["id"] == "s1" + assert db.search_messages("graphiti", limit=5) + + +def test_non_wal_uses_locked_path(db): + db._wal_active = False + assert db._get_read_conn() is None + # And queries still work via the legacy path. + assert db.get_session("s1")["id"] == "s1" + + +def test_read_conn_open_failure_marks_thread(db, monkeypatch, tmp_path): + """A failed read-conn open must not retry per query; fallback still works.""" + import sqlite3 as _sqlite3 + + calls = {"n": 0} + real_connect = _sqlite3.connect + + def failing_connect(*a, **k): + if a and isinstance(a[0], str) and a[0].startswith("file:") and "mode=ro" in a[0]: + calls["n"] += 1 + raise _sqlite3.OperationalError("simulated open failure") + return real_connect(*a, **k) + + fresh = SessionDB(db_path=tmp_path / "state2.db") + try: + fresh.create_session(session_id="x", source="cli", model="m") + monkeypatch.setattr("hermes_state.sqlite3.connect", failing_connect) + assert fresh.get_session("x")["id"] == "x" + assert fresh.get_session("x")["id"] == "x" + assert calls["n"] == 1, "open failure should be remembered per thread" + finally: + fresh.close() + + +def test_anchored_view_and_around_use_read_path(db): + msgs = db.get_messages("s1") + anchor = msgs[0]["id"] + acquired = db._lock.acquire() + try: + done = {} + + def reader(): + done["around"] = db.get_messages_around("s1", anchor, window=2) + done["view"] = db.get_anchored_view("s1", anchor, window=2, bookend=1) + + t = threading.Thread(target=reader) + t.start(); t.join(timeout=5.0) + assert not t.is_alive(), "anchored reads blocked on writer lock" + assert done["around"]["window"] + assert done["view"]["window"] + finally: + db._lock.release() From f228e145ba35cbbf785eded2021ae6682285b91b Mon Sep 17 00:00:00 2001 From: kshitij kapoor Date: Tue, 28 Jul 2026 17:45:17 +0500 Subject: [PATCH 004/156] =?UTF-8?q?fix:=20follow-up=20for=20PR=20#65541=20?= =?UTF-8?q?=E2=80=94=20track=20read=20conns,=20convert=20remaining=20read?= =?UTF-8?q?=20paths,=20mark=20WAL=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Route _get_read_conn through _connect_tracked_db so per-thread read-only connections are registered with the POSIX lock-safety guard (connect_tracked), matching the writer and existing read-only paths. Without this, byte-level probes of state.db could close() an fd that cancels locks held by an untracked read connection. - Convert _search_unindexed_gap, _run_trigram_search, CJK-bigram FTS search, and get_meta to _read_ctx — these are pure SELECT queries called from search_messages that were still taking self._lock, defeating the PR's contention fix for those paths. - Add @pytest.mark.requires_wal to the 5 tests that assume WAL is active. Hermes disables WAL on SQLite < 3.51.3 (WAL-reset bug), so these tests fail on the venv's SQLite 3.46.0 without the marker. - Remove unused 'time' import. --- hermes_state.py | 81 +++++++++++++++++++----- tests/test_session_db_read_path_split.py | 6 +- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/hermes_state.py b/hermes_state.py index 80cce2c5ef6f..0cb56250ac70 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -2005,6 +2005,16 @@ class SessionDB: # read-only connections so they never queue behind writer flushes on # self._lock. See _read_ctx(). self._read_local = threading.local() + # Strong set of all live read connections across all threads. We + # hold a reference so short-lived reader threads' connections are + # not GC'd without close() — that would leak tracked fds in + # _live_connections. close() drains this set. + self._read_conns: "set[sqlite3.Connection]" = set() + self._read_conns_lock = threading.Lock() + # Set when close() begins. _get_read_conn checks this under the + # lock so a reader that finishes opening after the drain finds the + # shutdown in progress and closes its own connection immediately. + self._read_conns_closed = False self._wal_active = False self._write_count = 0 # One-shot guard for the runtime FTS rebuild recovery on the write @@ -2183,13 +2193,28 @@ class SessionDB: if getattr(self._read_local, "failed", False): return None try: - conn = sqlite3.connect( + conn = _connect_tracked_db( f"file:{self.db_path}?mode=ro", + tracking_path=self.db_path, uri=True, timeout=5.0, isolation_level=None, ) conn.row_factory = sqlite3.Row + # Load the CJK tokenizer extension on this connection so + # messages_fts_cjk queries work on the read path. The .so + # registers the tokenizer in the connection's in-memory + # registry, not the database file, so mode=ro is fine. + if self._fts_cjk_loaded: + load_fts5_cjk_extension(conn) + with self._read_conns_lock: + if self._read_conns_closed: + # close() already drained — don't register; close + # immediately so no tracked fd leaks. + conn.close() + self._read_local.failed = True + return None + self._read_conns.add(conn) except sqlite3.Error: # Mark this thread failed so we don't retry the open on every # query; the locked writer connection still serves reads. @@ -2749,8 +2774,11 @@ class SessionDB: # connections live in threading.local() and would otherwise be GC'd # without calling close(), leaking tracked fds in _live_connections. # The strong set holds references so short-lived reader threads' - # connections survive until close() drains them. + # connections survive until close() drains them. Setting the closed + # flag under the lock prevents a reader from registering a new + # connection after the drain. with self._read_conns_lock: + self._read_conns_closed = True read_conns = list(self._read_conns) self._read_conns.clear() for conn in read_conns: @@ -2809,11 +2837,21 @@ class SessionDB: "indexed": , "percent": <0-100 int>}. Consumed by search_messages() notes and by status surfaces (dashboard/desktop can poll this to render a progress indicator). + + Reads state_meta directly via _read_ctx instead of calling + get_meta() (which takes self._lock) so search_messages doesn't + block on the writer lock when checking rebuild status. """ - high_water = self.get_meta("fts_rebuild_high_water") + with self._read_ctx() as conn: + row = conn.execute( + "SELECT key, value FROM state_meta WHERE key IN (?, ?)", + ("fts_rebuild_high_water", "fts_rebuild_progress"), + ).fetchall() + meta = {r["key"]: r["value"] for r in row} + high_water = meta.get("fts_rebuild_high_water") if high_water is None: return None - progress = int(self.get_meta("fts_rebuild_progress") or 0) + progress = int(meta.get("fts_rebuild_progress") or 0) total = int(high_water) if total <= 0: return None @@ -2986,10 +3024,16 @@ class SessionDB: def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]: """CJK-index backfill progress, or None when none is pending.""" - high_water = self.get_meta("fts_cjk_rebuild_high_water") + with self._read_ctx() as conn: + row = conn.execute( + "SELECT key, value FROM state_meta WHERE key IN (?, ?)", + ("fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress"), + ).fetchall() + meta = {r["key"]: r["value"] for r in row} + high_water = meta.get("fts_cjk_rebuild_high_water") if high_water is None: return None - progress = int(self.get_meta("fts_cjk_rebuild_progress") or 0) + progress = int(meta.get("fts_cjk_rebuild_progress") or 0) total = int(high_water) if total <= 0: return None @@ -8403,9 +8447,9 @@ class SessionDB: LIMIT ? OFFSET ? """ tri_params.extend([limit, offset]) - with self._lock: + with self._read_ctx() as conn: try: - tri_cursor = self._conn.execute(tri_sql, tri_params) + tri_cursor = conn.execute(tri_sql, tri_params) except sqlite3.OperationalError: # Query failed at runtime — let the caller fall back. return None @@ -8686,8 +8730,8 @@ class SessionDB: """ cjk_params.extend([limit, offset]) try: - with self._lock: - cjk_cursor = self._conn.execute(cjk_sql, cjk_params) + with self._read_ctx() as conn: + cjk_cursor = conn.execute(cjk_sql, cjk_params) matches = [dict(row) for row in cjk_cursor.fetchall()] _trigram_succeeded = True except sqlite3.OperationalError: @@ -8702,8 +8746,8 @@ class SessionDB: # in place once and retry; on refusal/failure fall back. if self._try_runtime_fts_rebuild(exc): try: - with self._lock: - cjk_cursor = self._conn.execute( + with self._read_ctx() as conn: + cjk_cursor = conn.execute( cjk_sql, cjk_params ) matches = [ @@ -8969,7 +9013,8 @@ class SessionDB: matches = tri_matches # Add surrounding context (1 message before + after each match). - # Done outside the lock so we don't hold it across N sequential queries. + # Each query takes its own fresh read transaction via _read_ctx, so + # we never hold a lock across N sequential queries. for match in matches: try: with self._read_ctx() as conn: @@ -9105,8 +9150,8 @@ class SessionDB: LIMIT ? """ params = [terms[0]] + params + [limit] - with self._lock: - rows = self._conn.execute(sql, params).fetchall() + with self._read_ctx() as conn: + rows = conn.execute(sql, params).fetchall() return [dict(r) for r in rows] def search_sessions_by_id( @@ -10515,6 +10560,12 @@ class SessionDB: def get_meta(self, key: str) -> Optional[str]: """Read a value from the state_meta key/value store.""" + # Kept on self._lock (not _read_ctx) because callers like + # fts_rebuild_step read progress before entering a write + # transaction, and the read-only WAL connection sees only + # committed data — a pending write transaction's uncommitted + # meta writes would be invisible. This is a cheap point lookup, + # not the convoy bottleneck the read-path split targets. with self._lock: row = self._conn.execute( "SELECT value FROM state_meta WHERE key = ?", (key,) diff --git a/tests/test_session_db_read_path_split.py b/tests/test_session_db_read_path_split.py index 160637b156fd..1ab8760c5115 100644 --- a/tests/test_session_db_read_path_split.py +++ b/tests/test_session_db_read_path_split.py @@ -9,7 +9,6 @@ back to the legacy locked path when WAL or the read connection is missing. """ import threading -import time import pytest @@ -26,6 +25,7 @@ def db(tmp_path): d.close() +@pytest.mark.requires_wal def test_read_conn_is_per_thread(db): conns = {} @@ -43,6 +43,7 @@ def test_read_conn_reused_within_thread(db): assert db._get_read_conn() is db._get_read_conn() +@pytest.mark.requires_wal def test_reads_do_not_take_writer_lock(db): """Reads must complete while another thread holds self._lock.""" acquired = db._lock.acquire() @@ -66,6 +67,7 @@ def test_reads_do_not_take_writer_lock(db): db._lock.release() +@pytest.mark.requires_wal def test_title_resolution_does_not_take_writer_lock(db): """Exact-title and numbered-variant resolution must not block on self._lock.""" db.create_session(session_id="t1", source="cli", model="m") @@ -112,6 +114,7 @@ def test_non_wal_uses_locked_path(db): assert db.get_session("s1")["id"] == "s1" +@pytest.mark.requires_wal def test_read_conn_open_failure_marks_thread(db, monkeypatch, tmp_path): """A failed read-conn open must not retry per query; fallback still works.""" import sqlite3 as _sqlite3 @@ -136,6 +139,7 @@ def test_read_conn_open_failure_marks_thread(db, monkeypatch, tmp_path): fresh.close() +@pytest.mark.requires_wal def test_anchored_view_and_around_use_read_path(db): msgs = db.get_messages("s1") anchor = msgs[0]["id"] From 2dcd7448d5ef28b76ccbac0f02ac42d2f19a2913 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Tue, 28 Jul 2026 08:21:05 -0700 Subject: [PATCH 005/156] test(relay): gate provider stream contracts Signed-off-by: Alex Fournier --- .github/workflows/tests.yml | 17 +++++ .../test_request_client_reuse_abort_races.py | 66 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cdae2e037a59..c81e7d84b4bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -101,6 +101,23 @@ jobs: # re-download, keeping the persisted cache small and fast to restore. run: uv cache prune --ci + - name: Run merge-sensitive provider contracts + if: matrix.slice.index == 1 + # Keep provider concurrency and client-lifecycle contracts visible + # outside the duration-balanced file shards. + run: | + source .venv/bin/activate + python -m pytest -q --tb=short \ + tests/agent/test_relay_llm.py::test_anthropic_stream_callbacks_do_not_reenter_captured_context \ + tests/agent/test_relay_llm.py::test_explicit_stream_close_surfaces_provider_close_failure \ + tests/run_agent/test_streaming.py::TestAnthropicStreamCallbacks::test_anthropic_sdk_stream_runs_through_relay_managed_execution \ + tests/run_agent/test_request_client_reuse_abort_races.py::test_relay_managed_close_failure_poisons_request_client + env: + ANTHROPIC_API_KEY: "" + OPENROUTER_API_KEY: "" + OPENAI_API_KEY: "" + NOUS_API_KEY: "" + - name: Run tests (slice ${{ matrix.slice.index }}/${{ inputs.slice_count }}) # Per-file isolation via scripts/run_tests.sh: each test file runs # in its own freshly-spawned `python -m pytest ` subprocess diff --git a/tests/run_agent/test_request_client_reuse_abort_races.py b/tests/run_agent/test_request_client_reuse_abort_races.py index 73a9c41887cc..76b3bc49946d 100644 --- a/tests/run_agent/test_request_client_reuse_abort_races.py +++ b/tests/run_agent/test_request_client_reuse_abort_races.py @@ -25,7 +25,12 @@ Invariants pinned here: 4. ``run_codex_stream``'s finally must poison the reuse slot when ``event_stream.close()`` fails; otherwise a failed close caches the client with a connection still checked out of the pool. + +5. Relay's managed stream wrapper must preserve the same close failure and + request-client identity rather than masking the signal used to poison the + slot. """ +from contextlib import contextmanager import threading import time from types import SimpleNamespace @@ -49,6 +54,33 @@ def _make_agent(): return agent +@contextmanager +def _managed_relay_turn(agent, tmp_path, monkeypatch): + pytest.importorskip("nemo_relay") + from agent import relay_runtime + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profile")) + relay_runtime._reset_for_tests() + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=agent.session_id, + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="request-client-reuse-turn", + task_id="request-client-reuse-task", + ) + lease.host.retain_managed_execution("test.request_client_reuse") + try: + yield + finally: + lease.host.release_managed_execution("test.request_client_reuse") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + + def _chunk(content=None, finish_reason=None): return SimpleNamespace( choices=[ @@ -154,6 +186,40 @@ def test_worker_interrupt_break_poisons_slot_when_stream_close_fails(): assert "interrupt_stream_close_failed" in abort_reasons +def test_relay_managed_close_failure_poisons_request_client(tmp_path, monkeypatch): + """Relay must preserve close failures needed by the reuse-slot guard.""" + agent = _make_agent() + + def chunks(): + yield _chunk(content="partial ") + agent._interrupt_requested = True + yield _chunk(content="never processed") + + stream = _FakeStream(chunks, close_raises=True) + request_client = _mock_wire_client(stream) + abort_reasons = [] + + with _managed_relay_turn(agent, tmp_path, monkeypatch), patch.object( + agent, "_create_request_openai_client", return_value=request_client + ), patch.object(agent, "_close_request_openai_client"), patch.object( + agent, + "_abort_request_openai_client", + side_effect=lambda client, *, reason: abort_reasons.append( + (client, reason) + ), + ): + with pytest.raises(InterruptedError): + agent._interruptible_streaming_api_call( + { + "model": "test/model", + "messages": [{"role": "user", "content": "hello"}], + } + ) + + assert stream.close_calls == 1 + assert abort_reasons == [(request_client, "interrupt_stream_close_failed")] + + def test_stale_abort_is_atomic_with_holder_read(monkeypatch): """The stranger-thread abort must complete before the worker's finally can pop + cache the client. From 956fc87eef9c165ebb7976531097b02555821a34 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Tue, 28 Jul 2026 08:25:25 -0700 Subject: [PATCH 006/156] test(relay): gate native Anthropic streaming in e2e Signed-off-by: Alex Fournier --- .github/workflows/tests.yml | 17 ---- .../e2e/test_relay_native_anthropic_stream.py | 92 +++++++++++++++++++ tests/run_agent/test_streaming.py | 89 ------------------ 3 files changed, 92 insertions(+), 106 deletions(-) create mode 100644 tests/e2e/test_relay_native_anthropic_stream.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c81e7d84b4bf..cdae2e037a59 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -101,23 +101,6 @@ jobs: # re-download, keeping the persisted cache small and fast to restore. run: uv cache prune --ci - - name: Run merge-sensitive provider contracts - if: matrix.slice.index == 1 - # Keep provider concurrency and client-lifecycle contracts visible - # outside the duration-balanced file shards. - run: | - source .venv/bin/activate - python -m pytest -q --tb=short \ - tests/agent/test_relay_llm.py::test_anthropic_stream_callbacks_do_not_reenter_captured_context \ - tests/agent/test_relay_llm.py::test_explicit_stream_close_surfaces_provider_close_failure \ - tests/run_agent/test_streaming.py::TestAnthropicStreamCallbacks::test_anthropic_sdk_stream_runs_through_relay_managed_execution \ - tests/run_agent/test_request_client_reuse_abort_races.py::test_relay_managed_close_failure_poisons_request_client - env: - ANTHROPIC_API_KEY: "" - OPENROUTER_API_KEY: "" - OPENAI_API_KEY: "" - NOUS_API_KEY: "" - - name: Run tests (slice ${{ matrix.slice.index }}/${{ inputs.slice_count }}) # Per-file isolation via scripts/run_tests.sh: each test file runs # in its own freshly-spawned `python -m pytest ` subprocess diff --git a/tests/e2e/test_relay_native_anthropic_stream.py b/tests/e2e/test_relay_native_anthropic_stream.py new file mode 100644 index 000000000000..3f1075071e03 --- /dev/null +++ b/tests/e2e/test_relay_native_anthropic_stream.py @@ -0,0 +1,92 @@ +"""Native Anthropic SDK streaming through Relay's managed execution path.""" + +import pytest + + +@pytest.mark.filterwarnings("ignore:Pydantic serializer warnings:UserWarning") +def test_anthropic_sdk_stream_runs_through_relay_managed_execution( + tmp_path, + monkeypatch, +): + anthropic = pytest.importorskip("anthropic") + httpx = pytest.importorskip("httpx") + pytest.importorskip("nemo_relay") + from agent import relay_runtime + from run_agent import AIAgent + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) + response_body = b"""event: message_start +data: {"type":"message_start","message":{"id":"msg_test","type":"message","role":"assistant","content":[],"model":"claude-test","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":1}} + +event: message_stop +data: {"type":"message_stop"} + +""" + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=response_body, + request=request, + ) + + client = anthropic.Anthropic( + api_key="test-key", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + relay_runtime._reset_for_tests() + agent = AIAgent( + api_key="test-key", + base_url="https://api.anthropic.com", + provider="anthropic", + model="claude-test", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "anthropic_messages" + agent.session_id = "anthropic-relay-session" + agent._interrupt_requested = False + agent._create_request_anthropic_client = lambda *args, **kwargs: client + lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( + profile_key=relay_runtime.current_profile_key(), + session_id=agent.session_id, + platform="cli", + ) + turn = relay_runtime.SESSION_COORDINATOR.begin_turn( + lease, + turn_id="anthropic-relay-turn", + task_id="anthropic-relay-task", + ) + lease.host.retain_managed_execution("test.anthropic_relay") + + try: + result = agent._interruptible_streaming_api_call( + { + "model": "claude-test", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + } + ) + finally: + lease.host.release_managed_execution("test.anthropic_relay") + relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") + relay_runtime.SESSION_COORDINATOR.release_conversation(lease) + relay_runtime._reset_for_tests() + client.close() + + assert result.content[0].text == "hello" + assert result.stop_reason == "end_turn" diff --git a/tests/run_agent/test_streaming.py b/tests/run_agent/test_streaming.py index 199d1ae355e1..0f2fc27a5158 100644 --- a/tests/run_agent/test_streaming.py +++ b/tests/run_agent/test_streaming.py @@ -1313,95 +1313,6 @@ class TestAnthropicStreamCallbacks: assert touch_calls.count("receiving stream response") == len(events) mock_stream.close.assert_called_once() - @pytest.mark.filterwarnings("ignore:Pydantic serializer warnings:UserWarning") - def test_anthropic_sdk_stream_runs_through_relay_managed_execution( - self, - tmp_path, - monkeypatch, - ): - anthropic = pytest.importorskip("anthropic") - httpx = pytest.importorskip("httpx") - pytest.importorskip("nemo_relay") - from agent import relay_runtime - from run_agent import AIAgent - - monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) - response_body = b"""event: message_start -data: {"type":"message_start","message":{"id":"msg_test","type":"message","role":"assistant","content":[],"model":"claude-test","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":0}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":1}} - -event: message_stop -data: {"type":"message_stop"} - -""" - - def respond(request): - return httpx.Response( - 200, - headers={"content-type": "text/event-stream"}, - content=response_body, - request=request, - ) - - client = anthropic.Anthropic( - api_key="test-key", - http_client=httpx.Client(transport=httpx.MockTransport(respond)), - ) - relay_runtime._reset_for_tests() - agent = AIAgent( - api_key="test-key", - base_url="https://api.anthropic.com", - provider="anthropic", - model="claude-test", - quiet_mode=True, - skip_context_files=True, - skip_memory=True, - ) - agent.api_mode = "anthropic_messages" - agent.session_id = "anthropic-relay-session" - agent._interrupt_requested = False - agent._create_request_anthropic_client = lambda *args, **kwargs: client - lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( - profile_key=relay_runtime.current_profile_key(), - session_id=agent.session_id, - platform="cli", - ) - turn = relay_runtime.SESSION_COORDINATOR.begin_turn( - lease, - turn_id="anthropic-relay-turn", - task_id="anthropic-relay-task", - ) - lease.host.retain_managed_execution("test.anthropic_relay") - - try: - result = agent._interruptible_streaming_api_call( - { - "model": "claude-test", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - } - ) - finally: - lease.host.release_managed_execution("test.anthropic_relay") - relay_runtime.SESSION_COORDINATOR.end_turn(turn, outcome="success") - relay_runtime.SESSION_COORDINATOR.release_conversation(lease) - relay_runtime._reset_for_tests() - client.close() - - assert result.content[0].text == "hello" - assert result.stop_reason == "end_turn" - @patch("run_agent.AIAgent._rebuild_anthropic_client") @patch("run_agent.AIAgent._replace_primary_openai_client") def test_anthropic_stream_parser_valueerror_retries_before_delivery( From 5e34fa2d5c32a4c1c45d658d7bb6039acd018cc9 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Tue, 28 Jul 2026 09:58:43 -0700 Subject: [PATCH 007/156] fix(relay): preserve provider stream errors Signed-off-by: Alex Fournier --- agent/relay_llm.py | 5 ++++ tests/agent/test_relay_llm.py | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 4508fce94144..3481ca072376 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -438,6 +438,11 @@ class ManagedLlmStream(Iterator[Any]): run_callback(self._on_chunk, _jsonable(chunk)) def relay_finalizer() -> Any: + # Relay can invoke the finalizer while unwinding a provider-stream + # failure. Preserve that original callback error instead of + # replacing it with a secondary "missing terminal response" error. + if self._callback_error is not None: + return None try: if self.final_response is not None: return _jsonable(self.final_response) diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py index ae4eac15826a..2e09923f327e 100644 --- a/tests/agent/test_relay_llm.py +++ b/tests/agent/test_relay_llm.py @@ -181,6 +181,49 @@ def test_deferred_stream_preserves_provider_error_and_logical_scope_for_retry( assert "request-2" in turn.logical_llm_calls +def test_stream_provider_error_is_not_replaced_by_finalizer_error(relay_turn): + _relay, turn = relay_turn + + class ProviderError(Exception): + pass + + provider_error = ProviderError("provider failed before first chunk") + finalizer_called = False + + def failing_stream(_request): + def generate(): + raise provider_error + yield # pragma: no cover + + return generate() + + def failing_finalizer(): + nonlocal finalizer_called + finalizer_called = True + raise RuntimeError("missing terminal response") + + stream = relay_llm.stream( + {"model": "test-model", "input": "hi"}, + failing_stream, + session_id="session-1", + name="test-provider", + model_name="test-model", + finalizer=failing_finalizer, + metadata={ + "api_mode": "codex_responses", + "api_request_id": "request-provider-before-finalizer", + }, + defer_logical_completion=True, + ) + + with pytest.raises(ProviderError) as caught: + list(stream) + + assert caught.value is provider_error + assert finalizer_called is False + assert "request-provider-before-finalizer" in turn.logical_llm_calls + + def test_non_deferred_partial_stream_close_cancels_logical_call( relay_turn, monkeypatch, From fbc878ee2ee56c76b2d9de49e7ffa3802d0e8ed8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:40:51 -0700 Subject: [PATCH 008/156] feat(models): swap Gemini catalog entries to 3.1 Pro + 3.6 Flash; drop retired Qwen models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - openrouter + nous curated lists: replace google/gemini-3-pro-preview with google/gemini-3.1-pro-preview as the sole Pro entry, and google/gemini-3.5-flash with google/gemini-3.6-flash - remove qwen/qwen3.7-plus and qwen/qwen3.6-35b-a3b from both lists - regenerate website/static/api/model-catalog.json - test fixture: swap qwen3.7-plus catalog-label fixture to qwen3.7-max (must be a model present in the nous curated list) Both new Gemini ids verified live on OpenRouter /api/v1/models and the Nous portal /v1/models (1,048,576 ctx — covered by the existing 'gemini' prefix in DEFAULT_CONTEXT_LENGTHS). --- hermes_cli/models.py | 10 ++-------- tests/test_empty_model_fallback.py | 8 ++++---- website/static/api/model-catalog.json | 27 +++------------------------ 3 files changed, 9 insertions(+), 36 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 617fa1906dc1..8fef8ad051bf 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -58,9 +58,8 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [ ("openai/gpt-5.5-pro", ""), ("openai/gpt-5.4-mini", ""), # Google - ("google/gemini-3-pro-preview", ""), ("google/gemini-3.1-pro-preview", ""), - ("google/gemini-3.5-flash", ""), + ("google/gemini-3.6-flash", ""), # xAI ("x-ai/grok-4.5", ""), # DeepSeek @@ -68,8 +67,6 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [ ("deepseek/deepseek-v4-flash", ""), # Qwen ("qwen/qwen3.7-max", ""), - ("qwen/qwen3.7-plus", ""), - ("qwen/qwen3.6-35b-a3b", ""), # MoonshotAI ("moonshotai/kimi-k3", "recommended"), # MiniMax @@ -209,9 +206,8 @@ _PROVIDER_MODELS: dict[str, list[str]] = { "openai/gpt-5.5-pro", "openai/gpt-5.4-mini", # Google - "google/gemini-3-pro-preview", "google/gemini-3.1-pro-preview", - "google/gemini-3.5-flash", + "google/gemini-3.6-flash", # xAI "x-ai/grok-4.5", # DeepSeek @@ -219,8 +215,6 @@ _PROVIDER_MODELS: dict[str, list[str]] = { "deepseek/deepseek-v4-flash", # Qwen "qwen/qwen3.7-max", - "qwen/qwen3.7-plus", - "qwen/qwen3.6-35b-a3b", # MoonshotAI "moonshotai/kimi-k3", # MiniMax diff --git a/tests/test_empty_model_fallback.py b/tests/test_empty_model_fallback.py index 53260f655ed9..20e88a9cf886 100644 --- a/tests/test_empty_model_fallback.py +++ b/tests/test_empty_model_fallback.py @@ -74,16 +74,16 @@ class TestGetDefaultModelForProvider: with patch( "hermes_cli.model_catalog.get_default_model_from_cache", - return_value="qwen/qwen3.7-plus", + return_value="qwen/qwen3.7-max", ): assert ( models_mod.get_preferred_silent_default_model("nous") - == "qwen/qwen3.7-plus" + == "qwen/qwen3.7-max" ) - # nous catalog carries qwen3.7-plus, so the full resolver follows. + # nous catalog carries qwen3.7-max, so the full resolver follows. assert ( models_mod.get_default_model_for_provider("nous") - == "qwen/qwen3.7-plus" + == "qwen/qwen3.7-max" ) def test_no_catalog_cache_falls_back_to_constant(self): diff --git a/website/static/api/model-catalog.json b/website/static/api/model-catalog.json index 3fa346ca4b39..630380a71093 100644 --- a/website/static/api/model-catalog.json +++ b/website/static/api/model-catalog.json @@ -1,6 +1,6 @@ { "version": 1, - "updated_at": "2026-07-24T19:06:06Z", + "updated_at": "2026-07-28T16:38:40Z", "metadata": { "source": "hermes-agent repo", "docs": "https://hermes-agent.nousresearch.com/docs/reference/model-catalog" @@ -76,16 +76,12 @@ "id": "openai/gpt-5.4-mini", "description": "" }, - { - "id": "google/gemini-3-pro-preview", - "description": "" - }, { "id": "google/gemini-3.1-pro-preview", "description": "" }, { - "id": "google/gemini-3.5-flash", + "id": "google/gemini-3.6-flash", "description": "" }, { @@ -104,14 +100,6 @@ "id": "qwen/qwen3.7-max", "description": "" }, - { - "id": "qwen/qwen3.7-plus", - "description": "" - }, - { - "id": "qwen/qwen3.6-35b-a3b", - "description": "" - }, { "id": "moonshotai/kimi-k3", "description": "recommended" @@ -227,14 +215,11 @@ { "id": "openai/gpt-5.4-mini" }, - { - "id": "google/gemini-3-pro-preview" - }, { "id": "google/gemini-3.1-pro-preview" }, { - "id": "google/gemini-3.5-flash" + "id": "google/gemini-3.6-flash" }, { "id": "x-ai/grok-4.5" @@ -248,12 +233,6 @@ { "id": "qwen/qwen3.7-max" }, - { - "id": "qwen/qwen3.7-plus" - }, - { - "id": "qwen/qwen3.6-35b-a3b" - }, { "id": "moonshotai/kimi-k3" }, From e35c2f6049633d3b6d897d63e637fc63b1b8b8c6 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Tue, 28 Jul 2026 12:10:53 -0500 Subject: [PATCH 009/156] fix(sessions): claim the cap slot on first turn, not on open An open chat window took a session-cap slot at session.create/resume time. Every desktop tile paint and every background reconnect-resume opens one, so on a websocket-flappy host they accumulated: five parked desktop tabs filled a 5-slot cap and locked the messaging gateway (which shares the cap) out for fourteen minutes while running no agents at all. A slot held that way is invisible everywhere. An unprompted draft has no DB row and the sidebar filters it out with min_messages=1, so the only way to diagnose it was reading runtime/active_sessions.json by hand. Claim on the first turn instead, mirroring the lazy contract _ensure_session_db_row already uses for the row itself. Capacity now means an agent can run rather than that a window exists, and anything holding a slot is something the user can see. Also reclaim leases whose session skipped teardown. _prune_dead only fires when the owning pid dies, and a dashboard/serve backend runs for days, so a leaked lease was held until restart. The owning process reconciles against the leases it still holds, which is exact and needs no heartbeat write on the turn path. --- hermes_cli/active_sessions.py | 77 ++++++++++++++++++++++-- tests/hermes_cli/test_active_sessions.py | 38 ++++++++++-- tests/test_tui_gateway_server.py | 30 +++++---- tui_gateway/server.py | 77 ++++++++++++++++-------- website/docs/user-guide/configuration.md | 10 ++- 5 files changed, 186 insertions(+), 46 deletions(-) diff --git a/hermes_cli/active_sessions.py b/hermes_cli/active_sessions.py index 7eba80e50242..a572c7409329 100644 --- a/hermes_cli/active_sessions.py +++ b/hermes_cli/active_sessions.py @@ -70,10 +70,47 @@ def resolve_max_concurrent_sessions(config: Any) -> Optional[int]: return coerce_max_concurrent_sessions(raw, key=key) -def active_session_limit_message(active_count: int, max_sessions: int) -> str: +def format_age(seconds: float) -> str: + minutes = max(0, int(seconds // 60)) + if minutes < 60: + return f"{minutes}m" + hours, minutes = divmod(minutes, 60) + return f"{hours}h" if not minutes else f"{hours}h{minutes}m" + + +def summarize_holders(entries: list[dict[str, Any]]) -> str: + """Compact "who is holding the slots" phrase, e.g. ``desktop x4, cli``.""" + if not entries: + return "" + counts: dict[str, int] = {} + for entry in entries: + surface = str(entry.get("surface") or "unknown") + counts[surface] = counts.get(surface, 0) + 1 + held = ", ".join( + f"{surface} x{n}" if n > 1 else surface + for surface, n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + ) + started = [t for t in (_optional_float(e.get("started_at")) for e in entries) if t] + if started: + held += f", oldest {format_age(time.time() - min(started))} ago" + return held + + +def active_session_limit_message( + active_count: int, + max_sessions: int, + entries: Optional[list[dict[str, Any]]] = None, +) -> str: + # Name the holders: the slots are shared across CLI, desktop/TUI and the + # messaging gateway, so the surface that gets rejected is usually NOT the + # one squatting on them (idle desktop chats starving a Discord bot, say). + # Without this the message is unactionable and the only way to find out is + # reading runtime/active_sessions.json by hand. + held = summarize_holders(entries or []) + detail = f" Held by: {held}." if held else "" return ( - f"Hermes is at the active session limit ({active_count}/{max_sessions}). " - "Try again when another session finishes." + f"Hermes is at the active session limit ({active_count}/{max_sessions})." + f"{detail} Try again when another session finishes." ) @@ -284,7 +321,9 @@ def try_acquire_active_session( max_sessions, surface, ) - return None, active_session_limit_message(active_count, max_sessions) + return None, active_session_limit_message( + active_count, max_sessions, entries + ) entries.append(entry) _write_entries(state_path, entries) @@ -348,6 +387,36 @@ def transfer_active_session( return updated +def release_orphaned_leases(live_lease_ids: set[str]) -> int: + """Drop this process's registry entries that no live session owns. + + ``_prune_dead`` only reclaims leases whose owning process died. A server + that runs for days (``hermes dashboard`` / ``serve``) never trips that + check, so a lease whose session skipped teardown is held until restart. + The owning process is the only authority on which of its own leases are + real, so it drops the rest itself — exact, with no heartbeat write on the + turn path and no staleness threshold to tune. + """ + pid = os.getpid() + state_path = _state_path() + # With the cap disabled the registry is never written, so don't take a lock + # (or create its file) on the idle-reaper tick for the majority of installs. + if not state_path.exists(): + return 0 + with _FileLock(_lock_path()): + entries = _prune_dead(_read_entries(state_path)) + kept = [ + entry + for entry in entries + if entry.get("pid") != pid + or str(entry.get("lease_id") or "") in live_lease_ids + ] + dropped = len(entries) - len(kept) + if dropped: + _write_entries(state_path, kept) + return dropped + + def active_session_registry_snapshot() -> list[dict[str, Any]]: """Return the pruned active-session registry for diagnostics/tests.""" state_path = _state_path() diff --git a/tests/hermes_cli/test_active_sessions.py b/tests/hermes_cli/test_active_sessions.py index 560803dc852e..64ab9e5377ce 100644 --- a/tests/hermes_cli/test_active_sessions.py +++ b/tests/hermes_cli/test_active_sessions.py @@ -57,10 +57,10 @@ def test_active_session_lease_blocks_until_release(tmp_path, monkeypatch): ) assert blocked_lease is None - assert blocked_message == ( - "Hermes is at the active session limit (1/1). " - "Try again when another session finishes." - ) + assert "active session limit (1/1)" in blocked_message + # The rejected surface is rarely the one holding the slots, so the message + # must name the holder — here the "cli" lease, not the blocked "tui" one. + assert "Held by: cli" in blocked_message lease.release() @@ -354,3 +354,33 @@ def test_pid_start_time_mismatch_prunes_reused_pid(tmp_path, monkeypatch): "new-session" ] lease.release() + + +def test_release_orphaned_leases_reclaims_only_unowned_own_pid_entries(tmp_path, monkeypatch): + """A long-lived server must reclaim leases whose session skipped teardown. + + ``_prune_dead`` only fires when the owning pid dies, so a ``hermes + dashboard`` running for days holds a leaked lease until restart. The + process reconciles against the leases it still owns instead. + """ + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + cfg = {"max_concurrent_sessions": 5} + kept, orphan = ( + active_sessions.try_acquire_active_session( + session_id=sid, surface="desktop", config=cfg + )[0] + for sid in ("kept", "orphaned") + ) + # Another live process's lease is not ours to reclaim. + active_sessions._write_entries( + active_sessions._state_path(), + active_sessions._read_entries(active_sessions._state_path()) + + [{"lease_id": "elsewhere", "session_id": "other", "surface": "cli", "pid": os.getpid() }], + ) + + assert active_sessions.release_orphaned_leases({kept.lease_id, "elsewhere"}) == 1 + assert sorted( + entry["session_id"] + for entry in active_sessions.active_session_registry_snapshot() + ) == ["kept", "other"] + assert orphan is not None diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index fdb146f08c39..8aafbb0f4049 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -37,7 +37,7 @@ def _neuter_agent_prewarm_timer(request, monkeypatch): yield -def test_session_create_rejects_at_active_session_limit(monkeypatch, tmp_path): +def test_session_slot_is_claimed_on_first_turn_not_on_create(monkeypatch, tmp_path): home = tmp_path / ".hermes" home.mkdir() (home / "config.yaml").write_text("max_concurrent_sessions: 1\n", encoding="utf-8") @@ -56,23 +56,31 @@ def test_session_create_rejects_at_active_session_limit(monkeypatch, tmp_path): monkeypatch.setattr(server, "_start_agent_build", lambda *args, **kwargs: None) monkeypatch.setattr(server, "_completion_cwd", lambda params=None: str(tmp_path)) + # Opening a chat must NOT take a slot. Every tile paint and every + # background reconnect-resume calls session.create, and an unprompted + # draft has no DB row and is filtered out of the sidebar — so a slot + # held here is invisible to the user while still starving the other + # surfaces that share this cap. first = server._methods["session.create"]("r1", {"cols": 80}) - assert "result" in first - sid = first["result"]["session_id"] - second = server._methods["session.create"]("r2", {"cols": 80}) - assert second["error"]["message"] == ( - "Hermes is at the active session limit (1/1). " - "Try again when another session finishes." - ) - assert list(server._sessions) == [sid] + assert "result" in first and "result" in second + sid = first["result"]["session_id"] + other = second["result"]["session_id"] + assert active_session_registry_snapshot() == [] + + # The first turn is what claims the slot, and is re-entrant. + assert server._ensure_active_session_slot(sid, server._sessions[sid]) is None + assert server._ensure_active_session_slot(sid, server._sessions[sid]) is None + assert len(active_session_registry_snapshot()) == 1 + + blocked = server._ensure_active_session_slot(other, server._sessions[other]) + assert "active session limit (1/1)" in blocked closed = server._methods["session.close"]("r3", {"session_id": sid}) assert closed["result"]["closed"] is True assert active_session_registry_snapshot() == [] - third = server._methods["session.create"]("r4", {"cols": 80}) - assert "result" in third + assert server._ensure_active_session_slot(other, server._sessions[other]) is None finally: _clear_server_sessions() server._cfg_cache = None diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 6227ab0b3a3d..ace39ce9e528 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -512,6 +512,33 @@ def _claim_active_session_slot( return None, None +def _ensure_active_session_slot(sid: str, session: dict) -> str | None: + """Claim this session's cap slot on its first real turn; None when ok. + + session.create / session.resume deliberately do NOT claim one. Every + desktop tile paint, background reconnect-resume and abandoned draft opens a + session just to paint a composer, and a slot held by one of those is + invisible everywhere: an unprompted draft has no DB row, and the sidebar + filters it out with min_messages=1. Idle desktop tabs therefore silently + starved the messaging gateway, which shares this cap — five parked tabs on + a websocket-flappy host locked a Discord bot out of a 5-slot cap while + running no agents at all. Claiming on the first turn mirrors the lazy + contract _ensure_session_db_row already uses for the row itself, and keeps + the invariant that anything holding a slot is something the user can see. + """ + if session.get("active_session_lease") is not None: + return None + lease, limit_message = _claim_active_session_slot( + str(session.get("session_key") or ""), + live_session_id=sid, + surface=_session_source(session), + ) + if limit_message is not None: + return limit_message + session["active_session_lease"] = lease + return None + + def _release_active_session_slot(session: dict | None) -> None: if not session: return @@ -980,6 +1007,24 @@ def _reap_idle_sessions() -> None: for sid in victims: _close_session_by_id(sid, end_reason="idle_timeout") _enforce_session_cap() + _reclaim_orphaned_leases() + + +def _reclaim_orphaned_leases() -> None: + """Hand the registry the lease ids we still own so it can drop the rest.""" + try: + from hermes_cli.active_sessions import release_orphaned_leases + + with _sessions_lock: + live = { + lease.lease_id + for session in _sessions.values() + if (lease := session.get("active_session_lease")) is not None + } + if dropped := release_orphaned_leases(live): + logger.info("Reclaimed %d orphaned active-session lease(s)", dropped) + except Exception: + logger.debug("orphaned lease reclaim failed", exc_info=True) # Soft LRU cap on in-memory sessions. The 6h TTL reaper above only frees @@ -6997,11 +7042,7 @@ def _(rid, params: dict) -> dict: ready = threading.Event() now = time.time() - lease, limit_message = _claim_active_session_slot( - key, live_session_id=sid, surface=source - ) - if limit_message is not None: - return _err(rid, 4090, limit_message) + lease = None # claimed lazily on the first turn (_ensure_active_session_slot) with _sessions_lock: _sessions[sid] = { @@ -7445,11 +7486,7 @@ def _(rid, params: dict) -> dict: if is_truthy_value(params.get("lazy", False)): sid = uuid.uuid4().hex[:8] source = _resolve_session_source(str(params.get("source") or "").strip() or None) - lease, limit_message = _claim_active_session_slot( - target, live_session_id=sid, surface=source - ) - if limit_message is not None: - return _err(rid, 4090, limit_message) + lease = None # claimed lazily on the first turn (_ensure_active_session_slot) try: db.reopen_session(target) # The child's OWN conversation only — include_ancestors would prepend @@ -7526,11 +7563,7 @@ def _(rid, params: dict) -> dict: if not is_truthy_value(params.get("eager_build", False)): sid = uuid.uuid4().hex[:8] source = _resolve_session_source(str(params.get("source") or "").strip() or None) - lease, limit_message = _claim_active_session_slot( - target, live_session_id=sid, surface=source - ) - if limit_message is not None: - return _err(rid, 4090, limit_message) + lease = None # claimed lazily on the first turn (_ensure_active_session_slot) # Interactive resume routes approvals/clarify through gateway prompts; # the deferred build wires the remaining per-session callbacks. _enable_gateway_prompts() @@ -7605,11 +7638,7 @@ def _(rid, params: dict) -> dict: # dispatch thread (it's not a _LONG_HANDLER), blocking fast-path RPCs. sid = uuid.uuid4().hex[:8] source = _resolve_session_source(str(params.get("source") or "").strip() or None) - lease, limit_message = _claim_active_session_slot( - target, live_session_id=sid, surface=source - ) - if limit_message is not None: - return _err(rid, 4090, limit_message) + lease = None # claimed lazily on the first turn (_ensure_active_session_slot) _enable_gateway_prompts() home_token = ( set_hermes_home_override(str(profile_home)) if profile_home is not None else None @@ -10423,11 +10452,7 @@ def _(rid, params: dict) -> dict: new_key = _new_session_key() new_sid = uuid.uuid4().hex[:8] source = _session_source(session) - lease, limit_message = _claim_active_session_slot( - new_key, live_session_id=new_sid, surface=source - ) - if limit_message is not None: - return _err(rid, 4090, limit_message) + lease = None # claimed lazily on the first turn (_ensure_active_session_slot) branch_name = params.get("name", "") try: if branch_name: @@ -10932,6 +10957,8 @@ def _(rid, params: dict) -> dict: session, err = _sess_nowait(params, rid) if err: return err + if (limit_message := _ensure_active_session_slot(sid, session)) is not None: + return _err(rid, 4090, limit_message) if truncate_user_ordinal is not None and isinstance(text, str): # A rewind/regenerate replays a turn from what the transcript shows. A # skill turn shows its invocation, so re-expand it here — otherwise diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 5d1214e2c493..086a09f5a646 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1808,8 +1808,14 @@ and messaging gateway: max_concurrent_sessions: null # null/0 = unlimited; positive integer = active session cap ``` -When the cap is reached, Hermes returns a direct limit message for new sessions. -Existing active sessions keep their normal behavior. +A slot is taken when a session runs its **first turn**, not when a chat window +is opened. Opening, resuming or reconnecting to a chat costs nothing until you +send a message, so idle desktop tabs (and the background resumes a flaky +websocket triggers) cannot starve the messaging gateway that shares this cap. + +When the cap is reached, Hermes returns a direct limit message naming which +surfaces hold the slots. Existing active sessions keep their normal behavior. +Run `hermes status` to see the current slot usage and every holder. The canonical key is top-level `max_concurrent_sessions`. Hermes also accepts `gateway.max_concurrent_sessions` as a fallback, but the top-level key wins when From 78aeeab8d1a53be636c2185f057dede774d57a3d Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Tue, 28 Jul 2026 12:10:58 -0500 Subject: [PATCH 010/156] feat(sessions): say which surfaces hold the active-session slots The cap is shared across CLI, desktop/TUI and the messaging gateway, so the surface that gets rejected is rarely the one holding the slots. The rejection read "Hermes is at the active session limit (5/5). Try again when another session finishes." while every slot was an idle desktop tab, which took filesystem access to work out. Name the holders in the message, and show slot usage plus each holder in `hermes status`. Both are inert when max_concurrent_sessions is unset, which is the default. The gateway's duplicate copy of the message now reuses the shared helper. --- gateway/run.py | 7 ++-- hermes_cli/status.py | 35 +++++++++++++++++++ .../test_cli_active_session_limit.py | 8 ++--- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index a3c36ea19310..cef8dc845fa6 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -5978,10 +5978,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew active_count = len(getattr(self, "_running_agents", {})) if active_count < max_sessions: return None - return ( - f"Hermes is at the active session limit ({active_count}/{max_sessions}). " - "Try again when another session finishes." - ) + from hermes_cli.active_sessions import active_session_limit_message + + return active_session_limit_message(active_count, max_sessions) def _claim_active_session_slot( self, diff --git a/hermes_cli/status.py b/hermes_cli/status.py index 7537b85a1d92..4a1b0e123793 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -6,6 +6,7 @@ Shows the status of all Hermes Agent components. import os import sys +import time import subprocess # noqa: F401 — re-exported for tests that monkeypatch status.subprocess to guard against regressions from pathlib import Path @@ -581,6 +582,40 @@ def show_status(args): else: print(f" Active: {_session_count if _session_count is not None else 0}") + # Slot usage, only when max_concurrent_sessions is set. The cap is shared + # across CLI, desktop/TUI and the messaging gateway, so the surface that + # gets rejected is rarely the one holding the slots — without this the only + # way to find out is reading runtime/active_sessions.json by hand. + try: + from hermes_cli.active_sessions import ( + active_session_registry_snapshot, + format_age, + resolve_max_concurrent_sessions, + ) + + _cap = resolve_max_concurrent_sessions(config) + except Exception: + _cap = None + if _cap: + try: + _held = active_session_registry_snapshot() + except Exception: + _held = [] + _full = len(_held) >= _cap + print( + " Slots: " + + color( + f"{len(_held)}/{_cap} in use", Colors.YELLOW if _full else Colors.GREEN + ) + ) + _now = time.time() + for _entry in sorted(_held, key=lambda e: e.get("started_at") or 0): + _age = format_age(_now - float(_entry.get("started_at") or _now)) + print( + f" {_entry.get('surface') or 'unknown':<17} " + f"{_entry.get('session_id') or '?':<24} {_age}" + ) + # ========================================================================= # Deep checks # ========================================================================= diff --git a/tests/hermes_cli/test_cli_active_session_limit.py b/tests/hermes_cli/test_cli_active_session_limit.py index 6e47954c8a0e..b62cef459325 100644 --- a/tests/hermes_cli/test_cli_active_session_limit.py +++ b/tests/hermes_cli/test_cli_active_session_limit.py @@ -25,10 +25,10 @@ def test_cli_claim_active_session_respects_global_limit(tmp_path, monkeypatch): try: assert cli._claim_active_session("cli") is False - assert printed == [ - "[bold red]Hermes is at the active session limit (1/1). " - "Try again when another session finishes.[/]" - ] + assert len(printed) == 1 + assert "active session limit (1/1)" in printed[0] + # Names the holding surface ("tui"), not the blocked one. + assert "Held by: tui" in printed[0] held.release() From 3b55419cb15b74024322b3a2ab3303b579a940d4 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Tue, 28 Jul 2026 10:19:53 -0700 Subject: [PATCH 011/156] fix(relay): close logical calls in stack order Signed-off-by: Alex Fournier --- agent/relay_runtime.py | 15 +++- .../test_relay_shared_metrics_runtime.py | 85 +++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py index 4e3654cf7780..533604791a86 100644 --- a/agent/relay_runtime.py +++ b/agent/relay_runtime.py @@ -722,7 +722,8 @@ class RelaySessionCoordinator: with turn.logical_llm_lock: logical_calls = list(turn.logical_llm_calls.items()) turn.logical_llm_calls.clear() - for request_id, logical_handle in logical_calls: + for index in range(len(logical_calls) - 1, -1, -1): + request_id, logical_handle = logical_calls[index] try: lease.host.run_in_session( lease.session, @@ -736,11 +737,21 @@ class RelaySessionCoordinator: ) except Exception: with turn.logical_llm_lock: - turn.logical_llm_calls.setdefault(request_id, logical_handle) + # Relay scopes are stack-owned. If the newest remaining + # handle cannot close, older handles cannot close safely + # either, so retain the unclosed prefix for diagnostics. + for pending_request_id, pending_handle in logical_calls[ + : index + 1 + ]: + turn.logical_llm_calls.setdefault( + pending_request_id, + pending_handle, + ) logger.warning( "Hermes Relay logical LLM finalization failed", exc_info=True, ) + break @staticmethod def _reset_turn_context(turn: RelayTurnContext) -> None: diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py index 23258def3385..ccda2abc1f4c 100644 --- a/tests/hermes_cli/test_relay_shared_metrics_runtime.py +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -1525,6 +1525,91 @@ def test_turn_cleanup_drains_logical_calls_after_turn_scope_start_failure( assert turn.logical_llm_calls == {} +def test_turn_cleanup_drains_logical_calls_in_lifo_order(direct_runtime): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="session-lifo", + platform="cli", + ) + assert lease.session is not None + turn = coordinator.begin_turn( + lease, + turn_id="turn-lifo", + task_id="task-lifo", + ) + assert turn.handle is not None + runtime = lease.host + + handles = [] + for request_id in ("request-1", "request-2"): + handle = runtime.run_in_session( + lease.session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=turn.handle, + input={}, + ) + turn.logical_llm_calls[request_id] = handle + handles.append(handle) + + coordinator.end_turn(turn, outcome="failed") + coordinator.release_conversation(lease) + + logical_closes = [ + event[1] + for event in direct_runtime.events + if event[0] == "scope.pop" and event[1] in handles + ] + assert logical_closes == list(reversed(handles)) + assert turn.logical_llm_calls == {} + + +def test_real_binding_drains_multiple_logical_calls_before_turn_close( + real_binding_runtime, + caplog, +): + coordinator = relay_runtime.SESSION_COORDINATOR + profile_key = relay_runtime.current_profile_key() + lease = coordinator.acquire_conversation( + profile_key=profile_key, + session_id="session-native-lifo", + platform="cli", + ) + assert lease.session is not None + turn = coordinator.begin_turn( + lease, + turn_id="turn-native-lifo", + task_id="task-native-lifo", + ) + assert turn.handle is not None + runtime = lease.host + + for request_id in ("request-1", "request-2"): + handle = runtime.run_in_session( + lease.session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=turn.handle, + input={}, + ) + turn.logical_llm_calls[request_id] = handle + + coordinator.end_turn(turn, outcome="failed") + coordinator.release_conversation(lease) + coordinator.finalize_conversation( + profile_key=profile_key, + session_id=lease.session_id, + ) + + assert turn.logical_llm_calls == {} + assert "finalization failed" not in caplog.text + assert "closed with errors" not in caplog.text + + def test_shared_metrics_creates_one_task_under_concurrent_access(direct_runtime): runtime = relay_shared_metrics._get_runtime() assert runtime is not None From 5cb0a6aec1b75ea0673d571e53e0851a9db22631 Mon Sep 17 00:00:00 2001 From: David Metcalfe <80915+DavidMetcalfe@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:46:04 -0700 Subject: [PATCH 012/156] =?UTF-8?q?feat(desktop):=20searchable=20timezone?= =?UTF-8?q?=20dropdown=20in=20Settings=20=E2=86=92=20Chat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timezone field was a free-text input with no guidance on format. Users had to know the exact IANA identifier (e.g. America/New_York) to configure it. Replace it with a searchable combobox built on Popover + cmdk Command — the same stack as Shadcn's Combobox. Backend: - Add `_timezone_options()` to web_server.py (cached at import time, returns sorted zoneinfo.available_timezones() — ~598 identifiers) - Add `"timezone"` to `_SCHEMA_OVERRIDES` with `type: "select"`, `options`, and `searchable: true` Frontend: - New `SearchableSelect` component (Popover + cmdk Command) — closed-world filterable dropdown for large option lists - `ConfigField` routes to `SearchableSelect` when `schema.searchable === true` (explicit opt-in, no threshold) - Add `searchable?: boolean` to `ConfigFieldSchema` type - Add i18n keys: `searchPlaceholder`, `noResults` The `searchable` flag is deterministic — no existing field is affected unless explicitly opted in. Future large-list fields can adopt the same pattern by adding `searchable: true` to their schema override. --- .../desktop/src/app/settings/config-field.tsx | 17 +++ apps/desktop/src/app/settings/constants.ts | 2 +- .../src/app/settings/searchable-select.tsx | 109 ++++++++++++++++++ apps/desktop/src/i18n/en.ts | 2 + apps/desktop/src/i18n/types.ts | 2 + apps/desktop/src/i18n/zh.ts | 2 + apps/desktop/src/types/hermes.ts | 3 + hermes_cli/web_server.py | 15 +++ 8 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/src/app/settings/searchable-select.tsx diff --git a/apps/desktop/src/app/settings/config-field.tsx b/apps/desktop/src/app/settings/config-field.tsx index aac75a2ee971..7e5418b30709 100644 --- a/apps/desktop/src/app/settings/config-field.tsx +++ b/apps/desktop/src/app/settings/config-field.tsx @@ -13,6 +13,7 @@ import { CONTROL_TEXT, EMPTY_SELECT_VALUE, FIELD_DESCRIPTIONS, FIELD_LABELS, FRE import { FallbackModelsField } from './fallback-models-field' import { fieldCopyForSchemaKey } from './field-copy' import { ListRow } from './primitives' +import { SearchableSelect } from './searchable-select' /** * One generic config row: label + description resolved from the i18n field @@ -93,6 +94,22 @@ export function ConfigField({ const selectOptions = enumOptions ?? (schema.type === 'select' ? (schema.options ?? []).map(String) : undefined) + // Large closed-world lists (e.g. ~590 IANA timezones) get a searchable + // Popover + cmdk combobox instead of a closed Select dropdown. The schema + // opt-in via `searchable: true` keeps this deterministic — no field + // accidentally triggers based on dynamic option count. + if (selectOptions && schema.searchable) { + return row( + onChange(next)} + options={selectOptions.filter(o => o !== '')} + placeholder={c.searchPlaceholder} + value={String(value ?? '')} + /> + ) + } + // Voice/model name fields are open-world (custom voice IDs, cloned voices, // brand-new model names) — render a free-input combobox where the known // options are datalist suggestions instead of a closed Select gate. diff --git a/apps/desktop/src/app/settings/constants.ts b/apps/desktop/src/app/settings/constants.ts index f041b2992470..3e12e08e70d7 100644 --- a/apps/desktop/src/app/settings/constants.ts +++ b/apps/desktop/src/app/settings/constants.ts @@ -564,7 +564,7 @@ export const FIELD_DESCRIPTIONS: Record = defineFieldCopy({ repoScanRoots: 'Folders to scan. Leave empty to scan your home directory.', repoScanExcludePaths: 'Folders and their descendants to skip during repository discovery.' }, - timezone: 'Used when Hermes needs local time context. Blank uses the system timezone.', + timezone: 'IANA timezone identifier. Blank uses the system timezone.', agent: { imageInputMode: 'Controls how image attachments are sent to the model.', maxTurns: 'Upper bound for tool-calling turns before Hermes stops a run.' diff --git a/apps/desktop/src/app/settings/searchable-select.tsx b/apps/desktop/src/app/settings/searchable-select.tsx new file mode 100644 index 000000000000..c2c55be0af30 --- /dev/null +++ b/apps/desktop/src/app/settings/searchable-select.tsx @@ -0,0 +1,109 @@ +import { useCallback, useRef, useState } from 'react' + +import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from '@/components/ui/command' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' +import { Codicon } from '@/components/ui/codicon' +import { controlVariants } from '@/components/ui/control' +import { cn } from '@/lib/utils' + +/** + * Searchable select for large option lists (e.g. ~590 IANA timezones). + * Built on Popover + cmdk Command — the same stack as Shadcn's Combobox. + * + * The trigger renders like the existing closed `` dropdown. For large option lists like IANA timezones. */ + searchable?: boolean type?: 'boolean' | 'list' | 'number' | 'select' | 'string' | 'text' } diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 674df896fc41..5c1de9df75a7 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -799,7 +799,22 @@ def _memory_provider_options() -> List[str]: return list(dict.fromkeys(options)) +def _timezone_options() -> List[str]: + """Return sorted IANA timezone identifiers, cached at import time.""" + try: + import zoneinfo + return sorted(zoneinfo.available_timezones()) + except Exception: # pragma: no cover + return ["UTC"] + + _SCHEMA_OVERRIDES: Dict[str, Dict[str, Any]] = { + "timezone": { + "type": "select", + "description": "IANA timezone (e.g. America/New_York). Blank uses the system timezone.", + "options": _timezone_options(), + "searchable": True, + }, "memory.provider": { "type": "select", "description": "Memory provider plugin", From b5b3ed6563ddb11163b2f47b7536499f7d68a5ae Mon Sep 17 00:00:00 2001 From: David Metcalfe <80915+DavidMetcalfe@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:48:26 -0700 Subject: [PATCH 013/156] fix: address cross-vendor review feedback - Fix focus: use autoFocus on CommandInput instead of e.preventDefault() - Fix filter: prioritize city segment match (return 2 for slash match) - Fix handleSelect: always select, don't toggle-deselect - Add aria-haspopup to trigger button - Use defensive placeholder logic --- .../src/app/settings/searchable-select.tsx | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/apps/desktop/src/app/settings/searchable-select.tsx b/apps/desktop/src/app/settings/searchable-select.tsx index c2c55be0af30..855b2c4d5688 100644 --- a/apps/desktop/src/app/settings/searchable-select.tsx +++ b/apps/desktop/src/app/settings/searchable-select.tsx @@ -34,18 +34,19 @@ export function SearchableSelect({ const handleSelect = useCallback( (selected: string) => { - onChange(selected === value ? '' : selected) + onChange(selected) setOpen(false) }, - [onChange, value] + [onChange] ) - const displayValue = value || placeholder + const displayValue = value !== '' && value !== undefined ? value : placeholder return ( - - { - // cmdk's default filter is case-insensitive substring match on - // the item's value. For timezone-like values we also want to - // match segments after "/" so "york" matches "America/New_York". - const lower = search.toLowerCase() - const itemLower = value.toLowerCase() - // Prioritize city/region segment (after last "/") so "york" ranks - // "America/New_York" above "America/New_York/Special". - const slash = itemLower.lastIndexOf('/') - if (slash !== -1 && itemLower.slice(slash + 1).includes(lower)) return 2 - if (itemLower.includes(lower)) return 1 - return 0 - }} - > + + {emptyMessage} {clearLabel && ( - handleSelect('')} - value={clearLabel} - > - + handleSelect('')} value={clearLabel}> + {clearLabel} )} {options.map(option => ( - handleSelect(option)} - value={option} - > - + handleSelect(option)} value={option}> + {option} ))} diff --git a/apps/desktop/src/i18n/ar.ts b/apps/desktop/src/i18n/ar.ts index 0f087a9dd723..4fa1e65e8ddc 100644 --- a/apps/desktop/src/i18n/ar.ts +++ b/apps/desktop/src/i18n/ar.ts @@ -610,6 +610,9 @@ export const ar = defineLocale({ noneParen: '(لا شيء)', notSet: 'غير مضبوط', commaSeparated: 'قيم مفصولة بفواصل', + searchPlaceholder: 'بحث…', + noResults: 'لا توجد نتائج', + systemDefault: 'إعداد النظام الافتراضي', loading: 'جار تحميل إعدادات Hermes...', emptyTitle: 'لا توجد إعدادات', emptyDesc: 'لا يحتوي هذا القسم على إعدادات قابلة للتعديل.', diff --git a/apps/desktop/src/i18n/ja.ts b/apps/desktop/src/i18n/ja.ts index 6ec6c4872001..533e68cd1799 100644 --- a/apps/desktop/src/i18n/ja.ts +++ b/apps/desktop/src/i18n/ja.ts @@ -639,6 +639,9 @@ export const ja = defineLocale({ builtinOnly: '内蔵のみ', notSet: '未設定', commaSeparated: 'カンマ区切りの値', + searchPlaceholder: '検索…', + noResults: '結果が見つかりません', + systemDefault: 'システムのデフォルト', loading: 'Hermes の設定を読み込み中...', emptyTitle: '設定項目がありません', emptyDesc: 'このセクションには調整できる設定がありません。', diff --git a/apps/desktop/src/i18n/zh-hant.ts b/apps/desktop/src/i18n/zh-hant.ts index 10f85b9940e9..bc8b5f83d519 100644 --- a/apps/desktop/src/i18n/zh-hant.ts +++ b/apps/desktop/src/i18n/zh-hant.ts @@ -627,6 +627,9 @@ export const zhHant = defineLocale({ builtinOnly: '僅內建', notSet: '未設定', commaSeparated: '逗號分隔的值', + searchPlaceholder: '搜尋…', + noResults: '找不到結果', + systemDefault: '系統預設', loading: '正在載入 Hermes 設定...', emptyTitle: '無可設定項目', emptyDesc: '此區段沒有可調整的設定。', diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 24ef3d0f8c40..08c0bfc6f238 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -5327,6 +5327,29 @@ class TestBuildSchemaFromConfig: missing = set(list_memory_provider_names()) - options assert missing == set(), f"discovered providers missing from schema options: {missing}" + def test_timezone_field_is_searchable_select(self): + """timezone must ship as a searchable, clearable select of IANA ids. + + Desktop renders this via SearchableSelect (Popover + cmdk); the old + free-text input let users type invalid timezone strings (#68970). + Invariants, not snapshots: valid IANA entries present, sorted, no + blank entry server-side (the clear item is client-side via + ``clearable``), and never empty even without tzdata (UTC fallback). + """ + from hermes_cli.web_server import CONFIG_SCHEMA, _timezone_options + + entry = CONFIG_SCHEMA["timezone"] + assert entry["type"] == "select" + assert entry.get("searchable") is True + assert entry.get("clearable") is True + options = entry["options"] + assert len(options) >= 1 + assert options == sorted(options) + assert "" not in options + assert "UTC" in options + # Fallback path: never returns an empty list. + assert len(_timezone_options()) >= 1 + def test_dynamic_merge_recomputes_memory_provider_options(self, monkeypatch): """The per-request schema merge re-discovers memory providers. From bff3aa3c1e788de514eb1c24682c6550a2821501 Mon Sep 17 00:00:00 2001 From: Bartok9 Date: Mon, 29 Jun 2026 03:38:43 -0400 Subject: [PATCH 016/156] fix(cli): resolve deferred platform plugin for its top-level CLI command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #54678 `hermes photon ...` could fail with argparse `invalid choice: 'photon'` even when the bundled Photon platform plugin is present. Photon registers its top-level CLI command from the platform adapter module via `ctx.register_cli_command(name="photon", ...)`, but bundled platform plugins are cheap-registered as *deferred* entries to avoid importing every gateway SDK during normal startup. On the unknown-top-level-command slow path, `discover_plugins()` records the deferred loader but never imports the matching platform module, so the CLI registration side effect doesn't run and `photon` stays absent from `_cli_commands` — argparse then rejects it. Fix: after `discover_plugins()` on that slow path, resolve only the deferred platform whose name matches the first positional token (via `platform_registry.get(name)`) before reading `_cli_commands`. This imports exactly the targeted platform, leaving normal startup cheap (a bare `hermes` or flags-only invocation has no positional token and touches nothing). The resolution is best-effort: registry/import failures are logged at debug and never crash startup. Added 3 tests in tests/hermes_cli/test_startup_plugin_gating.py: resolves the matching platform, ignores empty/None command, and swallows registry errors. Fails without the fix (symbol absent). --- hermes_cli/main.py | 34 +++++++++ .../hermes_cli/test_startup_plugin_gating.py | 71 +++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 2ccfe808d4fe..ce6ca6f2bbf8 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -15245,6 +15245,35 @@ def _plugin_cli_discovery_needed() -> bool: return True +def _resolve_deferred_platform_cli_command(command_name: str | None) -> None: + """Materialize the deferred platform whose top-level CLI command matches. + + Bundled platform plugins are cheap-registered as *deferred* entries to + avoid importing every gateway SDK during normal startup. A platform that + registers a top-level ``hermes `` command (e.g. Photon -> + ``ctx.register_cli_command(name="photon", ...)``) only runs that side + effect when its module is imported. On the unknown-top-level-command slow + path, ``discover_plugins()`` records the deferred loader but does not + import it, so the CLI registration never happens and ``hermes photon`` + fails with argparse ``invalid choice`` (issue #54678). + + Resolving only the platform whose name matches the first positional token + keeps normal startup cheap while making the targeted command available. + """ + if not command_name: + return + try: + from gateway.platform_registry import platform_registry + + platform_registry.get(command_name) + except Exception as exc: + logging.getLogger(__name__).debug( + "Deferred platform CLI resolution failed for %s: %s", + command_name, + exc, + ) + + _AGENT_COMMANDS = {None, "chat", "acp", "rl"} _AGENT_SUBCOMMANDS = { "cron": ("cron_command", {"run", "tick"}), @@ -16105,6 +16134,11 @@ def main(): seen_plugin_commands.add(cmd_info["name"]) discover_plugins() + # A bundled platform whose top-level CLI command is the one being + # invoked is still only a deferred entry at this point; import it + # so its register_cli_command side effect runs before we read + # _cli_commands (issue #54678). + _resolve_deferred_platform_cli_command(_first_positional_argv()) for cmd_info in get_plugin_manager()._cli_commands.values(): if cmd_info["name"] in seen_plugin_commands: continue diff --git a/tests/hermes_cli/test_startup_plugin_gating.py b/tests/hermes_cli/test_startup_plugin_gating.py index 6028b3ea2d16..813e40b51f6c 100644 --- a/tests/hermes_cli/test_startup_plugin_gating.py +++ b/tests/hermes_cli/test_startup_plugin_gating.py @@ -32,6 +32,7 @@ from hermes_cli.main import ( _BUILTIN_SUBCOMMANDS, _first_positional_argv, _plugin_cli_discovery_needed, + _resolve_deferred_platform_cli_command, ) @@ -178,3 +179,73 @@ def test_builtin_set_has_no_phantom_entries(): f"_BUILTIN_SUBCOMMANDS has entries that are not registered as " f"top-level subparsers: {sorted(phantom)}" ) + + +# ── _resolve_deferred_platform_cli_command (issue #54678) ────────────────── + + +def test_deferred_platform_cli_resolution_targets_matching_platform(): + """The slow path must import the deferred platform whose name matches the + invoked command, so its register_cli_command side effect fires. + + Photon registers ``hermes photon`` only when its adapter module is + imported; on the unknown-command slow path the platform is still a + deferred entry, so without this resolution step the CLI command stays + absent and argparse rejects ``photon`` (issue #54678). + """ + from hermes_cli import main as _main + + class _FakeRegistry: + def __init__(self): + self.resolved: list[str] = [] + + def get(self, name): + self.resolved.append(name) + return None + + fake = _FakeRegistry() + fake_module = type(sys)("gateway.platform_registry") + fake_module.platform_registry = fake + with patch.dict(sys.modules, {"gateway.platform_registry": fake_module}): + _resolve_deferred_platform_cli_command("photon") + + assert fake.resolved == ["photon"] + + +def test_deferred_platform_cli_resolution_ignores_empty_command(): + """A None/empty first token (bare ``hermes`` / flags only) must not touch + the registry — normal startup stays cheap.""" + from hermes_cli import main as _main + + class _FakeRegistry: + def __init__(self): + self.resolved: list[str] = [] + + def get(self, name): # pragma: no cover - must not be called + self.resolved.append(name) + return None + + fake = _FakeRegistry() + fake_module = type(sys)("gateway.platform_registry") + fake_module.platform_registry = fake + with patch.dict(sys.modules, {"gateway.platform_registry": fake_module}): + _resolve_deferred_platform_cli_command(None) + _resolve_deferred_platform_cli_command("") + + assert fake.resolved == [] + + +def test_deferred_platform_cli_resolution_swallows_registry_errors(): + """A registry/import failure must be logged-and-ignored, never crash the + CLI startup path.""" + from hermes_cli import main as _main + + class _BoomRegistry: + def get(self, name): + raise RuntimeError("boom") + + fake_module = type(sys)("gateway.platform_registry") + fake_module.platform_registry = _BoomRegistry() + with patch.dict(sys.modules, {"gateway.platform_registry": fake_module}): + # Must not raise. + _resolve_deferred_platform_cli_command("photon") From 585726ac2e08dcc2bd32c134e0124e0657ecc519 Mon Sep 17 00:00:00 2001 From: Bartok9 Date: Wed, 15 Jul 2026 04:27:46 -0400 Subject: [PATCH 017/156] test(cli): regression for deferred platform CLI registration path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Responds to hermes-sweeper review on #54717: existing tests only mocked platform_registry.get(). Add a hermetic fake deferred-loader test that runs real PlatformRegistry resolution → PluginContext.register_cli_command → argparse subparser/choices visibility, without Photon SDK imports. --- .../hermes_cli/test_startup_plugin_gating.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/hermes_cli/test_startup_plugin_gating.py b/tests/hermes_cli/test_startup_plugin_gating.py index 813e40b51f6c..359524004e1b 100644 --- a/tests/hermes_cli/test_startup_plugin_gating.py +++ b/tests/hermes_cli/test_startup_plugin_gating.py @@ -249,3 +249,79 @@ def test_deferred_platform_cli_resolution_swallows_registry_errors(): with patch.dict(sys.modules, {"gateway.platform_registry": fake_module}): # Must not raise. _resolve_deferred_platform_cli_command("photon") + + +def test_deferred_platform_loader_registers_cli_command_before_parser_table(): + """Fake deferred loader must resolve through real registry + register CLI + before argparse builds plugin command subparsers (issue #54678 review). + + Existing unit tests mock ``platform_registry.get`` and only assert the + helper calls it. This regression exercises the full chain hermes-sweeper + asked for: + + 1. a deferred platform loader is pending on a real ``PlatformRegistry`` + 2. that loader materializes a ``register_cli_command`` side effect on the + plugin manager (no Photon SDK import) + 3. ``_resolve_deferred_platform_cli_command`` runs the pending loader + 4. the registered command is present on the manager *and* visible as an + argparse subparser/choice after the same construction path ``main()`` + uses for plugin CLI commands + """ + import argparse + + from gateway.platform_registry import PlatformRegistry + from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest + + mgr = PluginManager() + manifest = PluginManifest(name="fake-photon-platform") + command_name = "fakephoton" + + def _fake_loader(): + # Mirrors what a real platform adapter does on import: register its + # top-level hermes CLI command via PluginContext. + ctx = PluginContext(manifest, mgr) + ctx.register_cli_command( + name=command_name, + help="Fake deferred platform CLI (test-only)", + setup_fn=lambda p: None, + description="Hermetic stand-in for deferred platform CLI registration", + ) + + registry = PlatformRegistry() + registry.register_deferred(command_name, _fake_loader) + + # Precondition: deferred is known, but CLI command is not yet registered. + assert registry.is_registered(command_name) + assert command_name not in mgr._cli_commands + assert command_name in registry._deferred + + fake_module = type(sys)("gateway.platform_registry") + fake_module.platform_registry = registry + with patch.dict(sys.modules, {"gateway.platform_registry": fake_module}): + _resolve_deferred_platform_cli_command(command_name) + + # Loader resolution must promote deferred -> concrete and fire CLI register. + assert command_name not in registry._deferred + assert command_name in mgr._cli_commands + cmd_info = mgr._cli_commands[command_name] + assert cmd_info["name"] == command_name + assert cmd_info["plugin"] == "fake-photon-platform" + + # Same parser-table assembly main() uses after reading _cli_commands. + parser = argparse.ArgumentParser(prog="hermes") + subparsers = parser.add_subparsers(dest="command") + for info in mgr._cli_commands.values(): + plugin_parser = subparsers.add_parser( + info["name"], + help=info["help"], + description=info.get("description", ""), + ) + info["setup_fn"](plugin_parser) + if info.get("handler_fn") is not None: + plugin_parser.set_defaults(func=info["handler_fn"]) + + # argparse surface: choices + parse success. + choices = getattr(subparsers, "choices", None) or {} + assert command_name in choices + parsed = parser.parse_args([command_name]) + assert parsed.command == command_name From 5dc6a14c1446656a83cb4bf5de20ddbf75daac29 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:59:36 -0700 Subject: [PATCH 018/156] =?UTF-8?q?fix(credits):=20remove=20the=20'Grant?= =?UTF-8?q?=20spent=20=C2=B7=20$X=20top-up=20left'=20notice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grant_spent notice fired for every subscription user with top-up funds the moment their cap was reached and camped in the CLI/TUI status bar and desktop toasts with no action to take — the account keeps working off top-up. Remove it everywhere: - agent/credits_tracker.py: drop grant_cond + the emit/clear block; the dev fixture state now (correctly) produces no notice - TUI: keep the turn-start clear of credits.grant_spent as back-compat for older backends that still emit the key - Desktop: drop the demo step and stale comment references - Docs/config comments: remove grant-spent from credits_notices text - Tests updated: policy/cold-start now assert the key never fires Usage bands, depleted, and restored notices are unchanged; /usage still reports the full balance breakdown. --- agent/credits_tracker.py | 34 +------ .../app/contrib/dev/credits-notice-demo.ts | 3 +- apps/desktop/src/store/agent-notices.test.ts | 16 ++-- apps/desktop/src/store/agent-notices.ts | 4 +- hermes_cli/config.py | 2 +- tests/agent/test_credits_cold_start.py | 16 ++-- tests/agent/test_credits_policy.py | 93 +++++++------------ tests/gateway/test_notice_rendering.py | 2 +- ui-tui/src/app/turnController.ts | 12 +-- website/docs/user-guide/configuration.md | 2 +- 10 files changed, 68 insertions(+), 116 deletions(-) diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 929bc34d3262..dfd39e4290fb 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -288,20 +288,12 @@ def evaluate_credits_notices( current_band = band # Top-up suppression: when the account holds purchased (top-up) credits, # the subscription-cap gauge is the wrong denominator — warning "90% used" - # at a user sitting on $50 of top-up is noise (and it previously stuck - # PERMANENTLY alongside grant_spent at >=100%). Suppress the usage band - # entirely; the cap-reached case is covered by the grant_spent info notice - # below, which already names the remaining top-up balance. A top-up landing - # mid-session flips current_band → None and the clear path below removes - # any showing band line. + # at a user sitting on $50 of top-up is noise. Suppress the usage band + # entirely; /usage still reports the full balance breakdown. A top-up + # landing mid-session flips current_band → None and the clear path below + # removes any showing band line. if state.purchased_micros > 0: current_band = None - grant_cond = ( - state.denominator_kind == "subscription_cap" - and uf is not None - and uf >= 1.0 - and state.purchased_micros > 0 - ) depleted_cond = not state.paid_access # ── usage gauge (escalating single notice: 50 → 75 → 90) ────────────────── @@ -340,22 +332,6 @@ def evaluate_credits_notices( active.add(CREDITS_USAGE_KEY) latch["usage_band"] = target_band - # ── grant_spent ────────────────────────────────────────────────────────── - if grant_cond and "credits.grant_spent" not in active: - to_show.append( - AgentNotice( - text=f"• Grant spent · ${state.purchased_usd} top-up left", - level="info", - kind=CREDITS_NOTICE_KIND, - key="credits.grant_spent", - id="credits.grant_spent", - ) - ) - active.add("credits.grant_spent") - elif "credits.grant_spent" in active and not grant_cond: - to_clear.append("credits.grant_spent") - active.discard("credits.grant_spent") - # ── depleted ───────────────────────────────────────────────────────────── # Suppressed while the active model is free: inference still works there, # so the error banner would just alarm users (free-tier users especially, @@ -627,7 +603,7 @@ _DEV_FIXTURES: dict[str, dict] = { subscription_limit_micros=20_000_000, subscription_limit_usd="20.00", denominator_kind="subscription_cap", paid_access=True, ), - "grant_exhausted": dict( # used_fraction == 1.0 + purchased>0 → credits.grant_spent + "grant_exhausted": dict( # cap reached + purchased>0 → no notice (gauge suppressed by top-up) remaining_micros=12_340_000, remaining_usd="12.34", subscription_micros=0, subscription_usd="0.00", subscription_limit_micros=20_000_000, subscription_limit_usd="20.00", diff --git a/apps/desktop/src/app/contrib/dev/credits-notice-demo.ts b/apps/desktop/src/app/contrib/dev/credits-notice-demo.ts index 1766489818b8..a2ba8080da1e 100644 --- a/apps/desktop/src/app/contrib/dev/credits-notice-demo.ts +++ b/apps/desktop/src/app/contrib/dev/credits-notice-demo.ts @@ -25,7 +25,7 @@ interface NoticeStep { } // Walks the same lifecycle the Nous credits tracker drives: usage escalates in -// place (50→75→90, one key), then grant-spent, then the depleted/restored pair. +// place (50→75→90, one key), then the depleted/restored pair. // Wraps around. These are all separate SHOW steps; the stepper auto-clears the // previous notice when the key changes, so the demo shows one toast at a time // (real usage CAN stack these, but that's noise when you're eyeballing a single @@ -34,7 +34,6 @@ const STEPS: readonly NoticeStep[] = [ { key: 'credits.usage', kind: 'sticky', level: 'info', text: "• You've used $110.00 of your $220.00 cap" }, { key: 'credits.usage', kind: 'sticky', level: 'warn', text: "⚠ You've used $165.00 of your $220.00 cap" }, { key: 'credits.usage', kind: 'sticky', level: 'warn', text: "⚠ You've used $198.00 of your $220.00 cap" }, - { key: 'credits.grant_spent', kind: 'sticky', level: 'info', text: '• Grant spent · $12.00 top-up left' }, { key: 'credits.depleted', kind: 'sticky', level: 'error', text: '✕ Credit access paused · run /topup to top up' }, { key: 'credits.restored', kind: 'ttl', level: 'success', text: '✓ Credit access restored', ttl_ms: 8000 } ] diff --git a/apps/desktop/src/store/agent-notices.test.ts b/apps/desktop/src/store/agent-notices.test.ts index 1d8122ecd3cc..6153203b0964 100644 --- a/apps/desktop/src/store/agent-notices.test.ts +++ b/apps/desktop/src/store/agent-notices.test.ts @@ -83,10 +83,10 @@ test('the leading severity glyph is stripped from the toast message', () => { }) test('the trailing "· detail" is split off as a secondary meta line, not inlined', () => { - // grant_spent still carries a `· detail` tail. - const grant = noticeToToast({ key: 'credits.grant_spent', level: 'info', text: '• Grant spent · $12.00 top-up left' }) - expect(grant?.message).toBe('Grant spent') - expect(grant?.meta).toBe('$12.00 top-up left') + // Detail-carrying notices split on the first ` · `. + const paused = noticeToToast({ key: 'credits.depleted', level: 'error', text: '✕ Credit access paused · run /topup to top up' }) + expect(paused?.message).toBe('Credit access paused') + expect(paused?.meta).toBe('run /topup to top up') // The usage line has no middot → whole line is the message, no meta. const plain = noticeToToast(usage()) @@ -95,7 +95,7 @@ test('the trailing "· detail" is split off as a secondary meta line, not inline }) test('splitMeta splits on the first space-middot-space only', () => { - expect(splitMeta('Grant spent · $12.00 top-up left')).toEqual(['Grant spent', '$12.00 top-up left']) + expect(splitMeta('Credit access paused · run /topup to top up')).toEqual(['Credit access paused', 'run /topup to top up']) expect(splitMeta('Credit access restored')).toEqual(['Credit access restored', undefined]) // Interior middots after the first split stay in the meta. expect(splitMeta('a · b · c')).toEqual(['a', 'b · c']) @@ -117,7 +117,7 @@ test('usageFraction derives $used / $cap from the notice text', () => { expect(usageFraction("You've used $15.00 of your $20.00 cap")).toBeCloseTo(0.75) expect(usageFraction("You've used $198.00 of your $220.00 cap")).toBeCloseTo(0.9) // Fewer than two amounts, or a zero cap → no fraction. - expect(usageFraction('Grant spent')).toBeNull() + expect(usageFraction('Credit access restored')).toBeNull() expect(usageFraction("You've used $5.00 of your $0.00 cap")).toBeNull() expect(usageFraction(undefined)).toBeNull() }) @@ -138,7 +138,7 @@ test('usage accent stays muted below 75%, then ramps orange → red', () => { test('terminal credit states carry their own accent; others stay default', () => { expect(noticeAccent({ key: 'credits.depleted', text: '✕ paused' })).toBe('var(--ui-red)') expect(noticeAccent({ key: 'credits.restored', text: '✓ restored' })).toBe('var(--ui-green)') - expect(noticeAccent({ key: 'credits.grant_spent', text: '• Grant spent' })).toBeUndefined() + expect(noticeAccent({ key: 'credits.usage', text: '• Credits update' })).toBeUndefined() expect(noticeAccent(undefined)).toBeUndefined() }) @@ -189,7 +189,7 @@ test('clearAgentNotice dismisses only the matching key', () => { test('only credits.depleted and credits.restored map to a native notification', () => { expect(nativeNoticeInput(usage({ key: 'credits.usage' }), 'Credits')).toBeNull() - expect(nativeNoticeInput(usage({ key: 'credits.grant_spent' }), 'Credits')).toBeNull() + expect(nativeNoticeInput(usage({ key: 'credits.other' }), 'Credits')).toBeNull() expect(nativeNoticeInput({ text: 'x', key: undefined }, 'Credits')).toBeNull() expect(nativeNoticeInput({ text: '', key: 'credits.depleted' }, 'Credits')).toBeNull() }) diff --git a/apps/desktop/src/store/agent-notices.ts b/apps/desktop/src/store/agent-notices.ts index 683855428ee2..ae6ccfe22bde 100644 --- a/apps/desktop/src/store/agent-notices.ts +++ b/apps/desktop/src/store/agent-notices.ts @@ -175,8 +175,8 @@ export function clearAgentNotice(key: string | undefined): void { // Only these two credit notices are urgent enough to break through as a native // OS notification (when Hermes is backgrounded). The escalating usage line -// (`credits.usage`) and the grant-spent notice stay in-app toasts only — they -// aren't worth interrupting the user's OS for. +// (`credits.usage`) stays an in-app toast only — it isn't worth interrupting +// the user's OS for. const NATIVE_NOTICE_KEYS = new Set(['credits.depleted', 'credits.restored']) /** diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 54ebd00df996..5a8ca4889c17 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1962,7 +1962,7 @@ DEFAULT_CONFIG = { # class of over-claim that otherwise forces users to run # `git status` to verify edits landed. Set false to suppress. "file_mutation_verifier": True, - # Nous credits status-bar notices (usage bands, grant-spent, depleted / + # Nous credits status-bar notices (usage bands, depleted / # restored). When false, no credits notices are emitted — balance data # is still captured and /usage keeps working. Off switch for sub + # top-up users who find the gauge noisy. diff --git a/tests/agent/test_credits_cold_start.py b/tests/agent/test_credits_cold_start.py index 751ee5f1c688..8140eac4af74 100644 --- a/tests/agent/test_credits_cold_start.py +++ b/tests/agent/test_credits_cold_start.py @@ -49,22 +49,20 @@ def test_cold_start_opens_already_at_90pct_warns(): assert "credits.usage" in _cold_start_notices(s) -def test_cold_start_grant_exhausted_grant_spent_only(): - """Cap reached but top-up funds remain → grant_spent info notice ONLY. +def test_cold_start_grant_exhausted_no_notice(): + """Cap reached but top-up funds remain → NO notice. - The usage band is suppressed whenever purchased (top-up) credits exist: - the sub-cap gauge is the wrong denominator for an account that can keep - spending, and previously the 90/100% warn banner stuck permanently - alongside grant_spent.""" + The usage band is suppressed whenever purchased (top-up) credits exist + (the sub-cap gauge is the wrong denominator for an account that can keep + spending), and the old 'Grant spent · $X top-up left' notice was removed — + it camped in the status bar with no action to take.""" s = _state( remaining_micros=12_340_000, subscription_micros=0, subscription_limit_micros=20_000_000, subscription_limit_usd="20.00", purchased_micros=12_340_000, denominator_kind="subscription_cap", paid_access=True, ) assert s.used_fraction == 1.0 - keys = _cold_start_notices(s) - assert "credits.usage" not in keys - assert "credits.grant_spent" in keys + assert _cold_start_notices(s) == [] def test_cold_start_depleted_warns(): diff --git a/tests/agent/test_credits_policy.py b/tests/agent/test_credits_policy.py index ba7142d14aed..81c72527115c 100644 --- a/tests/agent/test_credits_policy.py +++ b/tests/agent/test_credits_policy.py @@ -163,11 +163,10 @@ class TestBoundaryFractions: assert "credits.usage" in keys assert "credits.usage" not in to_clear - def test_just_below_1_0_does_not_fire_grant_spent(self): - """subscription_micros = limit - 1 (used_fraction just under 1.0) must NOT fire grant_spent. - - Locks the boundary so a future used_fraction clamp refactor cannot fire - grant_spent a micro early. + def test_just_below_1_0_boundary_no_notices_with_topup(self): + """subscription_micros = 1 (used_fraction just under 1.0) with top-up + funds → nothing fires: gauge suppressed by purchased>0, and the old + grant_spent notice no longer exists at any fraction. """ latch = fresh_latch() limit = 20_000_000 @@ -182,49 +181,42 @@ class TestBoundaryFractions: ) assert s.used_fraction is not None and s.used_fraction < 1.0 to_show, to_clear = evaluate_credits_notices(s, latch) - assert all(n.key != "credits.grant_spent" for n in to_show) - assert "credits.grant_spent" not in to_clear + assert to_show == [] + assert to_clear == [] -# ── Scenario 4: grant_spent ─────────────────────────────────────────────────── +# ── Scenario 4: grant_spent removed ────────────────────────────────────────── -class TestGrantSpent: - def _grant_state(self, purchased_micros: int = 12_340_000) -> CreditsState: +class TestGrantSpentRemoved: + """The 'Grant spent · $X top-up left' notice was removed (July 2026) — it + camped in the status bar for every sub+top-up user and carried no action. + The policy must never emit or clear the key, even for the state that used + to fire it (cap reached + purchased top-up funds).""" + + def _old_trigger_state(self) -> CreditsState: return state_with_fraction( 1.0, denominator_kind="subscription_cap", - purchased_micros=purchased_micros, + purchased_micros=12_340_000, purchased_usd="12.34", ) - def test_grant_spent_fires_on_first_obs(self): - """No crossing gate for grant_spent — fires immediately on first obs.""" + def test_never_fires_on_old_trigger_state(self): latch = fresh_latch() - to_show, to_clear = evaluate_credits_notices(self._grant_state(), latch) - keys = [n.key for n in to_show] - assert "credits.grant_spent" in keys - - def test_grant_spent_no_refire(self): - latch = fresh_latch() - evaluate_credits_notices(self._grant_state(), latch) - to_show, to_clear = evaluate_credits_notices(self._grant_state(), latch) + to_show, to_clear = evaluate_credits_notices(self._old_trigger_state(), latch) assert all(n.key != "credits.grant_spent" for n in to_show) assert "credits.grant_spent" not in to_clear + assert "credits.grant_spent" not in latch["active"] - def test_grant_spent_clears_when_purchased_zero(self): + def test_stale_active_key_is_left_alone(self): + """A latch persisted from an older build may still hold the key; the + policy neither re-fires nor clears it (frontends drop unknown keys).""" latch = fresh_latch() - evaluate_credits_notices(self._grant_state(), latch) - # Now purchased → 0: grant_cond becomes False - s_no_purchase = state_with_fraction( - 1.0, - denominator_kind="subscription_cap", - purchased_micros=0, - purchased_usd="0.00", - ) - to_show, to_clear = evaluate_credits_notices(s_no_purchase, latch) - assert "credits.grant_spent" in to_clear + latch["active"].add("credits.grant_spent") + to_show, to_clear = evaluate_credits_notices(self._old_trigger_state(), latch) assert all(n.key != "credits.grant_spent" for n in to_show) + assert "credits.grant_spent" not in to_clear # ── Scenario 5: depleted + recovery ────────────────────────────────────────── @@ -451,7 +443,9 @@ class TestNoticeCopy: assert "$20.00" in warn_notice.text assert "cap" in warn_notice.text - def test_grant_spent_contains_verbatim_purchased_usd(self): + def test_grant_spent_never_emitted(self): + """Old grant_spent trigger state produces no notice at all — the gauge + is suppressed by top-up and the grant_spent notice is gone.""" latch = fresh_latch() s = state_with_fraction( 1.0, @@ -460,9 +454,7 @@ class TestNoticeCopy: purchased_usd="12.34", ) to_show, _ = evaluate_credits_notices(s, latch) - grant_notice = next(n for n in to_show if n.key == "credits.grant_spent") - assert "$12.34" in grant_notice.text - assert "top-up left" in grant_notice.text + assert to_show == [] def test_depleted_mentions_credits_command(self): latch = fresh_latch() @@ -476,18 +468,11 @@ class TestNoticeCopy: class TestSeverityOrder: - def test_multiple_new_notices_ordered_ascending_severity(self): - """grant_spent < depleted in to_show when both fire in one call. - - (usage is suppressed here: purchased>0 — see TestTopUpSuppression. - usage + grant_spent are now mutually exclusive by design.) - """ + def test_depleted_only_notice_with_topup_at_cap(self): + """With top-up funds and cap reached while unpaid, only depleted fires: + the usage gauge is suppressed by purchased>0 and grant_spent is gone.""" latch = {"active": set(), "seen_below_90": True, "usage_band": None} - # Build state: subscription_cap, uf >= 1.0, purchased_micros > 0, NOT paid_access - # grant_cond: subscription_cap + uf >= 1.0 + purchased > 0 ✓ - # depleted_cond: not paid_access ✓ - # usage band: suppressed (purchased > 0) s = CreditsState( subscription_limit_micros=20_000_000, subscription_limit_usd="20.00", @@ -499,11 +484,7 @@ class TestSeverityOrder: ) to_show, _ = evaluate_credits_notices(s, latch) keys = [n.key for n in to_show] - assert "credits.usage" not in keys - assert "credits.grant_spent" in keys - assert "credits.depleted" in keys - # Ascending severity: grant_spent before depleted - assert keys.index("credits.grant_spent") < keys.index("credits.depleted") + assert keys == ["credits.depleted"] def test_usage_before_depleted_without_topup(self): """With no top-up funds, usage fires and precedes depleted.""" @@ -573,9 +554,9 @@ class TestTopUpSuppression: assert "$19.00" in n.text assert latch["usage_band"] == 90 - def test_grant_spent_still_fires_with_topup(self): - """Suppression only affects the gauge — grant_spent (which NEEDS purchased>0) - is untouched.""" + def test_no_notice_at_cap_with_topup(self): + """Cap reached with top-up funds → nothing fires: gauge suppressed, + grant_spent removed.""" latch = fresh_latch() s = state_with_fraction( 1.0, @@ -584,9 +565,7 @@ class TestTopUpSuppression: purchased_usd="12.34", ) to_show, _ = evaluate_credits_notices(s, latch) - keys = [n.key for n in to_show] - assert "credits.grant_spent" in keys - assert "credits.usage" not in keys + assert to_show == [] def test_depleted_unaffected_by_topup_suppression(self): latch = fresh_latch() diff --git a/tests/gateway/test_notice_rendering.py b/tests/gateway/test_notice_rendering.py index 3ba731f32e41..292155f1408f 100644 --- a/tests/gateway/test_notice_rendering.py +++ b/tests/gateway/test_notice_rendering.py @@ -84,7 +84,7 @@ def test_real_policy_notices_render_without_doubling(): notices = ( _emitted(uf=0.9) # band 90 (warn) + _emitted(uf=0.5) # band 50 (info) - + _emitted(uf=1.0, purchased=5_000_000) # band 90 + grant_spent + + _emitted(uf=1.0, purchased=5_000_000) # top-up at cap → no notices (empty) + _emitted(uf=None, paid=False) # depleted ) assert notices, "policy produced no notices to check" diff --git a/ui-tui/src/app/turnController.ts b/ui-tui/src/app/turnController.ts index 91380466487d..3c373ac5df81 100644 --- a/ui-tui/src/app/turnController.ts +++ b/ui-tui/src/app/turnController.ts @@ -989,12 +989,12 @@ class TurnController { this.interrupted = false this.persistedToolLabels.clear() // "Flash and yield" notices clear when a new turn starts: a usage-band heads-up - // (credits.usage, 50/75/90%) and the one-time "grant spent" transition - // (credits.grant_spent) should show once, then get out of the way — not camp the - // bar (e.g. "Grant spent · $990 top-up left" sitting there with plenty of top-up - // left). Depletion (credits.depleted) and other notices stay — they're explicitly - // sticky until the policy clears them. The Python `active` latch retains the key, - // so a yielded notice won't re-fire on the next turn. + // (credits.usage, 50/75/90%) should show once, then get out of the way — not + // camp the bar. Depletion (credits.depleted) and other notices stay — they're + // explicitly sticky until the policy clears them. The Python `active` latch + // retains the key, so a yielded notice won't re-fire on the next turn. + // (credits.grant_spent is also cleared here for back-compat with older + // backends that still emit it — the notice was removed in July 2026.) const yieldingNoticeKey = getUiState().notice?.key if (yieldingNoticeKey === 'credits.usage' || yieldingNoticeKey === 'credits.grant_spent') { diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 086a09f5a646..fb55d3f4cd32 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1542,7 +1542,7 @@ display: enabled: false fields: ["model", "context_pct", "cwd"] file_mutation_verifier: true # Append an advisory footer when write_file/patch calls failed this turn - credits_notices: true # Nous credits status-bar notices (usage bands, grant-spent, depleted). false = silence them; /usage still works + credits_notices: true # Nous credits status-bar notices (usage bands, depleted). false = silence them; /usage still works language: en # UI language for static messages (approval prompts, some gateway replies). en | zh | zh-hant | ja | de | es | fr | tr | uk | af | ko | it | ga | pt | ru | hu ``` From 2faac36866fcf11d581d6dc7c467c787173bca0b Mon Sep 17 00:00:00 2001 From: "hermes-seaeye[bot]" <307254004+hermes-seaeye[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:30:19 +0000 Subject: [PATCH 019/156] fmt(js): `npm run fix` on merge (#73552) Co-authored-by: github-actions[bot] --- apps/desktop/electron/main.ts | 4 ++-- .../app/session/hooks/use-session-actions/index.ts | 2 ++ apps/desktop/src/app/shell/model-menu-panel.tsx | 6 +++++- .../src/components/model-visibility-dialog.tsx | 13 ++++++++----- apps/desktop/src/store/agent-notices.test.ts | 11 +++++++++-- apps/desktop/src/store/model-visibility.ts | 1 + ui-tui/src/__tests__/createSlashHandler.test.ts | 1 + 7 files changed, 28 insertions(+), 10 deletions(-) diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index dfb4747f0392..b9e966067a38 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -9272,8 +9272,8 @@ ipcMain.handle('hermes:window:openInstance', async () => { // shortcuts and the View menu. Reads and writes target the asking window. ipcMain.handle('hermes:zoom:get', event => { const window = BrowserWindow.fromWebContents(event.sender) - const level = - window && !window.isDestroyed() ? window.webContents.getZoomLevel() : DEFAULT_ZOOM_LEVEL + + const level = window && !window.isDestroyed() ? window.webContents.getZoomLevel() : DEFAULT_ZOOM_LEVEL return { level, percent: zoomLevelToPercent(level) } }) diff --git a/apps/desktop/src/app/session/hooks/use-session-actions/index.ts b/apps/desktop/src/app/session/hooks/use-session-actions/index.ts index 9ec65d78d028..266a33b65fd2 100644 --- a/apps/desktop/src/app/session/hooks/use-session-actions/index.ts +++ b/apps/desktop/src/app/session/hooks/use-session-actions/index.ts @@ -558,6 +558,7 @@ export function useSessionActions({ resetViewSync() setSelectedStoredSessionId(storedSessionId) selectedStoredSessionIdRef.current = storedSessionId + // A session is EITHER the main thread OR a tile — never both. openSessionTile // enforces this from the tile side (it refuses to tile the selected session); // this enforces it from the main side. Loading an existing session into main @@ -570,6 +571,7 @@ export function useSessionActions({ if ($sessionTiles.get().some(t => t.storedSessionId === storedSessionId)) { closeSessionTile(storedSessionId) } + // Optimistically clear any prior resume-failure latch for this session: // we're attempting a fresh resume, so the self-heal in use-route-resume // must not keep treating it as stranded. It's re-armed below only if THIS diff --git a/apps/desktop/src/app/shell/model-menu-panel.tsx b/apps/desktop/src/app/shell/model-menu-panel.tsx index 5fd4c9691c8a..8ca63f5afd6d 100644 --- a/apps/desktop/src/app/shell/model-menu-panel.tsx +++ b/apps/desktop/src/app/shell/model-menu-panel.tsx @@ -259,7 +259,11 @@ export function ModelMenuPanel({ gateway, onSelectModel, profile = 'default', re textValue="" > {group.provider.name} - + {!collapsed && group.families.map(family => { diff --git a/apps/desktop/src/components/model-visibility-dialog.tsx b/apps/desktop/src/components/model-visibility-dialog.tsx index 995e1d9adbda..8b80c3d5d0d9 100644 --- a/apps/desktop/src/components/model-visibility-dialog.tsx +++ b/apps/desktop/src/components/model-visibility-dialog.tsx @@ -108,9 +108,11 @@ export function ModelVisibilityDialog({ } const allFamilies = collapseModelFamilies(provider.models ?? []) + const onCount = allFamilies.filter(family => visible.has(modelVisibilityKey(provider.slug, family.id)) ).length + const checkState = onCount === 0 ? false : onCount === allFamilies.length ? true : 'indeterminate' const collapsed = collapsedProviders.includes(provider.slug) && !q @@ -138,15 +140,16 @@ export function ModelVisibilityDialog({ const key = modelVisibilityKey(provider.slug, family.id) return ( -