"""Tests for the direct Hermes-to-Relay shared-metrics runtime.""" from __future__ import annotations import contextvars import asyncio import json import sqlite3 import threading from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace from typing import Any import pytest from hermes_cli import lifecycle, plugins from hermes_cli.observability import relay_runtime, relay_shared_metrics from hermes_cli.plugins import PluginManager class _Request: def __init__(self, headers: dict[str, Any], content: dict[str, Any]) -> None: self.headers = headers self.content = content class _Relay: def __init__(self) -> None: self.events: list[tuple[Any, ...]] = [] self._callbacks: dict[str, Any] = {} self._starts: dict[Any, dict[str, Any]] = {} self._scope_starts: dict[Any, dict[str, Any]] = {} self._scope = contextvars.ContextVar("relay_scope", default=None) self._scope_serial = 0 self.ScopeType = SimpleNamespace(Agent="agent", Function="function") self.LLMRequest = _Request self.scope = SimpleNamespace( push=self._scope_push, pop=self._scope_pop, event=self._scope_event, ) self.llm = SimpleNamespace(call=self._llm_call, call_end=self._llm_call_end) self.subscribers = SimpleNamespace( register=self._register, deregister=self._deregister, flush=self._flush, ) self.get_scope_stack = self._get_scope_stack def _scope_push(self, name: str, scope_type: Any, **kwargs: Any) -> Any: self._scope_serial += 1 handle = ("scope", name, self._scope_serial) self._scope.set(handle) self.events.append(("scope.push", name, scope_type, kwargs)) if scope_type == self.ScopeType.Function: self._scope_starts[handle] = kwargs event = SimpleNamespace( kind="scope", category="function", name=name, scope_category="start", category_profile=None, metadata=kwargs.get("metadata"), data=kwargs.get("input"), ) for callback in list(self._callbacks.values()): callback(event) return handle def _scope_pop(self, handle: Any, **kwargs: Any) -> None: self.events.append(("scope.pop", handle, kwargs)) start = self._scope_starts.pop(handle, None) if start is not None: event = SimpleNamespace( kind="scope", category="function", name=handle[1], scope_category="end", category_profile=None, metadata={ **(start.get("metadata") or {}), **(kwargs.get("metadata") or {}), }, data=kwargs.get("output"), ) for callback in list(self._callbacks.values()): callback(event) def _scope_event(self, name: str, **kwargs: Any) -> None: self.events.append(("scope.event", name, kwargs)) def _get_scope_stack(self) -> Any: current = self._scope.get() self.events.append(("scope.sync", current)) return current def _llm_call( self, name: str, request: _Request, **kwargs: Any, ) -> Any: handle = ("llm", name, len(self._starts)) self._starts[handle] = kwargs self.events.append(("llm.call", name, request.content, kwargs)) return handle def _llm_call_end( self, handle: Any, response: dict[str, Any], **kwargs: Any, ) -> None: start = self._starts.pop(handle) self.events.append(("llm.call_end", handle, response, kwargs)) event = SimpleNamespace( kind="scope", category="llm", name=handle[1], scope_category="end", category_profile={"model_name": start["model_name"]}, metadata={ **start["metadata"], **kwargs["metadata"], "otel.status_code": "OK", }, data=response, ) for callback in list(self._callbacks.values()): callback(event) def _register(self, name: str, callback: Any) -> None: self._callbacks[name] = callback self.events.append(("subscribers.register", name)) def _deregister(self, name: str) -> None: self._callbacks.pop(name, None) self.events.append(("subscribers.deregister", name)) def _flush(self) -> None: self.events.append(("subscribers.flush",)) @pytest.fixture def direct_runtime(tmp_path, monkeypatch): fake = _Relay() monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setattr( "hermes_cli.config.read_raw_config", lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, ) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) yield fake relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() @pytest.fixture def real_binding_runtime(tmp_path, monkeypatch): relay = pytest.importorskip("nemo_relay") if getattr(relay, "_native", None) is None: pytest.skip("NeMo Relay native binding is unavailable on this platform") monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) monkeypatch.setattr( "hermes_cli.config.read_raw_config", lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, ) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) yield relay relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_path): base = { "session_id": "sensitive-session", "task_id": "task-1", "api_request_id": "request-1", "platform": "cli", "provider": "custom", "model": "gpt-sensitive-model-id", "base_url": "http://127.0.0.1:11434/v1", } assert lifecycle.has_hook("pre_api_request") lifecycle.invoke_hook("on_session_start", **base) lifecycle.invoke_hook("pre_llm_call", **base) lifecycle.invoke_hook( "pre_api_request", **base, request={"body": {"messages": ["sensitive-prompt"]}}, ) lifecycle.invoke_hook( "post_tool_call", **base, tool_call_id="sensitive-tool-call", tool_name="terminal", args={"command": "sensitive-command"}, result={"output": "sensitive-tool-result"}, status="ok", ) lifecycle.invoke_hook( "api_request_error", **base, retryable=True, error={"message": "sensitive-error"}, ) lifecycle.invoke_hook( "pre_api_request", **{ **base, "provider": "anthropic", "model": "claude-sonnet", "base_url": "https://api.anthropic.com", }, request={"body": {"messages": ["sensitive-prompt"]}}, ) lifecycle.invoke_hook( "post_api_request", **{ **base, "provider": "anthropic", "model": "claude-sonnet", "base_url": "https://api.anthropic.com", }, response={"content": "sensitive-response"}, ) lifecycle.invoke_hook( "on_session_end", **base, completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) lifecycle.finalize_session(session_id=base["session_id"]) starts = [event for event in direct_runtime.events if event[0] == "llm.call"] ends = [event for event in direct_runtime.events if event[0] == "llm.call_end"] scope_starts = [ event for event in direct_runtime.events if event[0] == "scope.push" ] assert len(scope_starts) == 2 assert scope_starts[0][2] == direct_runtime.ScopeType.Agent assert scope_starts[1][1] == "hermes.task_run" assert scope_starts[1][2] == direct_runtime.ScopeType.Function assert scope_starts[1][3]["handle"][1] == relay_runtime.SESSION_SCOPE assert scope_starts[1][3]["input"] == { "entrypoint": "interactive", "execution_surface": "cli", } assert len(starts) == 1 assert len(ends) == 1 assert starts[0][2] == {} assert starts[0][3]["model_name"] == "gpt" assert ends[0][2] == { "call_role": "primary", "locality": "remote", "model_family": "claude", "outcome": "success", "provider_family": "direct", } serialized_events = json.dumps(direct_runtime.events) assert "sensitive-prompt" not in serialized_events assert "sensitive-response" not in serialized_events assert "sensitive-error" not in serialized_events assert "sensitive-command" not in serialized_events assert "sensitive-tool-result" not in serialized_events assert "sensitive-tool-call" not in serialized_events assert "gpt-sensitive-model-id" not in serialized_events assert plugins.get_plugin_manager().list_plugins() == [] root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" packages = list((root / "outbox").glob("*.json")) assert len(packages) == 1 package = json.loads(packages[0].read_text(encoding="utf-8")) metrics = {metric["name"]: metric for metric in package["metrics"]} assert set(metrics) == { "hermes.model_call.count", "hermes.task_run.finished", "hermes.task_run.started", } assert metrics["hermes.model_call.count"]["dimensions"]["model_family"] == "claude" assert metrics["hermes.model_call.count"]["value"] == 1 assert metrics["hermes.task_run.started"] == { "name": "hermes.task_run.started", "type": "counter", "dimensions": { "entrypoint": "interactive", "execution_surface": "cli", }, "value": 1, } terminal = metrics["hermes.task_run.finished"]["dimensions"] assert terminal["duration_bucket"] in { "lt_1s", "1s_to_5s", "5s_to_30s", "30s_to_2m", "2m_to_10m", "gte_10m", } assert { key: value for key, value in terminal.items() if key != "duration_bucket" } == { "end_reason": "completed", "entrypoint": "interactive", "execution_surface": "cli", "model_call_count_bucket": "1", "outcome": "success", "retry_count_bucket": "1", "termination": "none", "tool_call_count_bucket": "1", } def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot( real_binding_runtime, tmp_path, monkeypatch, ): assert real_binding_runtime._native is not None prompt_canary = "real-relay-sensitive-prompt" response_canary = "real-relay-sensitive-response" model_canary = "gpt-real-relay-sensitive-model" tool_canary = "real-relay-sensitive-tool-result" def base(index: int) -> dict[str, Any]: return { "session_id": f"sensitive-session-{index}", "task_id": f"sensitive-task-{index}", "turn_id": f"sensitive-turn-{index}", "api_request_id": f"sensitive-request-{index}", "platform": "cli", "provider": "custom", "model": model_canary, "base_url": "http://127.0.0.1:11434/v1", } success = base(1) lifecycle.invoke_hook("on_session_start", **success) lifecycle.invoke_hook("pre_llm_call", **success, messages=[prompt_canary]) lifecycle.invoke_hook("pre_api_request", **success, retry_count=0) lifecycle.invoke_hook( "api_request_error", **success, retry_count=0, retryable=True, error={"message": prompt_canary}, ) lifecycle.invoke_hook("pre_api_request", **success, retry_count=1) lifecycle.invoke_hook( "post_tool_call", **success, tool_call_id="sensitive-tool-call", tool_name="terminal", args={"command": prompt_canary}, result={"output": tool_canary}, status="ok", ) lifecycle.invoke_hook( "post_api_request", **success, retry_count=1, response={"content": response_canary}, ) lifecycle.invoke_hook( "on_session_end", **success, completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) lifecycle.finalize_session(session_id=success["session_id"]) failed = base(2) lifecycle.invoke_hook("on_session_start", **failed) lifecycle.invoke_hook("pre_llm_call", **failed, messages=[prompt_canary]) lifecycle.invoke_hook("pre_api_request", **failed, retry_count=0) lifecycle.invoke_hook( "api_request_error", **failed, retry_count=0, retryable=False, error={"message": response_canary}, ) lifecycle.invoke_hook( "on_session_end", **failed, completed=False, failed=True, interrupted=False, turn_exit_reason="system_aborted", ) lifecycle.finalize_session(session_id=failed["session_id"]) cancelled = base(3) lifecycle.invoke_hook("on_session_start", **cancelled) lifecycle.invoke_hook("pre_llm_call", **cancelled, messages=[prompt_canary]) lifecycle.invoke_hook("pre_api_request", **cancelled, retry_count=0) lifecycle.invoke_hook( "on_session_end", **cancelled, completed=False, failed=False, interrupted=True, turn_exit_reason="interrupted_by_user", ) lifecycle.finalize_session(session_id=cancelled["session_id"]) from hermes_cli.observability.shared_metrics import SharedMetricsStore root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") tomorrow = datetime.now(timezone.utc) + timedelta(days=1) monkeypatch.setattr( "hermes_cli.observability.shared_metrics._utc_now", lambda: tomorrow, ) assert len(store.create_and_export_package_if_due()) == 1 snapshot = store.counter_snapshot() by_metric: dict[str, list[dict[str, Any]]] = {} for counter in snapshot: by_metric.setdefault(counter["metric_name"], []).append(counter) assert len(by_metric["hermes.task_run.started"]) == 1 assert by_metric["hermes.task_run.started"][0]["value"] == 3 assert { counter["dimensions"]["outcome"] for counter in by_metric["hermes.model_call.count"] } == {"success", "failed", "cancelled"} terminal_by_outcome = { counter["dimensions"]["outcome"]: counter for counter in by_metric["hermes.task_run.finished"] } assert set(terminal_by_outcome) == {"success", "failed", "cancelled"} assert terminal_by_outcome["success"]["dimensions"]["retry_count_bucket"] == "1" assert terminal_by_outcome["success"]["dimensions"]["tool_call_count_bucket"] == "1" assert terminal_by_outcome["failed"]["dimensions"]["end_reason"] == ( "system_aborted" ) assert terminal_by_outcome["cancelled"]["dimensions"]["termination"] == ( "user_cancelled" ) assert all(counter["packaged_value"] == counter["value"] for counter in snapshot) snapshot_values = { ( counter["metric_name"], tuple(sorted(counter["dimensions"].items())), ): counter["value"] for counter in snapshot } package_values: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {} packages = sorted((root / "outbox").glob("*.json")) assert len(packages) == 2 package_payloads = [ json.loads(package.read_text(encoding="utf-8")) for package in packages ] for package in package_payloads: assert package["schema_version"] == "hermes.shared_metrics.v1" for metric in package["metrics"]: key = (metric["name"], tuple(sorted(metric["dimensions"].items()))) package_values[key] = package_values.get(key, 0) + metric["value"] assert package_values == snapshot_values serialized_analytics = json.dumps({ "snapshot": snapshot, "packages": package_payloads, }) for canary in ( prompt_canary, response_canary, model_canary, tool_canary, "sensitive-session", "sensitive-task", "sensitive-request", "sensitive-tool-call", ): assert canary not in serialized_analytics def test_direct_runtime_is_disabled_by_default(tmp_path, monkeypatch): fake = _Relay() monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {}) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) assert not plugins.has_hook("pre_api_request") lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") lifecycle.finalize_session(session_id="s1") assert fake.events == [] assert not (tmp_path / "hermes-home" / "telemetry").exists() relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() def test_tool_intercept_bypass_does_not_create_relay_host(monkeypatch): relay_runtime._reset_for_tests() imports = [] def load_relay(): imports.append("nemo_relay") raise AssertionError("disabled helper created Relay host") monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) args = {"command": "true"} assert ( relay_runtime.apply_tool_request_intercepts( session_id="s1", tool_name="terminal", args=args, ) is args ) assert relay_runtime.get_host(create=False) is None assert imports == [] def test_execution_adapters_do_not_create_relay_host_without_a_consumer( monkeypatch, ): from agent import relay_llm, relay_tools relay_runtime._reset_for_tests() imports = [] def load_relay(): imports.append("nemo_relay") raise AssertionError("disabled execution adapter created Relay host") monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) request = {"model": "test-model", "messages": []} response = object() tool_args = {"command": "true"} tool_result = object() assert ( relay_llm.execute( request, lambda observed: response if observed is request else None, session_id="llm-session", name="test-provider", model_name="test-model", ) is response ) result, observed_args = relay_tools.execute( "terminal", tool_args, lambda observed: tool_result if observed is tool_args else None, session_id="tool-session", ) assert result is tool_result assert observed_args is tool_args assert relay_runtime.get_host(create=False) is None assert imports == [] def test_profile_key_caches_absolute_path_resolution(monkeypatch): relay_runtime._reset_for_tests() class Home: def __init__(self): self.resolve_calls = 0 def expanduser(self): return self def is_absolute(self): return True def resolve(self): self.resolve_calls += 1 return self def __str__(self): return "/profiles/cached" home = Home() monkeypatch.setattr(relay_runtime, "get_hermes_home", lambda: home) assert relay_runtime.current_profile_key() == "/profiles/cached" assert relay_runtime.current_profile_key() == "/profiles/cached" assert home.resolve_calls == 1 def test_host_registry_reads_existing_host_without_lock(): registry = relay_runtime.RelayHostRegistry() host = relay_runtime.NoopRelayRuntime("profile", "test") registry._hosts["profile"] = host class UnexpectedLock: def __enter__(self): raise AssertionError("registry read acquired the write lock") def __exit__(self, *_args): return False registry._lock = UnexpectedLock() assert registry.for_profile("profile", create=False) is host assert registry.for_profile("missing", create=False) is None def test_core_runtime_is_fail_open_without_a_published_binding(monkeypatch, caplog): relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() def missing_relay(name: str): assert name == "nemo_relay" raise ModuleNotFoundError(name) monkeypatch.setattr(relay_runtime.importlib, "import_module", missing_relay) assert relay_runtime.get_runtime() is None host = relay_runtime.get_host() assert isinstance(host, relay_runtime.NoopRelayRuntime) assert host.profile_key == relay_runtime.current_profile_key() assert "nemo_relay" in host.reason assert host.apply_tool_request_intercepts( session_id="s1", tool_name="terminal", args={"command": "true"}, ) == {"command": "true"} assert not relay_runtime.emit_mark("hermes.probe", session_id="s1") assert "Hermes Relay runtime initialization failed" in caplog.text relay_runtime._reset_for_tests() def test_core_mark_uses_the_shared_session_handle_without_a_plugin(direct_runtime): lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli") handle = relay_runtime.get_session_handle("s1") assert handle is not None assert relay_runtime.emit_mark( "hermes.skill.created", session_id="s1", data={"provenance": "agent_created"}, metadata={"data_schema": "hermes.skill.lifecycle.v1"}, ) [mark] = [event for event in direct_runtime.events if event[0] == "scope.event"] assert mark[1] == "hermes.skill.created" assert mark[2]["handle"] == handle assert plugins.get_plugin_manager().list_plugins() == [] def test_core_task_instrumentation_preserves_prompt_history_and_tool_schema( direct_runtime, monkeypatch, ): from run_agent import AIAgent agent = object.__new__(AIAgent) agent.session_id = "cache-stable-session" agent.platform = "cli" agent._parent_session_id = None agent._session_db = None agent._cached_system_prompt = "byte-stable-system-prompt\nwith exact spacing" agent.tools = [ { "type": "function", "function": { "name": "probe", "parameters": { "type": "object", "properties": {"value": {"type": "string"}}, }, }, } ] history = [{"role": "user", "content": "sensitive-history-canary"}] prompt_before = agent._cached_system_prompt.encode("utf-8") history_before = json.dumps(history, ensure_ascii=False, sort_keys=True) tools_before = json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) def fake_run_conversation( active_agent, user_message, system_message, conversation_history, task_id, stream_callback, persist_user_message, **kwargs, ): del ( user_message, system_message, task_id, stream_callback, persist_user_message, kwargs, ) assert active_agent is agent assert conversation_history is history return {"final_response": "ok", "completed": True} monkeypatch.setattr( "agent.conversation_loop.run_conversation", fake_run_conversation, ) for task_id in ("cache-task-1", "cache-task-2"): result = AIAgent.run_conversation( agent, "hello", conversation_history=history, task_id=task_id, ) assert result["final_response"] == "ok" assert agent._cached_system_prompt.encode("utf-8") == prompt_before assert json.dumps(history, ensure_ascii=False, sort_keys=True) == history_before assert json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) == tools_before def test_session_coordinator_separates_turn_release_from_hard_finalize( direct_runtime, ): profile_key = relay_runtime.current_profile_key() coordinator = relay_runtime.SESSION_COORDINATOR lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="coordinated-session", platform="cli", ) turn = coordinator.begin_turn( lease, turn_id="turn-1", task_id="task-1", ) assert relay_runtime.current_turn() is turn assert turn.handle is not None session_handle = lease.session.handle turn_push = next( event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == relay_runtime.TURN_SCOPE ) assert turn_push[3]["handle"] == session_handle coordinator.end_turn(turn, outcome="success") coordinator.release_conversation(lease) assert relay_runtime.current_turn() is None runtime = relay_runtime.get_runtime(create=False) assert runtime is not None assert runtime.get_session("coordinated-session") is not None coordinator.finalize_conversation( profile_key=profile_key, session_id="coordinated-session", ) assert runtime.get_session("coordinated-session") is None def test_turn_cleanup_can_be_retried_from_its_originating_context(direct_runtime): del direct_runtime coordinator = relay_runtime.SESSION_COORDINATOR lease = coordinator.acquire_conversation( profile_key=relay_runtime.current_profile_key(), session_id="cross-context-session", platform="cli", ) turn = coordinator.begin_turn( lease, turn_id="turn-1", task_id="task-1", ) contextvars.Context().run(coordinator.end_turn, turn, outcome="success") assert turn.closed assert relay_runtime.active_turn("cross-context-session") is None assert relay_runtime.current_turn() is turn coordinator.end_turn(turn, outcome="success") assert relay_runtime.current_turn() is None coordinator.release_conversation(lease) def test_session_coordinator_prepares_subscribers_before_opening_scope( direct_runtime, ): coordinator = relay_runtime.SESSION_COORDINATOR initializer_name = "test.pre_scope_subscriber" def prepare(host, context): assert host.profile_key == relay_runtime.current_profile_key() direct_runtime.events.append(("session.prepare", dict(context))) coordinator.register_session_initializer(initializer_name, prepare) try: lease = coordinator.acquire_conversation( profile_key=relay_runtime.current_profile_key(), session_id="prepared-session", platform="cli", model="demo-model", ) finally: coordinator.unregister_session_initializer(initializer_name) prepared = next( index for index, event in enumerate(direct_runtime.events) if event[0] == "session.prepare" ) opened = next( index for index, event in enumerate(direct_runtime.events) if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE ) assert prepared < opened assert direct_runtime.events[prepared][1] == { "profile_key": relay_runtime.current_profile_key(), "session_id": "prepared-session", "platform": "cli", "parent_session_id": "", "model": "demo-model", } coordinator.finalize_conversation( profile_key=relay_runtime.current_profile_key(), session_id=lease.session_id, ) def test_session_initializer_failure_does_not_block_conversation( direct_runtime, caplog, ): coordinator = relay_runtime.SESSION_COORDINATOR initializer_name = "test.failing_subscriber" def fail(_host, _context): raise RuntimeError("subscriber setup failed") coordinator.register_session_initializer(initializer_name, fail) try: with caplog.at_level("WARNING"): lease = coordinator.acquire_conversation( profile_key=relay_runtime.current_profile_key(), session_id="initializer-failure", platform="cli", ) finally: coordinator.unregister_session_initializer(initializer_name) assert lease.session is not None assert "Hermes Relay session initializer failed" in caplog.text coordinator.finalize_conversation( profile_key=relay_runtime.current_profile_key(), session_id=lease.session_id, ) def test_profile_host_recreation_rebinds_shared_metrics_subscriber( direct_runtime, ): coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() first_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="before-restart", platform="cli", ) first_metrics = relay_shared_metrics._get_runtime() assert first_metrics is not None first_host = first_lease.host coordinator.finalize_conversation( profile_key=profile_key, session_id=first_lease.session_id, ) coordinator.shutdown_profile(profile_key) second_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="after-restart", platform="cli", ) second_metrics = relay_shared_metrics._get_runtime() assert second_metrics is not None assert second_lease.host is not first_host assert second_metrics is not first_metrics assert second_metrics.host is second_lease.host coordinator.finalize_conversation( profile_key=profile_key, session_id=second_lease.session_id, ) def test_core_mark_does_not_start_relay_without_metrics_or_a_plugin( tmp_path, monkeypatch, ): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home")) imports = [] def load_relay(): imports.append("nemo_relay") return _Relay() monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay) monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {}) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() monkeypatch.setattr(plugins, "_plugin_manager", PluginManager()) assert not relay_runtime.emit_mark( "hermes.skill.created", session_id="s1", data={"provenance": "agent_created"}, ) assert imports == [] assert relay_runtime.get_host(create=False) is None assert not (tmp_path / "hermes-home" / "telemetry").exists() relay_runtime._reset_for_tests() def test_core_runtime_creates_one_session_under_concurrent_access(direct_runtime): runtime = relay_runtime.get_runtime() assert runtime is not None ready = threading.Barrier(8) sessions: list[Any] = [] def ensure() -> None: ready.wait(timeout=5) sessions.append(runtime.ensure_session({"session_id": "shared"})) threads = [threading.Thread(target=ensure) for _ in range(8)] for thread in threads: thread.start() for thread in threads: thread.join(timeout=5) assert all(not thread.is_alive() for thread in threads) assert len({id(session) for session in sessions}) == 1 assert ( len([event for event in direct_runtime.events if event[0] == "scope.push"]) == 1 ) def test_core_runtime_isolates_same_session_id_by_profile(direct_runtime, tmp_path): from hermes_constants import ( reset_hermes_home_override, set_hermes_home_override, ) profile_a = tmp_path / "profile-a" profile_b = tmp_path / "profile-b" token = set_hermes_home_override(profile_a) try: runtime_a = relay_runtime.get_runtime() session_a = runtime_a.ensure_session({"session_id": "shared"}) finally: reset_hermes_home_override(token) token = set_hermes_home_override(profile_b) try: runtime_b = relay_runtime.get_runtime() session_b = runtime_b.ensure_session({"session_id": "shared"}) finally: reset_hermes_home_override(token) assert runtime_a is not None assert runtime_b is not None assert runtime_a is not runtime_b assert runtime_a.profile_key == str(profile_a.resolve()) assert runtime_b.profile_key == str(profile_b.resolve()) assert session_a is not session_b assert session_a.handle != session_b.handle @pytest.mark.parametrize( ("profile_enabled", "managed_enabled"), ((None, True), (False, True), (True, False)), ) def test_managed_config_cannot_override_shared_metrics_consent( tmp_path, monkeypatch, profile_enabled, managed_enabled, ): from hermes_cli import config, managed_scope from hermes_constants import ( reset_hermes_home_override, set_hermes_home_override, ) profile = tmp_path / "profile" managed = tmp_path / "managed" profile.mkdir() managed.mkdir() profile_config = "{}\n" if profile_enabled is not None: profile_config = ( "telemetry:\n" " shared_metrics:\n" f" enabled: {str(profile_enabled).lower()}\n" ) (profile / "config.yaml").write_text(profile_config, encoding="utf-8") (managed / "config.yaml").write_text( "telemetry:\n" " shared_metrics:\n" f" enabled: {str(managed_enabled).lower()}\n", encoding="utf-8", ) monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed)) config._LOAD_CONFIG_CACHE.clear() config._RAW_CONFIG_CACHE.clear() managed_scope.invalidate_managed_cache() token = set_hermes_home_override(profile) try: assert ( config.load_config_readonly()["telemetry"]["shared_metrics"]["enabled"] is managed_enabled ) assert relay_shared_metrics.enabled() is (profile_enabled is True) finally: reset_hermes_home_override(token) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() managed_scope.invalidate_managed_cache() def test_shared_metrics_policy_and_store_are_profile_scoped(tmp_path, monkeypatch): from hermes_constants import ( get_hermes_home, reset_hermes_home_override, set_hermes_home_override, ) fake = _Relay() profile_a = tmp_path / "profile-a" profile_b = tmp_path / "profile-b" monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setattr( "hermes_cli.config.read_raw_config", lambda: { "telemetry": { "shared_metrics": {"enabled": get_hermes_home() == profile_a} } }, ) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() token = set_hermes_home_override(profile_a) try: assert relay_shared_metrics.enabled() relay_shared_metrics.start_task_run( session_id="shared", task_id="task-a", platform="cli", ) relay_shared_metrics.finish_task_run( session_id="shared", task_id="task-a", platform="cli", result={"completed": True}, ) relay_shared_metrics._get_runtime().close_session({"session_id": "shared"}) finally: reset_hermes_home_override(token) token = set_hermes_home_override(profile_b) try: assert not relay_shared_metrics.enabled() relay_shared_metrics.start_task_run( session_id="shared", task_id="task-b", platform="cli", ) finally: reset_hermes_home_override(token) assert list((profile_a / "telemetry" / "shared_metrics" / "outbox").glob("*.json")) assert not (profile_b / "telemetry").exists() relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() def test_shared_metrics_subscribers_isolate_two_enabled_profiles(tmp_path, monkeypatch): from hermes_constants import ( reset_hermes_home_override, set_hermes_home_override, ) fake = _Relay() profile_a = tmp_path / "profile-a" profile_b = tmp_path / "profile-b" monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setattr( "hermes_cli.config.read_raw_config", lambda: {"telemetry": {"shared_metrics": {"enabled": True}}}, ) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() for profile, task_id in ((profile_a, "task-a"), (profile_b, "task-b")): token = set_hermes_home_override(profile) try: relay_shared_metrics.start_task_run( session_id="shared", task_id=task_id, platform="cli", ) relay_shared_metrics.finish_task_run( session_id="shared", task_id=task_id, platform="cli", result={"completed": True}, ) relay_shared_metrics._get_runtime().close_session( {"session_id": "shared"} ) finally: reset_hermes_home_override(token) for profile in (profile_a, profile_b): packages = list( (profile / "telemetry" / "shared_metrics" / "outbox").glob("*.json") ) assert len(packages) == 1 package = json.loads(packages[0].read_text(encoding="utf-8")) metrics = {metric["name"]: metric for metric in package["metrics"]} assert metrics["hermes.task_run.started"]["value"] == 1 assert metrics["hermes.task_run.finished"]["value"] == 1 relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() def test_shared_metrics_isolates_same_task_id_across_sessions(direct_runtime): runtime = relay_shared_metrics._get_runtime() assert runtime is not None task_a = runtime.start_task({ "session_id": "session-a", "task_id": "shared-task", "platform": "cli", }) task_b = runtime.start_task({ "session_id": "session-b", "task_id": "shared-task", "platform": "gateway", }) assert task_a is not None assert task_b is not None assert task_a is not task_b assert task_a.handle != task_b.handle runtime.finish_task({ "session_id": "session-a", "task_id": "shared-task", "platform": "cli", "completed": True, }) runtime.finish_task({ "session_id": "session-b", "task_id": "shared-task", "platform": "gateway", "completed": True, }) task_starts = [ event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == "hermes.task_run" ] task_ends = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] assert len(task_starts) == 2 assert len(task_ends) == 2 def test_shared_metrics_keys_turn_ownership_by_session(direct_runtime): runtime = relay_shared_metrics._get_runtime() assert runtime is not None event_a = { "session_id": "session-a", "task_id": "task-a", "turn_id": "shared-turn", "platform": "cli", } event_b = { "session_id": "session-b", "task_id": "task-b", "turn_id": "shared-turn", "platform": "gateway", } task_a = runtime.start_task(event_a) task_b = runtime.start_task(event_b) runtime.start_model_call({ **event_a, "api_request_id": "request-a", "provider": "anthropic", "model": "claude-sonnet", }) session_a = runtime._session(event_a) session_b = runtime._session(event_b) assert task_a is not None assert task_b is not None assert session_a is not None assert session_b is not None assert "request-a" in session_a.model_calls assert "request-a" not in session_b.model_calls [model_start] = [ event for event in direct_runtime.events if event[0] == "llm.call" ] assert model_start[3]["handle"] == task_a.handle def test_disabling_shared_metrics_stops_collection_and_shutdown_export( tmp_path, monkeypatch ): from hermes_cli.observability.shared_metrics import SharedMetricsStore fake = _Relay() profile = tmp_path / "profile" policy = {"enabled": True} monkeypatch.setenv("HERMES_HOME", str(profile)) monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake) monkeypatch.setattr( "hermes_cli.config.read_raw_config", lambda: {"telemetry": {"shared_metrics": dict(policy)}}, ) relay_shared_metrics._reset_for_tests() relay_runtime._reset_for_tests() relay_shared_metrics.start_task_run( session_id="session", task_id="task", platform="cli", ) runtime = relay_shared_metrics._get_runtime() assert runtime is not None policy["enabled"] = False assert not relay_shared_metrics.enabled() counters_before_stale_event = runtime.subscriber.store.counter_snapshot() runtime.subscriber(SimpleNamespace( kind="scope", category="function", category_profile=None, name="hermes.task_run", scope_category="start", metadata={ "hermes.metrics.schema_version": "hermes.metrics.event.v1", relay_runtime.RUNTIME_INSTANCE_KEY: runtime.host.runtime_id, }, data={"entrypoint": "interactive", "execution_surface": "cli"}, )) assert runtime.subscriber.store.counter_snapshot() == counters_before_stale_event assert runtime.start_task({ "session_id": "session", "task_id": "stale-runtime-task", "platform": "cli", }) is None relay_shared_metrics.finish_task_run( session_id="session", task_id="task", platform="cli", result={"completed": True}, ) relay_shared_metrics._reset_for_tests() root = profile / "telemetry" / "shared_metrics" store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox") assert [row["metric_name"] for row in store.counter_snapshot()] == [ "hermes.task_run.started" ] assert list((root / "outbox").glob("*.json")) == [] relay_runtime._reset_for_tests() def test_shared_metrics_retries_transient_initialization_failure( direct_runtime, monkeypatch ): real_store = relay_shared_metrics.SharedMetricsStore attempts = 0 def flaky_store(): nonlocal attempts attempts += 1 if attempts == 1: raise OSError("transient store failure") return real_store() monkeypatch.setattr(relay_shared_metrics, "SharedMetricsStore", flaky_store) relay_shared_metrics.start_task_run( session_id="session", task_id="first", platform="cli", ) relay_shared_metrics.start_task_run( session_id="session", task_id="second", platform="cli", ) assert attempts == 2 task_starts = [ event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == "hermes.task_run" ] assert len(task_starts) == 1 def test_async_session_runner_awaits_inside_saved_relay_context(direct_runtime): runtime = relay_runtime.get_runtime() assert runtime is not None session = runtime.ensure_session({"session_id": "async-session"}) assert session is not None async def probe() -> Any: await asyncio.sleep(0) return direct_runtime._scope.get() result = asyncio.run(runtime.run_in_session_async(session, probe)) assert result == session.handle def test_session_runners_preserve_caller_context_and_profile_override( direct_runtime, tmp_path, ): from hermes_constants import ( get_hermes_home_override, reset_hermes_home_override, set_hermes_home_override, ) runtime = relay_runtime.get_runtime() assert runtime is not None session = runtime.ensure_session({"session_id": "caller-context-session"}) assert session is not None caller_value = contextvars.ContextVar("caller_value", default="default") caller_value.set("caller") profile = tmp_path / "caller-profile" profile_token = set_hermes_home_override(profile) def sync_probe() -> tuple[Any, str, str | None]: return ( direct_runtime._scope.get(), caller_value.get(), get_hermes_home_override(), ) async def async_probe() -> tuple[Any, str, str | None]: await asyncio.sleep(0) return sync_probe() try: sync_result = runtime.run_in_session(session, sync_probe) async_result = asyncio.run(runtime.run_in_session_async(session, async_probe)) finally: reset_hermes_home_override(profile_token) expected = (session.handle, "caller", str(profile)) assert sync_result == expected assert async_result == expected @pytest.mark.asyncio async def test_async_session_runner_isolates_concurrent_caller_contexts( direct_runtime, ): runtime = relay_runtime.get_runtime() assert runtime is not None session = runtime.ensure_session({"session_id": "concurrent-context-session"}) assert session is not None caller_value = contextvars.ContextVar("concurrent_caller", default="default") ready = asyncio.Event() entered = 0 async def run(value: str) -> tuple[Any, str]: nonlocal entered caller_value.set(value) async def probe() -> tuple[Any, str]: nonlocal entered entered += 1 if entered == 2: ready.set() await ready.wait() return direct_runtime._scope.get(), caller_value.get() return await runtime.run_in_session_async(session, probe) results = await asyncio.gather(run("first"), run("second")) assert results == [ (session.handle, "first"), (session.handle, "second"), ] def test_sync_session_runner_releases_lock_before_callback(direct_runtime): runtime = relay_runtime.get_runtime() assert runtime is not None session = runtime.ensure_session({"session_id": "sync-session"}) assert session is not None acquired = threading.Event() contender = None def probe() -> Any: nonlocal contender def acquire_session_lock() -> None: with session.lock: acquired.set() contender = threading.Thread(target=acquire_session_lock) contender.start() assert acquired.wait(timeout=1) return direct_runtime._scope.get() result = runtime.run_in_session(session, probe) assert contender is not None contender.join(timeout=1) assert result == session.handle assert contender.is_alive() is False def test_active_turn_requires_matching_session_and_profile( direct_runtime, tmp_path, monkeypatch, ): del direct_runtime coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="session-1", platform="cli", ) turn = coordinator.begin_turn( lease, turn_id="turn-1", task_id="task-1", ) assert relay_runtime.active_turn("session-1") is turn assert relay_runtime.active_turn("session-2") is None monkeypatch.setenv("HERMES_HOME", str(tmp_path / "other-profile")) assert relay_runtime.active_turn("session-1") is None coordinator.end_turn(turn, outcome="success") coordinator.release_conversation(lease) assert relay_runtime.active_turn("session-1") is None def test_turn_cleanup_drains_logical_calls_after_turn_scope_start_failure( direct_runtime, ): coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="session-1", platform="cli", ) assert lease.session is not None original_push = direct_runtime.scope.push def fail_turn_push(name, *args, **kwargs): if name == relay_runtime.TURN_SCOPE: raise RuntimeError("simulated turn scope failure") return original_push(name, *args, **kwargs) direct_runtime.scope.push = fail_turn_push turn = coordinator.begin_turn( lease, turn_id="turn-1", task_id="task-1", ) direct_runtime.scope.push = original_push assert turn.handle is None runtime = lease.host logical_handle = runtime.run_in_session( lease.session, runtime.relay.scope.push, relay_runtime.LOGICAL_LLM_SCOPE, runtime.relay.ScopeType.Function, handle=lease.session.handle, input={}, ) turn.logical_llm_calls["request-1"] = logical_handle coordinator.end_turn(turn, outcome="failed") coordinator.release_conversation(lease) logical_closes = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1] == logical_handle ] assert len(logical_closes) == 1 assert turn.logical_llm_calls == {} def test_turn_cleanup_drains_logical_calls_in_lifo_order(direct_runtime): coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="session-lifo", platform="cli", ) assert lease.session is not None turn = coordinator.begin_turn( lease, turn_id="turn-lifo", task_id="task-lifo", ) assert turn.handle is not None runtime = lease.host handles = [] for request_id in ("request-1", "request-2"): handle = runtime.run_in_session( lease.session, runtime.relay.scope.push, relay_runtime.LOGICAL_LLM_SCOPE, runtime.relay.ScopeType.Function, handle=turn.handle, input={}, ) turn.logical_llm_calls[request_id] = handle handles.append(handle) coordinator.end_turn(turn, outcome="failed") coordinator.release_conversation(lease) logical_closes = [ event[1] for event in direct_runtime.events if event[0] == "scope.pop" and event[1] in handles ] assert logical_closes == list(reversed(handles)) assert turn.logical_llm_calls == {} def test_real_binding_drains_multiple_logical_calls_before_turn_close( real_binding_runtime, caplog, ): coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="session-native-lifo", platform="cli", ) assert lease.session is not None turn = coordinator.begin_turn( lease, turn_id="turn-native-lifo", task_id="task-native-lifo", ) assert turn.handle is not None runtime = lease.host for request_id in ("request-1", "request-2"): handle = runtime.run_in_session( lease.session, runtime.relay.scope.push, relay_runtime.LOGICAL_LLM_SCOPE, runtime.relay.ScopeType.Function, handle=turn.handle, input={}, ) turn.logical_llm_calls[request_id] = handle coordinator.end_turn(turn, outcome="failed") coordinator.release_conversation(lease) coordinator.finalize_conversation( profile_key=profile_key, session_id=lease.session_id, ) assert turn.logical_llm_calls == {} assert "finalization failed" not in caplog.text assert "closed with errors" not in caplog.text def test_shared_metrics_creates_one_task_under_concurrent_access(direct_runtime): runtime = relay_shared_metrics._get_runtime() assert runtime is not None ready = threading.Barrier(8) tasks: list[Any] = [] def start() -> None: ready.wait(timeout=5) tasks.append( runtime.start_task({"session_id": "s1", "task_id": "t1", "platform": "cli"}) ) threads = [threading.Thread(target=start) for _ in range(8)] for thread in threads: thread.start() for thread in threads: thread.join(timeout=5) assert all(not thread.is_alive() for thread in threads) assert len({id(task) for task in tasks}) == 1 task_starts = [ event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == "hermes.task_run" ] assert len(task_starts) == 1 def test_core_runtime_parents_subagent_session_without_exposing_ids( direct_runtime, ): coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() parent_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="parent", platform="cli", ) parent_turn = coordinator.begin_turn( parent_lease, turn_id="parent-turn", task_id="parent-task", ) child_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="sensitive-child", platform="subagent", parent_session_id="parent", ) runtime = relay_runtime.get_runtime() assert runtime is not None child = runtime.get_session("sensitive-child") assert child is not None assert child.parent_session_id == "parent" session_pushes = [ event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE ] assert len(session_pushes) == 2 child_kwargs = session_pushes[1][3] assert child_kwargs["handle"] == parent_turn.handle assert child_kwargs["metadata"] == { "hermes.execution_surface": "subagent", relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id, "nemo_relay_scope_role": "subagent", } assert "sensitive-child" not in json.dumps(session_pushes) coordinator.finalize_conversation( profile_key=profile_key, session_id="sensitive-child", ) coordinator.release_conversation(child_lease) coordinator.end_turn(parent_turn, outcome="success") coordinator.release_conversation(parent_lease) coordinator.finalize_conversation( profile_key=profile_key, session_id="parent", ) def test_subagent_stop_hook_does_not_own_child_session_lifetime(direct_runtime): runtime = relay_runtime.get_runtime() assert runtime is not None child = runtime.register_subagent( { "parent_session_id": "parent", "child_session_id": "child", } ) assert child is not None lifecycle.invoke_hook( "subagent_stop", parent_session_id="parent", child_session_id="child", child_status="completed", ) assert runtime.get_session("child") is child runtime.unregister_subagent({"child_session_id": "child"}) assert runtime.get_session("child") is None child_closes = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1] == child.handle ] assert len(child_closes) == 1 @pytest.mark.parametrize( "terminal", ["return", "exception", "cancelled", "timeout"], ) def test_subagent_agent_boundary_closes_its_own_scope( direct_runtime, monkeypatch, terminal, ): from run_agent import AIAgent coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() parent_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="parent", platform="cli", ) parent_turn = coordinator.begin_turn( parent_lease, turn_id="parent-turn", task_id="parent-task", ) child_agent = SimpleNamespace( session_id="child", platform="subagent", _parent_session_id="parent", _session_db=None, _conversation_root_id=lambda: "parent", ) if terminal == "return": monkeypatch.setattr( "agent.conversation_loop.run_conversation", lambda *_args, **_kwargs: { "final_response": "done", "completed": True, "interrupted": False, }, ) AIAgent.run_conversation(child_agent, "private", task_id="child-task") elif terminal == "exception": def fail(*_args, **_kwargs): raise RuntimeError("child failed") monkeypatch.setattr("agent.conversation_loop.run_conversation", fail) with pytest.raises(RuntimeError, match="child failed"): AIAgent.run_conversation(child_agent, "private", task_id="child-task") elif terminal == "cancelled": def cancel(*_args, **_kwargs): raise KeyboardInterrupt monkeypatch.setattr("agent.conversation_loop.run_conversation", cancel) with pytest.raises(KeyboardInterrupt): AIAgent.run_conversation(child_agent, "private", task_id="child-task") else: def time_out(*_args, **_kwargs): raise TimeoutError("child timed out") monkeypatch.setattr("agent.conversation_loop.run_conversation", time_out) with pytest.raises(TimeoutError, match="child timed out"): AIAgent.run_conversation(child_agent, "private", task_id="child-task") runtime = relay_runtime.get_runtime(create=False) assert runtime is not None assert runtime.get_session("child") is None child_push = next( event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent" ) assert child_push[3]["handle"] == parent_turn.handle child_closes = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == relay_runtime.SESSION_SCOPE ] assert len(child_closes) == 1 assert relay_runtime.current_turn() is parent_turn coordinator.end_turn(parent_turn, outcome="success") coordinator.release_conversation(parent_lease) coordinator.finalize_conversation( profile_key=profile_key, session_id="parent", ) def test_concurrent_subagents_inherit_parent_turn_and_close_independently( direct_runtime, ): from concurrent.futures import ThreadPoolExecutor coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() parent_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="parent", platform="cli", ) parent_turn = coordinator.begin_turn( parent_lease, turn_id="parent-turn", task_id="parent-task", ) def run_child(child_id): lease = coordinator.acquire_conversation( profile_key=profile_key, session_id=child_id, platform="subagent", parent_session_id="parent", ) turn = coordinator.begin_turn( lease, turn_id=f"{child_id}-turn", task_id=f"{child_id}-task", ) coordinator.end_turn(turn, outcome="success") coordinator.finalize_conversation( profile_key=profile_key, session_id=child_id, ) coordinator.release_conversation(lease) contexts = [contextvars.copy_context(), contextvars.copy_context()] with ThreadPoolExecutor(max_workers=2) as executor: futures = [ executor.submit(context.run, run_child, f"child-{index}") for index, context in enumerate(contexts) ] for future in futures: future.result() runtime = relay_runtime.get_runtime(create=False) assert runtime is not None assert runtime.get_session("child-0") is None assert runtime.get_session("child-1") is None child_pushes = [ event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent" ] assert len(child_pushes) == 2 assert all(event[3]["handle"] == parent_turn.handle for event in child_pushes) coordinator.end_turn(parent_turn, outcome="success") coordinator.release_conversation(parent_lease) coordinator.finalize_conversation( profile_key=profile_key, session_id="parent", ) def test_coordinator_tracks_active_turns_across_threads(direct_runtime): del direct_runtime coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="cross-thread-child", platform="subagent", ) assert not coordinator.has_active_turn( profile_key=profile_key, session_id="cross-thread-child", ) turn = coordinator.begin_turn( lease, turn_id="child-turn", task_id="child-task", ) observed = [] thread = threading.Thread( target=lambda: observed.append( coordinator.has_active_turn( profile_key=profile_key, session_id="cross-thread-child", ) ) ) thread.start() thread.join(timeout=5) assert observed == [True] coordinator.end_turn(turn, outcome="success") assert not coordinator.has_active_turn( profile_key=profile_key, session_id="cross-thread-child", ) coordinator.release_conversation(lease) coordinator.finalize_conversation( profile_key=profile_key, session_id="cross-thread-child", ) def test_child_session_closes_before_active_turn_guard_is_released( direct_runtime, monkeypatch, ): del direct_runtime coordinator = relay_runtime.SESSION_COORDINATOR profile_key = relay_runtime.current_profile_key() parent_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="parent", platform="cli", ) child_lease = coordinator.acquire_conversation( profile_key=profile_key, session_id="child", platform="subagent", parent_session_id="parent", ) child_turn = coordinator.begin_turn( child_lease, turn_id="child-turn", task_id="child-task", ) runtime = relay_runtime.get_runtime(create=False) assert runtime is not None close_observations = [] original_close = runtime.close_session def observe_close(event): close_observations.append( coordinator.has_active_turn( profile_key=profile_key, session_id="child", ) ) original_close(event) monkeypatch.setattr(runtime, "close_session", observe_close) coordinator.end_turn(child_turn, outcome="success") assert close_observations == [True] assert runtime.get_session("child") is None assert not coordinator.has_active_turn( profile_key=profile_key, session_id="child", ) coordinator.release_conversation(child_lease) coordinator.release_conversation(parent_lease) coordinator.finalize_conversation( profile_key=profile_key, session_id="parent", ) def test_core_runtime_ignores_self_parenting_subagent_event(direct_runtime): runtime = relay_runtime.get_runtime() assert runtime is not None runtime.register_subagent({"parent_session_id": "same", "child_session_id": "same"}) session = runtime.ensure_session({"session_id": "same"}) assert session is not None assert session.parent_session_id == "" def test_terminal_model_error_is_counted_as_failed(direct_runtime): base = { "session_id": "s1", "task_id": "t1", "api_request_id": "r1", "provider": "anthropic", "model": "claude-sonnet", } lifecycle.invoke_hook("pre_api_request", **base) lifecycle.invoke_hook("api_request_error", **base, retryable=False) lifecycle.finalize_session(session_id="s1") [end] = [event for event in direct_runtime.events if event[0] == "llm.call_end"] assert end[2]["outcome"] == "failed" def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runtime): base = { "session_id": "s1", "task_id": "t1", "api_request_id": "r1", "platform": "cli", "provider": "nvidia", "model": "nvidia/nemotron-3-super-120b-a12b", } lifecycle.invoke_hook("pre_llm_call", **base) lifecycle.invoke_hook("pre_api_request", **base) lifecycle.invoke_hook("api_request_error", **base, retryable=True) lifecycle.invoke_hook("pre_api_request", **base) lifecycle.invoke_hook("api_request_error", **base, retryable=True) lifecycle.invoke_hook("pre_api_request", **base) lifecycle.invoke_hook("api_request_error", **base, retryable=False) for tool_call_id in ("tool-1", "tool-1", "tool-2"): lifecycle.invoke_hook( "post_tool_call", **base, tool_call_id=tool_call_id, tool_name="terminal", result={"output": "private"}, status="ok", ) lifecycle.invoke_hook( "on_session_end", **base, completed=False, failed=True, interrupted=False, turn_exit_reason="all_retries_exhausted_no_response", ) lifecycle.finalize_session(session_id="s1") model_starts = [event for event in direct_runtime.events if event[0] == "llm.call"] model_ends = [ event for event in direct_runtime.events if event[0] == "llm.call_end" ] assert len(model_starts) == 1 assert len(model_ends) == 1 assert model_ends[0][2]["outcome"] == "failed" [task_end] = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] assert task_end[2]["output"] == { "duration_bucket": task_end[2]["output"]["duration_bucket"], "end_reason": "failed", "entrypoint": "interactive", "execution_surface": "cli", "model_call_count_bucket": "1", "outcome": "failed", "retry_count_bucket": "2", "termination": "none", "tool_call_count_bucket": "2", } def test_task_terminal_counts_explicit_retry_with_new_request_id(direct_runtime): base = { "session_id": "s1", "task_id": "t1", "platform": "cli", "provider": "nvidia", "model": "nvidia/nemotron-3-super-120b-a12b", } lifecycle.invoke_hook("pre_llm_call", **base) lifecycle.invoke_hook( "pre_api_request", **base, api_request_id="r1", retry_count=0, ) lifecycle.invoke_hook( "api_request_error", **base, api_request_id="r1", retryable=True, ) lifecycle.invoke_hook( "pre_api_request", **base, api_request_id="r2", retry_count=1, ) lifecycle.invoke_hook( "post_api_request", **base, api_request_id="r2", ) lifecycle.invoke_hook( "on_session_end", **base, completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) lifecycle.finalize_session(session_id="s1") [task_end] = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] assert task_end[2]["output"]["model_call_count_bucket"] == "2" assert task_end[2]["output"]["retry_count_bucket"] == "1" def test_outer_agent_boundary_closes_early_returns_and_exceptions( direct_runtime, monkeypatch, ): from run_agent import AIAgent agent = SimpleNamespace( session_id="s1", platform="cli", _parent_session_id=None, _session_db=None, _conversation_root_id=lambda: "s1", ) def early_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]: return { "final_response": "private failure detail", "completed": False, "failed": True, "interrupted": False, } monkeypatch.setattr( "agent.conversation_loop.run_conversation", early_failure, ) result = AIAgent.run_conversation(agent, "private prompt", task_id="early") assert result["failed"] is True def raise_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]: raise RuntimeError("private exception detail") monkeypatch.setattr( "agent.conversation_loop.run_conversation", raise_failure, ) with pytest.raises(RuntimeError, match="private exception detail"): AIAgent.run_conversation(agent, "private prompt", task_id="exception") def raise_interrupt(*_args: Any, **_kwargs: Any) -> dict[str, Any]: raise KeyboardInterrupt monkeypatch.setattr( "agent.conversation_loop.run_conversation", raise_interrupt, ) with pytest.raises(KeyboardInterrupt): AIAgent.run_conversation(agent, "private prompt", task_id="cancelled") def raise_timeout(*_args: Any, **_kwargs: Any) -> dict[str, Any]: raise TimeoutError("private timeout detail") monkeypatch.setattr( "agent.conversation_loop.run_conversation", raise_timeout, ) with pytest.raises(TimeoutError, match="private timeout detail"): AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") lifecycle.finalize_session(session_id="s1") task_ends = [ event[2]["output"] for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] assert len(task_ends) == 4 assert task_ends[0]["outcome"] == "failed" assert task_ends[0]["end_reason"] == "failed" assert task_ends[0]["termination"] == "none" assert task_ends[1]["outcome"] == "failed" assert task_ends[1]["end_reason"] == "system_aborted" assert task_ends[1]["termination"] == "system_aborted" assert task_ends[2]["outcome"] == "cancelled" assert task_ends[2]["end_reason"] == "user_cancelled" assert task_ends[2]["termination"] == "user_cancelled" assert task_ends[3]["outcome"] == "timed_out" assert task_ends[3]["end_reason"] == "timed_out" assert task_ends[3]["termination"] == "timed_out" serialized = json.dumps(direct_runtime.events) assert "private prompt" not in serialized assert "private failure detail" not in serialized assert "private exception detail" not in serialized assert "private timeout detail" not in serialized def test_outer_agent_boundary_preserves_a_returned_timeout_reason( direct_runtime, monkeypatch, ): from run_agent import AIAgent agent = SimpleNamespace( session_id="s1", platform="cli", _parent_session_id=None, _session_db=None, _conversation_root_id=lambda: "s1", ) monkeypatch.setattr( "agent.conversation_loop.run_conversation", lambda *_args, **_kwargs: { "final_response": "private timeout response", "completed": False, "failed": True, "failure_reason": "timeout", }, ) AIAgent.run_conversation(agent, "private prompt", task_id="timed-out") [task_end] = [ event[2]["output"] for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] assert task_end["outcome"] == "timed_out" assert task_end["end_reason"] == "timed_out" assert task_end["termination"] == "timed_out" serialized = json.dumps(direct_runtime.events) assert "private prompt" not in serialized assert "private timeout response" not in serialized def test_session_finalize_closes_a_pending_task_as_system_aborted(direct_runtime): lifecycle.invoke_hook( "pre_llm_call", session_id="s1", task_id="t1", platform="cli", ) lifecycle.finalize_session(session_id="s1") [task_end] = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] assert task_end[2]["output"] == { "duration_bucket": task_end[2]["output"]["duration_bucket"], "end_reason": "system_aborted", "entrypoint": "interactive", "execution_surface": "cli", "model_call_count_bucket": "0", "outcome": "failed", "retry_count_bucket": "0", "termination": "system_aborted", "tool_call_count_bucket": "0", } def test_desktop_task_completion_exports_once_per_utc_day( direct_runtime, tmp_path, monkeypatch ): current_time = datetime(2026, 7, 28, 9, tzinfo=timezone.utc) monkeypatch.setattr( "hermes_cli.observability.shared_metrics._utc_now", lambda: current_time, ) for task_id in ("t1", "t2"): lifecycle.invoke_hook( "pre_llm_call", session_id="s1", task_id=task_id, platform="desktop", ) lifecycle.invoke_hook( "on_session_end", session_id="s1", task_id=task_id, platform="desktop", completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) outbox = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" / "outbox" [first_package_path] = list(outbox.glob("*.json")) first_package = json.loads(first_package_path.read_text(encoding="utf-8")) first_metrics = {metric["name"]: metric for metric in first_package["metrics"]} assert first_metrics["hermes.task_run.started"]["value"] == 1 assert first_metrics["hermes.task_run.started"]["dimensions"] == { "entrypoint": "interactive", "execution_surface": "desktop", } assert first_metrics["hermes.task_run.finished"]["value"] == 1 lifecycle.finalize_session(session_id="s1") assert list(outbox.glob("*.json")) == [first_package_path] current_time = datetime(2026, 7, 29, 9, tzinfo=timezone.utc) lifecycle.invoke_hook( "pre_llm_call", session_id="s1", task_id="t3", platform="desktop", ) lifecycle.invoke_hook( "on_session_end", session_id="s1", task_id="t3", platform="desktop", completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) packages = [ json.loads(package_path.read_text(encoding="utf-8")) for package_path in outbox.glob("*.json") ] totals: dict[str, int] = {} for package in packages: for metric in package["metrics"]: totals[metric["name"]] = totals.get(metric["name"], 0) + metric["value"] assert totals["hermes.task_run.started"] == 3 assert totals["hermes.task_run.finished"] == 3 def test_failed_flush_keeps_daily_export_open_for_later_task( direct_runtime, tmp_path, monkeypatch, caplog ): current_time = datetime(2026, 7, 28, 9, tzinfo=timezone.utc) monkeypatch.setattr( "hermes_cli.observability.shared_metrics._utc_now", lambda: current_time, ) original_flush = direct_runtime.subscribers.flush flush_attempts = 0 def fail_first_flush() -> None: nonlocal flush_attempts flush_attempts += 1 if flush_attempts == 1: raise RuntimeError("simulated flush failure") original_flush() direct_runtime.subscribers.flush = fail_first_flush def finish_desktop_task(task_id: str) -> None: lifecycle.invoke_hook( "pre_llm_call", session_id="s1", task_id=task_id, platform="desktop", ) lifecycle.invoke_hook( "on_session_end", session_id="s1", task_id=task_id, platform="desktop", completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) finish_desktop_task("t1") root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" assert list((root / "outbox").glob("*.json")) == [] with sqlite3.connect(root / "metrics.sqlite3") as connection: [package_count] = connection.execute( "SELECT COUNT(*) FROM package_outbox" ).fetchone() assert package_count == 0 finish_desktop_task("t2") [package_path] = list((root / "outbox").glob("*.json")) package = json.loads(package_path.read_text(encoding="utf-8")) metrics = {metric["name"]: metric for metric in package["metrics"]} assert metrics["hermes.task_run.started"]["value"] == 2 assert metrics["hermes.task_run.finished"]["value"] == 2 assert flush_attempts == 2 assert "Hermes shared-metrics task flush failed" in caplog.text def test_task_ownership_survives_session_id_rotation(direct_runtime): lifecycle.invoke_hook( "pre_llm_call", session_id="before-compression", task_id="t1", platform="cli", ) lifecycle.invoke_hook( "pre_api_request", session_id="after-compression", task_id="t1", api_request_id="r1", platform="cli", provider="nvidia", model="nvidia/nemotron-3-super-120b-a12b", ) lifecycle.invoke_hook( "post_api_request", session_id="after-compression", task_id="t1", api_request_id="r1", platform="cli", provider="nvidia", model="nvidia/nemotron-3-super-120b-a12b", ) lifecycle.invoke_hook( "on_session_end", session_id="after-compression", task_id="t1", platform="cli", completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) lifecycle.finalize_session(session_id="before-compression") task_starts = [ event for event in direct_runtime.events if event[0] == "scope.push" and event[1] == "hermes.task_run" ] task_ends = [ event for event in direct_runtime.events if event[0] == "scope.pop" and event[1][1] == "hermes.task_run" ] model_ends = [ event for event in direct_runtime.events if event[0] == "llm.call_end" ] assert len(task_starts) == 1 assert len(task_ends) == 1 assert len(model_ends) == 1 assert model_ends[0][2]["outcome"] == "success" assert task_ends[0][2]["output"]["model_call_count_bucket"] == "1" assert task_ends[0][2]["output"]["outcome"] == "success" def test_gateway_and_delegated_entrypoints_flow_through_relay(direct_runtime): tasks = [ { "session_id": "gateway-session", "task_id": "gateway-task", "platform": "whatsapp_cloud", }, { "session_id": "child-session", "task_id": "delegated-task", "platform": "cli", "parent_session_id": "private-parent-session", }, ] for task in tasks: lifecycle.invoke_hook("pre_llm_call", **task) lifecycle.invoke_hook( "on_session_end", **task, completed=True, failed=False, interrupted=False, turn_exit_reason="text_response(stop)", ) starts = [ event[3]["input"] for event in direct_runtime.events if event[0] == "scope.push" and event[1] == "hermes.task_run" ] assert starts == [ {"entrypoint": "gateway_message", "execution_surface": "gateway"}, {"entrypoint": "delegated", "execution_surface": "cli"}, ] assert "private-parent-session" not in json.dumps(direct_runtime.events) def test_persistence_failure_does_not_escape_the_hook( direct_runtime, monkeypatch, caplog, ): runtime = relay_shared_metrics._get_runtime() assert runtime is not None def fail_record(*_args: Any, **_kwargs: Any) -> None: raise OSError("store unavailable") monkeypatch.setattr(runtime.subscriber.store, "record_counter", fail_record) lifecycle.invoke_hook( "pre_api_request", session_id="s1", task_id="t1", api_request_id="r1", provider="openai", model="gpt-5", ) lifecycle.invoke_hook( "post_api_request", session_id="s1", task_id="t1", api_request_id="r1", provider="openai", model="gpt-5", ) assert "Unable to persist the Hermes shared metric" in caplog.text def test_close_does_not_reopen_a_session_after_scope_start_failure( direct_runtime, monkeypatch, ): runtime = relay_runtime.get_runtime() assert runtime is not None original_push = direct_runtime.scope.push push_attempts = 0 def fail_first_push(*args: Any, **kwargs: Any) -> Any: nonlocal push_attempts push_attempts += 1 if push_attempts == 1: raise RuntimeError("simulated scope failure") return original_push(*args, **kwargs) direct_runtime.scope.push = fail_first_push with pytest.raises(RuntimeError, match="simulated scope failure"): runtime.ensure_session({"session_id": "s1"}) close_started = threading.Event() allow_close = threading.Event() original_flush = direct_runtime.subscribers.flush def block_flush(): session = runtime._sessions["s1"] assert session.closing is True close_started.set() assert allow_close.wait(timeout=5) original_flush() direct_runtime.subscribers.flush = block_flush close_thread = threading.Thread( target=runtime.close_session, args=({"session_id": "s1"},), ) close_thread.start() assert close_started.wait(timeout=5) ensure_thread = threading.Thread( target=runtime.ensure_session, args=({"session_id": "s1"},), ) ensure_thread.start() allow_close.set() close_thread.join(timeout=5) ensure_thread.join(timeout=5) assert not close_thread.is_alive() assert not ensure_thread.is_alive() assert push_attempts == 1 assert "s1" not in runtime._sessions