hermes-agent/tests/hermes_cli/test_relay_shared_metrics_runtime.py

"""Tests for the direct Hermes-to-Relay shared-metrics runtime."""

from __future__ import annotations

import contextvars
import asyncio
import json
import threading
from pathlib import Path
from types import SimpleNamespace
from typing import Any

import pytest

from hermes_cli import lifecycle, plugins
from hermes_cli.observability import relay_runtime, relay_shared_metrics
from hermes_cli.plugins import PluginManager


class _Request:
    def __init__(self, headers: dict[str, Any], content: dict[str, Any]) -> None:
        self.headers = headers
        self.content = content


class _Relay:
    def __init__(self) -> None:
        self.events: list[tuple[Any, ...]] = []
        self._callbacks: dict[str, Any] = {}
        self._starts: dict[Any, dict[str, Any]] = {}
        self._scope_starts: dict[Any, dict[str, Any]] = {}
        self._scope = contextvars.ContextVar("relay_scope", default=None)
        self._scope_serial = 0
        self.ScopeType = SimpleNamespace(Agent="agent", Function="function")
        self.LLMRequest = _Request
        self.scope = SimpleNamespace(
            push=self._scope_push,
            pop=self._scope_pop,
            event=self._scope_event,
        )
        self.llm = SimpleNamespace(call=self._llm_call, call_end=self._llm_call_end)
        self.subscribers = SimpleNamespace(
            register=self._register,
            deregister=self._deregister,
            flush=self._flush,
        )
        self.get_scope_stack = self._get_scope_stack

    def _scope_push(self, name: str, scope_type: Any, **kwargs: Any) -> Any:
        self._scope_serial += 1
        handle = ("scope", name, self._scope_serial)
        self._scope.set(handle)
        self.events.append(("scope.push", name, scope_type, kwargs))
        if scope_type == self.ScopeType.Function:
            self._scope_starts[handle] = kwargs
            event = SimpleNamespace(
                kind="scope",
                category="function",
                name=name,
                scope_category="start",
                category_profile=None,
                metadata=kwargs.get("metadata"),
                data=kwargs.get("input"),
            )
            for callback in list(self._callbacks.values()):
                callback(event)
        return handle

    def _scope_pop(self, handle: Any, **kwargs: Any) -> None:
        self.events.append(("scope.pop", handle, kwargs))
        start = self._scope_starts.pop(handle, None)
        if start is not None:
            event = SimpleNamespace(
                kind="scope",
                category="function",
                name=handle[1],
                scope_category="end",
                category_profile=None,
                metadata={
                    **(start.get("metadata") or {}),
                    **(kwargs.get("metadata") or {}),
                },
                data=kwargs.get("output"),
            )
            for callback in list(self._callbacks.values()):
                callback(event)

    def _scope_event(self, name: str, **kwargs: Any) -> None:
        self.events.append(("scope.event", name, kwargs))

    def _get_scope_stack(self) -> Any:
        current = self._scope.get()
        self.events.append(("scope.sync", current))
        return current

    def _llm_call(
        self,
        name: str,
        request: _Request,
        **kwargs: Any,
    ) -> Any:
        handle = ("llm", name, len(self._starts))
        self._starts[handle] = kwargs
        self.events.append(("llm.call", name, request.content, kwargs))
        return handle

    def _llm_call_end(
        self,
        handle: Any,
        response: dict[str, Any],
        **kwargs: Any,
    ) -> None:
        start = self._starts.pop(handle)
        self.events.append(("llm.call_end", handle, response, kwargs))
        event = SimpleNamespace(
            kind="scope",
            category="llm",
            name=handle[1],
            scope_category="end",
            category_profile={"model_name": start["model_name"]},
            metadata={
                **start["metadata"],
                **kwargs["metadata"],
                "otel.status_code": "OK",
            },
            data=response,
        )
        for callback in list(self._callbacks.values()):
            callback(event)

    def _register(self, name: str, callback: Any) -> None:
        self._callbacks[name] = callback
        self.events.append(("subscribers.register", name))

    def _deregister(self, name: str) -> None:
        self._callbacks.pop(name, None)
        self.events.append(("subscribers.deregister", name))

    def _flush(self) -> None:
        self.events.append(("subscribers.flush",))


@pytest.fixture
def direct_runtime(tmp_path, monkeypatch):
    fake = _Relay()
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
    monkeypatch.setattr(
        "hermes_cli.config.read_raw_config",
        lambda: {"telemetry": {"shared_metrics": {"enabled": True}}},
    )
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()
    monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())
    yield fake
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()


@pytest.fixture
def real_binding_runtime(tmp_path, monkeypatch):
    relay = pytest.importorskip("nemo_relay")
    if getattr(relay, "_native", None) is None:
        pytest.skip("NeMo Relay native binding is unavailable on this platform")
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
    monkeypatch.setattr(
        "hermes_cli.config.read_raw_config",
        lambda: {"telemetry": {"shared_metrics": {"enabled": True}}},
    )
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()
    monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())
    yield relay
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()


def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_path):
    base = {
        "session_id": "sensitive-session",
        "task_id": "task-1",
        "api_request_id": "request-1",
        "platform": "cli",
        "provider": "custom",
        "model": "gpt-sensitive-model-id",
        "base_url": "http://127.0.0.1:11434/v1",
    }

    assert lifecycle.has_hook("pre_api_request")
    lifecycle.invoke_hook("on_session_start", **base)
    lifecycle.invoke_hook("pre_llm_call", **base)
    lifecycle.invoke_hook(
        "pre_api_request",
        **base,
        request={"body": {"messages": ["sensitive-prompt"]}},
    )
    lifecycle.invoke_hook(
        "post_tool_call",
        **base,
        tool_call_id="sensitive-tool-call",
        tool_name="terminal",
        args={"command": "sensitive-command"},
        result={"output": "sensitive-tool-result"},
        status="ok",
    )
    lifecycle.invoke_hook(
        "api_request_error",
        **base,
        retryable=True,
        error={"message": "sensitive-error"},
    )
    lifecycle.invoke_hook(
        "pre_api_request",
        **{
            **base,
            "provider": "anthropic",
            "model": "claude-sonnet",
            "base_url": "https://api.anthropic.com",
        },
        request={"body": {"messages": ["sensitive-prompt"]}},
    )
    lifecycle.invoke_hook(
        "post_api_request",
        **{
            **base,
            "provider": "anthropic",
            "model": "claude-sonnet",
            "base_url": "https://api.anthropic.com",
        },
        response={"content": "sensitive-response"},
    )
    lifecycle.invoke_hook(
        "on_session_end",
        **base,
        completed=True,
        failed=False,
        interrupted=False,
        turn_exit_reason="text_response(stop)",
    )
    lifecycle.finalize_session(session_id=base["session_id"])

    starts = [event for event in direct_runtime.events if event[0] == "llm.call"]
    ends = [event for event in direct_runtime.events if event[0] == "llm.call_end"]
    scope_starts = [
        event for event in direct_runtime.events if event[0] == "scope.push"
    ]
    assert len(scope_starts) == 2
    assert scope_starts[0][2] == direct_runtime.ScopeType.Agent
    assert scope_starts[1][1] == "hermes.task_run"
    assert scope_starts[1][2] == direct_runtime.ScopeType.Function
    assert scope_starts[1][3]["handle"][1] == relay_runtime.SESSION_SCOPE
    assert scope_starts[1][3]["input"] == {
        "entrypoint": "interactive",
        "execution_surface": "cli",
    }
    assert len(starts) == 1
    assert len(ends) == 1
    assert starts[0][2] == {}
    assert starts[0][3]["model_name"] == "gpt"
    assert ends[0][2] == {
        "call_role": "primary",
        "locality": "remote",
        "model_family": "claude",
        "outcome": "success",
        "provider_family": "direct",
    }
    serialized_events = json.dumps(direct_runtime.events)
    assert "sensitive-prompt" not in serialized_events
    assert "sensitive-response" not in serialized_events
    assert "sensitive-error" not in serialized_events
    assert "sensitive-command" not in serialized_events
    assert "sensitive-tool-result" not in serialized_events
    assert "sensitive-tool-call" not in serialized_events
    assert "gpt-sensitive-model-id" not in serialized_events
    assert plugins.get_plugin_manager().list_plugins() == []

    root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics"
    packages = list((root / "outbox").glob("*.json"))
    assert len(packages) == 1
    package = json.loads(packages[0].read_text(encoding="utf-8"))
    metrics = {metric["name"]: metric for metric in package["metrics"]}
    assert set(metrics) == {
        "hermes.model_call.count",
        "hermes.task_run.finished",
        "hermes.task_run.started",
    }
    assert metrics["hermes.model_call.count"]["dimensions"]["model_family"] == "claude"
    assert metrics["hermes.model_call.count"]["value"] == 1
    assert metrics["hermes.task_run.started"] == {
        "name": "hermes.task_run.started",
        "type": "counter",
        "dimensions": {
            "entrypoint": "interactive",
            "execution_surface": "cli",
        },
        "value": 1,
    }
    terminal = metrics["hermes.task_run.finished"]["dimensions"]
    assert terminal["duration_bucket"] in {
        "lt_1s",
        "1s_to_5s",
        "5s_to_30s",
        "30s_to_2m",
        "2m_to_10m",
        "gte_10m",
    }
    assert {
        key: value for key, value in terminal.items() if key != "duration_bucket"
    } == {
        "end_reason": "completed",
        "entrypoint": "interactive",
        "execution_surface": "cli",
        "model_call_count_bucket": "1",
        "outcome": "success",
        "retry_count_bucket": "1",
        "termination": "none",
        "tool_call_count_bucket": "1",
    }


def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot(
    real_binding_runtime,
    tmp_path,
):
    assert real_binding_runtime._native is not None
    prompt_canary = "real-relay-sensitive-prompt"
    response_canary = "real-relay-sensitive-response"
    model_canary = "gpt-real-relay-sensitive-model"
    tool_canary = "real-relay-sensitive-tool-result"

    def base(index: int) -> dict[str, Any]:
        return {
            "session_id": f"sensitive-session-{index}",
            "task_id": f"sensitive-task-{index}",
            "turn_id": f"sensitive-turn-{index}",
            "api_request_id": f"sensitive-request-{index}",
            "platform": "cli",
            "provider": "custom",
            "model": model_canary,
            "base_url": "http://127.0.0.1:11434/v1",
        }

    success = base(1)
    lifecycle.invoke_hook("on_session_start", **success)
    lifecycle.invoke_hook("pre_llm_call", **success, messages=[prompt_canary])
    lifecycle.invoke_hook("pre_api_request", **success, retry_count=0)
    lifecycle.invoke_hook(
        "api_request_error",
        **success,
        retry_count=0,
        retryable=True,
        error={"message": prompt_canary},
    )
    lifecycle.invoke_hook("pre_api_request", **success, retry_count=1)
    lifecycle.invoke_hook(
        "post_tool_call",
        **success,
        tool_call_id="sensitive-tool-call",
        tool_name="terminal",
        args={"command": prompt_canary},
        result={"output": tool_canary},
        status="ok",
    )
    lifecycle.invoke_hook(
        "post_api_request",
        **success,
        retry_count=1,
        response={"content": response_canary},
    )
    lifecycle.invoke_hook(
        "on_session_end",
        **success,
        completed=True,
        failed=False,
        interrupted=False,
        turn_exit_reason="text_response(stop)",
    )
    lifecycle.finalize_session(session_id=success["session_id"])

    failed = base(2)
    lifecycle.invoke_hook("on_session_start", **failed)
    lifecycle.invoke_hook("pre_llm_call", **failed, messages=[prompt_canary])
    lifecycle.invoke_hook("pre_api_request", **failed, retry_count=0)
    lifecycle.invoke_hook(
        "api_request_error",
        **failed,
        retry_count=0,
        retryable=False,
        error={"message": response_canary},
    )
    lifecycle.invoke_hook(
        "on_session_end",
        **failed,
        completed=False,
        failed=True,
        interrupted=False,
        turn_exit_reason="system_aborted",
    )
    lifecycle.finalize_session(session_id=failed["session_id"])

    cancelled = base(3)
    lifecycle.invoke_hook("on_session_start", **cancelled)
    lifecycle.invoke_hook("pre_llm_call", **cancelled, messages=[prompt_canary])
    lifecycle.invoke_hook("pre_api_request", **cancelled, retry_count=0)
    lifecycle.invoke_hook(
        "on_session_end",
        **cancelled,
        completed=False,
        failed=False,
        interrupted=True,
        turn_exit_reason="interrupted_by_user",
    )
    lifecycle.finalize_session(session_id=cancelled["session_id"])

    from hermes_cli.observability.shared_metrics import SharedMetricsStore

    root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics"
    store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox")
    snapshot = store.counter_snapshot()
    by_metric: dict[str, list[dict[str, Any]]] = {}
    for counter in snapshot:
        by_metric.setdefault(counter["metric_name"], []).append(counter)

    assert len(by_metric["hermes.task_run.started"]) == 1
    assert by_metric["hermes.task_run.started"][0]["value"] == 3
    assert {
        counter["dimensions"]["outcome"]
        for counter in by_metric["hermes.model_call.count"]
    } == {"success", "failed", "cancelled"}
    terminal_by_outcome = {
        counter["dimensions"]["outcome"]: counter
        for counter in by_metric["hermes.task_run.finished"]
    }
    assert set(terminal_by_outcome) == {"success", "failed", "cancelled"}
    assert terminal_by_outcome["success"]["dimensions"]["retry_count_bucket"] == "1"
    assert terminal_by_outcome["success"]["dimensions"]["tool_call_count_bucket"] == "1"
    assert terminal_by_outcome["failed"]["dimensions"]["end_reason"] == (
        "system_aborted"
    )
    assert terminal_by_outcome["cancelled"]["dimensions"]["termination"] == (
        "user_cancelled"
    )
    assert all(counter["packaged_value"] == counter["value"] for counter in snapshot)

    snapshot_values = {
        (
            counter["metric_name"],
            tuple(sorted(counter["dimensions"].items())),
        ): counter["value"]
        for counter in snapshot
    }
    package_values: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {}
    packages = sorted((root / "outbox").glob("*.json"))
    assert len(packages) == 3
    package_payloads = [
        json.loads(package.read_text(encoding="utf-8")) for package in packages
    ]
    for package in package_payloads:
        assert package["schema_version"] == "hermes.shared_metrics.v1"
        for metric in package["metrics"]:
            key = (metric["name"], tuple(sorted(metric["dimensions"].items())))
            package_values[key] = package_values.get(key, 0) + metric["value"]
    assert package_values == snapshot_values

    serialized_analytics = json.dumps({
        "snapshot": snapshot,
        "packages": package_payloads,
    })
    for canary in (
        prompt_canary,
        response_canary,
        model_canary,
        tool_canary,
        "sensitive-session",
        "sensitive-task",
        "sensitive-request",
        "sensitive-tool-call",
    ):
        assert canary not in serialized_analytics


def test_direct_runtime_is_disabled_by_default(tmp_path, monkeypatch):
    fake = _Relay()
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
    monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {})
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()
    monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())

    assert not plugins.has_hook("pre_api_request")
    lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli")
    lifecycle.finalize_session(session_id="s1")

    assert fake.events == []
    assert not (tmp_path / "hermes-home" / "telemetry").exists()
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()


def test_tool_intercept_bypass_does_not_create_relay_host(monkeypatch):
    relay_runtime._reset_for_tests()
    imports = []

    def load_relay():
        imports.append("nemo_relay")
        raise AssertionError("disabled helper created Relay host")

    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay)
    args = {"command": "true"}

    assert (
        relay_runtime.apply_tool_request_intercepts(
            session_id="s1",
            tool_name="terminal",
            args=args,
        )
        is args
    )
    assert relay_runtime.get_host(create=False) is None
    assert imports == []


def test_execution_adapters_do_not_create_relay_host_without_a_consumer(
    monkeypatch,
):
    from agent import relay_llm, relay_tools

    relay_runtime._reset_for_tests()
    imports = []

    def load_relay():
        imports.append("nemo_relay")
        raise AssertionError("disabled execution adapter created Relay host")

    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay)
    request = {"model": "test-model", "messages": []}
    response = object()
    tool_args = {"command": "true"}
    tool_result = object()

    assert (
        relay_llm.execute(
            request,
            lambda observed: response if observed is request else None,
            session_id="llm-session",
            name="test-provider",
            model_name="test-model",
        )
        is response
    )
    result, observed_args = relay_tools.execute(
        "terminal",
        tool_args,
        lambda observed: tool_result if observed is tool_args else None,
        session_id="tool-session",
    )

    assert result is tool_result
    assert observed_args is tool_args
    assert relay_runtime.get_host(create=False) is None
    assert imports == []


def test_profile_key_caches_absolute_path_resolution(monkeypatch):
    relay_runtime._reset_for_tests()

    class Home:
        def __init__(self):
            self.resolve_calls = 0

        def expanduser(self):
            return self

        def is_absolute(self):
            return True

        def resolve(self):
            self.resolve_calls += 1
            return self

        def __str__(self):
            return "/profiles/cached"

    home = Home()
    monkeypatch.setattr(relay_runtime, "get_hermes_home", lambda: home)

    assert relay_runtime.current_profile_key() == "/profiles/cached"
    assert relay_runtime.current_profile_key() == "/profiles/cached"
    assert home.resolve_calls == 1


def test_host_registry_reads_existing_host_without_lock():
    registry = relay_runtime.RelayHostRegistry()
    host = relay_runtime.NoopRelayRuntime("profile", "test")
    registry._hosts["profile"] = host

    class UnexpectedLock:
        def __enter__(self):
            raise AssertionError("registry read acquired the write lock")

        def __exit__(self, *_args):
            return False

    registry._lock = UnexpectedLock()

    assert registry.for_profile("profile", create=False) is host
    assert registry.for_profile("missing", create=False) is None


def test_core_runtime_is_fail_open_without_a_published_binding(monkeypatch, caplog):
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()

    def missing_relay(name: str):
        assert name == "nemo_relay"
        raise ModuleNotFoundError(name)

    monkeypatch.setattr(relay_runtime.importlib, "import_module", missing_relay)

    assert relay_runtime.get_runtime() is None
    host = relay_runtime.get_host()
    assert isinstance(host, relay_runtime.NoopRelayRuntime)
    assert host.profile_key == relay_runtime.current_profile_key()
    assert "nemo_relay" in host.reason
    assert host.apply_tool_request_intercepts(
        session_id="s1",
        tool_name="terminal",
        args={"command": "true"},
    ) == {"command": "true"}
    assert not relay_runtime.emit_mark("hermes.probe", session_id="s1")
    assert "Hermes Relay runtime initialization failed" in caplog.text
    relay_runtime._reset_for_tests()


def test_core_mark_uses_the_shared_session_handle_without_a_plugin(direct_runtime):
    lifecycle.invoke_hook("on_session_start", session_id="s1", platform="cli")

    handle = relay_runtime.get_session_handle("s1")
    assert handle is not None
    assert relay_runtime.emit_mark(
        "hermes.skill.created",
        session_id="s1",
        data={"provenance": "agent_created"},
        metadata={"data_schema": "hermes.skill.lifecycle.v1"},
    )

    [mark] = [event for event in direct_runtime.events if event[0] == "scope.event"]
    assert mark[1] == "hermes.skill.created"
    assert mark[2]["handle"] == handle
    assert plugins.get_plugin_manager().list_plugins() == []


def test_core_task_instrumentation_preserves_prompt_history_and_tool_schema(
    direct_runtime,
    monkeypatch,
):
    from run_agent import AIAgent

    agent = object.__new__(AIAgent)
    agent.session_id = "cache-stable-session"
    agent.platform = "cli"
    agent._parent_session_id = None
    agent._session_db = None
    agent._cached_system_prompt = "byte-stable-system-prompt\nwith exact spacing"
    agent.tools = [
        {
            "type": "function",
            "function": {
                "name": "probe",
                "parameters": {
                    "type": "object",
                    "properties": {"value": {"type": "string"}},
                },
            },
        }
    ]
    history = [{"role": "user", "content": "sensitive-history-canary"}]
    prompt_before = agent._cached_system_prompt.encode("utf-8")
    history_before = json.dumps(history, ensure_ascii=False, sort_keys=True)
    tools_before = json.dumps(agent.tools, ensure_ascii=False, sort_keys=True)

    def fake_run_conversation(
        active_agent,
        user_message,
        system_message,
        conversation_history,
        task_id,
        stream_callback,
        persist_user_message,
        **kwargs,
    ):
        del (
            user_message,
            system_message,
            task_id,
            stream_callback,
            persist_user_message,
            kwargs,
        )
        assert active_agent is agent
        assert conversation_history is history
        return {"final_response": "ok", "completed": True}

    monkeypatch.setattr(
        "agent.conversation_loop.run_conversation",
        fake_run_conversation,
    )

    for task_id in ("cache-task-1", "cache-task-2"):
        result = AIAgent.run_conversation(
            agent,
            "hello",
            conversation_history=history,
            task_id=task_id,
        )
        assert result["final_response"] == "ok"

    assert agent._cached_system_prompt.encode("utf-8") == prompt_before
    assert json.dumps(history, ensure_ascii=False, sort_keys=True) == history_before
    assert json.dumps(agent.tools, ensure_ascii=False, sort_keys=True) == tools_before


def test_session_coordinator_separates_turn_release_from_hard_finalize(
    direct_runtime,
):
    profile_key = relay_runtime.current_profile_key()
    coordinator = relay_runtime.SESSION_COORDINATOR
    lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="coordinated-session",
        platform="cli",
    )
    turn = coordinator.begin_turn(
        lease,
        turn_id="turn-1",
        task_id="task-1",
    )

    assert relay_runtime.current_turn() is turn
    assert turn.handle is not None
    session_handle = lease.session.handle
    turn_push = next(
        event
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == relay_runtime.TURN_SCOPE
    )
    assert turn_push[3]["handle"] == session_handle

    coordinator.end_turn(turn, outcome="success")
    coordinator.release_conversation(lease)

    assert relay_runtime.current_turn() is None
    runtime = relay_runtime.get_runtime(create=False)
    assert runtime is not None
    assert runtime.get_session("coordinated-session") is not None

    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="coordinated-session",
    )
    assert runtime.get_session("coordinated-session") is None


def test_turn_cleanup_can_be_retried_from_its_originating_context(direct_runtime):
    del direct_runtime
    coordinator = relay_runtime.SESSION_COORDINATOR
    lease = coordinator.acquire_conversation(
        profile_key=relay_runtime.current_profile_key(),
        session_id="cross-context-session",
        platform="cli",
    )
    turn = coordinator.begin_turn(
        lease,
        turn_id="turn-1",
        task_id="task-1",
    )

    contextvars.Context().run(coordinator.end_turn, turn, outcome="success")

    assert turn.closed
    assert relay_runtime.active_turn("cross-context-session") is None
    assert relay_runtime.current_turn() is turn

    coordinator.end_turn(turn, outcome="success")
    assert relay_runtime.current_turn() is None
    coordinator.release_conversation(lease)


def test_session_coordinator_prepares_subscribers_before_opening_scope(
    direct_runtime,
):
    coordinator = relay_runtime.SESSION_COORDINATOR
    initializer_name = "test.pre_scope_subscriber"

    def prepare(host, context):
        assert host.profile_key == relay_runtime.current_profile_key()
        direct_runtime.events.append(("session.prepare", dict(context)))

    coordinator.register_session_initializer(initializer_name, prepare)
    try:
        lease = coordinator.acquire_conversation(
            profile_key=relay_runtime.current_profile_key(),
            session_id="prepared-session",
            platform="cli",
            model="demo-model",
        )
    finally:
        coordinator.unregister_session_initializer(initializer_name)

    prepared = next(
        index
        for index, event in enumerate(direct_runtime.events)
        if event[0] == "session.prepare"
    )
    opened = next(
        index
        for index, event in enumerate(direct_runtime.events)
        if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE
    )
    assert prepared < opened
    assert direct_runtime.events[prepared][1] == {
        "profile_key": relay_runtime.current_profile_key(),
        "session_id": "prepared-session",
        "platform": "cli",
        "parent_session_id": "",
        "model": "demo-model",
    }
    coordinator.finalize_conversation(
        profile_key=relay_runtime.current_profile_key(),
        session_id=lease.session_id,
    )


def test_session_initializer_failure_does_not_block_conversation(
    direct_runtime,
    caplog,
):
    coordinator = relay_runtime.SESSION_COORDINATOR
    initializer_name = "test.failing_subscriber"

    def fail(_host, _context):
        raise RuntimeError("subscriber setup failed")

    coordinator.register_session_initializer(initializer_name, fail)
    try:
        with caplog.at_level("WARNING"):
            lease = coordinator.acquire_conversation(
                profile_key=relay_runtime.current_profile_key(),
                session_id="initializer-failure",
                platform="cli",
            )
    finally:
        coordinator.unregister_session_initializer(initializer_name)

    assert lease.session is not None
    assert "Hermes Relay session initializer failed" in caplog.text
    coordinator.finalize_conversation(
        profile_key=relay_runtime.current_profile_key(),
        session_id=lease.session_id,
    )


def test_profile_host_recreation_rebinds_shared_metrics_subscriber(
    direct_runtime,
):
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    first_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="before-restart",
        platform="cli",
    )
    first_metrics = relay_shared_metrics._get_runtime()
    assert first_metrics is not None
    first_host = first_lease.host

    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id=first_lease.session_id,
    )
    coordinator.shutdown_profile(profile_key)

    second_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="after-restart",
        platform="cli",
    )
    second_metrics = relay_shared_metrics._get_runtime()

    assert second_metrics is not None
    assert second_lease.host is not first_host
    assert second_metrics is not first_metrics
    assert second_metrics.host is second_lease.host
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id=second_lease.session_id,
    )


def test_core_mark_does_not_start_relay_without_metrics_or_a_plugin(
    tmp_path,
    monkeypatch,
):
    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
    imports = []

    def load_relay():
        imports.append("nemo_relay")
        return _Relay()

    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", load_relay)
    monkeypatch.setattr("hermes_cli.config.read_raw_config", lambda: {})
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()
    monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())

    assert not relay_runtime.emit_mark(
        "hermes.skill.created",
        session_id="s1",
        data={"provenance": "agent_created"},
    )

    assert imports == []
    assert relay_runtime.get_host(create=False) is None
    assert not (tmp_path / "hermes-home" / "telemetry").exists()
    relay_runtime._reset_for_tests()


def test_core_runtime_creates_one_session_under_concurrent_access(direct_runtime):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    ready = threading.Barrier(8)
    sessions: list[Any] = []

    def ensure() -> None:
        ready.wait(timeout=5)
        sessions.append(runtime.ensure_session({"session_id": "shared"}))

    threads = [threading.Thread(target=ensure) for _ in range(8)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(timeout=5)

    assert all(not thread.is_alive() for thread in threads)
    assert len({id(session) for session in sessions}) == 1
    assert (
        len([event for event in direct_runtime.events if event[0] == "scope.push"]) == 1
    )


def test_core_runtime_isolates_same_session_id_by_profile(direct_runtime, tmp_path):
    from hermes_constants import (
        reset_hermes_home_override,
        set_hermes_home_override,
    )

    profile_a = tmp_path / "profile-a"
    profile_b = tmp_path / "profile-b"

    token = set_hermes_home_override(profile_a)
    try:
        runtime_a = relay_runtime.get_runtime()
        session_a = runtime_a.ensure_session({"session_id": "shared"})
    finally:
        reset_hermes_home_override(token)

    token = set_hermes_home_override(profile_b)
    try:
        runtime_b = relay_runtime.get_runtime()
        session_b = runtime_b.ensure_session({"session_id": "shared"})
    finally:
        reset_hermes_home_override(token)

    assert runtime_a is not None
    assert runtime_b is not None
    assert runtime_a is not runtime_b
    assert runtime_a.profile_key == str(profile_a.resolve())
    assert runtime_b.profile_key == str(profile_b.resolve())
    assert session_a is not session_b
    assert session_a.handle != session_b.handle


@pytest.mark.parametrize(
    ("profile_enabled", "managed_enabled"),
    ((None, True), (False, True), (True, False)),
)
def test_managed_config_cannot_override_shared_metrics_consent(
    tmp_path,
    monkeypatch,
    profile_enabled,
    managed_enabled,
):
    from hermes_cli import config, managed_scope
    from hermes_constants import (
        reset_hermes_home_override,
        set_hermes_home_override,
    )

    profile = tmp_path / "profile"
    managed = tmp_path / "managed"
    profile.mkdir()
    managed.mkdir()
    profile_config = "{}\n"
    if profile_enabled is not None:
        profile_config = (
            "telemetry:\n"
            "  shared_metrics:\n"
            f"    enabled: {str(profile_enabled).lower()}\n"
        )
    (profile / "config.yaml").write_text(profile_config, encoding="utf-8")
    (managed / "config.yaml").write_text(
        "telemetry:\n"
        "  shared_metrics:\n"
        f"    enabled: {str(managed_enabled).lower()}\n",
        encoding="utf-8",
    )
    monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed))
    config._LOAD_CONFIG_CACHE.clear()
    config._RAW_CONFIG_CACHE.clear()
    managed_scope.invalidate_managed_cache()

    token = set_hermes_home_override(profile)
    try:
        assert (
            config.load_config_readonly()["telemetry"]["shared_metrics"]["enabled"]
            is managed_enabled
        )
        assert relay_shared_metrics.enabled() is (profile_enabled is True)
    finally:
        reset_hermes_home_override(token)
        relay_shared_metrics._reset_for_tests()
        relay_runtime._reset_for_tests()
        managed_scope.invalidate_managed_cache()


def test_shared_metrics_policy_and_store_are_profile_scoped(tmp_path, monkeypatch):
    from hermes_constants import (
        get_hermes_home,
        reset_hermes_home_override,
        set_hermes_home_override,
    )

    fake = _Relay()
    profile_a = tmp_path / "profile-a"
    profile_b = tmp_path / "profile-b"
    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
    monkeypatch.setattr(
        "hermes_cli.config.read_raw_config",
        lambda: {
            "telemetry": {
                "shared_metrics": {"enabled": get_hermes_home() == profile_a}
            }
        },
    )
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()

    token = set_hermes_home_override(profile_a)
    try:
        assert relay_shared_metrics.enabled()
        relay_shared_metrics.start_task_run(
            session_id="shared",
            task_id="task-a",
            platform="cli",
        )
        relay_shared_metrics.finish_task_run(
            session_id="shared",
            task_id="task-a",
            platform="cli",
            result={"completed": True},
        )
        relay_shared_metrics._get_runtime().close_session({"session_id": "shared"})
    finally:
        reset_hermes_home_override(token)

    token = set_hermes_home_override(profile_b)
    try:
        assert not relay_shared_metrics.enabled()
        relay_shared_metrics.start_task_run(
            session_id="shared",
            task_id="task-b",
            platform="cli",
        )
    finally:
        reset_hermes_home_override(token)

    assert list((profile_a / "telemetry" / "shared_metrics" / "outbox").glob("*.json"))
    assert not (profile_b / "telemetry").exists()
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()


def test_shared_metrics_subscribers_isolate_two_enabled_profiles(tmp_path, monkeypatch):
    from hermes_constants import (
        reset_hermes_home_override,
        set_hermes_home_override,
    )

    fake = _Relay()
    profile_a = tmp_path / "profile-a"
    profile_b = tmp_path / "profile-b"
    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
    monkeypatch.setattr(
        "hermes_cli.config.read_raw_config",
        lambda: {"telemetry": {"shared_metrics": {"enabled": True}}},
    )
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()

    for profile, task_id in ((profile_a, "task-a"), (profile_b, "task-b")):
        token = set_hermes_home_override(profile)
        try:
            relay_shared_metrics.start_task_run(
                session_id="shared",
                task_id=task_id,
                platform="cli",
            )
            relay_shared_metrics.finish_task_run(
                session_id="shared",
                task_id=task_id,
                platform="cli",
                result={"completed": True},
            )
            relay_shared_metrics._get_runtime().close_session(
                {"session_id": "shared"}
            )
        finally:
            reset_hermes_home_override(token)

    for profile in (profile_a, profile_b):
        packages = list(
            (profile / "telemetry" / "shared_metrics" / "outbox").glob("*.json")
        )
        assert len(packages) == 1
        package = json.loads(packages[0].read_text(encoding="utf-8"))
        metrics = {metric["name"]: metric for metric in package["metrics"]}
        assert metrics["hermes.task_run.started"]["value"] == 1
        assert metrics["hermes.task_run.finished"]["value"] == 1

    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()


def test_shared_metrics_isolates_same_task_id_across_sessions(direct_runtime):
    runtime = relay_shared_metrics._get_runtime()
    assert runtime is not None

    task_a = runtime.start_task({
        "session_id": "session-a",
        "task_id": "shared-task",
        "platform": "cli",
    })
    task_b = runtime.start_task({
        "session_id": "session-b",
        "task_id": "shared-task",
        "platform": "gateway",
    })

    assert task_a is not None
    assert task_b is not None
    assert task_a is not task_b
    assert task_a.handle != task_b.handle

    runtime.finish_task({
        "session_id": "session-a",
        "task_id": "shared-task",
        "platform": "cli",
        "completed": True,
    })
    runtime.finish_task({
        "session_id": "session-b",
        "task_id": "shared-task",
        "platform": "gateway",
        "completed": True,
    })

    task_starts = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == "hermes.task_run"
    ]
    task_ends = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    assert len(task_starts) == 2
    assert len(task_ends) == 2


def test_shared_metrics_keys_turn_ownership_by_session(direct_runtime):
    runtime = relay_shared_metrics._get_runtime()
    assert runtime is not None
    event_a = {
        "session_id": "session-a",
        "task_id": "task-a",
        "turn_id": "shared-turn",
        "platform": "cli",
    }
    event_b = {
        "session_id": "session-b",
        "task_id": "task-b",
        "turn_id": "shared-turn",
        "platform": "gateway",
    }
    task_a = runtime.start_task(event_a)
    task_b = runtime.start_task(event_b)

    runtime.start_model_call({
        **event_a,
        "api_request_id": "request-a",
        "provider": "anthropic",
        "model": "claude-sonnet",
    })

    session_a = runtime._session(event_a)
    session_b = runtime._session(event_b)
    assert task_a is not None
    assert task_b is not None
    assert session_a is not None
    assert session_b is not None
    assert "request-a" in session_a.model_calls
    assert "request-a" not in session_b.model_calls
    [model_start] = [
        event for event in direct_runtime.events if event[0] == "llm.call"
    ]
    assert model_start[3]["handle"] == task_a.handle


def test_disabling_shared_metrics_stops_collection_and_shutdown_export(
    tmp_path, monkeypatch
):
    from hermes_cli.observability.shared_metrics import SharedMetricsStore

    fake = _Relay()
    profile = tmp_path / "profile"
    policy = {"enabled": True}
    monkeypatch.setenv("HERMES_HOME", str(profile))
    monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
    monkeypatch.setattr(
        "hermes_cli.config.read_raw_config",
        lambda: {"telemetry": {"shared_metrics": dict(policy)}},
    )
    relay_shared_metrics._reset_for_tests()
    relay_runtime._reset_for_tests()

    relay_shared_metrics.start_task_run(
        session_id="session",
        task_id="task",
        platform="cli",
    )
    runtime = relay_shared_metrics._get_runtime()
    assert runtime is not None
    policy["enabled"] = False

    assert not relay_shared_metrics.enabled()
    counters_before_stale_event = runtime.subscriber.store.counter_snapshot()
    runtime.subscriber(SimpleNamespace(
        kind="scope",
        category="function",
        category_profile=None,
        name="hermes.task_run",
        scope_category="start",
        metadata={
            "hermes.metrics.schema_version": "hermes.metrics.event.v1",
            relay_runtime.RUNTIME_INSTANCE_KEY: runtime.host.runtime_id,
        },
        data={"entrypoint": "interactive", "execution_surface": "cli"},
    ))
    assert runtime.subscriber.store.counter_snapshot() == counters_before_stale_event
    assert runtime.start_task({
        "session_id": "session",
        "task_id": "stale-runtime-task",
        "platform": "cli",
    }) is None
    relay_shared_metrics.finish_task_run(
        session_id="session",
        task_id="task",
        platform="cli",
        result={"completed": True},
    )
    relay_shared_metrics._reset_for_tests()

    root = profile / "telemetry" / "shared_metrics"
    store = SharedMetricsStore(root / "metrics.sqlite3", root / "outbox")
    assert [row["metric_name"] for row in store.counter_snapshot()] == [
        "hermes.task_run.started"
    ]
    assert list((root / "outbox").glob("*.json")) == []
    relay_runtime._reset_for_tests()


def test_shared_metrics_retries_transient_initialization_failure(
    direct_runtime, monkeypatch
):
    real_store = relay_shared_metrics.SharedMetricsStore
    attempts = 0

    def flaky_store():
        nonlocal attempts
        attempts += 1
        if attempts == 1:
            raise OSError("transient store failure")
        return real_store()

    monkeypatch.setattr(relay_shared_metrics, "SharedMetricsStore", flaky_store)

    relay_shared_metrics.start_task_run(
        session_id="session",
        task_id="first",
        platform="cli",
    )
    relay_shared_metrics.start_task_run(
        session_id="session",
        task_id="second",
        platform="cli",
    )

    assert attempts == 2
    task_starts = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == "hermes.task_run"
    ]
    assert len(task_starts) == 1


def test_async_session_runner_awaits_inside_saved_relay_context(direct_runtime):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    session = runtime.ensure_session({"session_id": "async-session"})
    assert session is not None

    async def probe() -> Any:
        await asyncio.sleep(0)
        return direct_runtime._scope.get()

    result = asyncio.run(runtime.run_in_session_async(session, probe))

    assert result == session.handle


def test_session_runners_preserve_caller_context_and_profile_override(
    direct_runtime,
    tmp_path,
):
    from hermes_constants import (
        get_hermes_home_override,
        reset_hermes_home_override,
        set_hermes_home_override,
    )

    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    session = runtime.ensure_session({"session_id": "caller-context-session"})
    assert session is not None
    caller_value = contextvars.ContextVar("caller_value", default="default")
    caller_value.set("caller")
    profile = tmp_path / "caller-profile"
    profile_token = set_hermes_home_override(profile)

    def sync_probe() -> tuple[Any, str, str | None]:
        return (
            direct_runtime._scope.get(),
            caller_value.get(),
            get_hermes_home_override(),
        )

    async def async_probe() -> tuple[Any, str, str | None]:
        await asyncio.sleep(0)
        return sync_probe()

    try:
        sync_result = runtime.run_in_session(session, sync_probe)
        async_result = asyncio.run(runtime.run_in_session_async(session, async_probe))
    finally:
        reset_hermes_home_override(profile_token)

    expected = (session.handle, "caller", str(profile))
    assert sync_result == expected
    assert async_result == expected


@pytest.mark.asyncio
async def test_async_session_runner_isolates_concurrent_caller_contexts(
    direct_runtime,
):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    session = runtime.ensure_session({"session_id": "concurrent-context-session"})
    assert session is not None
    caller_value = contextvars.ContextVar("concurrent_caller", default="default")
    ready = asyncio.Event()
    entered = 0

    async def run(value: str) -> tuple[Any, str]:
        nonlocal entered
        caller_value.set(value)

        async def probe() -> tuple[Any, str]:
            nonlocal entered
            entered += 1
            if entered == 2:
                ready.set()
            await ready.wait()
            return direct_runtime._scope.get(), caller_value.get()

        return await runtime.run_in_session_async(session, probe)

    results = await asyncio.gather(run("first"), run("second"))

    assert results == [
        (session.handle, "first"),
        (session.handle, "second"),
    ]


def test_sync_session_runner_releases_lock_before_callback(direct_runtime):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    session = runtime.ensure_session({"session_id": "sync-session"})
    assert session is not None
    acquired = threading.Event()
    contender = None

    def probe() -> Any:
        nonlocal contender

        def acquire_session_lock() -> None:
            with session.lock:
                acquired.set()

        contender = threading.Thread(target=acquire_session_lock)
        contender.start()
        assert acquired.wait(timeout=1)
        return direct_runtime._scope.get()

    result = runtime.run_in_session(session, probe)
    assert contender is not None
    contender.join(timeout=1)

    assert result == session.handle
    assert contender.is_alive() is False


def test_active_turn_requires_matching_session_and_profile(
    direct_runtime,
    tmp_path,
    monkeypatch,
):
    del direct_runtime
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="session-1",
        platform="cli",
    )
    turn = coordinator.begin_turn(
        lease,
        turn_id="turn-1",
        task_id="task-1",
    )

    assert relay_runtime.active_turn("session-1") is turn
    assert relay_runtime.active_turn("session-2") is None

    monkeypatch.setenv("HERMES_HOME", str(tmp_path / "other-profile"))
    assert relay_runtime.active_turn("session-1") is None

    coordinator.end_turn(turn, outcome="success")
    coordinator.release_conversation(lease)
    assert relay_runtime.active_turn("session-1") is None


def test_turn_cleanup_drains_logical_calls_after_turn_scope_start_failure(
    direct_runtime,
):
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="session-1",
        platform="cli",
    )
    assert lease.session is not None
    original_push = direct_runtime.scope.push

    def fail_turn_push(name, *args, **kwargs):
        if name == relay_runtime.TURN_SCOPE:
            raise RuntimeError("simulated turn scope failure")
        return original_push(name, *args, **kwargs)

    direct_runtime.scope.push = fail_turn_push
    turn = coordinator.begin_turn(
        lease,
        turn_id="turn-1",
        task_id="task-1",
    )
    direct_runtime.scope.push = original_push
    assert turn.handle is None

    runtime = lease.host
    logical_handle = runtime.run_in_session(
        lease.session,
        runtime.relay.scope.push,
        relay_runtime.LOGICAL_LLM_SCOPE,
        runtime.relay.ScopeType.Function,
        handle=lease.session.handle,
        input={},
    )
    turn.logical_llm_calls["request-1"] = logical_handle

    coordinator.end_turn(turn, outcome="failed")
    coordinator.release_conversation(lease)

    logical_closes = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1] == logical_handle
    ]
    assert len(logical_closes) == 1
    assert turn.logical_llm_calls == {}


def test_turn_cleanup_drains_logical_calls_in_lifo_order(direct_runtime):
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="session-lifo",
        platform="cli",
    )
    assert lease.session is not None
    turn = coordinator.begin_turn(
        lease,
        turn_id="turn-lifo",
        task_id="task-lifo",
    )
    assert turn.handle is not None
    runtime = lease.host

    handles = []
    for request_id in ("request-1", "request-2"):
        handle = runtime.run_in_session(
            lease.session,
            runtime.relay.scope.push,
            relay_runtime.LOGICAL_LLM_SCOPE,
            runtime.relay.ScopeType.Function,
            handle=turn.handle,
            input={},
        )
        turn.logical_llm_calls[request_id] = handle
        handles.append(handle)

    coordinator.end_turn(turn, outcome="failed")
    coordinator.release_conversation(lease)

    logical_closes = [
        event[1]
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1] in handles
    ]
    assert logical_closes == list(reversed(handles))
    assert turn.logical_llm_calls == {}


def test_real_binding_drains_multiple_logical_calls_before_turn_close(
    real_binding_runtime,
    caplog,
):
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="session-native-lifo",
        platform="cli",
    )
    assert lease.session is not None
    turn = coordinator.begin_turn(
        lease,
        turn_id="turn-native-lifo",
        task_id="task-native-lifo",
    )
    assert turn.handle is not None
    runtime = lease.host

    for request_id in ("request-1", "request-2"):
        handle = runtime.run_in_session(
            lease.session,
            runtime.relay.scope.push,
            relay_runtime.LOGICAL_LLM_SCOPE,
            runtime.relay.ScopeType.Function,
            handle=turn.handle,
            input={},
        )
        turn.logical_llm_calls[request_id] = handle

    coordinator.end_turn(turn, outcome="failed")
    coordinator.release_conversation(lease)
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id=lease.session_id,
    )

    assert turn.logical_llm_calls == {}
    assert "finalization failed" not in caplog.text
    assert "closed with errors" not in caplog.text


def test_shared_metrics_creates_one_task_under_concurrent_access(direct_runtime):
    runtime = relay_shared_metrics._get_runtime()
    assert runtime is not None
    ready = threading.Barrier(8)
    tasks: list[Any] = []

    def start() -> None:
        ready.wait(timeout=5)
        tasks.append(
            runtime.start_task({"session_id": "s1", "task_id": "t1", "platform": "cli"})
        )

    threads = [threading.Thread(target=start) for _ in range(8)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(timeout=5)

    assert all(not thread.is_alive() for thread in threads)
    assert len({id(task) for task in tasks}) == 1
    task_starts = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == "hermes.task_run"
    ]
    assert len(task_starts) == 1


def test_core_runtime_parents_subagent_session_without_exposing_ids(
    direct_runtime,
):
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    parent_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="parent",
        platform="cli",
    )
    parent_turn = coordinator.begin_turn(
        parent_lease,
        turn_id="parent-turn",
        task_id="parent-task",
    )
    child_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="sensitive-child",
        platform="subagent",
        parent_session_id="parent",
    )

    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    child = runtime.get_session("sensitive-child")
    assert child is not None
    assert child.parent_session_id == "parent"
    session_pushes = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == relay_runtime.SESSION_SCOPE
    ]
    assert len(session_pushes) == 2
    child_kwargs = session_pushes[1][3]
    assert child_kwargs["handle"] == parent_turn.handle
    assert child_kwargs["metadata"] == {
        "hermes.execution_surface": "subagent",
        relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION,
        relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id,
        "nemo_relay_scope_role": "subagent",
    }
    assert "sensitive-child" not in json.dumps(session_pushes)

    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="sensitive-child",
    )
    coordinator.release_conversation(child_lease)
    coordinator.end_turn(parent_turn, outcome="success")
    coordinator.release_conversation(parent_lease)
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="parent",
    )


def test_subagent_stop_hook_does_not_own_child_session_lifetime(direct_runtime):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    child = runtime.register_subagent(
        {
            "parent_session_id": "parent",
            "child_session_id": "child",
        }
    )
    assert child is not None

    lifecycle.invoke_hook(
        "subagent_stop",
        parent_session_id="parent",
        child_session_id="child",
        child_status="completed",
    )

    assert runtime.get_session("child") is child

    runtime.unregister_subagent({"child_session_id": "child"})

    assert runtime.get_session("child") is None
    child_closes = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1] == child.handle
    ]
    assert len(child_closes) == 1


@pytest.mark.parametrize(
    "terminal",
    ["return", "exception", "cancelled", "timeout"],
)
def test_subagent_agent_boundary_closes_its_own_scope(
    direct_runtime,
    monkeypatch,
    terminal,
):
    from run_agent import AIAgent

    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    parent_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="parent",
        platform="cli",
    )
    parent_turn = coordinator.begin_turn(
        parent_lease,
        turn_id="parent-turn",
        task_id="parent-task",
    )
    child_agent = SimpleNamespace(
        session_id="child",
        platform="subagent",
        _parent_session_id="parent",
        _session_db=None,
        _conversation_root_id=lambda: "parent",
    )

    if terminal == "return":
        monkeypatch.setattr(
            "agent.conversation_loop.run_conversation",
            lambda *_args, **_kwargs: {
                "final_response": "done",
                "completed": True,
                "interrupted": False,
            },
        )
        AIAgent.run_conversation(child_agent, "private", task_id="child-task")
    elif terminal == "exception":
        def fail(*_args, **_kwargs):
            raise RuntimeError("child failed")

        monkeypatch.setattr("agent.conversation_loop.run_conversation", fail)
        with pytest.raises(RuntimeError, match="child failed"):
            AIAgent.run_conversation(child_agent, "private", task_id="child-task")
    elif terminal == "cancelled":
        def cancel(*_args, **_kwargs):
            raise KeyboardInterrupt

        monkeypatch.setattr("agent.conversation_loop.run_conversation", cancel)
        with pytest.raises(KeyboardInterrupt):
            AIAgent.run_conversation(child_agent, "private", task_id="child-task")
    else:
        def time_out(*_args, **_kwargs):
            raise TimeoutError("child timed out")

        monkeypatch.setattr("agent.conversation_loop.run_conversation", time_out)
        with pytest.raises(TimeoutError, match="child timed out"):
            AIAgent.run_conversation(child_agent, "private", task_id="child-task")

    runtime = relay_runtime.get_runtime(create=False)
    assert runtime is not None
    assert runtime.get_session("child") is None
    child_push = next(
        event
        for event in direct_runtime.events
        if event[0] == "scope.push"
        and event[1] == relay_runtime.SESSION_SCOPE
        and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent"
    )
    assert child_push[3]["handle"] == parent_turn.handle
    child_closes = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == relay_runtime.SESSION_SCOPE
    ]
    assert len(child_closes) == 1
    assert relay_runtime.current_turn() is parent_turn

    coordinator.end_turn(parent_turn, outcome="success")
    coordinator.release_conversation(parent_lease)
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="parent",
    )


def test_concurrent_subagents_inherit_parent_turn_and_close_independently(
    direct_runtime,
):
    from concurrent.futures import ThreadPoolExecutor

    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    parent_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="parent",
        platform="cli",
    )
    parent_turn = coordinator.begin_turn(
        parent_lease,
        turn_id="parent-turn",
        task_id="parent-task",
    )

    def run_child(child_id):
        lease = coordinator.acquire_conversation(
            profile_key=profile_key,
            session_id=child_id,
            platform="subagent",
            parent_session_id="parent",
        )
        turn = coordinator.begin_turn(
            lease,
            turn_id=f"{child_id}-turn",
            task_id=f"{child_id}-task",
        )
        coordinator.end_turn(turn, outcome="success")
        coordinator.finalize_conversation(
            profile_key=profile_key,
            session_id=child_id,
        )
        coordinator.release_conversation(lease)

    contexts = [contextvars.copy_context(), contextvars.copy_context()]
    with ThreadPoolExecutor(max_workers=2) as executor:
        futures = [
            executor.submit(context.run, run_child, f"child-{index}")
            for index, context in enumerate(contexts)
        ]
        for future in futures:
            future.result()

    runtime = relay_runtime.get_runtime(create=False)
    assert runtime is not None
    assert runtime.get_session("child-0") is None
    assert runtime.get_session("child-1") is None
    child_pushes = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.push"
        and event[1] == relay_runtime.SESSION_SCOPE
        and event[3]["metadata"].get("nemo_relay_scope_role") == "subagent"
    ]
    assert len(child_pushes) == 2
    assert all(event[3]["handle"] == parent_turn.handle for event in child_pushes)

    coordinator.end_turn(parent_turn, outcome="success")
    coordinator.release_conversation(parent_lease)
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="parent",
    )


def test_coordinator_tracks_active_turns_across_threads(direct_runtime):
    del direct_runtime
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="cross-thread-child",
        platform="subagent",
    )

    assert not coordinator.has_active_turn(
        profile_key=profile_key,
        session_id="cross-thread-child",
    )

    turn = coordinator.begin_turn(
        lease,
        turn_id="child-turn",
        task_id="child-task",
    )

    observed = []
    thread = threading.Thread(
        target=lambda: observed.append(
            coordinator.has_active_turn(
                profile_key=profile_key,
                session_id="cross-thread-child",
            )
        )
    )
    thread.start()
    thread.join(timeout=5)

    assert observed == [True]

    coordinator.end_turn(turn, outcome="success")

    assert not coordinator.has_active_turn(
        profile_key=profile_key,
        session_id="cross-thread-child",
    )
    coordinator.release_conversation(lease)
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="cross-thread-child",
    )


def test_child_session_closes_before_active_turn_guard_is_released(
    direct_runtime,
    monkeypatch,
):
    del direct_runtime
    coordinator = relay_runtime.SESSION_COORDINATOR
    profile_key = relay_runtime.current_profile_key()
    parent_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="parent",
        platform="cli",
    )
    child_lease = coordinator.acquire_conversation(
        profile_key=profile_key,
        session_id="child",
        platform="subagent",
        parent_session_id="parent",
    )
    child_turn = coordinator.begin_turn(
        child_lease,
        turn_id="child-turn",
        task_id="child-task",
    )
    runtime = relay_runtime.get_runtime(create=False)
    assert runtime is not None
    close_observations = []
    original_close = runtime.close_session

    def observe_close(event):
        close_observations.append(
            coordinator.has_active_turn(
                profile_key=profile_key,
                session_id="child",
            )
        )
        original_close(event)

    monkeypatch.setattr(runtime, "close_session", observe_close)

    coordinator.end_turn(child_turn, outcome="success")

    assert close_observations == [True]
    assert runtime.get_session("child") is None
    assert not coordinator.has_active_turn(
        profile_key=profile_key,
        session_id="child",
    )
    coordinator.release_conversation(child_lease)
    coordinator.release_conversation(parent_lease)
    coordinator.finalize_conversation(
        profile_key=profile_key,
        session_id="parent",
    )


def test_core_runtime_ignores_self_parenting_subagent_event(direct_runtime):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None

    runtime.register_subagent({"parent_session_id": "same", "child_session_id": "same"})
    session = runtime.ensure_session({"session_id": "same"})

    assert session is not None
    assert session.parent_session_id == ""


def test_terminal_model_error_is_counted_as_failed(direct_runtime):
    base = {
        "session_id": "s1",
        "task_id": "t1",
        "api_request_id": "r1",
        "provider": "anthropic",
        "model": "claude-sonnet",
    }

    lifecycle.invoke_hook("pre_api_request", **base)
    lifecycle.invoke_hook("api_request_error", **base, retryable=False)
    lifecycle.finalize_session(session_id="s1")

    [end] = [event for event in direct_runtime.events if event[0] == "llm.call_end"]
    assert end[2]["outcome"] == "failed"


def test_task_terminal_counts_logical_calls_retries_and_unique_tools(direct_runtime):
    base = {
        "session_id": "s1",
        "task_id": "t1",
        "api_request_id": "r1",
        "platform": "cli",
        "provider": "nvidia",
        "model": "nvidia/nemotron-3-super-120b-a12b",
    }

    lifecycle.invoke_hook("pre_llm_call", **base)
    lifecycle.invoke_hook("pre_api_request", **base)
    lifecycle.invoke_hook("api_request_error", **base, retryable=True)
    lifecycle.invoke_hook("pre_api_request", **base)
    lifecycle.invoke_hook("api_request_error", **base, retryable=True)
    lifecycle.invoke_hook("pre_api_request", **base)
    lifecycle.invoke_hook("api_request_error", **base, retryable=False)
    for tool_call_id in ("tool-1", "tool-1", "tool-2"):
        lifecycle.invoke_hook(
            "post_tool_call",
            **base,
            tool_call_id=tool_call_id,
            tool_name="terminal",
            result={"output": "private"},
            status="ok",
        )
    lifecycle.invoke_hook(
        "on_session_end",
        **base,
        completed=False,
        failed=True,
        interrupted=False,
        turn_exit_reason="all_retries_exhausted_no_response",
    )
    lifecycle.finalize_session(session_id="s1")

    model_starts = [event for event in direct_runtime.events if event[0] == "llm.call"]
    model_ends = [
        event for event in direct_runtime.events if event[0] == "llm.call_end"
    ]
    assert len(model_starts) == 1
    assert len(model_ends) == 1
    assert model_ends[0][2]["outcome"] == "failed"
    [task_end] = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    assert task_end[2]["output"] == {
        "duration_bucket": task_end[2]["output"]["duration_bucket"],
        "end_reason": "failed",
        "entrypoint": "interactive",
        "execution_surface": "cli",
        "model_call_count_bucket": "1",
        "outcome": "failed",
        "retry_count_bucket": "2",
        "termination": "none",
        "tool_call_count_bucket": "2",
    }


def test_task_terminal_counts_explicit_retry_with_new_request_id(direct_runtime):
    base = {
        "session_id": "s1",
        "task_id": "t1",
        "platform": "cli",
        "provider": "nvidia",
        "model": "nvidia/nemotron-3-super-120b-a12b",
    }

    lifecycle.invoke_hook("pre_llm_call", **base)
    lifecycle.invoke_hook(
        "pre_api_request",
        **base,
        api_request_id="r1",
        retry_count=0,
    )
    lifecycle.invoke_hook(
        "api_request_error",
        **base,
        api_request_id="r1",
        retryable=True,
    )
    lifecycle.invoke_hook(
        "pre_api_request",
        **base,
        api_request_id="r2",
        retry_count=1,
    )
    lifecycle.invoke_hook(
        "post_api_request",
        **base,
        api_request_id="r2",
    )
    lifecycle.invoke_hook(
        "on_session_end",
        **base,
        completed=True,
        failed=False,
        interrupted=False,
        turn_exit_reason="text_response(stop)",
    )
    lifecycle.finalize_session(session_id="s1")

    [task_end] = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    assert task_end[2]["output"]["model_call_count_bucket"] == "2"
    assert task_end[2]["output"]["retry_count_bucket"] == "1"


def test_outer_agent_boundary_closes_early_returns_and_exceptions(
    direct_runtime,
    monkeypatch,
):
    from run_agent import AIAgent

    agent = SimpleNamespace(
        session_id="s1",
        platform="cli",
        _parent_session_id=None,
        _session_db=None,
        _conversation_root_id=lambda: "s1",
    )

    def early_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
        return {
            "final_response": "private failure detail",
            "completed": False,
            "failed": True,
            "interrupted": False,
        }

    monkeypatch.setattr(
        "agent.conversation_loop.run_conversation",
        early_failure,
    )
    result = AIAgent.run_conversation(agent, "private prompt", task_id="early")
    assert result["failed"] is True

    def raise_failure(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
        raise RuntimeError("private exception detail")

    monkeypatch.setattr(
        "agent.conversation_loop.run_conversation",
        raise_failure,
    )
    with pytest.raises(RuntimeError, match="private exception detail"):
        AIAgent.run_conversation(agent, "private prompt", task_id="exception")

    def raise_interrupt(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
        raise KeyboardInterrupt

    monkeypatch.setattr(
        "agent.conversation_loop.run_conversation",
        raise_interrupt,
    )
    with pytest.raises(KeyboardInterrupt):
        AIAgent.run_conversation(agent, "private prompt", task_id="cancelled")

    def raise_timeout(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
        raise TimeoutError("private timeout detail")

    monkeypatch.setattr(
        "agent.conversation_loop.run_conversation",
        raise_timeout,
    )
    with pytest.raises(TimeoutError, match="private timeout detail"):
        AIAgent.run_conversation(agent, "private prompt", task_id="timed-out")

    lifecycle.finalize_session(session_id="s1")

    task_ends = [
        event[2]["output"]
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    assert len(task_ends) == 4
    assert task_ends[0]["outcome"] == "failed"
    assert task_ends[0]["end_reason"] == "failed"
    assert task_ends[0]["termination"] == "none"
    assert task_ends[1]["outcome"] == "failed"
    assert task_ends[1]["end_reason"] == "system_aborted"
    assert task_ends[1]["termination"] == "system_aborted"
    assert task_ends[2]["outcome"] == "cancelled"
    assert task_ends[2]["end_reason"] == "user_cancelled"
    assert task_ends[2]["termination"] == "user_cancelled"
    assert task_ends[3]["outcome"] == "timed_out"
    assert task_ends[3]["end_reason"] == "timed_out"
    assert task_ends[3]["termination"] == "timed_out"
    serialized = json.dumps(direct_runtime.events)
    assert "private prompt" not in serialized
    assert "private failure detail" not in serialized
    assert "private exception detail" not in serialized
    assert "private timeout detail" not in serialized


def test_outer_agent_boundary_preserves_a_returned_timeout_reason(
    direct_runtime,
    monkeypatch,
):
    from run_agent import AIAgent

    agent = SimpleNamespace(
        session_id="s1",
        platform="cli",
        _parent_session_id=None,
        _session_db=None,
        _conversation_root_id=lambda: "s1",
    )

    monkeypatch.setattr(
        "agent.conversation_loop.run_conversation",
        lambda *_args, **_kwargs: {
            "final_response": "private timeout response",
            "completed": False,
            "failed": True,
            "failure_reason": "timeout",
        },
    )
    AIAgent.run_conversation(agent, "private prompt", task_id="timed-out")

    [task_end] = [
        event[2]["output"]
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    assert task_end["outcome"] == "timed_out"
    assert task_end["end_reason"] == "timed_out"
    assert task_end["termination"] == "timed_out"
    serialized = json.dumps(direct_runtime.events)
    assert "private prompt" not in serialized
    assert "private timeout response" not in serialized


def test_session_finalize_closes_a_pending_task_as_system_aborted(direct_runtime):
    lifecycle.invoke_hook(
        "pre_llm_call",
        session_id="s1",
        task_id="t1",
        platform="cli",
    )

    lifecycle.finalize_session(session_id="s1")

    [task_end] = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    assert task_end[2]["output"] == {
        "duration_bucket": task_end[2]["output"]["duration_bucket"],
        "end_reason": "system_aborted",
        "entrypoint": "interactive",
        "execution_surface": "cli",
        "model_call_count_bucket": "0",
        "outcome": "failed",
        "retry_count_bucket": "0",
        "termination": "system_aborted",
        "tool_call_count_bucket": "0",
    }


def test_sequential_tasks_in_one_session_aggregate_once_each(direct_runtime, tmp_path):
    for task_id in ("t1", "t2"):
        lifecycle.invoke_hook(
            "pre_llm_call",
            session_id="s1",
            task_id=task_id,
            platform="cli",
        )
        lifecycle.invoke_hook(
            "on_session_end",
            session_id="s1",
            task_id=task_id,
            platform="cli",
            completed=True,
            failed=False,
            interrupted=False,
            turn_exit_reason="text_response(stop)",
        )
    lifecycle.finalize_session(session_id="s1")

    outbox = tmp_path / "hermes-home" / "telemetry" / "shared_metrics" / "outbox"
    [package_path] = list(outbox.glob("*.json"))
    package = json.loads(package_path.read_text(encoding="utf-8"))
    metrics = {metric["name"]: metric for metric in package["metrics"]}
    assert metrics["hermes.task_run.started"]["value"] == 2
    assert metrics["hermes.task_run.finished"]["value"] == 2


def test_task_ownership_survives_session_id_rotation(direct_runtime):
    lifecycle.invoke_hook(
        "pre_llm_call",
        session_id="before-compression",
        task_id="t1",
        platform="cli",
    )
    lifecycle.invoke_hook(
        "pre_api_request",
        session_id="after-compression",
        task_id="t1",
        api_request_id="r1",
        platform="cli",
        provider="nvidia",
        model="nvidia/nemotron-3-super-120b-a12b",
    )
    lifecycle.invoke_hook(
        "post_api_request",
        session_id="after-compression",
        task_id="t1",
        api_request_id="r1",
        platform="cli",
        provider="nvidia",
        model="nvidia/nemotron-3-super-120b-a12b",
    )
    lifecycle.invoke_hook(
        "on_session_end",
        session_id="after-compression",
        task_id="t1",
        platform="cli",
        completed=True,
        failed=False,
        interrupted=False,
        turn_exit_reason="text_response(stop)",
    )
    lifecycle.finalize_session(session_id="before-compression")

    task_starts = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == "hermes.task_run"
    ]
    task_ends = [
        event
        for event in direct_runtime.events
        if event[0] == "scope.pop" and event[1][1] == "hermes.task_run"
    ]
    model_ends = [
        event for event in direct_runtime.events if event[0] == "llm.call_end"
    ]
    assert len(task_starts) == 1
    assert len(task_ends) == 1
    assert len(model_ends) == 1
    assert model_ends[0][2]["outcome"] == "success"
    assert task_ends[0][2]["output"]["model_call_count_bucket"] == "1"
    assert task_ends[0][2]["output"]["outcome"] == "success"


def test_gateway_and_delegated_entrypoints_flow_through_relay(direct_runtime):
    tasks = [
        {
            "session_id": "gateway-session",
            "task_id": "gateway-task",
            "platform": "whatsapp_cloud",
        },
        {
            "session_id": "child-session",
            "task_id": "delegated-task",
            "platform": "cli",
            "parent_session_id": "private-parent-session",
        },
    ]
    for task in tasks:
        lifecycle.invoke_hook("pre_llm_call", **task)
        lifecycle.invoke_hook(
            "on_session_end",
            **task,
            completed=True,
            failed=False,
            interrupted=False,
            turn_exit_reason="text_response(stop)",
        )

    starts = [
        event[3]["input"]
        for event in direct_runtime.events
        if event[0] == "scope.push" and event[1] == "hermes.task_run"
    ]
    assert starts == [
        {"entrypoint": "gateway_message", "execution_surface": "gateway"},
        {"entrypoint": "delegated", "execution_surface": "cli"},
    ]
    assert "private-parent-session" not in json.dumps(direct_runtime.events)


def test_persistence_failure_does_not_escape_the_hook(
    direct_runtime,
    monkeypatch,
    caplog,
):
    runtime = relay_shared_metrics._get_runtime()
    assert runtime is not None

    def fail_record(*_args: Any, **_kwargs: Any) -> None:
        raise OSError("store unavailable")

    monkeypatch.setattr(runtime.subscriber.store, "record_counter", fail_record)
    lifecycle.invoke_hook(
        "pre_api_request",
        session_id="s1",
        task_id="t1",
        api_request_id="r1",
        provider="openai",
        model="gpt-5",
    )
    lifecycle.invoke_hook(
        "post_api_request",
        session_id="s1",
        task_id="t1",
        api_request_id="r1",
        provider="openai",
        model="gpt-5",
    )

    assert "Unable to persist the Hermes shared metric" in caplog.text


def test_close_does_not_reopen_a_session_after_scope_start_failure(
    direct_runtime,
    monkeypatch,
):
    runtime = relay_runtime.get_runtime()
    assert runtime is not None
    original_push = direct_runtime.scope.push
    push_attempts = 0

    def fail_first_push(*args: Any, **kwargs: Any) -> Any:
        nonlocal push_attempts
        push_attempts += 1
        if push_attempts == 1:
            raise RuntimeError("simulated scope failure")
        return original_push(*args, **kwargs)

    direct_runtime.scope.push = fail_first_push
    with pytest.raises(RuntimeError, match="simulated scope failure"):
        runtime.ensure_session({"session_id": "s1"})

    close_started = threading.Event()
    allow_close = threading.Event()
    original_flush = direct_runtime.subscribers.flush

    def block_flush():
        session = runtime._sessions["s1"]
        assert session.closing is True
        close_started.set()
        assert allow_close.wait(timeout=5)
        original_flush()

    direct_runtime.subscribers.flush = block_flush
    close_thread = threading.Thread(
        target=runtime.close_session,
        args=({"session_id": "s1"},),
    )
    close_thread.start()
    assert close_started.wait(timeout=5)

    ensure_thread = threading.Thread(
        target=runtime.ensure_session,
        args=({"session_id": "s1"},),
    )
    ensure_thread.start()
    allow_close.set()
    close_thread.join(timeout=5)
    ensure_thread.join(timeout=5)

    assert not close_thread.is_alive()
    assert not ensure_thread.is_alive()
    assert push_attempts == 1
    assert "s1" not in runtime._sessions