mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-29 18:46:59 +00:00
refactor(observability): move Relay runtime ownership into core
Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
Alex Fournier
parent
3bd338d2a9
commit
64faff6768
20 changed files with 1826 additions and 1234 deletions
|
|
@ -7,6 +7,9 @@ graft locales
|
|||
# built from the sdist (e.g. Homebrew, downstream packagers). package-data
|
||||
# below covers the wheel; this covers the sdist. See #34034 / #28149.
|
||||
recursive-include plugins plugin.yaml plugin.yml
|
||||
# The built-in shared-metrics package validates exported JSON against this
|
||||
# closed schema. Include it when downstream packagers build from the sdist.
|
||||
recursive-include hermes_cli/observability/schemas *.json
|
||||
# Gateway assets include images plus YAML catalogs such as status_phrases.yaml.
|
||||
recursive-include gateway/assets *
|
||||
global-exclude __pycache__
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ Behavior-changing request or execution wrappers are outside this observer
|
|||
contract. Observer hooks should report what happened; they should not replace
|
||||
provider requests, tool arguments, or execution callbacks.
|
||||
|
||||
Hermes also has a first-party NeMo Relay shared-metrics path. It uses these
|
||||
lifecycle boundaries directly and does not require enabling an observability
|
||||
plugin. See [Relay shared metrics](relay-shared-metrics.md).
|
||||
|
||||
## Contract
|
||||
|
||||
Plugins register observer callbacks from `register(ctx)`:
|
||||
|
|
|
|||
72
docs/observability/relay-shared-metrics.md
Normal file
72
docs/observability/relay-shared-metrics.md
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
# NeMo Relay Shared Metrics
|
||||
|
||||
Hermes includes NeMo Relay as a normal runtime dependency on platforms for
|
||||
which Relay publishes a native wheel. The shared-metrics integration is built
|
||||
into Hermes and does not require `hermes plugins enable
|
||||
observability/nemo_relay`. Hermes remains importable without Relay on other
|
||||
native targets, but Relay-backed instrumentation is unavailable there.
|
||||
|
||||
Collection remains off unless Hermes policy enables it:
|
||||
|
||||
```yaml
|
||||
telemetry:
|
||||
shared_metrics:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
The existing `observability/nemo_relay` plugin remains separate. Enable that
|
||||
plugin only for its opt-in rich observability exporters, adaptive execution,
|
||||
or dynamic Relay plugins.
|
||||
|
||||
Hermes core owns one Relay host and one isolated Relay session scope per Hermes
|
||||
session. Core lifecycle producers use
|
||||
`hermes_cli.observability.relay_runtime` to obtain the shared session handle or
|
||||
run Relay scope, LLM, tool, and mark APIs in that session context. New product
|
||||
marks do not require Hermes plugin registration. Shared-metrics marks must
|
||||
still contain only fields approved by the versioned allowlist; the hard
|
||||
dependency does not change the collection or privacy policy.
|
||||
|
||||
## Current Slice
|
||||
|
||||
The first vertical slice records one logical model-call counter:
|
||||
|
||||
```text
|
||||
Hermes API hooks
|
||||
-> Relay session and LLM lifecycle
|
||||
-> Hermes shared-metrics subscriber
|
||||
-> SQLite counter
|
||||
-> immutable JSON delta package
|
||||
```
|
||||
|
||||
Hermes sends an empty `LLMRequest` into this metrics lifecycle. The terminal
|
||||
event contains only bounded model family, provider family, locality, call role,
|
||||
and outcome values. Prompts, responses, exact model IDs, endpoints, errors,
|
||||
session IDs, task IDs, and request IDs are not included in the metrics event or
|
||||
package.
|
||||
|
||||
Local state is written under:
|
||||
|
||||
```text
|
||||
$HERMES_HOME/telemetry/shared_metrics/metrics.sqlite3
|
||||
$HERMES_HOME/telemetry/shared_metrics/outbox/*.json
|
||||
```
|
||||
|
||||
The database keeps transactional aggregate and package-outbox state. Package
|
||||
files are immutable delta documents that conform to a closed JSON schema and
|
||||
are written with atomic replacement.
|
||||
|
||||
## Smoke Test
|
||||
|
||||
Run a real Hermes CLI turn against the deterministic local model server:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/smoke_nemo_relay_shared_metrics.py
|
||||
```
|
||||
|
||||
The script uses the installed `nemo-relay` dependency by default. Pass
|
||||
`--relay-python ../nemo-relay/python` only when testing a locally built Relay
|
||||
binding.
|
||||
|
||||
The smoke verifies the model request reached the local server, one counter was
|
||||
stored, one package was exported, and prompt, response, and exact-model
|
||||
canaries are absent from the package.
|
||||
51
hermes_cli/observability/__init__.py
Normal file
51
hermes_cli/observability/__init__.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"""First-party Hermes observability integrations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def prepare_lifecycle(hook_name: str, **kwargs: Any) -> None:
|
||||
"""Prepare subscribers that must observe a session's start event."""
|
||||
from . import relay_runtime, relay_shared_metrics
|
||||
|
||||
if hook_name in relay_runtime.SESSION_START_HOOKS:
|
||||
try:
|
||||
relay_shared_metrics.prepare_session_start()
|
||||
except Exception:
|
||||
logger.warning("Built-in observability preparation failed", exc_info=True)
|
||||
|
||||
|
||||
def observe_lifecycle(hook_name: str, **kwargs: Any) -> None:
|
||||
"""Dispatch a Hermes lifecycle event to built-in observability features."""
|
||||
from . import relay_runtime, relay_shared_metrics
|
||||
|
||||
# Session-start plugin callbacks register optional per-session subscribers
|
||||
# before this completion step opens the shared core scope. On teardown,
|
||||
# metrics finish child LLM scopes before the neutral host closes the owner.
|
||||
if hook_name not in relay_runtime.SESSION_CLOSE_HOOKS:
|
||||
_safe_observe(relay_runtime.observe_lifecycle, hook_name, kwargs)
|
||||
_safe_observe(relay_shared_metrics.observe_lifecycle, hook_name, kwargs)
|
||||
if hook_name in relay_runtime.SESSION_CLOSE_HOOKS:
|
||||
_safe_observe(relay_runtime.observe_lifecycle, hook_name, kwargs)
|
||||
|
||||
|
||||
def handles_hook(hook_name: str) -> bool:
|
||||
"""Return whether any built-in observability feature handles a hook."""
|
||||
from . import relay_runtime, relay_shared_metrics
|
||||
|
||||
return relay_runtime.handles_hook(hook_name) or relay_shared_metrics.handles_hook(
|
||||
hook_name
|
||||
)
|
||||
|
||||
|
||||
def _safe_observe(callback: Any, hook_name: str, kwargs: dict[str, Any]) -> None:
|
||||
try:
|
||||
callback(hook_name, **kwargs)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Built-in observability hook failed: %s", hook_name, exc_info=True
|
||||
)
|
||||
372
hermes_cli/observability/relay_runtime.py
Normal file
372
hermes_cli/observability/relay_runtime.py
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
"""Process-wide NeMo Relay runtime owned by Hermes core."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import contextvars
|
||||
import importlib
|
||||
import logging
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SESSION_SCOPE = "hermes.session"
|
||||
RUNTIME_SCHEMA_KEY = "hermes.relay.schema_version"
|
||||
RUNTIME_SCHEMA_VERSION = "hermes.relay.runtime.v1"
|
||||
|
||||
SESSION_START_HOOKS = frozenset({"on_session_start"})
|
||||
SESSION_CLOSE_HOOKS = frozenset({"on_session_finalize", "on_session_reset"})
|
||||
SUBAGENT_START_HOOKS = frozenset({"subagent_start"})
|
||||
SUBAGENT_STOP_HOOKS = frozenset({"subagent_stop"})
|
||||
HANDLED_HOOKS = (
|
||||
SESSION_START_HOOKS
|
||||
| SESSION_CLOSE_HOOKS
|
||||
| SUBAGENT_START_HOOKS
|
||||
| SUBAGENT_STOP_HOOKS
|
||||
)
|
||||
|
||||
_RUNTIME_FAILED = object()
|
||||
_RUNTIME: RelayRuntime | object | None = None
|
||||
_RUNTIME_LOCK = threading.RLock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RelaySession:
|
||||
"""One isolated Relay scope stack owned by a Hermes session."""
|
||||
|
||||
session_id: str
|
||||
parent_session_id: str = ""
|
||||
lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
|
||||
closing: bool = False
|
||||
handle: Any = None
|
||||
context: contextvars.Context | None = None
|
||||
|
||||
|
||||
class RelayRuntime:
|
||||
"""Own Relay session scopes independently of any exporter or plugin."""
|
||||
|
||||
def __init__(self, relay: Any = None) -> None:
|
||||
self.relay = relay or _load_nemo_relay()
|
||||
self._sessions_lock = threading.RLock()
|
||||
self._sessions: dict[str, RelaySession] = {}
|
||||
self._subagent_parents: dict[str, str] = {}
|
||||
self._shutdown_registered = True
|
||||
atexit.register(self.shutdown)
|
||||
|
||||
def ensure_session(
|
||||
self,
|
||||
event: dict[str, Any],
|
||||
*,
|
||||
data: Any = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> RelaySession | None:
|
||||
"""Return the existing session scope or create it once."""
|
||||
session_id = _session_id(event)
|
||||
if not session_id:
|
||||
return None
|
||||
with self._sessions_lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
parent_session_id = self._subagent_parents.get(session_id, "")
|
||||
session = RelaySession(
|
||||
session_id=session_id,
|
||||
parent_session_id=parent_session_id,
|
||||
)
|
||||
self._sessions[session_id] = session
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return None
|
||||
if session.handle is None:
|
||||
parent_handle = None
|
||||
scope_metadata = {
|
||||
**(metadata or {}),
|
||||
RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION,
|
||||
}
|
||||
if session.parent_session_id:
|
||||
parent = self.ensure_session({
|
||||
"session_id": session.parent_session_id
|
||||
})
|
||||
if parent is not None:
|
||||
parent_handle = parent.handle
|
||||
scope_metadata["nemo_relay_scope_role"] = "subagent"
|
||||
context = contextvars.Context()
|
||||
try:
|
||||
session.handle = context.run(
|
||||
self.relay.scope.push,
|
||||
SESSION_SCOPE,
|
||||
self.relay.ScopeType.Agent,
|
||||
handle=parent_handle,
|
||||
data=data,
|
||||
input={},
|
||||
metadata=scope_metadata,
|
||||
)
|
||||
except Exception:
|
||||
session.context = None
|
||||
raise
|
||||
session.context = context
|
||||
return session
|
||||
|
||||
def register_subagent(self, event: dict[str, Any]) -> None:
|
||||
"""Record the parent used when a delegated Hermes session starts."""
|
||||
parent_session_id = str(event.get("parent_session_id") or "")
|
||||
child_session_id = str(event.get("child_session_id") or "")
|
||||
if (
|
||||
not parent_session_id
|
||||
or not child_session_id
|
||||
or parent_session_id == child_session_id
|
||||
):
|
||||
return
|
||||
self.ensure_session({"session_id": parent_session_id})
|
||||
with self._sessions_lock:
|
||||
self._subagent_parents[child_session_id] = parent_session_id
|
||||
|
||||
def unregister_subagent(self, event: dict[str, Any]) -> None:
|
||||
"""Forget a delegated-session relationship after its terminal hook."""
|
||||
child_session_id = str(event.get("child_session_id") or "")
|
||||
if not child_session_id:
|
||||
return
|
||||
with self._sessions_lock:
|
||||
self._subagent_parents.pop(child_session_id, None)
|
||||
|
||||
def get_session(self, session_id: str) -> RelaySession | None:
|
||||
"""Return an active Hermes Relay session without creating one."""
|
||||
with self._sessions_lock:
|
||||
session = self._sessions.get(str(session_id or ""))
|
||||
if session is None:
|
||||
return None
|
||||
with session.lock:
|
||||
return None if session.closing else session
|
||||
|
||||
def get_session_handle(self, session_id: str) -> Any:
|
||||
"""Return the Relay parent handle for a Hermes session, if active."""
|
||||
session = self.get_session(session_id)
|
||||
return None if session is None else session.handle
|
||||
|
||||
def run_in_session(
|
||||
self,
|
||||
session: RelaySession,
|
||||
callback: Callable[..., Any],
|
||||
*args: Any,
|
||||
allow_closing: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Run a Relay operation against a session's isolated scope stack."""
|
||||
with session.lock:
|
||||
if session.closing and not allow_closing:
|
||||
raise RuntimeError("Hermes Relay session is closing")
|
||||
if session.context is None or session.handle is None:
|
||||
raise RuntimeError("Hermes Relay session context is unavailable")
|
||||
|
||||
def invoke() -> Any:
|
||||
self.relay.get_scope_stack()
|
||||
return callback(*args, **kwargs)
|
||||
|
||||
# A copy permits a helper called by an existing Relay callback to
|
||||
# re-enter the same logical session without re-entering Context.
|
||||
return session.context.copy().run(invoke)
|
||||
|
||||
def emit_mark(
|
||||
self,
|
||||
name: str,
|
||||
event: dict[str, Any],
|
||||
*,
|
||||
data: Any = None,
|
||||
metadata: Any = None,
|
||||
) -> bool:
|
||||
"""Emit a mark parented to the Hermes session identified by ``event``."""
|
||||
session = self.ensure_session(event)
|
||||
if session is None:
|
||||
return False
|
||||
self.run_in_session(
|
||||
session,
|
||||
self.relay.scope.event,
|
||||
name,
|
||||
handle=session.handle,
|
||||
data=data,
|
||||
metadata=metadata,
|
||||
)
|
||||
return True
|
||||
|
||||
def close_session(self, event: dict[str, Any]) -> None:
|
||||
"""Close one session scope and remove it from the core registry."""
|
||||
session_id = _session_id(event)
|
||||
with self._sessions_lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
return
|
||||
failures: list[str] = []
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return
|
||||
session.closing = True
|
||||
if session.handle is not None:
|
||||
try:
|
||||
self.run_in_session(
|
||||
session,
|
||||
self.relay.scope.pop,
|
||||
session.handle,
|
||||
output={},
|
||||
metadata={RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION},
|
||||
allow_closing=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
failures.append(f"session scope close failed: {exc}")
|
||||
try:
|
||||
self.relay.subscribers.flush()
|
||||
except Exception as exc:
|
||||
failures.append(f"subscriber flush failed: {exc}")
|
||||
with self._sessions_lock:
|
||||
if self._sessions.get(session_id) is session:
|
||||
self._sessions.pop(session_id, None)
|
||||
self._subagent_parents.pop(session_id, None)
|
||||
if failures:
|
||||
logger.warning(
|
||||
"Hermes Relay session %s closed with errors: %s",
|
||||
session_id,
|
||||
"; ".join(failures),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Close all core-owned Relay session scopes."""
|
||||
with self._sessions_lock:
|
||||
session_ids = list(self._sessions)
|
||||
for session_id in session_ids:
|
||||
self._safe(self.close_session, {"session_id": session_id})
|
||||
if self._shutdown_registered:
|
||||
try:
|
||||
atexit.unregister(self.shutdown)
|
||||
except Exception:
|
||||
pass
|
||||
self._shutdown_registered = False
|
||||
|
||||
@staticmethod
|
||||
def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
|
||||
try:
|
||||
return callback(*args, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Hermes Relay runtime operation failed", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def handles_hook(hook_name: str) -> bool:
|
||||
"""Return whether the core Relay host consumes this lifecycle hook."""
|
||||
return hook_name in HANDLED_HOOKS
|
||||
|
||||
|
||||
def observe_lifecycle(hook_name: str, **kwargs: Any) -> None:
|
||||
"""Apply session lifecycle events to the core Relay host."""
|
||||
if not handles_hook(hook_name):
|
||||
return
|
||||
# Session hooks do not activate Relay by themselves. A direct core
|
||||
# producer or an enabled built-in consumer creates the host lazily, after
|
||||
# which these hooks keep its session lifetime correct.
|
||||
runtime = get_runtime(create=False)
|
||||
if runtime is None:
|
||||
return
|
||||
try:
|
||||
if hook_name in SESSION_START_HOOKS:
|
||||
runtime.ensure_session(kwargs)
|
||||
elif hook_name in SESSION_CLOSE_HOOKS:
|
||||
runtime.close_session(kwargs)
|
||||
elif hook_name in SUBAGENT_START_HOOKS:
|
||||
runtime.register_subagent(kwargs)
|
||||
else:
|
||||
runtime.unregister_subagent(kwargs)
|
||||
except Exception:
|
||||
logger.warning("Hermes Relay lifecycle failed: %s", hook_name, exc_info=True)
|
||||
|
||||
|
||||
def emit_mark(
|
||||
name: str,
|
||||
*,
|
||||
session_id: str,
|
||||
data: Any = None,
|
||||
metadata: Any = None,
|
||||
) -> bool:
|
||||
"""Emit a fail-open Relay mark under a Hermes session."""
|
||||
runtime = get_runtime()
|
||||
if runtime is None:
|
||||
return False
|
||||
try:
|
||||
return runtime.emit_mark(
|
||||
name,
|
||||
{"session_id": session_id},
|
||||
data=data,
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Hermes Relay mark failed: %s", name, exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def ensure_session(*, session_id: str, **context: Any) -> RelaySession | None:
|
||||
"""Create or return the shared Relay session used by Hermes core."""
|
||||
runtime = get_runtime()
|
||||
if runtime is None:
|
||||
return None
|
||||
try:
|
||||
return runtime.ensure_session({"session_id": session_id, **context})
|
||||
except Exception:
|
||||
logger.warning("Hermes Relay session initialization failed", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def run_in_session(
|
||||
session_id: str,
|
||||
callback: Callable[..., Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Run a scope, LLM, or tool API against a shared Hermes session."""
|
||||
runtime = get_runtime()
|
||||
if runtime is None:
|
||||
raise RuntimeError("Hermes Relay runtime is unavailable")
|
||||
session = runtime.get_session(session_id)
|
||||
if session is None:
|
||||
session = runtime.ensure_session({"session_id": session_id})
|
||||
if session is None:
|
||||
raise RuntimeError("Hermes Relay session is unavailable")
|
||||
return runtime.run_in_session(session, callback, *args, **kwargs)
|
||||
|
||||
|
||||
def get_session_handle(session_id: str) -> Any:
|
||||
"""Return the shared Relay handle for direct core instrumentation."""
|
||||
runtime = get_runtime(create=False)
|
||||
return None if runtime is None else runtime.get_session_handle(session_id)
|
||||
|
||||
|
||||
def get_runtime(*, create: bool = True) -> RelayRuntime | None:
|
||||
"""Return the process-wide Hermes Relay host."""
|
||||
global _RUNTIME
|
||||
with _RUNTIME_LOCK:
|
||||
if isinstance(_RUNTIME, RelayRuntime):
|
||||
return _RUNTIME
|
||||
if _RUNTIME is _RUNTIME_FAILED or not create:
|
||||
return None
|
||||
try:
|
||||
_RUNTIME = RelayRuntime()
|
||||
except Exception:
|
||||
logger.warning("Hermes Relay runtime initialization failed", exc_info=True)
|
||||
_RUNTIME = _RUNTIME_FAILED
|
||||
return None
|
||||
return _RUNTIME
|
||||
|
||||
|
||||
def _load_nemo_relay() -> Any:
|
||||
"""Load the binding only when a producer or consumer needs Relay."""
|
||||
return importlib.import_module("nemo_relay")
|
||||
|
||||
|
||||
def _session_id(event: dict[str, Any]) -> str:
|
||||
return str(event.get("session_id") or "")
|
||||
|
||||
|
||||
def _reset_for_tests() -> None:
|
||||
"""Reset process-global core Relay state for isolated tests."""
|
||||
global _RUNTIME
|
||||
with _RUNTIME_LOCK:
|
||||
if isinstance(_RUNTIME, RelayRuntime):
|
||||
_RUNTIME.shutdown()
|
||||
_RUNTIME = None
|
||||
342
hermes_cli/observability/relay_shared_metrics.py
Normal file
342
hermes_cli/observability/relay_shared_metrics.py
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
"""Direct NeMo Relay integration for Hermes shared client metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import Any, Callable
|
||||
|
||||
from hermes_cli import __version__
|
||||
|
||||
from . import relay_runtime
|
||||
from .shared_metrics import SharedMetricsStore
|
||||
from .shared_metrics_contract import (
|
||||
MODEL_CALL_SCOPE,
|
||||
SCHEMA_KEY,
|
||||
SCHEMA_VERSION,
|
||||
SUBSCRIBER_NAME,
|
||||
model_call_fields,
|
||||
model_call_outcome,
|
||||
)
|
||||
from .shared_metrics_subscriber import SharedMetricsSubscriber
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HANDLED_HOOKS = frozenset({
|
||||
"on_session_start",
|
||||
"on_session_end",
|
||||
"on_session_finalize",
|
||||
"on_session_reset",
|
||||
"pre_api_request",
|
||||
"post_api_request",
|
||||
"api_request_error",
|
||||
})
|
||||
|
||||
_RUNTIME_FAILED = object()
|
||||
_RUNTIME: _Runtime | object | None = None
|
||||
_RUNTIME_LOCK = threading.RLock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ModelCall:
|
||||
handle: Any
|
||||
task_id: str
|
||||
fields: dict[str, str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MetricsSession:
|
||||
session_id: str
|
||||
relay_session: relay_runtime.RelaySession
|
||||
lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
|
||||
closing: bool = False
|
||||
model_calls: dict[str, _ModelCall] = field(default_factory=dict)
|
||||
|
||||
|
||||
class _Runtime:
|
||||
"""Own shared-metrics state layered on the Hermes core Relay host."""
|
||||
|
||||
def __init__(self, host: relay_runtime.RelayRuntime | None = None) -> None:
|
||||
resolved_host = host or relay_runtime.get_runtime()
|
||||
if resolved_host is None:
|
||||
raise RuntimeError("Hermes core Relay runtime is unavailable")
|
||||
self.host: relay_runtime.RelayRuntime = resolved_host
|
||||
self.relay = self.host.relay
|
||||
self._sessions_lock = threading.RLock()
|
||||
self._sessions: dict[str, _MetricsSession] = {}
|
||||
self.subscriber = SharedMetricsSubscriber(SharedMetricsStore(), __version__)
|
||||
self.relay.subscribers.register(SUBSCRIBER_NAME, self.subscriber)
|
||||
self._registered = True
|
||||
atexit.register(self.shutdown)
|
||||
|
||||
def ensure_session(self, event: dict[str, Any]) -> _MetricsSession | None:
|
||||
session_id = str(event.get("session_id") or "")
|
||||
if not session_id:
|
||||
return None
|
||||
relay_session = self.host.ensure_session(event)
|
||||
if relay_session is None:
|
||||
return None
|
||||
with self._sessions_lock:
|
||||
session = self._sessions.get(session_id)
|
||||
if session is None:
|
||||
session = _MetricsSession(
|
||||
session_id=session_id,
|
||||
relay_session=relay_session,
|
||||
)
|
||||
self._sessions[session_id] = session
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return None
|
||||
return session
|
||||
|
||||
def _run_in_session(
|
||||
self,
|
||||
session: _MetricsSession,
|
||||
callback: Callable[..., Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
return self.host.run_in_session(
|
||||
session.relay_session,
|
||||
callback,
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def start_model_call(self, event: dict[str, Any]) -> None:
|
||||
session = self.ensure_session(event)
|
||||
if session is None:
|
||||
return
|
||||
request_id = str(event.get("api_request_id") or "")
|
||||
if not request_id:
|
||||
return
|
||||
fields = model_call_fields(event)
|
||||
model_family = fields["model_family"]
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return
|
||||
existing = session.model_calls.get(request_id)
|
||||
if existing is not None:
|
||||
existing.fields = fields
|
||||
return
|
||||
handle = self._run_in_session(
|
||||
session,
|
||||
self.relay.llm.call,
|
||||
MODEL_CALL_SCOPE,
|
||||
self.relay.LLMRequest({}, {}),
|
||||
handle=session.relay_session.handle,
|
||||
metadata={SCHEMA_KEY: SCHEMA_VERSION},
|
||||
model_name=model_family,
|
||||
)
|
||||
session.model_calls[request_id] = _ModelCall(
|
||||
handle=handle,
|
||||
task_id=str(event.get("task_id") or ""),
|
||||
fields=fields,
|
||||
)
|
||||
|
||||
def end_model_call(self, event: dict[str, Any], outcome: str | None = None) -> None:
|
||||
session = self._session(event)
|
||||
if session is None:
|
||||
return
|
||||
request_id = str(event.get("api_request_id") or "")
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return
|
||||
model_call = session.model_calls.get(request_id)
|
||||
if model_call is None:
|
||||
return
|
||||
fields = model_call_fields(event)
|
||||
model_call.fields = fields
|
||||
self._finish_model_call(
|
||||
session,
|
||||
request_id,
|
||||
outcome or model_call_outcome(event),
|
||||
)
|
||||
|
||||
def end_pending_model_calls(self, event: dict[str, Any]) -> None:
|
||||
session = self._session(event)
|
||||
if session is None:
|
||||
return
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return
|
||||
self._end_pending_model_calls(session, event)
|
||||
|
||||
def close_session(self, event: dict[str, Any]) -> None:
|
||||
session = self._session(event)
|
||||
if session is None:
|
||||
return
|
||||
failures: list[str] = []
|
||||
with session.lock:
|
||||
if session.closing:
|
||||
return
|
||||
session.closing = True
|
||||
self._end_pending_model_calls(session, event)
|
||||
try:
|
||||
self.relay.subscribers.flush()
|
||||
except Exception as exc:
|
||||
failures.append(f"subscriber flush failed: {exc}")
|
||||
self._export()
|
||||
with self._sessions_lock:
|
||||
if self._sessions.get(session.session_id) is session:
|
||||
self._sessions.pop(session.session_id, None)
|
||||
if failures:
|
||||
logger.warning(
|
||||
"Hermes shared-metrics session %s closed with errors: %s",
|
||||
session.session_id,
|
||||
"; ".join(failures),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._sessions_lock:
|
||||
session_ids = list(self._sessions)
|
||||
for session_id in session_ids:
|
||||
self._safe(self.close_session, {"session_id": session_id})
|
||||
if not self._registered:
|
||||
return
|
||||
self._safe(self.relay.subscribers.flush)
|
||||
self._export()
|
||||
self._safe(self.relay.subscribers.deregister, SUBSCRIBER_NAME)
|
||||
self._registered = False
|
||||
try:
|
||||
atexit.unregister(self.shutdown)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _session(self, event: dict[str, Any]) -> _MetricsSession | None:
|
||||
session_id = str(event.get("session_id") or "")
|
||||
with self._sessions_lock:
|
||||
return self._sessions.get(session_id)
|
||||
|
||||
def _finish_model_call(
|
||||
self,
|
||||
session: _MetricsSession,
|
||||
request_id: str,
|
||||
outcome: str,
|
||||
) -> None:
|
||||
model_call = session.model_calls.pop(request_id, None)
|
||||
if model_call is None:
|
||||
return
|
||||
try:
|
||||
self._run_in_session(
|
||||
session,
|
||||
self.relay.llm.call_end,
|
||||
model_call.handle,
|
||||
{**model_call.fields, "outcome": outcome},
|
||||
metadata={SCHEMA_KEY: SCHEMA_VERSION},
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Hermes shared-metrics model call close failed", exc_info=True
|
||||
)
|
||||
|
||||
def _end_pending_model_calls(
|
||||
self,
|
||||
session: _MetricsSession,
|
||||
event: dict[str, Any],
|
||||
) -> None:
|
||||
task_id = str(event.get("task_id") or "")
|
||||
request_ids = [
|
||||
request_id
|
||||
for request_id, model_call in session.model_calls.items()
|
||||
if not task_id or model_call.task_id == task_id
|
||||
]
|
||||
outcome = "cancelled" if event.get("interrupted") else "failed"
|
||||
for request_id in request_ids:
|
||||
self._finish_model_call(session, request_id, outcome)
|
||||
|
||||
def _export(self) -> None:
|
||||
self._safe(self.subscriber.store.create_and_export_package)
|
||||
|
||||
@staticmethod
|
||||
def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
|
||||
try:
|
||||
return callback(*args, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Hermes shared metrics operation failed", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def enabled() -> bool:
|
||||
"""Return the process-lifetime Hermes shared-metrics policy."""
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
config = load_config_readonly() or {}
|
||||
except Exception:
|
||||
logger.debug("Unable to read Hermes shared-metrics policy", exc_info=True)
|
||||
return False
|
||||
if not isinstance(config, dict):
|
||||
return False
|
||||
telemetry = config.get("telemetry")
|
||||
if not isinstance(telemetry, dict):
|
||||
return False
|
||||
shared_metrics = telemetry.get("shared_metrics")
|
||||
return isinstance(shared_metrics, dict) and shared_metrics.get("enabled") is True
|
||||
|
||||
|
||||
def handles_hook(hook_name: str) -> bool:
|
||||
return hook_name in HANDLED_HOOKS and enabled()
|
||||
|
||||
|
||||
def observe_lifecycle(hook_name: str, **kwargs: Any) -> None:
|
||||
"""Project one Hermes lifecycle event into the core Relay integration."""
|
||||
if not handles_hook(hook_name):
|
||||
return
|
||||
runtime = _get_runtime()
|
||||
if runtime is None:
|
||||
return
|
||||
try:
|
||||
if hook_name == "on_session_start":
|
||||
runtime.ensure_session(kwargs)
|
||||
elif hook_name == "pre_api_request":
|
||||
runtime.start_model_call(kwargs)
|
||||
elif hook_name == "post_api_request":
|
||||
runtime.end_model_call(kwargs, "success")
|
||||
elif hook_name == "api_request_error":
|
||||
if kwargs.get("retryable") is False:
|
||||
runtime.end_model_call(kwargs, "failed")
|
||||
elif hook_name == "on_session_end":
|
||||
runtime.end_pending_model_calls(kwargs)
|
||||
elif hook_name in {"on_session_finalize", "on_session_reset"}:
|
||||
runtime.close_session(kwargs)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Hermes shared metrics hook failed: %s", hook_name, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
def prepare_session_start() -> None:
|
||||
"""Register the subscriber before any producer opens the session scope."""
|
||||
if enabled():
|
||||
_get_runtime()
|
||||
|
||||
|
||||
def _get_runtime() -> _Runtime | None:
|
||||
global _RUNTIME
|
||||
with _RUNTIME_LOCK:
|
||||
if isinstance(_RUNTIME, _Runtime):
|
||||
return _RUNTIME
|
||||
if _RUNTIME is _RUNTIME_FAILED:
|
||||
return None
|
||||
try:
|
||||
_RUNTIME = _Runtime()
|
||||
except Exception:
|
||||
logger.warning("Hermes shared metrics initialization failed", exc_info=True)
|
||||
_RUNTIME = _RUNTIME_FAILED
|
||||
return None
|
||||
return _RUNTIME
|
||||
|
||||
|
||||
def _reset_for_tests() -> None:
|
||||
"""Reset process-global state for isolated tests."""
|
||||
global _RUNTIME
|
||||
with _RUNTIME_LOCK:
|
||||
if isinstance(_RUNTIME, _Runtime):
|
||||
_RUNTIME.shutdown()
|
||||
_RUNTIME = None
|
||||
enabled.cache_clear()
|
||||
|
|
@ -8,12 +8,11 @@ from typing import Any
|
|||
|
||||
SCHEMA_KEY = "hermes.metrics.schema_version"
|
||||
SCHEMA_VERSION = "hermes.metrics.event.v1"
|
||||
SESSION_SCOPE = "hermes.session"
|
||||
MODEL_CALL_SCOPE = "hermes.model_call"
|
||||
SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics"
|
||||
PRIMARY_MODEL_CALL_ROLE = "primary"
|
||||
|
||||
EXECUTION_SURFACES = frozenset({
|
||||
EXECUTION_SURFACES: frozenset[str] = frozenset({
|
||||
"api",
|
||||
"batch",
|
||||
"cli",
|
||||
|
|
@ -25,14 +24,20 @@ EXECUTION_SURFACES = frozenset({
|
|||
"other",
|
||||
"unknown",
|
||||
})
|
||||
PROVIDER_FAMILIES = frozenset({"aggregator", "custom", "direct", "local", "unknown"})
|
||||
MODEL_LOCALITIES = frozenset({"local", "remote", "unknown"})
|
||||
MODEL_OUTCOMES = frozenset({"cancelled", "failed", "success"})
|
||||
PROVIDER_FAMILIES: frozenset[str] = frozenset({
|
||||
"aggregator",
|
||||
"custom",
|
||||
"direct",
|
||||
"local",
|
||||
"unknown",
|
||||
})
|
||||
MODEL_LOCALITIES: frozenset[str] = frozenset({"local", "remote", "unknown"})
|
||||
MODEL_OUTCOMES: frozenset[str] = frozenset({"cancelled", "failed", "success"})
|
||||
|
||||
# Shared metrics use an explicit family allowlist rather than raw model IDs or
|
||||
# dynamically sourced catalog values. The latter would make the exported schema
|
||||
# drift independently of this contract.
|
||||
MODEL_FAMILIES = frozenset({
|
||||
MODEL_FAMILIES: frozenset[str] = frozenset({
|
||||
"claude",
|
||||
"deepseek",
|
||||
"gemini",
|
||||
|
|
@ -60,7 +65,11 @@ _MODEL_FAMILY_PATTERN = re.compile(
|
|||
r"(?:^|[/_.:-])("
|
||||
+ "|".join(
|
||||
re.escape(family)
|
||||
for family in sorted(MODEL_FAMILIES - {"unknown"}, key=len, reverse=True)
|
||||
for family in sorted(
|
||||
MODEL_FAMILIES - {"unknown"},
|
||||
key=lambda value: len(value),
|
||||
reverse=True,
|
||||
)
|
||||
)
|
||||
+ r")(?=$|[/_.:-]|\d)"
|
||||
)
|
||||
|
|
@ -109,6 +118,7 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None:
|
|||
expected_fields = {
|
||||
"call_role",
|
||||
"locality",
|
||||
"model_family",
|
||||
"outcome",
|
||||
"provider_family",
|
||||
}
|
||||
|
|
@ -117,6 +127,7 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None:
|
|||
if (
|
||||
data.get("call_role") != PRIMARY_MODEL_CALL_ROLE
|
||||
or data.get("locality") not in MODEL_LOCALITIES
|
||||
or data.get("model_family") not in MODEL_FAMILIES
|
||||
or data.get("outcome") not in MODEL_OUTCOMES
|
||||
or data.get("provider_family") not in PROVIDER_FAMILIES
|
||||
):
|
||||
|
|
@ -124,7 +135,7 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None:
|
|||
return {
|
||||
"call_role": PRIMARY_MODEL_CALL_ROLE,
|
||||
"locality": data["locality"],
|
||||
"model_family": event_model_family,
|
||||
"model_family": data["model_family"],
|
||||
"outcome": data["outcome"],
|
||||
"provider_family": data["provider_family"],
|
||||
}
|
||||
|
|
@ -240,7 +251,7 @@ def model_family(kwargs: dict[str, Any]) -> str:
|
|||
declared_family = str(kwargs.get("model_family") or "").strip().lower()
|
||||
if declared_family in MODEL_FAMILIES - {"unknown"}:
|
||||
return declared_family
|
||||
model = str(kwargs.get("model") or "").lower()
|
||||
model = str(kwargs.get("response_model") or kwargs.get("model") or "").lower()
|
||||
match = _MODEL_FAMILY_PATTERN.search(model)
|
||||
return match.group(1) if match is not None else "unknown"
|
||||
|
||||
|
|
@ -2047,10 +2047,32 @@ def discover_plugins(force: bool = False) -> None:
|
|||
|
||||
|
||||
def invoke_hook(hook_name: str, **kwargs: Any) -> List[Any]:
|
||||
"""Invoke a lifecycle hook on all loaded plugins.
|
||||
"""Invoke a lifecycle hook on built-in observers and loaded plugins.
|
||||
|
||||
Returns a list of non-``None`` return values from plugin callbacks.
|
||||
"""
|
||||
if hook_name == "on_session_start":
|
||||
try:
|
||||
from hermes_cli.observability import prepare_lifecycle
|
||||
|
||||
prepare_lifecycle(hook_name, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Built-in observability preparation failed", exc_info=True)
|
||||
results = get_plugin_manager().invoke_hook(hook_name, **kwargs)
|
||||
try:
|
||||
from hermes_cli.observability import observe_lifecycle
|
||||
|
||||
observe_lifecycle(hook_name, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Built-in observability hook failed", exc_info=True)
|
||||
return results
|
||||
|
||||
try:
|
||||
from hermes_cli.observability import observe_lifecycle
|
||||
|
||||
observe_lifecycle(hook_name, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Built-in observability hook failed", exc_info=True)
|
||||
return get_plugin_manager().invoke_hook(hook_name, **kwargs)
|
||||
|
||||
|
||||
|
|
@ -2072,7 +2094,14 @@ def has_middleware(kind: str) -> bool:
|
|||
|
||||
|
||||
def has_hook(hook_name: str) -> bool:
|
||||
"""Return True when a hook has registered callbacks."""
|
||||
"""Return True when a built-in observer or plugin handles a hook."""
|
||||
try:
|
||||
from hermes_cli.observability import handles_hook
|
||||
|
||||
if handles_hook(hook_name):
|
||||
return True
|
||||
except Exception:
|
||||
logger.warning("Unable to inspect built-in shared-metrics hooks", exc_info=True)
|
||||
return get_plugin_manager().has_hook(hook_name)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -73,89 +73,24 @@ checkout that contains this plugin. A globally installed older CLI will not see
|
|||
new bundled plugins from your working tree.
|
||||
|
||||
```bash
|
||||
uv sync --extra nemo-relay
|
||||
uv sync
|
||||
uv run hermes plugins enable observability/nemo_relay
|
||||
uv run hermes chat --query 'Reply exactly ok' --provider custom --model qwen3.6:35b
|
||||
```
|
||||
|
||||
To ship the updated CLI into another environment, build and install a fresh
|
||||
wheel from this checkout, then install the official NeMo Relay runtime extra:
|
||||
wheel from this checkout. On platforms for which Relay publishes a native
|
||||
wheel, Hermes installs its supported NeMo Relay runtime as a normal dependency:
|
||||
|
||||
```bash
|
||||
uv build --wheel
|
||||
python -m pip install --force-reinstall dist/hermes_agent-*.whl
|
||||
python -m pip install "nemo-relay>=0.5.0,<0.6.0"
|
||||
hermes plugins enable observability/nemo_relay
|
||||
```
|
||||
|
||||
The plugin fails open when `nemo-relay` is not installed. Install a supported
|
||||
NeMo Relay 0.5.x distribution:
|
||||
|
||||
```bash
|
||||
pip install "nemo-relay>=0.5.0,<0.6.0"
|
||||
```
|
||||
|
||||
## Shared Metrics Proof Mode
|
||||
|
||||
The Phase 1 telemetry proof adds a separately gated metrics-only mode. Enable
|
||||
the bundled plugin and the mode in the same `config.yaml`:
|
||||
|
||||
```yaml
|
||||
plugins:
|
||||
enabled:
|
||||
- observability/nemo_relay
|
||||
entries:
|
||||
observability/nemo_relay:
|
||||
shared_metrics:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
The proof reads this setting when the plugin runtime initializes. Restart a
|
||||
long-running Hermes agent or gateway after changing it. Live revocation and
|
||||
local-state reset controls are follow-on requirements before production
|
||||
rollout.
|
||||
|
||||
This first vertical slice maps Hermes's existing API-request hooks to one Relay
|
||||
LLM lifecycle per logical model call and aggregates terminal primary calls into
|
||||
`hermes.model_call.count`. Retries sharing an API request ID remain one logical
|
||||
call. The counter uses bounded provider family, model family, locality, and
|
||||
outcome dimensions; raw model IDs and request or response content are never
|
||||
included in the metrics event.
|
||||
|
||||
The subscriber stores cumulative counters and a random opaque install ID in
|
||||
`$HERMES_HOME/telemetry/shared_metrics/metrics.sqlite3`. After Relay's flush
|
||||
barrier, Hermes commits un-packaged deltas to an immutable outbox record and
|
||||
atomically writes a `hermes.shared_metrics.v1` JSON package under
|
||||
`$HERMES_HOME/telemetry/shared_metrics/outbox/`. Repeating export without new
|
||||
model calls reuses pending package IDs and does not package a count twice. The
|
||||
proof mode does not perform network I/O. Task, tool, approval, and skill metrics
|
||||
remain follow-on slices. The closed package contract ships with the plugin at
|
||||
`schemas/hermes.shared_metrics.v1.schema.json`.
|
||||
|
||||
When shared metrics are enabled without a separate rich-observability
|
||||
configuration, the plugin does not emit its existing content-bearing turn,
|
||||
model, tool, approval, or subagent events. ATOF, ATIF, adaptive components, and
|
||||
dynamic plugins remain separate explicit configuration choices.
|
||||
|
||||
### End-to-End Smoke Test
|
||||
|
||||
The repository includes a deterministic smoke runner that exercises the real
|
||||
Hermes CLI and native NeMo Relay binding without external model credentials. It
|
||||
starts a loopback OpenAI-compatible model server, creates an isolated
|
||||
`HERMES_HOME`, runs one `hermes chat` turn, and validates the resulting SQLite
|
||||
counter and schema-conformant JSON package:
|
||||
|
||||
```bash
|
||||
./.venv/bin/python scripts/smoke_nemo_relay_shared_metrics.py \
|
||||
--relay-python ../nemo-relay/python
|
||||
```
|
||||
|
||||
The NeMo Relay Python binding must already be built. By default, the runner
|
||||
expects the Hermes checkout as its working directory and looks for a sibling
|
||||
`nemo-relay` checkout. Use `--hermes-repo`, `--relay-python`, or `--output-dir`
|
||||
to override those locations. The generated profile, captured CLI output,
|
||||
SQLite database, and immutable package are retained in the artifact directory
|
||||
printed after a successful run.
|
||||
The plugin remains opt-in even though the runtime dependency is installed by
|
||||
default. Enabling this plugin controls rich observability and adaptive
|
||||
behavior; it does not control Hermes shared client metrics.
|
||||
|
||||
## Export Configuration
|
||||
|
||||
|
|
@ -315,13 +250,10 @@ For the full generic Hermes middleware contract, see
|
|||
|
||||
## Canonical Local Examples
|
||||
|
||||
The observe-only examples in this section use a supported NeMo Relay 0.5.x
|
||||
distribution and a local Ollama model served through the
|
||||
OpenAI-compatible API.
|
||||
The observe-only examples in this section use the NeMo Relay runtime installed
|
||||
with Hermes and a local Ollama model served through the OpenAI-compatible API.
|
||||
|
||||
```bash
|
||||
pip install "nemo-relay>=0.5.0,<0.6.0"
|
||||
|
||||
export HERMES_HOME=/tmp/hermes-nemo-relay-docs/hermes-home
|
||||
mkdir -p "$HERMES_HOME"
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -138,6 +138,10 @@ dependencies = [
|
|||
# Hence the ``sys_platform == 'win32'`` marker: the dep (and its portalocker
|
||||
# / pywin32 tree) ships only where it's actually used.
|
||||
"concurrent-log-handler==0.9.29; sys_platform == 'win32'",
|
||||
# First-party lifecycle and shared-metrics runtime. Relay 0.5 publishes
|
||||
# native wheels for these targets only; keep unsupported Hermes targets
|
||||
# installable without moving Relay back behind a user-selected extra.
|
||||
"nemo-relay==0.5.0; (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'linux' and platform_machine == 'aarch64') or (sys_platform == 'win32' and platform_machine == 'AMD64') or (sys_platform == 'win32' and platform_machine == 'ARM64')",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
|
@ -205,7 +209,6 @@ vision = []
|
|||
# extra that exposes a Starlette-backed server surface so pip/uv can't resolve
|
||||
# a vulnerable pre-1.0.1 transitive. Bump in lockstep with uv.lock.
|
||||
mcp = ["mcp==1.26.0", "starlette==1.0.1"] # starlette: CVE-2026-48710
|
||||
nemo-relay = ["nemo-relay>=0.5.0,<0.6.0"]
|
||||
homeassistant = ["aiohttp==3.14.1"]
|
||||
sms = ["aiohttp==3.14.1"]
|
||||
teams = ["microsoft-teams-apps==2.0.13.4", "aiohttp==3.14.1"] # aiohttp 3.14.1: CVE-2026-34993(RCE)/47265 + 34513/34518/34519/34520/34525
|
||||
|
|
@ -336,7 +339,7 @@ locales = ["locales/*.yaml"]
|
|||
"optional-mcps/n8n" = ["optional-mcps/n8n/manifest.yaml"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
hermes_cli = ["web_dist/**/*", "tui_dist/**/*", "scripts/install.sh", "scripts/install.ps1"]
|
||||
hermes_cli = ["web_dist/**/*", "tui_dist/**/*", "scripts/install.sh", "scripts/install.ps1", "observability/schemas/*.json"]
|
||||
gateway = ["assets/**/*"]
|
||||
plugins = [
|
||||
"*/dashboard/manifest.json",
|
||||
|
|
@ -351,7 +354,6 @@ plugins = [
|
|||
"**/plugin.yaml",
|
||||
"**/plugin.yml",
|
||||
"**/README.md",
|
||||
"**/schemas/*.json",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ def _arguments() -> argparse.Namespace:
|
|||
"--relay-python",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="NeMo Relay checkout's python directory",
|
||||
help="Optional NeMo Relay checkout's python directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
|
|
@ -174,13 +174,9 @@ def _write_config(home: Path, port: int) -> None:
|
|||
api_key: no-key-required
|
||||
security:
|
||||
tirith_enabled: false
|
||||
plugins:
|
||||
enabled:
|
||||
- observability/nemo_relay
|
||||
entries:
|
||||
observability/nemo_relay:
|
||||
shared_metrics:
|
||||
enabled: true
|
||||
telemetry:
|
||||
shared_metrics:
|
||||
enabled: true
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
|
@ -270,15 +266,13 @@ def _validate_package(outbox: Path, schema_path: Path) -> tuple[Path, dict[str,
|
|||
def main() -> int:
|
||||
args = _arguments()
|
||||
hermes_repo = args.hermes_repo.resolve()
|
||||
relay_python = (
|
||||
args.relay_python.resolve()
|
||||
if args.relay_python
|
||||
else (hermes_repo.parent / "nemo-relay" / "python").resolve()
|
||||
)
|
||||
relay_python = args.relay_python.resolve() if args.relay_python else None
|
||||
hermes = hermes_repo / ".venv" / "bin" / "hermes"
|
||||
if not hermes.is_file():
|
||||
raise SystemExit(f"Hermes executable not found: {hermes}")
|
||||
if not any((relay_python / "nemo_relay").glob("_native.*")):
|
||||
if relay_python is not None and not any(
|
||||
(relay_python / "nemo_relay").glob("_native.*")
|
||||
):
|
||||
raise SystemExit(
|
||||
"Built NeMo Relay Python binding not found under "
|
||||
f"{relay_python}; run the Relay Python build first"
|
||||
|
|
@ -305,10 +299,11 @@ def main() -> int:
|
|||
_write_config(home, server.server_port)
|
||||
env = os.environ.copy()
|
||||
env["HERMES_HOME"] = str(home)
|
||||
env["PYTHONPATH"] = os.pathsep.join([
|
||||
str(relay_python),
|
||||
env.get("PYTHONPATH", ""),
|
||||
]).rstrip(os.pathsep)
|
||||
if relay_python is not None:
|
||||
env["PYTHONPATH"] = os.pathsep.join([
|
||||
str(relay_python),
|
||||
env.get("PYTHONPATH", ""),
|
||||
]).rstrip(os.pathsep)
|
||||
result = subprocess.run(
|
||||
[
|
||||
str(hermes),
|
||||
|
|
@ -359,9 +354,8 @@ def main() -> int:
|
|||
package_path, package = _validate_package(
|
||||
telemetry / "outbox",
|
||||
hermes_repo
|
||||
/ "plugins"
|
||||
/ "hermes_cli"
|
||||
/ "observability"
|
||||
/ "nemo_relay"
|
||||
/ "schemas"
|
||||
/ "hermes.shared_metrics.v1.schema.json",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ from types import SimpleNamespace
|
|||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from plugins.observability.nemo_relay.shared_metrics import SharedMetricsStore
|
||||
from plugins.observability.nemo_relay.shared_metrics_contract import (
|
||||
from hermes_cli.observability.shared_metrics import SharedMetricsStore
|
||||
from hermes_cli.observability.shared_metrics_contract import (
|
||||
MODEL_FAMILIES,
|
||||
MODEL_LOCALITIES,
|
||||
MODEL_OUTCOMES,
|
||||
|
|
@ -33,9 +33,8 @@ from plugins.observability.nemo_relay.shared_metrics_contract import (
|
|||
|
||||
SCHEMA_PATH = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "plugins"
|
||||
/ "hermes_cli"
|
||||
/ "observability"
|
||||
/ "nemo_relay"
|
||||
/ "schemas"
|
||||
/ "hermes.shared_metrics.v1.schema.json"
|
||||
)
|
||||
|
|
@ -211,6 +210,12 @@ def test_model_family_accepts_only_allowlisted_declared_metadata():
|
|||
assert model_family({"model": "private", "model_family": "private"}) == "unknown"
|
||||
|
||||
|
||||
def test_model_family_prefers_the_provider_reported_terminal_model():
|
||||
assert (
|
||||
model_family({"model": "gpt-5", "response_model": "claude-sonnet"}) == "claude"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "expected"),
|
||||
[
|
||||
|
|
@ -245,6 +250,7 @@ def test_subscriber_contract_rejects_unknown_fields_and_dimension_values():
|
|||
data={
|
||||
"call_role": "primary",
|
||||
"locality": "remote",
|
||||
"model_family": "gpt",
|
||||
"outcome": "success",
|
||||
"provider_family": "direct",
|
||||
},
|
||||
463
tests/hermes_cli/test_relay_shared_metrics_runtime.py
Normal file
463
tests/hermes_cli/test_relay_shared_metrics_runtime.py
Normal file
|
|
@ -0,0 +1,463 @@
|
|||
"""Tests for the direct Hermes-to-Relay shared-metrics runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli import plugins
|
||||
from hermes_cli.observability import relay_runtime, relay_shared_metrics
|
||||
from hermes_cli.plugins import PluginManager
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, headers: dict[str, Any], content: dict[str, Any]) -> None:
|
||||
self.headers = headers
|
||||
self.content = content
|
||||
|
||||
|
||||
class _Relay:
|
||||
def __init__(self) -> None:
|
||||
self.events: list[tuple[Any, ...]] = []
|
||||
self._callbacks: dict[str, Any] = {}
|
||||
self._starts: dict[Any, dict[str, Any]] = {}
|
||||
self._scope = contextvars.ContextVar("relay_scope", default=None)
|
||||
self._scope_serial = 0
|
||||
self.ScopeType = SimpleNamespace(Agent="agent")
|
||||
self.LLMRequest = _Request
|
||||
self.scope = SimpleNamespace(
|
||||
push=self._scope_push,
|
||||
pop=self._scope_pop,
|
||||
event=self._scope_event,
|
||||
)
|
||||
self.llm = SimpleNamespace(call=self._llm_call, call_end=self._llm_call_end)
|
||||
self.subscribers = SimpleNamespace(
|
||||
register=self._register,
|
||||
deregister=self._deregister,
|
||||
flush=self._flush,
|
||||
)
|
||||
self.get_scope_stack = self._get_scope_stack
|
||||
|
||||
def _scope_push(self, name: str, scope_type: Any, **kwargs: Any) -> Any:
|
||||
self._scope_serial += 1
|
||||
handle = ("scope", name, self._scope_serial)
|
||||
self._scope.set(handle)
|
||||
self.events.append(("scope.push", name, scope_type, kwargs))
|
||||
return handle
|
||||
|
||||
def _scope_pop(self, handle: Any, **kwargs: Any) -> None:
|
||||
self.events.append(("scope.pop", handle, kwargs))
|
||||
|
||||
def _scope_event(self, name: str, **kwargs: Any) -> None:
|
||||
self.events.append(("scope.event", name, kwargs))
|
||||
|
||||
def _get_scope_stack(self) -> Any:
|
||||
current = self._scope.get()
|
||||
self.events.append(("scope.sync", current))
|
||||
return current
|
||||
|
||||
def _llm_call(
|
||||
self,
|
||||
name: str,
|
||||
request: _Request,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
handle = ("llm", name, len(self._starts))
|
||||
self._starts[handle] = kwargs
|
||||
self.events.append(("llm.call", name, request.content, kwargs))
|
||||
return handle
|
||||
|
||||
def _llm_call_end(
|
||||
self,
|
||||
handle: Any,
|
||||
response: dict[str, Any],
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
start = self._starts.pop(handle)
|
||||
self.events.append(("llm.call_end", handle, response, kwargs))
|
||||
event = SimpleNamespace(
|
||||
kind="scope",
|
||||
category="llm",
|
||||
name=handle[1],
|
||||
scope_category="end",
|
||||
category_profile={"model_name": start["model_name"]},
|
||||
metadata={
|
||||
**start["metadata"],
|
||||
**kwargs["metadata"],
|
||||
"otel.status_code": "OK",
|
||||
},
|
||||
data=response,
|
||||
)
|
||||
for callback in list(self._callbacks.values()):
|
||||
callback(event)
|
||||
|
||||
def _register(self, name: str, callback: Any) -> None:
|
||||
self._callbacks[name] = callback
|
||||
self.events.append(("subscribers.register", name))
|
||||
|
||||
def _deregister(self, name: str) -> None:
|
||||
self._callbacks.pop(name, None)
|
||||
self.events.append(("subscribers.deregister", name))
|
||||
|
||||
def _flush(self) -> None:
|
||||
self.events.append(("subscribers.flush",))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def direct_runtime(tmp_path, monkeypatch):
|
||||
fake = _Relay()
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
|
||||
monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.load_config_readonly",
|
||||
lambda: {"telemetry": {"shared_metrics": {"enabled": True}}},
|
||||
)
|
||||
relay_shared_metrics._reset_for_tests()
|
||||
relay_runtime._reset_for_tests()
|
||||
monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())
|
||||
yield fake
|
||||
relay_shared_metrics._reset_for_tests()
|
||||
relay_runtime._reset_for_tests()
|
||||
|
||||
|
||||
def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_path):
|
||||
base = {
|
||||
"session_id": "sensitive-session",
|
||||
"task_id": "task-1",
|
||||
"api_request_id": "request-1",
|
||||
"platform": "cli",
|
||||
"provider": "custom",
|
||||
"model": "gpt-sensitive-model-id",
|
||||
"base_url": "http://127.0.0.1:11434/v1",
|
||||
}
|
||||
|
||||
assert plugins.has_hook("pre_api_request")
|
||||
plugins.invoke_hook("on_session_start", **base)
|
||||
plugins.invoke_hook(
|
||||
"pre_api_request",
|
||||
**base,
|
||||
request={"body": {"messages": ["sensitive-prompt"]}},
|
||||
)
|
||||
plugins.invoke_hook(
|
||||
"api_request_error",
|
||||
**base,
|
||||
retryable=True,
|
||||
error={"message": "sensitive-error"},
|
||||
)
|
||||
plugins.invoke_hook(
|
||||
"pre_api_request",
|
||||
**{
|
||||
**base,
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet",
|
||||
"base_url": "https://api.anthropic.com",
|
||||
},
|
||||
request={"body": {"messages": ["sensitive-prompt"]}},
|
||||
)
|
||||
plugins.invoke_hook(
|
||||
"post_api_request",
|
||||
**{
|
||||
**base,
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet",
|
||||
"base_url": "https://api.anthropic.com",
|
||||
},
|
||||
response={"content": "sensitive-response"},
|
||||
)
|
||||
plugins.invoke_hook("on_session_finalize", session_id=base["session_id"])
|
||||
|
||||
starts = [event for event in direct_runtime.events if event[0] == "llm.call"]
|
||||
ends = [event for event in direct_runtime.events if event[0] == "llm.call_end"]
|
||||
session_starts = [
|
||||
event for event in direct_runtime.events if event[0] == "scope.push"
|
||||
]
|
||||
assert len(session_starts) == 1
|
||||
assert len(starts) == 1
|
||||
assert len(ends) == 1
|
||||
assert starts[0][2] == {}
|
||||
assert starts[0][3]["model_name"] == "gpt"
|
||||
assert ends[0][2] == {
|
||||
"call_role": "primary",
|
||||
"locality": "remote",
|
||||
"model_family": "claude",
|
||||
"outcome": "success",
|
||||
"provider_family": "direct",
|
||||
}
|
||||
serialized_events = json.dumps(direct_runtime.events)
|
||||
assert "sensitive-prompt" not in serialized_events
|
||||
assert "sensitive-response" not in serialized_events
|
||||
assert "sensitive-error" not in serialized_events
|
||||
assert "gpt-sensitive-model-id" not in serialized_events
|
||||
assert plugins.get_plugin_manager().list_plugins() == []
|
||||
|
||||
root = tmp_path / "hermes-home" / "telemetry" / "shared_metrics"
|
||||
packages = list((root / "outbox").glob("*.json"))
|
||||
assert len(packages) == 1
|
||||
package = json.loads(packages[0].read_text(encoding="utf-8"))
|
||||
assert package["metrics"][0]["name"] == "hermes.model_call.count"
|
||||
assert package["metrics"][0]["dimensions"]["model_family"] == "claude"
|
||||
assert package["metrics"][0]["value"] == 1
|
||||
|
||||
|
||||
def test_direct_runtime_is_disabled_by_default(tmp_path, monkeypatch):
|
||||
fake = _Relay()
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
|
||||
monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
|
||||
monkeypatch.setattr("hermes_cli.config.load_config_readonly", lambda: {})
|
||||
relay_shared_metrics._reset_for_tests()
|
||||
relay_runtime._reset_for_tests()
|
||||
monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())
|
||||
|
||||
assert not plugins.has_hook("pre_api_request")
|
||||
plugins.invoke_hook("on_session_start", session_id="s1", platform="cli")
|
||||
plugins.invoke_hook("on_session_finalize", session_id="s1")
|
||||
|
||||
assert fake.events == []
|
||||
assert not (tmp_path / "hermes-home" / "telemetry").exists()
|
||||
relay_shared_metrics._reset_for_tests()
|
||||
relay_runtime._reset_for_tests()
|
||||
|
||||
|
||||
def test_core_runtime_is_fail_open_without_a_published_binding(monkeypatch, caplog):
|
||||
relay_shared_metrics._reset_for_tests()
|
||||
relay_runtime._reset_for_tests()
|
||||
def missing_relay(name: str):
|
||||
assert name == "nemo_relay"
|
||||
raise ModuleNotFoundError(name)
|
||||
|
||||
monkeypatch.setattr(relay_runtime.importlib, "import_module", missing_relay)
|
||||
|
||||
assert relay_runtime.get_runtime() is None
|
||||
assert not relay_runtime.emit_mark("hermes.probe", session_id="s1")
|
||||
assert "Hermes Relay runtime initialization failed" in caplog.text
|
||||
relay_runtime._reset_for_tests()
|
||||
|
||||
|
||||
def test_core_mark_uses_the_shared_session_handle_without_a_plugin(direct_runtime):
|
||||
plugins.invoke_hook("on_session_start", session_id="s1", platform="cli")
|
||||
|
||||
handle = relay_runtime.get_session_handle("s1")
|
||||
assert handle is not None
|
||||
assert relay_runtime.emit_mark(
|
||||
"hermes.skill.created",
|
||||
session_id="s1",
|
||||
data={"provenance": "agent_created"},
|
||||
metadata={"data_schema": "hermes.skill.lifecycle.v1"},
|
||||
)
|
||||
|
||||
[mark] = [event for event in direct_runtime.events if event[0] == "scope.event"]
|
||||
assert mark[1] == "hermes.skill.created"
|
||||
assert mark[2]["handle"] == handle
|
||||
assert plugins.get_plugin_manager().list_plugins() == []
|
||||
|
||||
|
||||
def test_core_mark_lazily_starts_relay_without_metrics_or_a_plugin(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
fake = _Relay()
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes-home"))
|
||||
monkeypatch.setattr(relay_runtime, "_load_nemo_relay", lambda: fake)
|
||||
monkeypatch.setattr("hermes_cli.config.load_config_readonly", lambda: {})
|
||||
relay_shared_metrics._reset_for_tests()
|
||||
relay_runtime._reset_for_tests()
|
||||
monkeypatch.setattr(plugins, "_plugin_manager", PluginManager())
|
||||
|
||||
assert relay_runtime.emit_mark(
|
||||
"hermes.skill.created",
|
||||
session_id="s1",
|
||||
data={"provenance": "agent_created"},
|
||||
)
|
||||
plugins.invoke_hook("on_session_finalize", session_id="s1")
|
||||
|
||||
assert [event[0] for event in fake.events] == [
|
||||
"scope.push",
|
||||
"scope.sync",
|
||||
"scope.event",
|
||||
"scope.sync",
|
||||
"scope.pop",
|
||||
"subscribers.flush",
|
||||
]
|
||||
assert not any(event[0] == "subscribers.register" for event in fake.events)
|
||||
assert not (tmp_path / "hermes-home" / "telemetry").exists()
|
||||
relay_runtime._reset_for_tests()
|
||||
|
||||
|
||||
def test_core_runtime_creates_one_session_under_concurrent_access(direct_runtime):
|
||||
runtime = relay_runtime.get_runtime()
|
||||
assert runtime is not None
|
||||
ready = threading.Barrier(8)
|
||||
sessions: list[Any] = []
|
||||
|
||||
def ensure() -> None:
|
||||
ready.wait(timeout=5)
|
||||
sessions.append(runtime.ensure_session({"session_id": "shared"}))
|
||||
|
||||
threads = [threading.Thread(target=ensure) for _ in range(8)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(timeout=5)
|
||||
|
||||
assert all(not thread.is_alive() for thread in threads)
|
||||
assert len({id(session) for session in sessions}) == 1
|
||||
assert (
|
||||
len([event for event in direct_runtime.events if event[0] == "scope.push"]) == 1
|
||||
)
|
||||
|
||||
|
||||
def test_core_runtime_parents_subagent_session_without_exposing_ids(
|
||||
direct_runtime,
|
||||
):
|
||||
plugins.invoke_hook("on_session_start", session_id="parent", platform="cli")
|
||||
parent_handle = relay_runtime.get_session_handle("parent")
|
||||
|
||||
plugins.invoke_hook(
|
||||
"subagent_start",
|
||||
parent_session_id="parent",
|
||||
child_session_id="sensitive-child",
|
||||
child_subagent_id="sensitive-subagent",
|
||||
)
|
||||
plugins.invoke_hook(
|
||||
"on_session_start",
|
||||
session_id="sensitive-child",
|
||||
platform="cli",
|
||||
)
|
||||
|
||||
runtime = relay_runtime.get_runtime()
|
||||
assert runtime is not None
|
||||
child = runtime.get_session("sensitive-child")
|
||||
assert child is not None
|
||||
assert child.parent_session_id == "parent"
|
||||
pushes = [event for event in direct_runtime.events if event[0] == "scope.push"]
|
||||
assert len(pushes) == 2
|
||||
child_kwargs = pushes[1][3]
|
||||
assert child_kwargs["handle"] == parent_handle
|
||||
assert child_kwargs["metadata"] == {
|
||||
relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION,
|
||||
"nemo_relay_scope_role": "subagent",
|
||||
}
|
||||
assert "sensitive-child" not in json.dumps(pushes)
|
||||
assert "sensitive-subagent" not in json.dumps(pushes)
|
||||
|
||||
|
||||
def test_core_runtime_ignores_self_parenting_subagent_event(direct_runtime):
|
||||
runtime = relay_runtime.get_runtime()
|
||||
assert runtime is not None
|
||||
|
||||
runtime.register_subagent(
|
||||
{"parent_session_id": "same", "child_session_id": "same"}
|
||||
)
|
||||
session = runtime.ensure_session({"session_id": "same"})
|
||||
|
||||
assert session is not None
|
||||
assert session.parent_session_id == ""
|
||||
|
||||
|
||||
def test_terminal_model_error_is_counted_as_failed(direct_runtime):
|
||||
base = {
|
||||
"session_id": "s1",
|
||||
"task_id": "t1",
|
||||
"api_request_id": "r1",
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet",
|
||||
}
|
||||
|
||||
plugins.invoke_hook("pre_api_request", **base)
|
||||
plugins.invoke_hook("api_request_error", **base, retryable=False)
|
||||
plugins.invoke_hook("on_session_finalize", session_id="s1")
|
||||
|
||||
[end] = [event for event in direct_runtime.events if event[0] == "llm.call_end"]
|
||||
assert end[2]["outcome"] == "failed"
|
||||
|
||||
|
||||
def test_persistence_failure_does_not_escape_the_hook(
|
||||
direct_runtime,
|
||||
monkeypatch,
|
||||
caplog,
|
||||
):
|
||||
runtime = relay_shared_metrics._get_runtime()
|
||||
assert runtime is not None
|
||||
|
||||
def fail_record(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise OSError("store unavailable")
|
||||
|
||||
monkeypatch.setattr(runtime.subscriber.store, "record_model_call", fail_record)
|
||||
plugins.invoke_hook(
|
||||
"pre_api_request",
|
||||
session_id="s1",
|
||||
task_id="t1",
|
||||
api_request_id="r1",
|
||||
provider="openai",
|
||||
model="gpt-5",
|
||||
)
|
||||
plugins.invoke_hook(
|
||||
"post_api_request",
|
||||
session_id="s1",
|
||||
task_id="t1",
|
||||
api_request_id="r1",
|
||||
provider="openai",
|
||||
model="gpt-5",
|
||||
)
|
||||
|
||||
assert "Unable to persist the Hermes model-call metric" in caplog.text
|
||||
|
||||
|
||||
def test_close_does_not_reopen_a_session_after_scope_start_failure(
|
||||
direct_runtime,
|
||||
monkeypatch,
|
||||
):
|
||||
runtime = relay_runtime.get_runtime()
|
||||
assert runtime is not None
|
||||
original_push = direct_runtime.scope.push
|
||||
push_attempts = 0
|
||||
|
||||
def fail_first_push(*args: Any, **kwargs: Any) -> Any:
|
||||
nonlocal push_attempts
|
||||
push_attempts += 1
|
||||
if push_attempts == 1:
|
||||
raise RuntimeError("simulated scope failure")
|
||||
return original_push(*args, **kwargs)
|
||||
|
||||
direct_runtime.scope.push = fail_first_push
|
||||
with pytest.raises(RuntimeError, match="simulated scope failure"):
|
||||
runtime.ensure_session({"session_id": "s1"})
|
||||
|
||||
close_started = threading.Event()
|
||||
allow_close = threading.Event()
|
||||
original_flush = direct_runtime.subscribers.flush
|
||||
|
||||
def block_flush():
|
||||
session = runtime._sessions["s1"]
|
||||
assert session.closing is True
|
||||
close_started.set()
|
||||
assert allow_close.wait(timeout=5)
|
||||
original_flush()
|
||||
|
||||
direct_runtime.subscribers.flush = block_flush
|
||||
close_thread = threading.Thread(
|
||||
target=runtime.close_session,
|
||||
args=({"session_id": "s1"},),
|
||||
)
|
||||
close_thread.start()
|
||||
assert close_started.wait(timeout=5)
|
||||
|
||||
ensure_thread = threading.Thread(
|
||||
target=runtime.ensure_session,
|
||||
args=({"session_id": "s1"},),
|
||||
)
|
||||
ensure_thread.start()
|
||||
allow_close.set()
|
||||
close_thread.join(timeout=5)
|
||||
ensure_thread.join(timeout=5)
|
||||
|
||||
assert not close_thread.is_alive()
|
||||
assert not ensure_thread.is_alive()
|
||||
assert push_attempts == 1
|
||||
assert "s1" not in runtime._sessions
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -3,6 +3,8 @@
|
|||
from pathlib import Path
|
||||
import tomllib
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
|
||||
def _load_optional_dependencies():
|
||||
pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
|
|
@ -11,6 +13,12 @@ def _load_optional_dependencies():
|
|||
return project["optional-dependencies"]
|
||||
|
||||
|
||||
def _load_project():
|
||||
pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
with pyproject_path.open("rb") as handle:
|
||||
return tomllib.load(handle)["project"]
|
||||
|
||||
|
||||
def _load_package_data():
|
||||
pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
with pyproject_path.open("rb") as handle:
|
||||
|
|
@ -222,14 +230,40 @@ def test_feishu_extra_includes_qrcode_for_qr_login():
|
|||
assert any(dep.startswith("qrcode") for dep in feishu_extra)
|
||||
|
||||
|
||||
def test_nemo_relay_extra_uses_supported_official_distribution_range():
|
||||
optional_dependencies = _load_optional_dependencies()
|
||||
def test_nemo_relay_is_a_pinned_core_dependency():
|
||||
metadata = _load_project()
|
||||
|
||||
assert optional_dependencies["nemo-relay"] == ["nemo-relay>=0.5,<1.0"]
|
||||
assert not any(
|
||||
spec == "hermes-agent[nemo-relay]"
|
||||
for spec in optional_dependencies["all"]
|
||||
relay_dependencies = [
|
||||
dependency
|
||||
for dependency in metadata["dependencies"]
|
||||
if dependency.startswith("nemo-relay==")
|
||||
]
|
||||
assert len(relay_dependencies) == 1
|
||||
requirement = Requirement(relay_dependencies[0])
|
||||
assert str(requirement.specifier) == "==0.5.0"
|
||||
assert requirement.marker is not None
|
||||
assert requirement.marker.evaluate(
|
||||
{"sys_platform": "darwin", "platform_machine": "arm64"}
|
||||
)
|
||||
assert requirement.marker.evaluate(
|
||||
{"sys_platform": "linux", "platform_machine": "x86_64"}
|
||||
)
|
||||
assert requirement.marker.evaluate(
|
||||
{"sys_platform": "linux", "platform_machine": "aarch64"}
|
||||
)
|
||||
assert requirement.marker.evaluate(
|
||||
{"sys_platform": "win32", "platform_machine": "AMD64"}
|
||||
)
|
||||
assert requirement.marker.evaluate(
|
||||
{"sys_platform": "win32", "platform_machine": "ARM64"}
|
||||
)
|
||||
assert not requirement.marker.evaluate(
|
||||
{"sys_platform": "android", "platform_machine": "aarch64"}
|
||||
)
|
||||
assert not requirement.marker.evaluate(
|
||||
{"sys_platform": "darwin", "platform_machine": "x86_64"}
|
||||
)
|
||||
assert "nemo-relay" not in metadata["optional-dependencies"]
|
||||
|
||||
|
||||
def test_dashboard_plugin_manifests_and_assets_are_packaged():
|
||||
|
|
@ -244,6 +278,12 @@ def test_dashboard_plugin_manifests_and_assets_are_packaged():
|
|||
assert "*/dashboard/dist/**/*" in plugin_data
|
||||
|
||||
|
||||
def test_shared_metrics_schema_is_packaged():
|
||||
package_data = _load_package_data()
|
||||
|
||||
assert "observability/schemas/*.json" in package_data["hermes_cli"]
|
||||
|
||||
|
||||
def test_nested_bundled_plugin_metadata_is_packaged():
|
||||
"""Nested opt-in plugins need manifests and READMEs in wheel installs."""
|
||||
package_data = _load_package_data()
|
||||
|
|
|
|||
8
uv.lock
generated
8
uv.lock
generated
|
|
@ -1528,6 +1528,7 @@ dependencies = [
|
|||
{ name = "httpx", extra = ["socks"] },
|
||||
{ name = "jinja2" },
|
||||
{ name = "markdown" },
|
||||
{ name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')" },
|
||||
{ name = "openai" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pathspec" },
|
||||
|
|
@ -1662,9 +1663,6 @@ mistral = [
|
|||
modal = [
|
||||
{ name = "modal" },
|
||||
]
|
||||
nemo-relay = [
|
||||
{ name = "nemo-relay" },
|
||||
]
|
||||
parallel-web = [
|
||||
{ name = "parallel-web" },
|
||||
]
|
||||
|
|
@ -1804,7 +1802,7 @@ requires-dist = [
|
|||
{ name = "microsoft-teams-apps", marker = "extra == 'teams'", specifier = "==2.0.13.4" },
|
||||
{ name = "mistralai", marker = "extra == 'mistral'", specifier = "==2.4.8" },
|
||||
{ name = "modal", marker = "extra == 'modal'", specifier = "==1.3.4" },
|
||||
{ name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5.0,<0.6.0" },
|
||||
{ name = "nemo-relay", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')", specifier = "==0.5.0" },
|
||||
{ name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" },
|
||||
{ name = "openai", specifier = "==2.24.0" },
|
||||
{ name = "packaging", specifier = "==26.0" },
|
||||
|
|
@ -1853,7 +1851,7 @@ requires-dist = [
|
|||
{ name = "websockets", specifier = "==15.0.1" },
|
||||
{ name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" },
|
||||
]
|
||||
provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"]
|
||||
provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"]
|
||||
|
||||
[[package]]
|
||||
name = "hf-xet"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue