mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-20 15:33:54 +00:00
tests: pin ink engine in _make_tui_argv npm-bootstrap tests (post-merge semantic fix)
Main's rewritten test_tui_npm_install.py tests call _make_tui_argv expecting the Ink/npm flow unconditionally; with the dual-engine dispatch merged in, _resolve_tui_engine() auto-selects opentui whenever ui-opentui/dist is built in the repo, routing the call away from the path under test (first subprocess became 'node --version' instead of 'npm run build'). Pin the engine to ink via an autouse fixture, mirroring the existing pinning precedent in test_tui_resume_flow.py.
This commit is contained in:
parent
ab37440ce6
commit
e1067dbbe5
756 changed files with 79874 additions and 19585 deletions
|
|
@ -2,13 +2,17 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import builtins
|
||||
import gc
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from hermes_cli.plugins import PluginManager
|
||||
|
|
@ -37,7 +41,7 @@ class _FakeNemoRelay:
|
|||
call_end=self._tool_call_end,
|
||||
execute=self._tool_execute,
|
||||
)
|
||||
self.plugin = SimpleNamespace(initialize=self._plugin_initialize)
|
||||
self.plugin = SimpleNamespace(initialize=self._plugin_initialize, clear=self._plugin_clear)
|
||||
self.LLMRequest = _FakeLLMRequest
|
||||
self.AtofExporterConfig = _FakeAtofExporterConfig
|
||||
self.AtofExporterMode = SimpleNamespace(Append="append", Overwrite="overwrite")
|
||||
|
|
@ -93,6 +97,9 @@ class _FakeNemoRelay:
|
|||
self.events.append(("plugin.initialize", config))
|
||||
return {"diagnostics": []}
|
||||
|
||||
async def _plugin_clear(self):
|
||||
self.events.append(("plugin.clear",))
|
||||
|
||||
|
||||
class _FakeLLMRequest:
|
||||
def __init__(self, headers, content):
|
||||
|
|
@ -115,6 +122,10 @@ class _FakeAtofExporter:
|
|||
def register(self, name):
|
||||
self.events.append(("atof.register", name, self.config.output_directory, self.config.filename))
|
||||
|
||||
def deregister(self, name):
|
||||
self.events.append(("atof.deregister", name, self.config.output_directory, self.config.filename))
|
||||
return True
|
||||
|
||||
|
||||
class _FakeAtifExporter:
|
||||
def __init__(self, events, session_id, agent_name, agent_version, kwargs):
|
||||
|
|
@ -143,6 +154,33 @@ def _fresh_plugin(monkeypatch, fake):
|
|||
return plugin
|
||||
|
||||
|
||||
def _wrapped_downstream_error(original):
|
||||
class _DownstreamExecutionError(Exception):
|
||||
def __init__(self, original):
|
||||
super().__init__(str(original))
|
||||
self.original = original
|
||||
|
||||
return _DownstreamExecutionError(original)
|
||||
|
||||
|
||||
def _enable_adaptive_plugin(tmp_path, monkeypatch) -> None:
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config.tool_parallelism]
|
||||
mode = "observe_only"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
|
||||
|
||||
def test_manifest_fields():
|
||||
data = yaml.safe_load((PLUGIN_DIR / "plugin.yaml").read_text())
|
||||
assert data["name"] == "nemo_relay"
|
||||
|
|
@ -445,6 +483,252 @@ output_directory = "{atif_dir}"
|
|||
assert atif_dir.is_dir()
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_clears_plugins_toml_on_final_session_finalize_and_reinitializes(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
|
||||
plugin.on_session_start(session_id="s1")
|
||||
plugin.on_session_finalize(session_id="s1", reason="shutdown")
|
||||
plugin.on_session_start(session_id="s2")
|
||||
|
||||
event_names = [event[0] for event in fake.events]
|
||||
assert event_names.count("plugin.initialize") == 2
|
||||
assert event_names.count("plugin.clear") == 1
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_keeps_plugins_toml_active_while_other_sessions_remain(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
|
||||
plugin.on_session_start(session_id="parent")
|
||||
plugin.on_session_start(session_id="child")
|
||||
plugin.on_session_finalize(session_id="child", reason="shutdown")
|
||||
plugin.on_session_finalize(session_id="parent", reason="shutdown")
|
||||
|
||||
event_names = [event[0] for event in fake.events]
|
||||
assert event_names.count("plugin.initialize") == 1
|
||||
assert event_names.count("plugin.clear") == 1
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_reinitializes_plugins_toml_inside_active_event_loop(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
|
||||
async def _drive() -> None:
|
||||
plugin.on_session_start(session_id="s1")
|
||||
plugin.on_session_finalize(session_id="s1", reason="shutdown")
|
||||
plugin.on_session_start(session_id="s2")
|
||||
await asyncio.sleep(0)
|
||||
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
asyncio.run(_drive())
|
||||
gc.collect()
|
||||
|
||||
assert not any("was never awaited" in str(w.message) for w in caught)
|
||||
runtime = plugin._get_runtime()
|
||||
assert runtime is not None
|
||||
assert runtime._plugin_config_initialized is True
|
||||
scope_push_names = [event[1] for event in fake.events if event[0] == "scope.push"]
|
||||
assert "hermes-session-s2" in scope_push_names
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_retries_plugins_toml_after_clear_failure(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
initialize_calls = 0
|
||||
|
||||
async def _counting_initialize(config):
|
||||
nonlocal initialize_calls
|
||||
initialize_calls += 1
|
||||
fake.events.append(("plugin.initialize.attempt", initialize_calls, config))
|
||||
return {"diagnostics": []}
|
||||
|
||||
async def _failing_clear():
|
||||
fake.events.append(("plugin.clear.failed",))
|
||||
raise RuntimeError("boom")
|
||||
|
||||
fake.plugin.initialize = _counting_initialize
|
||||
fake.plugin.clear = _failing_clear
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
|
||||
plugin.on_session_start(session_id="s1")
|
||||
plugin.on_session_finalize(session_id="s1", reason="shutdown")
|
||||
plugin.on_session_start(session_id="s2")
|
||||
|
||||
event_names = [event[0] for event in fake.events]
|
||||
assert event_names.count("plugin.initialize.attempt") == 2
|
||||
assert event_names.count("plugin.clear.failed") == 1
|
||||
scope_push_names = [event[1] for event in fake.events if event[0] == "scope.push"]
|
||||
assert "hermes-session-s2" in scope_push_names
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_disables_direct_atif_when_plugins_toml_owns_atif(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
f"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
|
||||
[components.config.atif]
|
||||
enabled = true
|
||||
output_directory = "{(tmp_path / "managed-atif").as_posix()}"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1")
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "direct-atif"))
|
||||
|
||||
plugin.on_session_start(session_id="s1")
|
||||
plugin.on_session_finalize(session_id="s1", reason="shutdown")
|
||||
|
||||
event_names = [event[0] for event in fake.events]
|
||||
assert "plugin.initialize" in event_names
|
||||
assert "plugin.clear" in event_names
|
||||
assert "atif.register" not in event_names
|
||||
assert not (tmp_path / "direct-atif" / "hermes-atif-s1.json").exists()
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_keeps_direct_atif_when_plugins_toml_init_fails(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
async def _failing_initialize(config):
|
||||
fake.events.append(("plugin.initialize.failed", config))
|
||||
raise RuntimeError("boom")
|
||||
|
||||
fake.plugin.initialize = _failing_initialize
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
f"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
|
||||
[components.config.atif]
|
||||
enabled = true
|
||||
output_directory = "{(tmp_path / "managed-atif").as_posix()}"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_ENABLED", "1")
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_ATIF_OUTPUT_DIRECTORY", str(tmp_path / "direct-atif"))
|
||||
|
||||
plugin.on_session_start(session_id="s1")
|
||||
plugin.on_session_finalize(session_id="s1", reason="shutdown")
|
||||
|
||||
event_names = [event[0] for event in fake.events]
|
||||
assert "plugin.initialize.failed" in event_names
|
||||
assert "plugin.clear" not in event_names
|
||||
assert "atif.register" in event_names
|
||||
assert (tmp_path / "direct-atif" / "hermes-atif-s1.json").exists()
|
||||
|
||||
|
||||
def test_nemo_relay_plugin_retries_plugins_toml_after_fallback_only_session_and_clears_direct_atof(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
fake = _FakeNemoRelay()
|
||||
initialize_calls = 0
|
||||
|
||||
async def _flaky_initialize(config):
|
||||
nonlocal initialize_calls
|
||||
initialize_calls += 1
|
||||
fake.events.append(("plugin.initialize.attempt", initialize_calls, config))
|
||||
if initialize_calls == 1:
|
||||
raise RuntimeError("boom")
|
||||
return {"diagnostics": []}
|
||||
|
||||
fake.plugin.initialize = _flaky_initialize
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(
|
||||
f"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "observability"
|
||||
enabled = true
|
||||
|
||||
[components.config.atof]
|
||||
enabled = true
|
||||
output_directory = "{(tmp_path / "managed-atof").as_posix()}"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_ATOF_ENABLED", "1")
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_ATOF_OUTPUT_DIRECTORY", str(tmp_path / "direct-atof"))
|
||||
|
||||
plugin.on_session_start(session_id="s1")
|
||||
plugin.on_session_finalize(session_id="s1", reason="shutdown")
|
||||
plugin.on_session_start(session_id="s2")
|
||||
|
||||
runtime = plugin._get_runtime()
|
||||
assert runtime is not None
|
||||
assert runtime._plugin_config_initialized is True
|
||||
event_names = [event[0] for event in fake.events]
|
||||
assert event_names.count("plugin.initialize.attempt") == 2
|
||||
assert event_names.count("atof.register") == 1
|
||||
assert event_names.count("atof.deregister") == 1
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_middleware_preserves_raw_response(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
|
|
@ -457,8 +741,8 @@ version = 1
|
|||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config]
|
||||
mode = "route"
|
||||
[components.config.tool_parallelism]
|
||||
mode = "observe_only"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
|
@ -506,7 +790,7 @@ mode = "route"
|
|||
assert response.choices == [raw_choice]
|
||||
assert seen_request["intercepted"] is True
|
||||
execute_start = next(event for event in fake.events if event[0] == "llm.execute.start")
|
||||
assert execute_start[3]["data"]["mode"] == "route"
|
||||
assert execute_start[3]["data"]["mode"] == "observe_only"
|
||||
execute_end = next(event for event in fake.events if event[0] == "llm.execute.end")
|
||||
assert execute_end[2] == {
|
||||
"model": "demo-model",
|
||||
|
|
@ -527,6 +811,298 @@ mode = "route"
|
|||
}
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
def native_like_execute(name, request, func, **kwargs):
|
||||
fake.events.append(("llm.execute.start", name, request.content, kwargs))
|
||||
try:
|
||||
return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content}))
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None
|
||||
|
||||
fake.llm.execute = native_like_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
class ProviderAuthError(Exception):
|
||||
status_code = 403
|
||||
|
||||
provider_error = ProviderAuthError("provider auth failed")
|
||||
|
||||
def next_call(request):
|
||||
raise _wrapped_downstream_error(provider_error)
|
||||
|
||||
with pytest.raises(ProviderAuthError) as caught:
|
||||
plugin.on_llm_execution_middleware(
|
||||
session_id="s1",
|
||||
provider="anthropic",
|
||||
model="demo-model",
|
||||
request={"messages": [{"role": "user", "content": "hi"}]},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is provider_error
|
||||
assert caught.value.status_code == 403
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_preserves_downstream_error_with_relay_suffix(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
# Guards the startswith (vs exact ==) match in _is_relay_wrapped_callback_error:
|
||||
# Relay re-wraps the callback failure with its canonical prefix but APPENDS a
|
||||
# trailing suffix. Exact equality would miss this and surface Relay's wrapper;
|
||||
# prefix matching must still recover the original downstream error.
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
def native_like_execute(name, request, func, **kwargs):
|
||||
try:
|
||||
return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content}))
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"internal error: {type(exc).__name__}: {exc} (retried 3x)") from None
|
||||
|
||||
fake.llm.execute = native_like_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
class ProviderAuthError(Exception):
|
||||
status_code = 403
|
||||
|
||||
provider_error = ProviderAuthError("provider auth failed")
|
||||
|
||||
def next_call(request):
|
||||
raise _wrapped_downstream_error(provider_error)
|
||||
|
||||
with pytest.raises(ProviderAuthError) as caught:
|
||||
plugin.on_llm_execution_middleware(
|
||||
session_id="s1",
|
||||
provider="anthropic",
|
||||
model="demo-model",
|
||||
request={"messages": [{"role": "user", "content": "hi"}]},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is provider_error
|
||||
assert caught.value.status_code == 403
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
relay_error = RuntimeError("internal error: relay setup failed")
|
||||
|
||||
def internal_error_execute(name, request, func, **kwargs):
|
||||
raise relay_error
|
||||
|
||||
fake.llm.execute = internal_error_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
plugin.on_llm_execution_middleware(
|
||||
session_id="s1",
|
||||
provider="anthropic",
|
||||
model="demo-model",
|
||||
request={"messages": [{"role": "user", "content": "hi"}]},
|
||||
next_call=lambda request: {"raw": request},
|
||||
)
|
||||
|
||||
assert caught.value is relay_error
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_keeps_wrapped_relay_error_after_downstream_failure(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
fake = _FakeNemoRelay()
|
||||
relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream")
|
||||
|
||||
def translated_execute(name, request, func, **kwargs):
|
||||
try:
|
||||
return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content}))
|
||||
except Exception:
|
||||
raise relay_error
|
||||
|
||||
fake.llm.execute = translated_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
def next_call(request):
|
||||
raise _wrapped_downstream_error(RuntimeError("provider failed"))
|
||||
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
plugin.on_llm_execution_middleware(
|
||||
session_id="s1",
|
||||
provider="anthropic",
|
||||
model="demo-model",
|
||||
request={"messages": [{"role": "user", "content": "hi"}]},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is relay_error
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_keeps_relay_translated_error(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
class RelayPolicyError(Exception):
|
||||
pass
|
||||
|
||||
relay_error = RelayPolicyError("relay policy blocked")
|
||||
|
||||
def translated_execute(name, request, func, **kwargs):
|
||||
try:
|
||||
return func(_FakeLLMRequest(request.headers, {"intercepted": True, **request.content}))
|
||||
except Exception:
|
||||
raise relay_error
|
||||
|
||||
fake.llm.execute = translated_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
provider_error = RuntimeError("provider failed")
|
||||
|
||||
def next_call(request):
|
||||
raise _wrapped_downstream_error(provider_error)
|
||||
|
||||
with pytest.raises(RelayPolicyError) as caught:
|
||||
plugin.on_llm_execution_middleware(
|
||||
session_id="s1",
|
||||
provider="anthropic",
|
||||
model="demo-model",
|
||||
request={"messages": [{"role": "user", "content": "hi"}]},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is relay_error
|
||||
|
||||
|
||||
def test_nemo_relay_downstream_unwrap_matches_real_middleware_wrapper_shape(monkeypatch):
|
||||
# Regression guard against core/plugin drift. The synthetic tests above model
|
||||
# the downstream-error wrapper with a local class, so they keep passing even
|
||||
# if core middleware renames its private ``_DownstreamExecutionError`` or drops
|
||||
# ``.original`` -- the exact shape the plugin matches by name at
|
||||
# ``_original_downstream_error``. Capture the wrapper the REAL
|
||||
# ``hermes_cli.middleware._run_execution_chain`` hands to a middleware
|
||||
# callback's ``next_call`` and assert the plugin's detector unwraps it to the
|
||||
# original exception. If core middleware changes the wrapper shape, this fails
|
||||
# here instead of silently defeating the unwrap in production.
|
||||
from hermes_cli import middleware
|
||||
|
||||
from plugins.observability.nemo_relay import _original_downstream_error
|
||||
|
||||
class ProviderError(Exception):
|
||||
status_code = 403
|
||||
|
||||
provider_error = ProviderError("provider auth failed")
|
||||
captured: dict[str, Exception] = {}
|
||||
|
||||
def terminal_call(payload):
|
||||
raise provider_error
|
||||
|
||||
def capturing_callback(**kwargs):
|
||||
next_call = kwargs["next_call"]
|
||||
try:
|
||||
return next_call(kwargs.get("request"))
|
||||
except Exception as exc:
|
||||
captured["wrapper"] = exc
|
||||
# Surface the original so the chain unwinds without re-wrapping noise.
|
||||
raise _original_downstream_error(exc) from None
|
||||
|
||||
with pytest.raises(ProviderError) as caught:
|
||||
middleware._run_execution_chain(
|
||||
"llm",
|
||||
[capturing_callback],
|
||||
terminal_call,
|
||||
request={"messages": []},
|
||||
)
|
||||
|
||||
wrapper = captured["wrapper"]
|
||||
# The wrapper the plugin sees must match what _original_downstream_error keys on.
|
||||
assert wrapper.__class__.__name__ == "_DownstreamExecutionError"
|
||||
assert isinstance(getattr(wrapper, "original", None), BaseException)
|
||||
assert _original_downstream_error(wrapper) is provider_error
|
||||
assert caught.value is provider_error
|
||||
assert caught.value.status_code == 403
|
||||
|
||||
|
||||
def _adaptive_llm_execute_mode(tmp_path, monkeypatch, plugins_toml_text: str) -> str:
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
plugins_toml = tmp_path / "plugins.toml"
|
||||
plugins_toml.write_text(plugins_toml_text, encoding="utf-8")
|
||||
monkeypatch.setenv("HERMES_NEMO_RELAY_PLUGINS_TOML", str(plugins_toml))
|
||||
|
||||
plugin.on_llm_execution_middleware(
|
||||
session_id="s1",
|
||||
provider="anthropic",
|
||||
model="demo-model",
|
||||
request={"messages": [{"role": "user", "content": "hi"}]},
|
||||
next_call=lambda request: {"raw": request},
|
||||
)
|
||||
|
||||
execute_start = next(event for event in fake.events if event[0] == "llm.execute.start")
|
||||
return execute_start[3]["data"]["mode"]
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_middleware_defaults_to_observe_only_when_mode_is_unset(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
mode = _adaptive_llm_execute_mode(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config]
|
||||
version = 1
|
||||
""",
|
||||
)
|
||||
assert mode == "observe_only"
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_middleware_accepts_legacy_top_level_mode(tmp_path, monkeypatch):
|
||||
mode = _adaptive_llm_execute_mode(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config]
|
||||
mode = "route"
|
||||
""",
|
||||
)
|
||||
assert mode == "route"
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_llm_execution_middleware_prefers_tool_parallelism_mode(tmp_path, monkeypatch):
|
||||
mode = _adaptive_llm_execute_mode(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
version = 1
|
||||
|
||||
[[components]]
|
||||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config]
|
||||
mode = "route"
|
||||
|
||||
[components.config.tool_parallelism]
|
||||
mode = "schedule"
|
||||
""",
|
||||
)
|
||||
assert mode == "schedule"
|
||||
|
||||
|
||||
def test_nemo_relay_llm_execution_middleware_calls_through_without_adaptive(monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
|
|
@ -555,8 +1131,8 @@ version = 1
|
|||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config]
|
||||
mode = "route"
|
||||
[components.config.tool_parallelism]
|
||||
mode = "observe_only"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
|
@ -582,10 +1158,131 @@ mode = "route"
|
|||
assert response == {"raw": True, "args": {"command": "pwd", "intercepted": True}}
|
||||
assert seen_args["intercepted"] is True
|
||||
execute_start = next(event for event in fake.events if event[0] == "tool.execute.start")
|
||||
assert execute_start[3]["data"]["mode"] == "route"
|
||||
assert execute_start[3]["data"]["mode"] == "observe_only"
|
||||
assert execute_start[3]["data"]["tool_call_id"] == "tool-1"
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_tool_execution_preserves_downstream_error(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
def native_like_execute(name, args, func, **kwargs):
|
||||
fake.events.append(("tool.execute.start", name, args, kwargs))
|
||||
try:
|
||||
return func({"intercepted": True, **args})
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"internal error: {type(exc).__name__}: {exc}") from None
|
||||
|
||||
fake.tools.execute = native_like_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
class ToolAuthError(Exception):
|
||||
status_code = 403
|
||||
|
||||
tool_error = ToolAuthError("tool auth failed")
|
||||
|
||||
def next_call(args):
|
||||
raise _wrapped_downstream_error(tool_error)
|
||||
|
||||
with pytest.raises(ToolAuthError) as caught:
|
||||
plugin.on_tool_execution_middleware(
|
||||
session_id="s1",
|
||||
tool_name="terminal",
|
||||
args={"command": "pwd"},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is tool_error
|
||||
assert caught.value.status_code == 403
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_tool_execution_keeps_unrelated_internal_error(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
relay_error = RuntimeError("internal error: relay setup failed")
|
||||
|
||||
def internal_error_execute(name, args, func, **kwargs):
|
||||
raise relay_error
|
||||
|
||||
fake.tools.execute = internal_error_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
plugin.on_tool_execution_middleware(
|
||||
session_id="s1",
|
||||
tool_name="terminal",
|
||||
args={"command": "pwd"},
|
||||
next_call=lambda args: {"raw": args},
|
||||
)
|
||||
|
||||
assert caught.value is relay_error
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_tool_execution_keeps_wrapped_relay_error_after_downstream_failure(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
fake = _FakeNemoRelay()
|
||||
relay_error = RuntimeError("internal error: RuntimeError: relay policy blocked after downstream")
|
||||
|
||||
def translated_execute(name, args, func, **kwargs):
|
||||
try:
|
||||
return func({"intercepted": True, **args})
|
||||
except Exception:
|
||||
raise relay_error
|
||||
|
||||
fake.tools.execute = translated_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
def next_call(args):
|
||||
raise _wrapped_downstream_error(RuntimeError("tool failed"))
|
||||
|
||||
with pytest.raises(RuntimeError) as caught:
|
||||
plugin.on_tool_execution_middleware(
|
||||
session_id="s1",
|
||||
tool_name="terminal",
|
||||
args={"command": "pwd"},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is relay_error
|
||||
|
||||
|
||||
def test_nemo_relay_adaptive_tool_execution_keeps_relay_translated_error(tmp_path, monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
|
||||
class RelayPolicyError(Exception):
|
||||
pass
|
||||
|
||||
relay_error = RelayPolicyError("relay policy blocked")
|
||||
|
||||
def translated_execute(name, args, func, **kwargs):
|
||||
try:
|
||||
return func({"intercepted": True, **args})
|
||||
except Exception:
|
||||
raise relay_error
|
||||
|
||||
fake.tools.execute = translated_execute
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
_enable_adaptive_plugin(tmp_path, monkeypatch)
|
||||
|
||||
tool_error = RuntimeError("tool failed")
|
||||
|
||||
def next_call(args):
|
||||
raise _wrapped_downstream_error(tool_error)
|
||||
|
||||
with pytest.raises(RelayPolicyError) as caught:
|
||||
plugin.on_tool_execution_middleware(
|
||||
session_id="s1",
|
||||
tool_name="terminal",
|
||||
args={"command": "pwd"},
|
||||
next_call=next_call,
|
||||
)
|
||||
|
||||
assert caught.value is relay_error
|
||||
|
||||
|
||||
def test_nemo_relay_tool_execution_middleware_calls_through_without_adaptive(monkeypatch):
|
||||
fake = _FakeNemoRelay()
|
||||
plugin = _fresh_plugin(monkeypatch, fake)
|
||||
|
|
@ -613,8 +1310,8 @@ version = 1
|
|||
kind = "adaptive"
|
||||
enabled = true
|
||||
|
||||
[components.config]
|
||||
mode = "route"
|
||||
[components.config.tool_parallelism]
|
||||
mode = "observe_only"
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue