mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-08 03:01:47 +00:00
test+docs: cover transform_llm_output hook + release author map
- tests/test_transform_llm_output_hook.py: dispatch semantics (kwargs contract, first-non-empty-string-wins, empty-string pass-through, raising-plugin fail-open, no-plugins = no-op) - tests/hermes_cli/test_plugins.py: assert the new hook name is in VALID_HOOKS alongside the other transform_* hooks - website/docs/user-guide/features/hooks.md: summary-table entry + full section mirroring transform_tool_result / transform_terminal_output - scripts/release.py: map barnacleboy.jezzahehn@agentmail.to -> JezzaHehn (existing entry only covers the gmail address)
This commit is contained in:
parent
c3be6ec184
commit
47bf5d7ecb
4 changed files with 205 additions and 0 deletions
|
|
@ -265,6 +265,7 @@ AUTHOR_MAP = {
|
||||||
"36056348+sirEven@users.noreply.github.com": "sirEven",
|
"36056348+sirEven@users.noreply.github.com": "sirEven",
|
||||||
"70424851+insecurejezza@users.noreply.github.com": "insecurejezza",
|
"70424851+insecurejezza@users.noreply.github.com": "insecurejezza",
|
||||||
"jezzahehn@gmail.com": "JezzaHehn",
|
"jezzahehn@gmail.com": "JezzaHehn",
|
||||||
|
"barnacleboy.jezzahehn@agentmail.to": "JezzaHehn",
|
||||||
"254021826+dodo-reach@users.noreply.github.com": "dodo-reach",
|
"254021826+dodo-reach@users.noreply.github.com": "dodo-reach",
|
||||||
"259807879+Bartok9@users.noreply.github.com": "Bartok9",
|
"259807879+Bartok9@users.noreply.github.com": "Bartok9",
|
||||||
"270082434+crayfish-ai@users.noreply.github.com": "crayfish-ai",
|
"270082434+crayfish-ai@users.noreply.github.com": "crayfish-ai",
|
||||||
|
|
|
||||||
|
|
@ -330,6 +330,7 @@ class TestPluginHooks:
|
||||||
assert "post_api_request" in VALID_HOOKS
|
assert "post_api_request" in VALID_HOOKS
|
||||||
assert "transform_terminal_output" in VALID_HOOKS
|
assert "transform_terminal_output" in VALID_HOOKS
|
||||||
assert "transform_tool_result" in VALID_HOOKS
|
assert "transform_tool_result" in VALID_HOOKS
|
||||||
|
assert "transform_llm_output" in VALID_HOOKS
|
||||||
|
|
||||||
def test_valid_hooks_include_pre_gateway_dispatch(self):
|
def test_valid_hooks_include_pre_gateway_dispatch(self):
|
||||||
assert "pre_gateway_dispatch" in VALID_HOOKS
|
assert "pre_gateway_dispatch" in VALID_HOOKS
|
||||||
|
|
|
||||||
159
tests/test_transform_llm_output_hook.py
Normal file
159
tests/test_transform_llm_output_hook.py
Normal file
|
|
@ -0,0 +1,159 @@
|
||||||
|
"""Tests for the ``transform_llm_output`` plugin hook.
|
||||||
|
|
||||||
|
The hook fires inside ``AIAgent.run_conversation`` once the tool-calling
|
||||||
|
loop has produced a final response. Driving the full agent loop from a
|
||||||
|
unit test would be prohibitively heavy, so these tests exercise the
|
||||||
|
invoke_hook dispatch semantics that the wiring in ``run_agent.py``
|
||||||
|
depends on:
|
||||||
|
|
||||||
|
for _hook_result in _transform_results:
|
||||||
|
if isinstance(_hook_result, str) and _hook_result:
|
||||||
|
final_response = _hook_result
|
||||||
|
break # First non-empty string wins
|
||||||
|
|
||||||
|
Mirrors ``test_transform_tool_result_hook.py`` which tests the equivalent
|
||||||
|
contract for the generic tool-result seam.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
import hermes_cli.plugins as plugins_mod
|
||||||
|
from hermes_cli.plugins import PluginManager, VALID_HOOKS
|
||||||
|
|
||||||
|
|
||||||
|
def _make_enabled_plugin(hermes_home: Path, name: str, register_body: str) -> Path:
|
||||||
|
"""Create a plugin under <hermes_home>/plugins/<name> and opt it in."""
|
||||||
|
plugin_dir = hermes_home / "plugins" / name
|
||||||
|
plugin_dir.mkdir(parents=True)
|
||||||
|
(plugin_dir / "plugin.yaml").write_text(
|
||||||
|
yaml.safe_dump({"name": name, "version": "0.1.0"}), encoding="utf-8",
|
||||||
|
)
|
||||||
|
(plugin_dir / "__init__.py").write_text(
|
||||||
|
"def register(ctx):\n"
|
||||||
|
f" {register_body}\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
cfg_path = hermes_home / "config.yaml"
|
||||||
|
cfg = {}
|
||||||
|
if cfg_path.exists():
|
||||||
|
cfg = yaml.safe_load(cfg_path.read_text()) or {}
|
||||||
|
cfg.setdefault("plugins", {}).setdefault("enabled", []).append(name)
|
||||||
|
cfg_path.write_text(yaml.safe_dump(cfg), encoding="utf-8")
|
||||||
|
return plugin_dir
|
||||||
|
|
||||||
|
|
||||||
|
def test_transform_llm_output_in_valid_hooks():
|
||||||
|
assert "transform_llm_output" in VALID_HOOKS
|
||||||
|
|
||||||
|
|
||||||
|
def test_hook_receives_expected_kwargs(tmp_path, monkeypatch):
|
||||||
|
"""Hook callback should see response_text + session_id + model + platform."""
|
||||||
|
hermes_home = tmp_path / "hermes_test"
|
||||||
|
hermes_home.mkdir(exist_ok=True)
|
||||||
|
_make_enabled_plugin(
|
||||||
|
hermes_home, "capture_hook",
|
||||||
|
register_body=(
|
||||||
|
'ctx.register_hook("transform_llm_output", '
|
||||||
|
'lambda **kw: f"{kw[\'response_text\']}|{kw[\'session_id\']}|'
|
||||||
|
'{kw[\'model\']}|{kw[\'platform\']}")'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||||
|
|
||||||
|
mgr = PluginManager()
|
||||||
|
mgr.discover_and_load()
|
||||||
|
|
||||||
|
results = mgr.invoke_hook(
|
||||||
|
"transform_llm_output",
|
||||||
|
response_text="hello world",
|
||||||
|
session_id="s1",
|
||||||
|
model="anthropic/claude-sonnet-4.6",
|
||||||
|
platform="cli",
|
||||||
|
)
|
||||||
|
assert results == ["hello world|s1|anthropic/claude-sonnet-4.6|cli"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_first_non_empty_string_wins_semantics():
|
||||||
|
"""Simulate the run_agent.py loop: first non-empty string replaces text."""
|
||||||
|
# The dispatch contract: invoke_hook returns a list; the caller walks
|
||||||
|
# it and stops at the first isinstance(_, str) and _.
|
||||||
|
hook_returns = [None, "", {"bad": True}, 123, "first-winner", "second"]
|
||||||
|
|
||||||
|
final_response = "original"
|
||||||
|
for _hook_result in hook_returns:
|
||||||
|
if isinstance(_hook_result, str) and _hook_result:
|
||||||
|
final_response = _hook_result
|
||||||
|
break
|
||||||
|
|
||||||
|
assert final_response == "first-winner"
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_string_return_leaves_response_unchanged():
|
||||||
|
"""Empty string must not replace the response (pass-through signal)."""
|
||||||
|
hook_returns = [""]
|
||||||
|
|
||||||
|
final_response = "original"
|
||||||
|
for _hook_result in hook_returns:
|
||||||
|
if isinstance(_hook_result, str) and _hook_result:
|
||||||
|
final_response = _hook_result
|
||||||
|
break
|
||||||
|
|
||||||
|
assert final_response == "original"
|
||||||
|
|
||||||
|
|
||||||
|
def test_hook_exception_does_not_replace_response(tmp_path, monkeypatch):
|
||||||
|
"""A plugin raising an exception must not break hook dispatch.
|
||||||
|
|
||||||
|
PluginManager.invoke_hook catches per-callback exceptions, logs a
|
||||||
|
warning, and continues — so a raising plugin contributes no entry
|
||||||
|
to the results list, and the walk in run_agent.py finds nothing to
|
||||||
|
replace with.
|
||||||
|
"""
|
||||||
|
hermes_home = tmp_path / "hermes_test"
|
||||||
|
hermes_home.mkdir(exist_ok=True)
|
||||||
|
_make_enabled_plugin(
|
||||||
|
hermes_home, "raising_hook",
|
||||||
|
register_body=(
|
||||||
|
'def _boom(**kw):\n'
|
||||||
|
' raise RuntimeError("boom")\n'
|
||||||
|
' ctx.register_hook("transform_llm_output", _boom)'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||||
|
|
||||||
|
mgr = PluginManager()
|
||||||
|
mgr.discover_and_load()
|
||||||
|
|
||||||
|
results = mgr.invoke_hook(
|
||||||
|
"transform_llm_output",
|
||||||
|
response_text="keep me",
|
||||||
|
session_id="s1",
|
||||||
|
model="m",
|
||||||
|
platform="cli",
|
||||||
|
)
|
||||||
|
|
||||||
|
final_response = "keep me"
|
||||||
|
for _hook_result in results:
|
||||||
|
if isinstance(_hook_result, str) and _hook_result:
|
||||||
|
final_response = _hook_result
|
||||||
|
break
|
||||||
|
|
||||||
|
assert final_response == "keep me"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_plugins_returns_empty_results(tmp_path, monkeypatch):
|
||||||
|
"""With no plugins loaded, invoke_hook returns [] and the response is unchanged."""
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes_empty"))
|
||||||
|
plugins_mod._plugin_manager = PluginManager()
|
||||||
|
|
||||||
|
mgr = plugins_mod._plugin_manager
|
||||||
|
results = mgr.invoke_hook(
|
||||||
|
"transform_llm_output",
|
||||||
|
response_text="unchanged",
|
||||||
|
session_id="",
|
||||||
|
model="m",
|
||||||
|
platform="",
|
||||||
|
)
|
||||||
|
assert results == []
|
||||||
|
|
@ -387,6 +387,7 @@ def register(ctx):
|
||||||
| [`post_approval_response`](#post_approval_response) | User responded to an approval prompt (or it timed out) | ignored |
|
| [`post_approval_response`](#post_approval_response) | User responded to an approval prompt (or it timed out) | ignored |
|
||||||
| [`transform_tool_result`](#transform_tool_result) | After any tool returns, before the result is handed back to the model | `str` to replace the result, `None` to leave unchanged |
|
| [`transform_tool_result`](#transform_tool_result) | After any tool returns, before the result is handed back to the model | `str` to replace the result, `None` to leave unchanged |
|
||||||
| [`transform_terminal_output`](#transform_terminal_output) | Inside the `terminal` tool, before truncation/ANSI-strip/redact | `str` to replace the raw output, `None` to leave unchanged |
|
| [`transform_terminal_output`](#transform_terminal_output) | Inside the `terminal` tool, before truncation/ANSI-strip/redact | `str` to replace the raw output, `None` to leave unchanged |
|
||||||
|
| [`transform_llm_output`](#transform_llm_output) | After the tool-calling loop completes, before the final response is delivered | `str` to replace the response text, `None`/empty to leave unchanged |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -1093,6 +1094,49 @@ Pairs well with `transform_tool_result` (which covers every other tool).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
### `transform_llm_output`
|
||||||
|
|
||||||
|
Fires **once per turn** after the tool-calling loop completes and the model has produced a final response, **before** that response is delivered to the user (CLI, gateway, or programmatic caller). Lets a plugin rewrite the assistant's final text using classical-programming methods — no extra inference tokens burned on SOUL flavor text or a skill-driven transform.
|
||||||
|
|
||||||
|
**Callback signature:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def my_callback(
|
||||||
|
response_text: str,
|
||||||
|
session_id: str,
|
||||||
|
model: str,
|
||||||
|
platform: str,
|
||||||
|
**kwargs,
|
||||||
|
) -> str | None:
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Type | Description |
|
||||||
|
|-----------|------|-------------|
|
||||||
|
| `response_text` | `str` | The assistant's final response text for this turn. |
|
||||||
|
| `session_id` | `str` | Session ID for this conversation (may be empty for one-shot runs). |
|
||||||
|
| `model` | `str` | Model name that produced the response (e.g. `anthropic/claude-sonnet-4.6`). |
|
||||||
|
| `platform` | `str` | Delivery platform (`cli`, `telegram`, `discord`, …; empty when unset). |
|
||||||
|
|
||||||
|
**Return value:** Non-empty `str` to replace the response text, `None` or empty string to leave it unchanged. **First non-empty string wins** when multiple plugins register — mirroring `transform_tool_result`.
|
||||||
|
|
||||||
|
**Use cases:** Apply a personality/vocabulary transform (pirate-speak, Spongebob), redact user-specific identifiers from the final text, append a project-specific signature footer, enforce a house style guide without burning tokens on SOUL instructions.
|
||||||
|
|
||||||
|
```python
|
||||||
|
import os, re
|
||||||
|
|
||||||
|
def spongebob(response_text, **kwargs):
|
||||||
|
if os.environ.get("SPONGEBOB_MODE") != "on":
|
||||||
|
return None # pass through unchanged
|
||||||
|
return re.sub(r"!", "!! Tartar sauce!", response_text)
|
||||||
|
|
||||||
|
def register(ctx):
|
||||||
|
ctx.register_hook("transform_llm_output", spongebob)
|
||||||
|
```
|
||||||
|
|
||||||
|
The hook is guarded on a non-empty, non-interrupted response — it will not fire on stop-button interrupts or empty turns. Exceptions are logged as warnings and do not break agent execution.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Shell Hooks
|
## Shell Hooks
|
||||||
|
|
||||||
Declare shell-script hooks in your `cli-config.yaml` and Hermes will run them as subprocesses whenever the corresponding plugin-hook event fires — in both CLI and gateway sessions. No Python plugin authoring required.
|
Declare shell-script hooks in your `cli-config.yaml` and Hermes will run them as subprocesses whenever the corresponding plugin-hook event fires — in both CLI and gateway sessions. No Python plugin authoring required.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue