mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
319 lines
11 KiB
Python
319 lines
11 KiB
Python
"""Tests for agent.title_generator — auto-generated session titles."""
|
|
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
|
|
from agent.title_generator import (
|
|
generate_title,
|
|
auto_title_session,
|
|
maybe_auto_title,
|
|
_title_language,
|
|
)
|
|
from hermes_state import SessionDB
|
|
|
|
|
|
class TestGenerateTitle:
|
|
"""Unit tests for generate_title()."""
|
|
|
|
|
|
|
|
|
|
def test_title_language_reads_config(self):
|
|
cfg = {"auxiliary": {"title_generation": {"language": " French "}}}
|
|
|
|
with patch("hermes_cli.config.load_config", return_value=cfg):
|
|
assert _title_language() == "French"
|
|
with patch("hermes_cli.config.load_config", return_value={}):
|
|
assert _title_language() == ""
|
|
with patch("hermes_cli.config.load_config", side_effect=RuntimeError("bad config")):
|
|
assert _title_language() == ""
|
|
|
|
def test_default_timeout_delegates_to_auxiliary_config(self):
|
|
captured_kwargs = {}
|
|
|
|
def mock_call_llm(**kwargs):
|
|
captured_kwargs.update(kwargs)
|
|
resp = MagicMock()
|
|
resp.choices = [MagicMock()]
|
|
resp.choices[0].message.content = "Configured Timeout"
|
|
return resp
|
|
|
|
with patch("agent.title_generator.call_llm", side_effect=mock_call_llm):
|
|
assert generate_title("question", "answer") == "Configured Timeout"
|
|
|
|
assert captured_kwargs["task"] == "title_generation"
|
|
assert captured_kwargs["timeout"] is None
|
|
|
|
|
|
|
|
def test_strips_think_blocks(self):
|
|
"""Reasoning-model output wrapped in <think>...</think> must not
|
|
leak into the session title."""
|
|
mock_response = MagicMock()
|
|
mock_response.choices = [MagicMock()]
|
|
mock_response.choices[0].message.content = (
|
|
"<think>The user wants a title. I'll summarize the topic "
|
|
"concisely.</think>Debugging Python Import Errors"
|
|
)
|
|
|
|
with patch("agent.title_generator.call_llm", return_value=mock_response):
|
|
title = generate_title("help me fix this import", "Sure...")
|
|
assert title == "Debugging Python Import Errors"
|
|
assert "<think>" not in title
|
|
assert "summarize" not in title
|
|
|
|
def test_strips_unterminated_think_block(self):
|
|
"""An unterminated <think> block (no close tag) must still be
|
|
stripped so the leaked reasoning doesn't become the title."""
|
|
mock_response = MagicMock()
|
|
mock_response.choices = [MagicMock()]
|
|
mock_response.choices[0].message.content = (
|
|
"<think>Let me reason about a good title for this session"
|
|
)
|
|
|
|
with patch("agent.title_generator.call_llm", return_value=mock_response):
|
|
title = generate_title("hello", "hi there")
|
|
# Everything from the unterminated open tag onward is stripped,
|
|
# leaving nothing → None.
|
|
assert title is None
|
|
|
|
|
|
def test_truncates_long_titles(self):
|
|
mock_response = MagicMock()
|
|
mock_response.choices = [MagicMock()]
|
|
mock_response.choices[0].message.content = "A" * 100
|
|
|
|
with patch("agent.title_generator.call_llm", return_value=mock_response):
|
|
title = generate_title("question", "answer")
|
|
assert len(title) == 80
|
|
assert title.endswith("...")
|
|
|
|
|
|
|
|
def test_invokes_failure_callback_on_exception(self):
|
|
"""failure_callback must fire so the user sees a warning (issue #15775)."""
|
|
captured = []
|
|
|
|
def _cb(task, exc):
|
|
captured.append((task, exc))
|
|
|
|
exc = RuntimeError("openrouter 402: credits exhausted")
|
|
with patch("agent.title_generator.call_llm", side_effect=exc):
|
|
result = generate_title("question", "answer", failure_callback=_cb)
|
|
|
|
assert result is None
|
|
assert len(captured) == 1
|
|
assert captured[0][0] == "title generation"
|
|
assert captured[0][1] is exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestAutoTitleSession:
|
|
"""Tests for auto_title_session() — the sync worker function."""
|
|
|
|
|
|
|
|
|
|
def test_does_not_overwrite_title_set_immediately_before_conditional_write(
|
|
self, tmp_path
|
|
):
|
|
db = SessionDB(tmp_path / "state.db")
|
|
db.create_session(session_id="sess-1", source="cli")
|
|
seen = []
|
|
|
|
def generate_after_manual_title(*_args, **_kwargs):
|
|
db.set_session_title("sess-1", "Manual Title")
|
|
return "Auto Title"
|
|
|
|
with patch(
|
|
"agent.title_generator.generate_title",
|
|
side_effect=generate_after_manual_title,
|
|
):
|
|
auto_title_session(
|
|
db,
|
|
"sess-1",
|
|
"hi",
|
|
"hello",
|
|
title_callback=seen.append,
|
|
)
|
|
|
|
assert db.get_session_title("sess-1") == "Manual Title"
|
|
assert seen == []
|
|
|
|
def test_invokes_title_callback_after_setting_title(self):
|
|
db = MagicMock()
|
|
db.get_session_title.return_value = None
|
|
db.set_auto_title_if_empty.return_value = True
|
|
seen = []
|
|
with patch("agent.title_generator.generate_title", return_value="Readable Session"):
|
|
auto_title_session(
|
|
db,
|
|
"sess-1",
|
|
"hello",
|
|
"hi there",
|
|
title_callback=seen.append,
|
|
)
|
|
db.set_auto_title_if_empty.assert_called_once_with("sess-1", "Readable Session")
|
|
assert seen == ["Readable Session"]
|
|
|
|
|
|
|
|
def test_body_exception_routed_to_failure_callback(self):
|
|
db = MagicMock()
|
|
db.get_session_title.return_value = None
|
|
seen = []
|
|
|
|
boom = ImportError("stale module")
|
|
with patch("agent.title_generator._auto_title_session", side_effect=boom):
|
|
auto_title_session(
|
|
db,
|
|
"sess-1",
|
|
"hi",
|
|
"hello",
|
|
failure_callback=lambda task, exc: seen.append((task, exc)),
|
|
)
|
|
assert seen == [("title generation", boom)]
|
|
|
|
|
|
|
|
class TestMaybeAutoTitle:
|
|
"""Tests for maybe_auto_title() — the fire-and-forget entry point."""
|
|
|
|
def test_skips_if_not_first_exchange(self):
|
|
"""Should not fire for conversations with more than 2 user messages."""
|
|
db = MagicMock()
|
|
history = [
|
|
{"role": "user", "content": "first"},
|
|
{"role": "assistant", "content": "response 1"},
|
|
{"role": "user", "content": "second"},
|
|
{"role": "assistant", "content": "response 2"},
|
|
{"role": "user", "content": "third"},
|
|
{"role": "assistant", "content": "response 3"},
|
|
]
|
|
|
|
with patch("agent.title_generator.auto_title_session") as mock_auto:
|
|
maybe_auto_title(db, "sess-1", "third", "response 3", history)
|
|
# Wait briefly for any thread to start
|
|
import time
|
|
time.sleep(0.1)
|
|
mock_auto.assert_not_called()
|
|
|
|
def test_fires_on_first_exchange(self):
|
|
"""Should fire a background thread for the first exchange."""
|
|
db = MagicMock()
|
|
db.get_session_title.return_value = None
|
|
history = [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": "hi there"},
|
|
]
|
|
|
|
with patch("agent.title_generator.auto_title_session") as mock_auto:
|
|
import threading
|
|
called = threading.Event()
|
|
mock_auto.side_effect = lambda *a, **k: called.set()
|
|
maybe_auto_title(db, "sess-1", "hello", "hi there", history)
|
|
# Event-based wait: sleep-sync flaked when the daemon thread
|
|
# wasn't scheduled within the fixed nap on a loaded runner.
|
|
assert called.wait(timeout=10), "auto_title thread never ran"
|
|
mock_auto.assert_called_once_with(
|
|
db,
|
|
"sess-1",
|
|
"hello",
|
|
"hi there",
|
|
failure_callback=None,
|
|
main_runtime=None,
|
|
title_callback=None,
|
|
runtime_validator=None,
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestAutoTitleDuplicateHandling:
|
|
"""Duplicate auto-title handling and not-found hardening (#50537)."""
|
|
|
|
def test_dedupes_duplicate_title_via_lineage(self):
|
|
db = MagicMock()
|
|
db.get_session_title.return_value = None
|
|
# Atomic write path: collision raises ValueError, retry persists.
|
|
db.set_auto_title_if_empty.side_effect = [ValueError("in use"), True]
|
|
db.get_next_title_in_lineage.return_value = "Debugging Import Error #2"
|
|
with patch(
|
|
"agent.title_generator.generate_title",
|
|
return_value="Debugging Import Error",
|
|
):
|
|
seen = []
|
|
auto_title_session(db, "sess-1", "hi", "hello", title_callback=seen.append)
|
|
db.get_next_title_in_lineage.assert_called_once_with("Debugging Import Error")
|
|
assert db.set_auto_title_if_empty.call_args_list[-1][0] == (
|
|
"sess-1",
|
|
"Debugging Import Error #2",
|
|
)
|
|
# callback fires with the actually-persisted (deduped) title
|
|
assert seen == ["Debugging Import Error #2"]
|
|
|
|
|
|
|
|
def test_manual_title_race_skips_without_callback(self):
|
|
# Atomic predicate fails (manual /title landed while generation was in
|
|
# flight) -> nothing persisted, no callback fired.
|
|
from agent.title_generator import _persist_session_title
|
|
db = MagicMock()
|
|
db.set_auto_title_if_empty.return_value = False
|
|
assert _persist_session_title(db, "sess-1", "Some Title") is None
|
|
db.set_session_title.assert_not_called()
|
|
|
|
|
|
|
|
class TestRuntimeValidator:
|
|
"""runtime_validator gating (#19027): a stale background title request
|
|
must not fire when the session's model/provider changed after spawn."""
|
|
|
|
|
|
|
|
def test_broken_validator_fails_open(self):
|
|
mock_response = MagicMock()
|
|
mock_response.choices = [MagicMock()]
|
|
mock_response.choices[0].message.content = "Resilient Title"
|
|
|
|
def _bad_validator():
|
|
raise RuntimeError("validator gone")
|
|
|
|
with patch("agent.title_generator.call_llm", return_value=mock_response) as mock_llm:
|
|
title = generate_title(
|
|
"question", "answer",
|
|
runtime_validator=_bad_validator,
|
|
)
|
|
assert title == "Resilient Title"
|
|
mock_llm.assert_called_once()
|
|
|
|
def test_forwards_runtime_validator_to_worker(self):
|
|
db = MagicMock()
|
|
db.get_session_title.return_value = None
|
|
history = [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": "hi there"},
|
|
]
|
|
|
|
def _v():
|
|
return True
|
|
|
|
with patch("agent.title_generator.auto_title_session") as mock_auto:
|
|
import threading
|
|
called = threading.Event()
|
|
mock_auto.side_effect = lambda *a, **k: called.set()
|
|
maybe_auto_title(db, "sess-1", "hello", "hi there", history, runtime_validator=_v)
|
|
assert called.wait(timeout=10), "auto_title thread never ran"
|
|
kwargs = mock_auto.call_args.kwargs
|
|
assert kwargs["runtime_validator"] is _v
|