mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-26 17:38:36 +00:00
fix(sessions): stop titling a session after the skill it invoked
generate_title() sent the first 500 characters of the user turn to the auxiliary model. On a /skill invocation those characters are the skill's own opening prose, so the session got named after the skill instead of the request — /work sessions came back as "Isolated Git Worktree Setup". Route the turn through describe_skill_invocation() first, so the titler sees what the user typed. Also keep only the first line of the response: a model that ignores "return ONLY the title" and answers the prompt would otherwise have a shell transcript stored as the title, truncated mid-command.
This commit is contained in:
parent
e4724ea455
commit
dcf5fc0eea
2 changed files with 114 additions and 1 deletions
|
|
@ -71,6 +71,27 @@ def _auto_title_enabled() -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _summarize_user_message(user_message: str) -> str:
|
||||
"""Collapse a slash-skill-expanded turn back to what the user typed.
|
||||
|
||||
A ``/skill`` invocation expands into a message that embeds the whole skill
|
||||
body, so feeding it to the titler verbatim titles the session after the
|
||||
*skill's* prose — "Kick off a task in a fresh isolated git worktree" — not
|
||||
after the user's request. Reuse the canonical scaffolding parser so the
|
||||
model sees ``/work — fix the title leak`` instead.
|
||||
"""
|
||||
if not user_message:
|
||||
return ""
|
||||
try:
|
||||
from agent.skill_commands import describe_skill_invocation
|
||||
|
||||
described = describe_skill_invocation(user_message)
|
||||
except Exception:
|
||||
logger.debug("Skill-scaffolding summary failed; titling raw", exc_info=True)
|
||||
return user_message
|
||||
return described if described is not None else user_message
|
||||
|
||||
|
||||
def generate_title(
|
||||
user_message: str,
|
||||
assistant_response: str,
|
||||
|
|
@ -110,7 +131,7 @@ def generate_title(
|
|||
logger.debug("Title runtime validator raised; proceeding", exc_info=True)
|
||||
|
||||
# Truncate long messages to keep the request small
|
||||
user_snippet = user_message[:500] if user_message else ""
|
||||
user_snippet = _summarize_user_message(user_message)[:500]
|
||||
assistant_snippet = assistant_response[:500] if assistant_response else ""
|
||||
|
||||
language = _title_language()
|
||||
|
|
@ -143,6 +164,11 @@ def generate_title(
|
|||
title = title.strip('"\'')
|
||||
if title.lower().startswith("title:"):
|
||||
title = title[6:].strip()
|
||||
# A title is one line. A model that ignores "return ONLY the title" and
|
||||
# answers the prompt instead (a shell transcript, a bulleted plan) would
|
||||
# otherwise be stored verbatim and truncated mid-command. Keep the first
|
||||
# non-empty line — the closest thing to a title in that response.
|
||||
title = next((line.strip() for line in title.splitlines() if line.strip()), "")
|
||||
# Enforce reasonable length
|
||||
if len(title) > 80:
|
||||
title = title[:77] + "..."
|
||||
|
|
|
|||
|
|
@ -212,6 +212,93 @@ class TestGenerateTitle:
|
|||
user_content = captured_kwargs["messages"][1]["content"]
|
||||
assert len(user_content) < 1100 # 500 + 500 + formatting
|
||||
|
||||
def test_skill_invocation_is_titled_from_what_the_user_typed(self):
|
||||
"""A /skill turn embeds the whole skill body — the titler must never
|
||||
see it, or the session gets named after the SKILL, not the request."""
|
||||
skill_body = "Kick off a task in a fresh isolated git worktree. " * 20
|
||||
expanded = (
|
||||
'[IMPORTANT: The user has invoked the "work" skill, indicating they want '
|
||||
"you to follow its instructions. The full skill content is loaded below.]\n\n"
|
||||
f"{skill_body}\n\n"
|
||||
"The user has provided the following instruction alongside the skill "
|
||||
"invocation: fix the session title leak"
|
||||
)
|
||||
captured_kwargs = {}
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
resp = MagicMock()
|
||||
resp.choices = [MagicMock()]
|
||||
resp.choices[0].message.content = "Fixing The Session Title Leak"
|
||||
return resp
|
||||
|
||||
with patch("agent.title_generator.call_llm", side_effect=mock_call_llm):
|
||||
generate_title(expanded, "On it.")
|
||||
|
||||
sent = captured_kwargs["messages"][1]["content"]
|
||||
assert "/work — fix the session title leak" in sent
|
||||
assert "worktree" not in sent
|
||||
assert "IMPORTANT" not in sent
|
||||
|
||||
def test_bare_skill_invocation_still_titles(self):
|
||||
"""No typed instruction — the titler gets the command, not the body."""
|
||||
expanded = (
|
||||
'[IMPORTANT: The user has invoked the "weather-forecast-lookup" skill, '
|
||||
"indicating they want you to follow its instructions. The full skill "
|
||||
"content is loaded below.]\n\nPull clean multi-day forecasts."
|
||||
)
|
||||
captured_kwargs = {}
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
resp = MagicMock()
|
||||
resp.choices = [MagicMock()]
|
||||
resp.choices[0].message.content = "Weather Forecast Lookup"
|
||||
return resp
|
||||
|
||||
with patch("agent.title_generator.call_llm", side_effect=mock_call_llm):
|
||||
assert generate_title(expanded, "Here you go.") == "Weather Forecast Lookup"
|
||||
|
||||
sent = captured_kwargs["messages"][1]["content"]
|
||||
assert "/weather-forecast-lookup" in sent
|
||||
assert "Pull clean multi-day forecasts" not in sent
|
||||
|
||||
def test_plain_message_reaches_the_titler_unchanged(self):
|
||||
"""The scaffolding summary must not touch an ordinary user turn."""
|
||||
captured_kwargs = {}
|
||||
|
||||
def mock_call_llm(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
resp = MagicMock()
|
||||
resp.choices = [MagicMock()]
|
||||
resp.choices[0].message.content = "A Title"
|
||||
return resp
|
||||
|
||||
with patch("agent.title_generator.call_llm", side_effect=mock_call_llm):
|
||||
generate_title("fix the session title leak", "On it.")
|
||||
|
||||
assert "fix the session title leak" in captured_kwargs["messages"][1]["content"]
|
||||
|
||||
def test_multiline_answer_collapses_to_its_first_line(self):
|
||||
"""A model that answers the prompt instead of titling it must not have
|
||||
a shell transcript stored as the session title."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = (
|
||||
"macOS Disk Cleanup\n\n$ df -h /\nFilesystem Size Used Avail\n"
|
||||
)
|
||||
|
||||
with patch("agent.title_generator.call_llm", return_value=mock_response):
|
||||
assert generate_title("clean my disk", "Sure.") == "macOS Disk Cleanup"
|
||||
|
||||
def test_leading_blank_lines_do_not_empty_the_title(self):
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "\n\n Real Title \nmore prose"
|
||||
|
||||
with patch("agent.title_generator.call_llm", return_value=mock_response):
|
||||
assert generate_title("q", "a") == "Real Title"
|
||||
|
||||
def test_skips_when_title_generation_disabled(self):
|
||||
"""auxiliary.title_generation.enabled=false disables automatic titles."""
|
||||
config = {"auxiliary": {"title_generation": {"enabled": False}}}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue