diff --git a/tests/tools/test_clarify_tool.py b/tests/tools/test_clarify_tool.py index 0c38961dd8d..d77214ee3d0 100644 --- a/tests/tools/test_clarify_tool.py +++ b/tests/tools/test_clarify_tool.py @@ -10,6 +10,7 @@ from tools.clarify_tool import ( MAX_CHOICES, CLARIFY_SCHEMA, _flatten_choice, + _sanitize_clarify_text, ) @@ -257,3 +258,57 @@ class TestClarifySchema: def test_max_choices_is_four(self): """MAX_CHOICES constant should be 4.""" assert MAX_CHOICES == 4 + + +class TestSanitizeClarifyText: + """Tests for _sanitize_clarify_text — stripping Discord/chat markup.""" + + def test_strips_user_mentions(self): + """Should strip <@123> user mentions.""" + assert _sanitize_clarify_text("Hello <@123> there") == "Hello there" + + def test_strips_nickname_mentions(self): + """Should strip <@!123> nickname mentions.""" + assert _sanitize_clarify_text("Hey <@!456>!") == "Hey !" + + def test_strips_role_mentions(self): + """Should strip <@&123> role mentions.""" + assert _sanitize_clarify_text("<@&789> check this") == "check this" + + def test_strips_channel_mentions(self): + """Should strip <#123> channel mentions.""" + assert _sanitize_clarify_text("Join <#123>?") == "Join ?" + assert _sanitize_clarify_text("<#123456789012345678>") == "" + + def test_strips_everyone(self): + """Should strip standalone @everyone.""" + assert _sanitize_clarify_text("@everyone please read") == "please read" + + def test_strips_here(self): + """Should strip standalone @here.""" + assert _sanitize_clarify_text("@here quick question") == "quick question" + + def test_preserves_plain_text(self): + """Normal text should pass through unchanged.""" + assert _sanitize_clarify_text("What is your name?") == "What is your name?" + + def test_preserves_valid_at_mentions_in_context(self): + """Legitimate @-prefixed words should be preserved.""" + assert _sanitize_clarify_text("Use @property decorator") == "Use @property decorator" + assert _sanitize_clarify_text("The @pytest.fixture syntax") == "The @pytest.fixture syntax" + + def test_does_not_partial_match_here(self): + """Should NOT strip @here from inside longer words (word boundary regression).""" + assert _sanitize_clarify_text("@hereditary disease") == "@hereditary disease" + assert _sanitize_clarify_text("contact@hereford.org") == "contact@hereford.org" + + def test_strips_multiple_mentions(self): + """Should strip all mention types from a single string.""" + result = _sanitize_clarify_text("<@123> and <@&456> and <#789> @everyone @here") + assert result == "and and" + + def test_question_with_mention_example(self): + """Real-world example: @bot mention with natural language.""" + result = _sanitize_clarify_text("<@&1234567890> how is the provided information?") + assert "how is the provided information?" in result + assert "<@&" not in result diff --git a/tools/clarify_tool.py b/tools/clarify_tool.py index e831d38fb4d..54c06e6cb03 100644 --- a/tools/clarify_tool.py +++ b/tools/clarify_tool.py @@ -12,6 +12,7 @@ a thin dispatcher that delegates to a platform-provided callback. """ import json +import re from typing import List, Optional, Callable @@ -19,6 +20,19 @@ from typing import List, Optional, Callable # A 5th "Other (type your answer)" option is always appended by the UI. MAX_CHOICES = 4 +# Discord/chat-platform markup that has no meaning in clarify questions. +# LLMs sometimes hallucinate these into question text. +_DISCORD_MENTION_RE = re.compile( + r"<@[!&]?\d+>" # user / role mentions: <@123>, <@!123>, <@&123> + r"|<#\d+>" # channel mentions: <#123> + r"|@(?:everyone|here)\b" # mass mentions (word boundary prevents partial match) +) + + +def _sanitize_clarify_text(text: str) -> str: + """Strip platform-specific markup that LLMs may hallucinate into question text.""" + return _DISCORD_MENTION_RE.sub("", text).strip() + def _flatten_choice(c) -> str: """Coerce a single choice into its user-facing display string. @@ -75,7 +89,9 @@ def clarify_tool( if not question or not question.strip(): return tool_error("Question text is required.") - question = question.strip() + question = _sanitize_clarify_text(question) + if not question: + return tool_error("Question text is required (empty after sanitizing).") # Validate and trim choices if choices is not None: @@ -86,7 +102,12 @@ def clarify_tool( # user-facing text here — the single platform-agnostic entry point — # so the CLI panel, Discord buttons, and Telegram list all render clean # text and the resolved answer is never a raw Python dict repr. - choices = [s for s in (_flatten_choice(c) for c in choices) if s] + # _sanitize_clarify_text then strips hallucinated Discord mentions. + choices = [ + s + for s in (_sanitize_clarify_text(_flatten_choice(c)) for c in choices) + if s + ] if len(choices) > MAX_CHOICES: choices = choices[:MAX_CHOICES] if not choices: @@ -154,7 +175,9 @@ CLARIFY_SCHEMA = { "description": ( "The question itself, and ONLY the question (e.g. 'Which " "deployment target?'). Do NOT embed the answer options here " - "— pass them as separate elements in `choices`." + "— pass them as separate elements in `choices`. Use plain " + "text only — no Discord/Telegram markup, no @mentions, no " + "platform-specific formatting." ), }, "choices": {