fix(tts): unify TTS text preprocessing behind one shared cleaner

Consolidates all TTS text-preparation paths onto
tools/tts_text_normalize.prepare_spoken_text:

- strip_nonspoken_blocks: removes <think> reasoning blocks (#34213,
  incl. unterminated streaming blocks) and the end-of-turn
  file-mutation verifier footer emitted by run_agent.py (#40772).
- flatten_newlines_for_payload: collapses newlines into sentence
  breaks so newline-sensitive OpenAI-compatible providers (Kokoro)
  speak the whole script instead of truncating at the first newline
  (#9004).
- tools/tts_tool._strip_markdown_for_tts (voice-mode streaming + web
  dashboard path) now delegates to the shared cleaner, with the legacy
  regex pipeline kept as a best-effort fallback.
- hermes_cli/voice.py speak_text and cli.py _voice_speak_response now
  use the shared cleaner instead of their own duplicated regex
  pipelines.
- gateway auto-TTS fallback also strips think blocks.

Tests: tests/tools/test_tts_prepare_spoken.py covers think blocks,
verifier footer, emoji, newline flattening, and the shared-cleaner
wiring on the tool/streaming/gateway paths. Updated the header
expectation in test_voice_cli_integration.py for the heading-fold
behavior of the shared cleaner.

Closes #34213, #9004, #40772

cli.py

@@ -11950,19 +11950,26 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
             from tools.tts_tool import text_to_speech_tool
             from tools.voice_mode import play_audio_file
 
-            # Strip markdown and non-speech content for cleaner TTS
-            tts_text = text[:4000] if len(text) > 4000 else text
-            tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text)   # fenced code blocks
-            tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text)  # [text](url) -> text
-            tts_text = re.sub(r'https?://\S+', '', tts_text)      # URLs
-            tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text)  # bold
-            tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text)      # italic
-            tts_text = re.sub(r'`(.+?)`', r'\1', tts_text)        # inline code
-            tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE)  # headers
-            tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE)  # list items
-            tts_text = re.sub(r'---+', '', tts_text)              # horizontal rules
-            tts_text = re.sub(r'\n{3,}', '\n\n', tts_text)        # excessive newlines
-            tts_text = tts_text.strip()
+            # Strip markdown and non-speech content for cleaner TTS via the
+            # shared cleaner (tools/tts_text_normalize): markdown, emoji,
+            # <think> blocks, verifier footer, units, newline flattening.
+            try:
+                from tools.tts_text_normalize import prepare_spoken_text
+                tts_text = prepare_spoken_text(text, max_chars=4000)
+            except Exception:
+                # Legacy fallback pipeline — keep voice replies best-effort.
+                tts_text = text[:4000] if len(text) > 4000 else text
+                tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text)   # fenced code blocks
+                tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text)  # [text](url) -> text
+                tts_text = re.sub(r'https?://\S+', '', tts_text)      # URLs
+                tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text)  # bold
+                tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text)      # italic
+                tts_text = re.sub(r'`(.+?)`', r'\1', tts_text)        # inline code
+                tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE)  # headers
+                tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE)  # list items
+                tts_text = re.sub(r'---+', '', tts_text)              # horizontal rules
+                tts_text = re.sub(r'\n{3,}', '\n\n', tts_text)        # excessive newlines
+                tts_text = tts_text.strip()
             if not tts_text:
                 return
 

contributors/emails/alcibiades.eth@protonmail.com

@@ -0,0 +1 @@
+0xAlcibiades

contributors/emails/alex-secure@tuta.io

@@ -0,0 +1 @@
+AlexxRussell

contributors/emails/gabriel@gabotronics.com

@@ -0,0 +1 @@
+ganzziani

contributors/emails/hubin-ll@foxmail.com

@@ -0,0 +1 @@
+LLQWQ

contributors/emails/johann@Mac.lan

@@ -0,0 +1 @@
+ousiaresearch

hermes_cli/voice.py

@@ -801,18 +801,25 @@ def speak_text(text: str) -> None:
     try:
         from tools.tts_tool import text_to_speech_tool
 
-        tts_text = text[:4000] if len(text) > 4000 else text
-        tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text)             # fenced code blocks
-        tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text)    # [text](url) → text
-        tts_text = re.sub(r'https?://\S+', '', tts_text)                # bare URLs
-        tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text)            # bold
-        tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text)                # italic
-        tts_text = re.sub(r'`(.+?)`', r'\1', tts_text)                  # inline code
-        tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE)  # headers
-        tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE)  # list bullets
-        tts_text = re.sub(r'---+', '', tts_text)                        # horizontal rules
-        tts_text = re.sub(r'\n{3,}', '\n\n', tts_text)                  # excess newlines
-        tts_text = tts_text.strip()
+        # Shared cleaner (tools/tts_text_normalize): markdown, emoji,
+        # <think> blocks, verifier footer, units, newline flattening.
+        try:
+            from tools.tts_text_normalize import prepare_spoken_text
+            tts_text = prepare_spoken_text(text, max_chars=4000)
+        except Exception:
+            # Legacy fallback pipeline — keep speak_text best-effort.
+            tts_text = text[:4000] if len(text) > 4000 else text
+            tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text)             # fenced code blocks
+            tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text)    # [text](url) → text
+            tts_text = re.sub(r'https?://\S+', '', tts_text)                # bare URLs
+            tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text)            # bold
+            tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text)                # italic
+            tts_text = re.sub(r'`(.+?)`', r'\1', tts_text)                  # inline code
+            tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE)  # headers
+            tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE)  # list bullets
+            tts_text = re.sub(r'---+', '', tts_text)                        # horizontal rules
+            tts_text = re.sub(r'\n{3,}', '\n\n', tts_text)                  # excess newlines
+            tts_text = tts_text.strip()
         if not tts_text:
             return
 

tests/tools/test_tts_instructions.py

@@ -30,7 +30,7 @@ class TestOpenaiBackendInstructions:
 
         with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \
              patch("tools.tts_tool._resolve_openai_audio_client_config",
-                   return_value=("test-key", None)):
+                   return_value=("test-key", None, False)):
             from tools.tts_tool import _generate_openai_tts
             kwargs = {}
             if instructions is not None:
@@ -83,7 +83,7 @@ class TestToolLevelInstructions:
 
         with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \
              patch("tools.tts_tool._resolve_openai_audio_client_config",
-                   return_value=("test-key", None)), \
+                   return_value=("test-key", None, False)), \
              patch("tools.tts_tool._load_tts_config",
                    return_value={"provider": "openai"}):
             from tools.tts_tool import text_to_speech_tool

tests/tools/test_tts_prepare_spoken.py

@@ -0,0 +1,148 @@
+"""Unit tests for the shared TTS text cleaner (tools/tts_text_normalize).
+
+Covers the consolidated preprocessing pipeline: <think> reasoning blocks
+(#34213), emoji strip (#13311/#18598), file-mutation verifier footer
+(#40772), newline flattening for newline-sensitive providers (#9004), and
+the wiring of the ONE shared cleaner into both the text_to_speech tool
+path and the voice-mode paths.
+"""
+
+import json
+
+from tools.tts_text_normalize import (
+    flatten_newlines_for_payload,
+    prepare_spoken_text,
+    strip_nonspoken_blocks,
+)
+
+
+class TestThinkBlockStrip:
+    def test_think_block_removed(self):
+        raw = "<think>\nsecret reasoning here\n</think>\nThe answer is 42."
+        spoken = prepare_spoken_text(raw)
+        assert "secret reasoning" not in spoken
+        assert "42" in spoken
+
+    def test_think_block_with_attributes_removed(self):
+        raw = "<think budget=high>chain of thought</think>Visible."
+        spoken = prepare_spoken_text(raw)
+        assert "chain of thought" not in spoken
+        assert "Visible" in spoken
+
+    def test_unterminated_think_block_removed(self):
+        raw = "Answer first. <think>\ntruncated reasoning stream"
+        spoken = prepare_spoken_text(raw)
+        assert "truncated reasoning" not in spoken
+        assert "Answer first" in spoken
+
+    def test_multiple_think_blocks(self):
+        raw = "<think>a</think>one<think>b</think> two"
+        spoken = strip_nonspoken_blocks(raw)
+        assert "a" not in spoken.replace("one", "").replace("two", "")
+        assert "one" in spoken and "two" in spoken
+
+
+class TestVerifierFooterStrip:
+    FOOTER = (
+        "⚠️ File-mutation verifier: 2 file(s) were NOT modified this turn "
+        "despite any wording above that may suggest otherwise. Run `git "
+        "status` or `read_file` to confirm.\n"
+        "  • `tools/foo.py` — [patch] old_string not found\n"
+        "  • `bar.md` — [write_file] failed"
+    )
+
+    def test_footer_removed(self):
+        raw = "I fixed the file.\n\n" + self.FOOTER
+        spoken = prepare_spoken_text(raw)
+        assert "File-mutation verifier" not in spoken
+        assert "NOT modified" not in spoken
+        assert "fixed the file" in spoken
+
+    def test_footer_bullets_removed(self):
+        spoken = strip_nonspoken_blocks("Reply.\n" + self.FOOTER)
+        assert "old_string" not in spoken
+        assert "write_file" not in spoken
+
+    def test_text_without_footer_untouched(self):
+        raw = "Just a normal reply about files."
+        assert strip_nonspoken_blocks(raw).strip() == raw
+
+
+class TestEmojiStrip:
+    def test_emoji_removed(self):
+        spoken = prepare_spoken_text("Done! 🎉🚀 All tests pass ✅")
+        assert "🎉" not in spoken
+        assert "🚀" not in spoken
+        assert "✅" not in spoken
+        assert "All tests pass" in spoken
+
+
+class TestNewlineFlattening:
+    def test_no_newlines_in_output(self):
+        raw = "First line\nSecond line\n\nThird paragraph"
+        spoken = prepare_spoken_text(raw)
+        assert "\n" not in spoken
+        assert "First line" in spoken
+        assert "Third paragraph" in spoken
+
+    def test_newlines_become_sentence_breaks(self):
+        out = flatten_newlines_for_payload("Alpha\nBeta")
+        assert out == "Alpha. Beta"
+
+    def test_existing_punctuation_not_doubled(self):
+        out = flatten_newlines_for_payload("Alpha.\nBeta!")
+        assert ".." not in out
+        assert "Alpha." in out and "Beta!" in out
+
+
+class TestSharedCleanerWiring:
+    """The ONE cleaner must be applied on every TTS entry path."""
+
+    def test_tool_path_strips_think_blocks(self):
+        from tools.tts_tool import _strip_markdown_for_tts
+
+        cleaned = _strip_markdown_for_tts("<think>hidden</think>**Loud** and clear 🎉")
+        assert "hidden" not in cleaned
+        assert "**" not in cleaned
+        assert "🎉" not in cleaned
+        assert "Loud and clear" in cleaned
+
+    def test_tool_rejects_text_empty_after_cleanup(self):
+        from tools.tts_tool import text_to_speech_tool
+
+        result = json.loads(text_to_speech_tool(text="<think>only reasoning</think>"))
+        assert result["success"] is False
+
+    def test_streaming_helper_uses_shared_cleaner(self):
+        from tools.tts_tool import _strip_markdown_for_tts
+
+        cleaned = _strip_markdown_for_tts("Temp is 14°C today\nand rising")
+        assert "degrees Celsius" in cleaned
+        assert "\n" not in cleaned
+
+    def test_gateway_prepare_tts_text_strips_think_blocks(self):
+        from gateway.config import Platform, PlatformConfig
+        from gateway.platforms.base import BasePlatformAdapter
+
+        class _DummyAdapter(BasePlatformAdapter):
+            def __init__(self):
+                super().__init__(
+                    PlatformConfig(enabled=True, token="test"), Platform.TELEGRAM
+                )
+
+            async def connect(self):
+                return True
+
+            async def disconnect(self):
+                pass
+
+            async def send(self, chat_id, content, **kwargs):
+                raise AssertionError("not used")
+
+            async def get_chat_info(self, chat_id):
+                return {"id": chat_id, "type": "dm"}
+
+        adapter = _DummyAdapter()
+        spoken = adapter.prepare_tts_text("<think>plan</think>Hello there")
+        assert "plan" not in spoken
+        assert "Hello there" in spoken

tests/tools/test_tts_speed.py

@@ -128,7 +128,7 @@ class TestOpenaiTtsLangCode:
 
         with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \
              patch("tools.tts_tool._resolve_openai_audio_client_config",
-                   return_value=("test-key", None)):
+                   return_value=("test-key", None, False)):
             from tools.tts_tool import _generate_openai_tts
             _generate_openai_tts("Hola", str(tmp_path / "out.mp3"), tts_config)
         return mock_client.audio.speech.create
@@ -309,7 +309,7 @@ class TestToolLevelSpeed:
 
         with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \
              patch("tools.tts_tool._resolve_openai_audio_client_config",
-                   return_value=("test-key", None)), \
+                   return_value=("test-key", None, False)), \
              patch("tools.tts_tool._load_tts_config", return_value={"provider": "openai", "openai": {}}), \
              patch("tools.tts_tool._get_provider", return_value="openai"), \
              patch("tools.tts_tool._resolve_command_provider_config", return_value=None), \

tests/tools/test_voice_cli_integration.py

@@ -64,7 +64,9 @@ class TestMarkdownStripping:
         assert "Done." in result
 
     def test_strips_headers(self):
-        assert _strip_markdown_for_tts("## Summary\nSome text") == "Summary\nSome text"
+        # The shared cleaner folds a heading into the following sentence as a
+        # spoken lead-in ("Summary, Some text.") instead of a bare label.
+        assert _strip_markdown_for_tts("## Summary\nSome text") == "Summary, Some text."
 
     def test_strips_list_markers(self):
         text = "- item one\n- item two\n* item three"

tools/tts_text_normalize.py

@@ -209,16 +209,70 @@ def smooth_whitespace_for_tts(text: str) -> str:
     return text.strip()
 
 
+# Reasoning blocks: models with ``/reasoning show`` enabled emit
+# ``<think>...</think>`` blocks in the final assistant message.  Users want to
+# SEE reasoning, not hear it read aloud (#34213).
+_THINK_BLOCK_RE = re.compile(r"<think[\s>].*?</think>", flags=re.DOTALL | re.IGNORECASE)
+# An unterminated block (streaming cut-off) should still not be spoken.
+_THINK_BLOCK_OPEN_RE = re.compile(r"<think[\s>].*\Z", flags=re.DOTALL | re.IGNORECASE)
+
+# Turn-end file-mutation verifier footer appended by run_agent.py
+# (``_format_file_mutation_failure_footer``).  It's a UI affordance — reading
+# "warning file mutation verifier, 2 files were NOT modified..." aloud is
+# noise (#40772).  The footer is a ``⚠️ File-mutation verifier:`` header line
+# followed by indented ``•`` bullet lines; strip the whole block.
+_VERIFIER_FOOTER_RE = re.compile(
+    r"^\s*⚠️?\s*File-mutation verifier:.*(?:\n[ \t]+•.*)*",
+    flags=re.MULTILINE,
+)
+
+
+def strip_nonspoken_blocks(text: str) -> str:
+    """Remove blocks that must never reach a speech provider.
+
+    Currently: ``<think>`` reasoning blocks and the end-of-turn
+    file-mutation verifier footer.
+    """
+    if not text:
+        return ""
+    text = _THINK_BLOCK_RE.sub(" ", text)
+    text = _THINK_BLOCK_OPEN_RE.sub(" ", text)
+    text = _VERIFIER_FOOTER_RE.sub(" ", text)
+    return text
+
+
+def flatten_newlines_for_payload(text: str) -> str:
+    """Collapse newlines into sentence breaks for single-line TTS payloads.
+
+    Some OpenAI-compatible backends (e.g. Kokoro) truncate synthesis at the
+    first newline (#9004).  The smoothing pass already terminates each line
+    with punctuation, so newlines can safely become plain spaces.
+    """
+    if not text:
+        return ""
+    text = re.sub(r"\n{2,}", ". ", text)
+    text = re.sub(r"(?<=[.!?;:,])\n", " ", text)
+    text = text.replace("\n", ". ")
+    text = re.sub(r"\.\s*\.", ".", text)
+    text = re.sub(r"[ \t]{2,}", " ", text)
+    return text.strip()
+
+
 def prepare_spoken_text(text: str, max_chars: int | None = 4000) -> str:
     """Return a TTS-friendly script from assistant text.
 
-    Deterministic cleanup, not a semantic rewrite: it removes Markdown, expands
-    common symbols such as a degree-Celsius sign to "degrees Celsius", and turns
-    visual line formatting into speakable sentence pauses.
+    Deterministic cleanup, not a semantic rewrite: it removes ``<think>``
+    reasoning blocks and the file-mutation verifier footer, removes Markdown,
+    expands common symbols such as a degree-Celsius sign to "degrees Celsius",
+    turns visual line formatting into speakable sentence pauses, and flattens
+    the result to a single line so newline-sensitive providers (Kokoro) speak
+    the whole script.
     """
-    spoken = strip_markdown_for_tts(text)
+    spoken = strip_nonspoken_blocks(text)
+    spoken = strip_markdown_for_tts(spoken)
     spoken = normalize_symbols_for_tts(spoken)
     spoken = smooth_whitespace_for_tts(spoken)
+    spoken = flatten_newlines_for_payload(spoken)
     if max_chars is not None and max_chars > 0 and len(spoken) > max_chars:
         spoken = spoken[:max_chars].rstrip()
     return spoken

tools/tts_tool.py

@@ -3115,7 +3115,20 @@ _THINK_BLOCK = re.compile(r'<think[\s>].*?</think>', flags=re.DOTALL)
 
 
 def _strip_markdown_for_tts(text: str) -> str:
-    """Remove markdown, think blocks, and emoji that shouldn't be spoken."""
+    """Prepare text for speech via the shared cleaner in tts_text_normalize.
+
+    One cleaner for every TTS path (tool, gateway auto-TTS, voice-mode
+    streaming, web dashboard): strips <think> reasoning blocks, the
+    file-mutation verifier footer, markdown, and emoji; expands units and
+    symbols; and flattens newlines to sentence breaks so newline-sensitive
+    providers (Kokoro) speak the whole script.  Falls back to the legacy
+    regex pipeline if the normalizer ever fails.
+    """
+    try:
+        from tools.tts_text_normalize import prepare_spoken_text
+        return prepare_spoken_text(text, max_chars=None)
+    except Exception:
+        pass
     text = _THINK_BLOCK.sub(' ', text)
     text = _MD_CODE_BLOCK.sub(' ', text)
     text = _MD_LINK.sub(r'\1', text)