From 4aac89b42925f2f5a5ef5a6f1fb04026e528851c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:44:07 -0700 Subject: [PATCH] fix(tts): unify TTS text preprocessing behind one shared cleaner Consolidates all TTS text-preparation paths onto tools/tts_text_normalize.prepare_spoken_text: - strip_nonspoken_blocks: removes reasoning blocks (#34213, incl. unterminated streaming blocks) and the end-of-turn file-mutation verifier footer emitted by run_agent.py (#40772). - flatten_newlines_for_payload: collapses newlines into sentence breaks so newline-sensitive OpenAI-compatible providers (Kokoro) speak the whole script instead of truncating at the first newline (#9004). - tools/tts_tool._strip_markdown_for_tts (voice-mode streaming + web dashboard path) now delegates to the shared cleaner, with the legacy regex pipeline kept as a best-effort fallback. - hermes_cli/voice.py speak_text and cli.py _voice_speak_response now use the shared cleaner instead of their own duplicated regex pipelines. - gateway auto-TTS fallback also strips think blocks. Tests: tests/tools/test_tts_prepare_spoken.py covers think blocks, verifier footer, emoji, newline flattening, and the shared-cleaner wiring on the tool/streaming/gateway paths. Updated the header expectation in test_voice_cli_integration.py for the heading-fold behavior of the shared cleaner. Closes #34213, #9004, #40772 --- cli.py | 33 ++-- .../emails/alcibiades.eth@protonmail.com | 1 + contributors/emails/alex-secure@tuta.io | 1 + contributors/emails/gabriel@gabotronics.com | 1 + contributors/emails/hubin-ll@foxmail.com | 1 + contributors/emails/johann@Mac.lan | 1 + hermes_cli/voice.py | 31 ++-- tests/tools/test_tts_instructions.py | 4 +- tests/tools/test_tts_prepare_spoken.py | 148 ++++++++++++++++++ tests/tools/test_tts_speed.py | 4 +- tests/tools/test_voice_cli_integration.py | 4 +- tools/tts_text_normalize.py | 62 +++++++- tools/tts_tool.py | 15 +- 13 files changed, 271 insertions(+), 35 deletions(-) create mode 100644 contributors/emails/alcibiades.eth@protonmail.com create mode 100644 contributors/emails/alex-secure@tuta.io create mode 100644 contributors/emails/gabriel@gabotronics.com create mode 100644 contributors/emails/hubin-ll@foxmail.com create mode 100644 contributors/emails/johann@Mac.lan create mode 100644 tests/tools/test_tts_prepare_spoken.py diff --git a/cli.py b/cli.py index a1728537e69..7338aa1fbea 100644 --- a/cli.py +++ b/cli.py @@ -11950,19 +11950,26 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): from tools.tts_tool import text_to_speech_tool from tools.voice_mode import play_audio_file - # Strip markdown and non-speech content for cleaner TTS - tts_text = text[:4000] if len(text) > 4000 else text - tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text) # fenced code blocks - tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text) # [text](url) -> text - tts_text = re.sub(r'https?://\S+', '', tts_text) # URLs - tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text) # bold - tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text) # italic - tts_text = re.sub(r'`(.+?)`', r'\1', tts_text) # inline code - tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE) # headers - tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE) # list items - tts_text = re.sub(r'---+', '', tts_text) # horizontal rules - tts_text = re.sub(r'\n{3,}', '\n\n', tts_text) # excessive newlines - tts_text = tts_text.strip() + # Strip markdown and non-speech content for cleaner TTS via the + # shared cleaner (tools/tts_text_normalize): markdown, emoji, + # blocks, verifier footer, units, newline flattening. + try: + from tools.tts_text_normalize import prepare_spoken_text + tts_text = prepare_spoken_text(text, max_chars=4000) + except Exception: + # Legacy fallback pipeline — keep voice replies best-effort. + tts_text = text[:4000] if len(text) > 4000 else text + tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text) # fenced code blocks + tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text) # [text](url) -> text + tts_text = re.sub(r'https?://\S+', '', tts_text) # URLs + tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text) # bold + tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text) # italic + tts_text = re.sub(r'`(.+?)`', r'\1', tts_text) # inline code + tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE) # headers + tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE) # list items + tts_text = re.sub(r'---+', '', tts_text) # horizontal rules + tts_text = re.sub(r'\n{3,}', '\n\n', tts_text) # excessive newlines + tts_text = tts_text.strip() if not tts_text: return diff --git a/contributors/emails/alcibiades.eth@protonmail.com b/contributors/emails/alcibiades.eth@protonmail.com new file mode 100644 index 00000000000..f16d53e07f9 --- /dev/null +++ b/contributors/emails/alcibiades.eth@protonmail.com @@ -0,0 +1 @@ +0xAlcibiades diff --git a/contributors/emails/alex-secure@tuta.io b/contributors/emails/alex-secure@tuta.io new file mode 100644 index 00000000000..ed43717f35d --- /dev/null +++ b/contributors/emails/alex-secure@tuta.io @@ -0,0 +1 @@ +AlexxRussell diff --git a/contributors/emails/gabriel@gabotronics.com b/contributors/emails/gabriel@gabotronics.com new file mode 100644 index 00000000000..1ec2b4a1f03 --- /dev/null +++ b/contributors/emails/gabriel@gabotronics.com @@ -0,0 +1 @@ +ganzziani diff --git a/contributors/emails/hubin-ll@foxmail.com b/contributors/emails/hubin-ll@foxmail.com new file mode 100644 index 00000000000..cb8816983d3 --- /dev/null +++ b/contributors/emails/hubin-ll@foxmail.com @@ -0,0 +1 @@ +LLQWQ diff --git a/contributors/emails/johann@Mac.lan b/contributors/emails/johann@Mac.lan new file mode 100644 index 00000000000..008cd4983e1 --- /dev/null +++ b/contributors/emails/johann@Mac.lan @@ -0,0 +1 @@ +ousiaresearch diff --git a/hermes_cli/voice.py b/hermes_cli/voice.py index c0617a168e3..e0bd3efcfe6 100644 --- a/hermes_cli/voice.py +++ b/hermes_cli/voice.py @@ -801,18 +801,25 @@ def speak_text(text: str) -> None: try: from tools.tts_tool import text_to_speech_tool - tts_text = text[:4000] if len(text) > 4000 else text - tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text) # fenced code blocks - tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text) # [text](url) → text - tts_text = re.sub(r'https?://\S+', '', tts_text) # bare URLs - tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text) # bold - tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text) # italic - tts_text = re.sub(r'`(.+?)`', r'\1', tts_text) # inline code - tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE) # headers - tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE) # list bullets - tts_text = re.sub(r'---+', '', tts_text) # horizontal rules - tts_text = re.sub(r'\n{3,}', '\n\n', tts_text) # excess newlines - tts_text = tts_text.strip() + # Shared cleaner (tools/tts_text_normalize): markdown, emoji, + # blocks, verifier footer, units, newline flattening. + try: + from tools.tts_text_normalize import prepare_spoken_text + tts_text = prepare_spoken_text(text, max_chars=4000) + except Exception: + # Legacy fallback pipeline — keep speak_text best-effort. + tts_text = text[:4000] if len(text) > 4000 else text + tts_text = re.sub(r'```[\s\S]*?```', ' ', tts_text) # fenced code blocks + tts_text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', tts_text) # [text](url) → text + tts_text = re.sub(r'https?://\S+', '', tts_text) # bare URLs + tts_text = re.sub(r'\*\*(.+?)\*\*', r'\1', tts_text) # bold + tts_text = re.sub(r'\*(.+?)\*', r'\1', tts_text) # italic + tts_text = re.sub(r'`(.+?)`', r'\1', tts_text) # inline code + tts_text = re.sub(r'^#+\s*', '', tts_text, flags=re.MULTILINE) # headers + tts_text = re.sub(r'^\s*[-*]\s+', '', tts_text, flags=re.MULTILINE) # list bullets + tts_text = re.sub(r'---+', '', tts_text) # horizontal rules + tts_text = re.sub(r'\n{3,}', '\n\n', tts_text) # excess newlines + tts_text = tts_text.strip() if not tts_text: return diff --git a/tests/tools/test_tts_instructions.py b/tests/tools/test_tts_instructions.py index cdad64e238c..3bb26aea8f1 100644 --- a/tests/tools/test_tts_instructions.py +++ b/tests/tools/test_tts_instructions.py @@ -30,7 +30,7 @@ class TestOpenaiBackendInstructions: with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \ patch("tools.tts_tool._resolve_openai_audio_client_config", - return_value=("test-key", None)): + return_value=("test-key", None, False)): from tools.tts_tool import _generate_openai_tts kwargs = {} if instructions is not None: @@ -83,7 +83,7 @@ class TestToolLevelInstructions: with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \ patch("tools.tts_tool._resolve_openai_audio_client_config", - return_value=("test-key", None)), \ + return_value=("test-key", None, False)), \ patch("tools.tts_tool._load_tts_config", return_value={"provider": "openai"}): from tools.tts_tool import text_to_speech_tool diff --git a/tests/tools/test_tts_prepare_spoken.py b/tests/tools/test_tts_prepare_spoken.py new file mode 100644 index 00000000000..1a807183380 --- /dev/null +++ b/tests/tools/test_tts_prepare_spoken.py @@ -0,0 +1,148 @@ +"""Unit tests for the shared TTS text cleaner (tools/tts_text_normalize). + +Covers the consolidated preprocessing pipeline: reasoning blocks +(#34213), emoji strip (#13311/#18598), file-mutation verifier footer +(#40772), newline flattening for newline-sensitive providers (#9004), and +the wiring of the ONE shared cleaner into both the text_to_speech tool +path and the voice-mode paths. +""" + +import json + +from tools.tts_text_normalize import ( + flatten_newlines_for_payload, + prepare_spoken_text, + strip_nonspoken_blocks, +) + + +class TestThinkBlockStrip: + def test_think_block_removed(self): + raw = "\nsecret reasoning here\n\nThe answer is 42." + spoken = prepare_spoken_text(raw) + assert "secret reasoning" not in spoken + assert "42" in spoken + + def test_think_block_with_attributes_removed(self): + raw = "chain of thoughtVisible." + spoken = prepare_spoken_text(raw) + assert "chain of thought" not in spoken + assert "Visible" in spoken + + def test_unterminated_think_block_removed(self): + raw = "Answer first. \ntruncated reasoning stream" + spoken = prepare_spoken_text(raw) + assert "truncated reasoning" not in spoken + assert "Answer first" in spoken + + def test_multiple_think_blocks(self): + raw = "aoneb two" + spoken = strip_nonspoken_blocks(raw) + assert "a" not in spoken.replace("one", "").replace("two", "") + assert "one" in spoken and "two" in spoken + + +class TestVerifierFooterStrip: + FOOTER = ( + "⚠️ File-mutation verifier: 2 file(s) were NOT modified this turn " + "despite any wording above that may suggest otherwise. Run `git " + "status` or `read_file` to confirm.\n" + " • `tools/foo.py` — [patch] old_string not found\n" + " • `bar.md` — [write_file] failed" + ) + + def test_footer_removed(self): + raw = "I fixed the file.\n\n" + self.FOOTER + spoken = prepare_spoken_text(raw) + assert "File-mutation verifier" not in spoken + assert "NOT modified" not in spoken + assert "fixed the file" in spoken + + def test_footer_bullets_removed(self): + spoken = strip_nonspoken_blocks("Reply.\n" + self.FOOTER) + assert "old_string" not in spoken + assert "write_file" not in spoken + + def test_text_without_footer_untouched(self): + raw = "Just a normal reply about files." + assert strip_nonspoken_blocks(raw).strip() == raw + + +class TestEmojiStrip: + def test_emoji_removed(self): + spoken = prepare_spoken_text("Done! 🎉🚀 All tests pass ✅") + assert "🎉" not in spoken + assert "🚀" not in spoken + assert "✅" not in spoken + assert "All tests pass" in spoken + + +class TestNewlineFlattening: + def test_no_newlines_in_output(self): + raw = "First line\nSecond line\n\nThird paragraph" + spoken = prepare_spoken_text(raw) + assert "\n" not in spoken + assert "First line" in spoken + assert "Third paragraph" in spoken + + def test_newlines_become_sentence_breaks(self): + out = flatten_newlines_for_payload("Alpha\nBeta") + assert out == "Alpha. Beta" + + def test_existing_punctuation_not_doubled(self): + out = flatten_newlines_for_payload("Alpha.\nBeta!") + assert ".." not in out + assert "Alpha." in out and "Beta!" in out + + +class TestSharedCleanerWiring: + """The ONE cleaner must be applied on every TTS entry path.""" + + def test_tool_path_strips_think_blocks(self): + from tools.tts_tool import _strip_markdown_for_tts + + cleaned = _strip_markdown_for_tts("hidden**Loud** and clear 🎉") + assert "hidden" not in cleaned + assert "**" not in cleaned + assert "🎉" not in cleaned + assert "Loud and clear" in cleaned + + def test_tool_rejects_text_empty_after_cleanup(self): + from tools.tts_tool import text_to_speech_tool + + result = json.loads(text_to_speech_tool(text="only reasoning")) + assert result["success"] is False + + def test_streaming_helper_uses_shared_cleaner(self): + from tools.tts_tool import _strip_markdown_for_tts + + cleaned = _strip_markdown_for_tts("Temp is 14°C today\nand rising") + assert "degrees Celsius" in cleaned + assert "\n" not in cleaned + + def test_gateway_prepare_tts_text_strips_think_blocks(self): + from gateway.config import Platform, PlatformConfig + from gateway.platforms.base import BasePlatformAdapter + + class _DummyAdapter(BasePlatformAdapter): + def __init__(self): + super().__init__( + PlatformConfig(enabled=True, token="test"), Platform.TELEGRAM + ) + + async def connect(self): + return True + + async def disconnect(self): + pass + + async def send(self, chat_id, content, **kwargs): + raise AssertionError("not used") + + async def get_chat_info(self, chat_id): + return {"id": chat_id, "type": "dm"} + + adapter = _DummyAdapter() + spoken = adapter.prepare_tts_text("planHello there") + assert "plan" not in spoken + assert "Hello there" in spoken diff --git a/tests/tools/test_tts_speed.py b/tests/tools/test_tts_speed.py index d64eab8c6af..7ce0cb5edfd 100644 --- a/tests/tools/test_tts_speed.py +++ b/tests/tools/test_tts_speed.py @@ -128,7 +128,7 @@ class TestOpenaiTtsLangCode: with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \ patch("tools.tts_tool._resolve_openai_audio_client_config", - return_value=("test-key", None)): + return_value=("test-key", None, False)): from tools.tts_tool import _generate_openai_tts _generate_openai_tts("Hola", str(tmp_path / "out.mp3"), tts_config) return mock_client.audio.speech.create @@ -309,7 +309,7 @@ class TestToolLevelSpeed: with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \ patch("tools.tts_tool._resolve_openai_audio_client_config", - return_value=("test-key", None)), \ + return_value=("test-key", None, False)), \ patch("tools.tts_tool._load_tts_config", return_value={"provider": "openai", "openai": {}}), \ patch("tools.tts_tool._get_provider", return_value="openai"), \ patch("tools.tts_tool._resolve_command_provider_config", return_value=None), \ diff --git a/tests/tools/test_voice_cli_integration.py b/tests/tools/test_voice_cli_integration.py index 1e566f74806..d098000452d 100644 --- a/tests/tools/test_voice_cli_integration.py +++ b/tests/tools/test_voice_cli_integration.py @@ -64,7 +64,9 @@ class TestMarkdownStripping: assert "Done." in result def test_strips_headers(self): - assert _strip_markdown_for_tts("## Summary\nSome text") == "Summary\nSome text" + # The shared cleaner folds a heading into the following sentence as a + # spoken lead-in ("Summary, Some text.") instead of a bare label. + assert _strip_markdown_for_tts("## Summary\nSome text") == "Summary, Some text." def test_strips_list_markers(self): text = "- item one\n- item two\n* item three" diff --git a/tools/tts_text_normalize.py b/tools/tts_text_normalize.py index 1f653ba8d0a..5693e74388f 100644 --- a/tools/tts_text_normalize.py +++ b/tools/tts_text_normalize.py @@ -209,16 +209,70 @@ def smooth_whitespace_for_tts(text: str) -> str: return text.strip() +# Reasoning blocks: models with ``/reasoning show`` enabled emit +# ``...`` blocks in the final assistant message. Users want to +# SEE reasoning, not hear it read aloud (#34213). +_THINK_BLOCK_RE = re.compile(r"].*?", flags=re.DOTALL | re.IGNORECASE) +# An unterminated block (streaming cut-off) should still not be spoken. +_THINK_BLOCK_OPEN_RE = re.compile(r"].*\Z", flags=re.DOTALL | re.IGNORECASE) + +# Turn-end file-mutation verifier footer appended by run_agent.py +# (``_format_file_mutation_failure_footer``). It's a UI affordance — reading +# "warning file mutation verifier, 2 files were NOT modified..." aloud is +# noise (#40772). The footer is a ``⚠️ File-mutation verifier:`` header line +# followed by indented ``•`` bullet lines; strip the whole block. +_VERIFIER_FOOTER_RE = re.compile( + r"^\s*⚠️?\s*File-mutation verifier:.*(?:\n[ \t]+•.*)*", + flags=re.MULTILINE, +) + + +def strip_nonspoken_blocks(text: str) -> str: + """Remove blocks that must never reach a speech provider. + + Currently: ```` reasoning blocks and the end-of-turn + file-mutation verifier footer. + """ + if not text: + return "" + text = _THINK_BLOCK_RE.sub(" ", text) + text = _THINK_BLOCK_OPEN_RE.sub(" ", text) + text = _VERIFIER_FOOTER_RE.sub(" ", text) + return text + + +def flatten_newlines_for_payload(text: str) -> str: + """Collapse newlines into sentence breaks for single-line TTS payloads. + + Some OpenAI-compatible backends (e.g. Kokoro) truncate synthesis at the + first newline (#9004). The smoothing pass already terminates each line + with punctuation, so newlines can safely become plain spaces. + """ + if not text: + return "" + text = re.sub(r"\n{2,}", ". ", text) + text = re.sub(r"(?<=[.!?;:,])\n", " ", text) + text = text.replace("\n", ". ") + text = re.sub(r"\.\s*\.", ".", text) + text = re.sub(r"[ \t]{2,}", " ", text) + return text.strip() + + def prepare_spoken_text(text: str, max_chars: int | None = 4000) -> str: """Return a TTS-friendly script from assistant text. - Deterministic cleanup, not a semantic rewrite: it removes Markdown, expands - common symbols such as a degree-Celsius sign to "degrees Celsius", and turns - visual line formatting into speakable sentence pauses. + Deterministic cleanup, not a semantic rewrite: it removes ```` + reasoning blocks and the file-mutation verifier footer, removes Markdown, + expands common symbols such as a degree-Celsius sign to "degrees Celsius", + turns visual line formatting into speakable sentence pauses, and flattens + the result to a single line so newline-sensitive providers (Kokoro) speak + the whole script. """ - spoken = strip_markdown_for_tts(text) + spoken = strip_nonspoken_blocks(text) + spoken = strip_markdown_for_tts(spoken) spoken = normalize_symbols_for_tts(spoken) spoken = smooth_whitespace_for_tts(spoken) + spoken = flatten_newlines_for_payload(spoken) if max_chars is not None and max_chars > 0 and len(spoken) > max_chars: spoken = spoken[:max_chars].rstrip() return spoken diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 9e10cb3ef81..947fe60ed74 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -3115,7 +3115,20 @@ _THINK_BLOCK = re.compile(r'].*?', flags=re.DOTALL) def _strip_markdown_for_tts(text: str) -> str: - """Remove markdown, think blocks, and emoji that shouldn't be spoken.""" + """Prepare text for speech via the shared cleaner in tts_text_normalize. + + One cleaner for every TTS path (tool, gateway auto-TTS, voice-mode + streaming, web dashboard): strips reasoning blocks, the + file-mutation verifier footer, markdown, and emoji; expands units and + symbols; and flattens newlines to sentence breaks so newline-sensitive + providers (Kokoro) speak the whole script. Falls back to the legacy + regex pipeline if the normalizer ever fails. + """ + try: + from tools.tts_text_normalize import prepare_spoken_text + return prepare_spoken_text(text, max_chars=None) + except Exception: + pass text = _THINK_BLOCK.sub(' ', text) text = _MD_CODE_BLOCK.sub(' ', text) text = _MD_LINK.sub(r'\1', text)