From ba131322988a3f8742386bc96f28028fa0955533 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:25:19 -0700 Subject: [PATCH] fix(voice): bare stop phrase ends the voice chat on every surface, spoken or typed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Saying OR typing a configured stop phrase (voice.stop_phrases, default "stop") now ends the voice chat everywhere, not just classic CLI PTT: - hermes_cli/voice.py: new explicit on_stop_phrase callback through start_continuous/stop_continuous. The force-transcribe path previously DISCARDED the stop phrase silently — with auto_restart=False the client re-arms the next capture, so the conversation never ended. Both halt paths now fire on_stop_phrase (fallback: on_silent_limit for legacy callers) as user intent, distinct from the no-speech timeout. - tui_gateway/server.py: voice.record wires on_stop_phrase and emits voice.transcript {stop_phrase: true} after flipping HERMES_VOICE(_TTS) off and stopping streaming TTS — same teardown as /voice off. The TTS barge-in monitor stop-checks its transcript too. prompt.submit consumes a TYPED bare stop phrase at the server-side choke point when voice mode is on (returns {voice_stopped: true}, no turn starts). - ui-tui: voice.transcript {stop_phrase} ends voice mode with a clear 'voice chat ended' notice (distinct from the no-speech-limit message); submitPrompt releases the busy latch on a consumed voice_stopped reply. - cli.py: _typed_voice_stop in process_loop — typing a bare stop phrase while voice mode/continuous is active ends voice mode instead of sending 'stop' to the agent; typed 'stop' outside voice mode is unchanged. Voice transcripts skip the check (already stop-checked). - desktop: interceptsTypedVoiceStop — the composer's onSubmit ends the live voice conversation (same path as clicking end on the pill) when a bare stop command is typed with no attachments; renderer-owned loop, so handled client-side like the existing spoken isVoiceStopCommand. - tools/voice_mode.py: transcribe_recording never lets the Whisper hallucination filter swallow a configured stop phrase (e.g. 'bye' configured as a stop phrase is both a hallucination-blocklist entry and a stop phrase — stop-phrase check now wins). Tests: continuous-loop signal (sabotage-verified), force-transcribe stop signal + legacy fallback, hallucination-filter ordering, typed-stop CLI unit tests (voice on/off/longer text), prompt.submit typed-stop gateway tests, TUI vitest for stop_phrase event handling, desktop vitest for the typed-stop interceptor. --- apps/desktop/src/app/chat/composer/index.tsx | 27 +++ apps/desktop/src/lib/voice-stop-word.test.ts | 26 ++- apps/desktop/src/lib/voice-stop-word.ts | 11 ++ cli.py | 35 ++++ hermes_cli/voice.py | 56 +++++- tests/test_tui_gateway_server.py | 94 ++++++++++ tests/tools/test_voice_cli_integration.py | 48 +++++ tests/tools/test_voice_stop_phrase.py | 170 ++++++++++++++++++ tools/voice_mode.py | 16 +- tui_gateway/server.py | 67 ++++++- .../createGatewayEventHandler.test.ts | 36 ++++ ui-tui/src/app/createGatewayEventHandler.ts | 13 ++ ui-tui/src/app/submissionCore.ts | 8 + ui-tui/src/gatewayTypes.ts | 9 +- .../docs/user-guide/features/voice-mode.md | 2 + 15 files changed, 605 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src/app/chat/composer/index.tsx b/apps/desktop/src/app/chat/composer/index.tsx index 633d70fc7798..567159662aac 100644 --- a/apps/desktop/src/app/chat/composer/index.tsx +++ b/apps/desktop/src/app/chat/composer/index.tsx @@ -11,6 +11,7 @@ import { sanitizeComposerInput } from '@/lib/composer-input-sanitize' import { DATA_IMAGE_URL_RE } from '@/lib/embedded-images' import { triggerHaptic } from '@/lib/haptics' import { cn } from '@/lib/utils' +import { interceptsTypedVoiceStop } from '@/lib/voice-stop-word' import { sessionCompacting } from '@/store/compaction' import { browseBackward, browseForward, deriveUserHistory, isBrowsingHistory } from '@/store/composer-input-history' import { POPOUT_WIDTH_REM } from '@/store/composer-popout' @@ -95,11 +96,32 @@ export function ChatBar({ onSubmit: onSubmitProp, onTranscribeAudio }: ChatBarProps) { + // Typed stop phrase during an active voice conversation ends it — same + // semantics as SAYING "stop" (voice-stop-word.ts) or clicking the pill's + // end control. Populated after useComposerVoice below (the submit wrapper + // is created first); render-time assignment keeps the ref current. + const voiceStopRef = useRef<{ active: boolean; end: () => void }>({ active: false, end: () => {} }) + // Every send (typed, queued, voice) passes through the contributed // middleware chain first — rewrite / pass-through / cancel. Empty chain = // exact pass-through, so surfaces without contributions are byte-identical. const onSubmit = useCallback( async (value, options) => { + // Bare stop phrase typed while the voice conversation is live: end the + // conversation (mic off, pill dismissed) instead of sending "stop" to + // the agent. Spoken transcripts are already stop-checked inside + // use-voice-conversation, so this only catches typed/queued sends. + // Outside a voice conversation, typed "stop" is a normal message. + const voiceStop = voiceStopRef.current + + if (interceptsTypedVoiceStop(voiceStop.active, value, options?.attachments?.length ?? 0)) { + voiceStop.end() + + // Consumed (not rejected): report accepted so the submit engine + // clears the draft instead of restoring "stop" into the composer. + return true + } + const draft = await runComposerMiddleware({ text: value, attachments: options?.attachments }) if (!draft) { @@ -840,6 +862,11 @@ export function ChatBar({ target: scope.target }) + // Keep the typed-stop interceptor (see onSubmit above) in sync with the + // live conversation state. Render-time ref assignment, same pattern as + // dispatchSubmitRef — no effect needed for a plain mirror. + voiceStopRef.current = { active: voiceConversationActive, end: endConversation } + const contextMenu = ( { it('matches bare stop commands', () => { @@ -60,3 +60,27 @@ describe('isVoiceStopCommand', () => { } }) }) + +describe('interceptsTypedVoiceStop', () => { + it('intercepts a typed bare stop command while the conversation is active', () => { + for (const text of ['stop', 'Stop.', 'never mind', 'hey hermes, stop']) { + expect(interceptsTypedVoiceStop(true, text)).toBe(true) + } + }) + + it('never intercepts when the voice conversation is inactive', () => { + for (const text of ['stop', 'never mind', 'goodbye']) { + expect(interceptsTypedVoiceStop(false, text)).toBe(false) + } + }) + + it('passes through substantive messages during a conversation', () => { + for (const text of ['stop the docker container', 'how do I stop a process', 'hello']) { + expect(interceptsTypedVoiceStop(true, text)).toBe(false) + } + }) + + it('passes through when attachments ride along (real payload)', () => { + expect(interceptsTypedVoiceStop(true, 'stop', 1)).toBe(false) + }) +}) diff --git a/apps/desktop/src/lib/voice-stop-word.ts b/apps/desktop/src/lib/voice-stop-word.ts index e59c9da83afb..2ed2e3769aa4 100644 --- a/apps/desktop/src/lib/voice-stop-word.ts +++ b/apps/desktop/src/lib/voice-stop-word.ts @@ -92,3 +92,14 @@ export function isVoiceStopCommand(transcript: string): boolean { return false } + +/** + * Typed-stop interception decision for the composer: a bare stop command + * typed while the voice conversation is live ends the conversation instead of + * being sent as a turn. Attachments mean the message is a real payload — + * never intercepted. Outside a voice conversation typed text always passes + * through unchanged. + */ +export function interceptsTypedVoiceStop(conversationActive: boolean, text: string, attachmentCount = 0): boolean { + return conversationActive && attachmentCount === 0 && isVoiceStopCommand(text) +} diff --git a/cli.py b/cli.py index 521bc2608a3e..86463a9b89cc 100644 --- a/cli.py +++ b/cli.py @@ -12250,6 +12250,33 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): _cprint(f" {_DIM}/voice tts to toggle speech output{_RST}") _cprint(f" {_DIM}/voice off to disable voice mode{_RST}") + def _typed_voice_stop(self, user_input) -> bool: + """Typed bare stop phrase during an active voice chat ends the chat. + + Saying "stop" ends the voice chat (PR #73106); TYPING the same bare + stop phrase while voice mode is on must behave identically instead of + sending "stop" to the agent as a turn. Guarded on voice mode being ON + — typed "stop" outside voice chat passes through to the agent exactly + as before. Reuses ``is_voice_stop_phrase`` (same config + ``voice.stop_phrases``, same exact-match semantics), so longer typed + messages containing "stop" are never swallowed. + """ + if not isinstance(user_input, str): + return False + with self._voice_lock: + voice_on = self._voice_mode or self._voice_continuous + if not voice_on: + return False + try: + from tools.voice_mode import is_voice_stop_phrase + if not is_voice_stop_phrase(user_input): + return False + except Exception: + return False + _cprint(f"\n{_DIM}Stop phrase typed — ending voice chat.{_RST}") + self._disable_voice_mode() + return True + def _disable_voice_mode(self): """Disable voice mode, cancel any active recording, and stop TTS.""" recorder = None @@ -16522,6 +16549,14 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): user_input, _had_mouse_reports = _strip_leaked_terminal_responses_with_meta(user_input) if _had_mouse_reports: self._recover_terminal_input_modes(reason="mouse reports leaked into submitted input") + + # Typed bare stop phrase while a voice chat is active ends + # the voice chat (same semantics as SAYING "stop") instead + # of sending the word to the agent. Voice transcripts are + # already stop-checked at the transcription points, so this + # only intercepts typed input. + if not is_voice_input and self._typed_voice_stop(user_input): + continue # Check for commands — but detect dragged/pasted file paths first. # See _detect_file_drop() for details. diff --git a/hermes_cli/voice.py b/hermes_cli/voice.py index 2c2fa8b8a104..83854a015fb0 100644 --- a/hermes_cli/voice.py +++ b/hermes_cli/voice.py @@ -304,6 +304,12 @@ _tts_playing.set() # initially "not playing" _continuous_on_transcript: Optional[Callable[[str], None]] = None _continuous_on_status: Optional[Callable[[str], None]] = None _continuous_on_silent_limit: Optional[Callable[[], None]] = None +# Explicit user-intent stop signal: fired when the user SAYS a bare stop +# phrase ("stop"). Distinct from on_silent_limit (a timeout) so consumers +# (TUI, desktop) can end the conversation like a manual stop instead of +# reporting "no speech detected". When unset, on_silent_limit fires as a +# fallback so older callers still turn voice off. +_continuous_on_stop_phrase: Optional[Callable[[str], None]] = None _continuous_no_speech_count = 0 _CONTINUOUS_NO_SPEECH_LIMIT = 3 @@ -379,6 +385,7 @@ def start_continuous( silence_duration: float = 3.0, auto_restart: bool = True, max_recording_seconds: float = 0.0, + on_stop_phrase: Optional[Callable[[str], None]] = None, ) -> bool: """Start a VAD-driven continuous recording loop. @@ -400,9 +407,16 @@ def start_continuous( ``max_recording_seconds`` is the hard cap on a single recording's length (``voice.max_recording_seconds``); any non-positive or non-numeric value disables the cap, preserving the previous unbounded behaviour. + + ``on_stop_phrase`` is called with the (stripped) transcript when the user + utters a bare voice stop phrase (``voice.stop_phrases``, default "stop"). + The loop halts first, so the consumer only needs to reflect "voice off" — + exactly like the user pressing the manual stop control. When omitted, + ``on_silent_limit`` fires instead so legacy callers still turn voice off. """ global _continuous_active, _continuous_recorder, _continuous_auto_restart global _continuous_on_transcript, _continuous_on_status, _continuous_on_silent_limit + global _continuous_on_stop_phrase global _continuous_no_speech_count with _continuous_lock: @@ -417,6 +431,7 @@ def start_continuous( _continuous_on_transcript = on_transcript _continuous_on_status = on_status _continuous_on_silent_limit = on_silent_limit + _continuous_on_stop_phrase = on_stop_phrase if auto_restart: _continuous_no_speech_count = 0 @@ -473,6 +488,7 @@ def stop_continuous(force_transcribe: bool = False) -> None: """ global _continuous_active, _continuous_on_transcript, _continuous_stopping global _continuous_on_status, _continuous_on_silent_limit + global _continuous_on_stop_phrase global _continuous_recorder, _continuous_no_speech_count with _continuous_lock: @@ -483,12 +499,14 @@ def stop_continuous(force_transcribe: bool = False) -> None: on_status = _continuous_on_status on_transcript = _continuous_on_transcript on_silent_limit = _continuous_on_silent_limit + on_stop_phrase = _continuous_on_stop_phrase auto_restart = _continuous_auto_restart track_no_speech = force_transcribe and not auto_restart _continuous_stopping = rec is not None _continuous_on_transcript = None _continuous_on_status = None _continuous_on_silent_limit = None + _continuous_on_stop_phrase = None if not track_no_speech: _continuous_no_speech_count = 0 @@ -530,10 +548,25 @@ def stop_continuous(force_transcribe: bool = False) -> None: finally: stop_phrase = bool(transcript and is_voice_stop_phrase(transcript)) if stop_phrase: - # Bare stop phrase — the loop is already stopping; - # swallow it instead of sending "stop" to the agent. - _debug("stop_continuous: stop phrase — transcript discarded") + # Bare stop phrase — explicit user intent to end the + # voice chat. Never sent to the agent; fire the + # dedicated signal so the consumer (TUI / desktop) + # ends the conversation instead of silently re-arming + # the next capture (with auto_restart=False the CLIENT + # drives the loop, so discarding the transcript alone + # would leave the conversation running forever). + _debug( + f"stop_continuous: stop phrase {transcript!r} — ending voice chat" + ) + stop_text = transcript or "" transcript = None + try: + if on_stop_phrase is not None: + on_stop_phrase(stop_text) + elif on_silent_limit is not None: + on_silent_limit() + except Exception: + pass if transcript: try: on_transcript(transcript) @@ -616,6 +649,7 @@ def _continuous_on_silence() -> None: on_transcript = _continuous_on_transcript on_status = _continuous_on_status on_silent_limit = _continuous_on_silent_limit + on_stop_phrase = _continuous_on_stop_phrase if rec is None: _debug("_continuous_on_silence: no recorder — abort") @@ -669,10 +703,12 @@ def _continuous_on_silence() -> None: pass stop_phrase = bool(transcript and is_voice_stop_phrase(transcript)) + stop_text = (transcript or "") if stop_phrase else "" if stop_phrase: # User said a bare stop phrase ("stop") — end the voice chat. - # Not delivered to the agent; the loop halts exactly like the - # silent-cycle limit so every UI (TUI, desktop) turns voice off. + # Not delivered to the agent; the loop halts and the explicit + # on_stop_phrase signal (fallback: on_silent_limit) tells every UI + # (TUI, desktop) to end the conversation like a manual stop. _debug(f"_continuous_on_silence: stop phrase {transcript!r} — ending loop") transcript = None @@ -704,7 +740,15 @@ def _continuous_on_silence() -> None: with _continuous_lock: _continuous_active = False _continuous_no_speech_count = 0 - if on_silent_limit: + if stop_phrase and on_stop_phrase is not None: + # Explicit user-intent stop — distinct from the no-speech timeout + # so consumers can report "voice chat ended" instead of "no + # speech detected". + try: + on_stop_phrase(stop_text) + except Exception: + pass + elif on_silent_limit: try: on_silent_limit() except Exception: diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index 5ac3ef247c54..d4eea3039900 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -1315,6 +1315,100 @@ def test_voice_record_start_handles_non_dict_voice_cfg(monkeypatch): assert captured["auto_restart"] is False +def test_prompt_submit_typed_stop_phrase_ends_voice_chat(monkeypatch): + """Typed bare stop phrase during an active voice chat is consumed at the + prompt.submit choke point: voice mode flips off, a distinct + voice.transcript {stop_phrase, typed} event fires, and NO turn starts. + """ + calls = {"stop_continuous": 0} + emitted = [] + monkeypatch.setattr( + server, "_emit", lambda event, sid, payload=None: emitted.append((event, payload)) + ) + monkeypatch.setitem( + sys.modules, + "tools.voice_mode", + types.SimpleNamespace( + is_voice_stop_phrase=lambda t: t.strip().lower().strip(".!?") == "stop" + ), + ) + monkeypatch.setitem( + sys.modules, + "hermes_cli.voice", + types.SimpleNamespace( + stop_continuous=lambda force_transcribe=False: calls.__setitem__( + "stop_continuous", calls["stop_continuous"] + 1 + ) + ), + ) + monkeypatch.setattr(server, "_tts_stream_stop", lambda user_barge=False: None) + monkeypatch.setenv("HERMES_VOICE", "1") + monkeypatch.setenv("HERMES_VOICE_TTS", "1") + + resp = server.dispatch( + { + "id": "typed-stop", + "method": "prompt.submit", + "params": {"session_id": "any-sid", "text": "Stop."}, + } + ) + + assert resp["result"] == {"voice_stopped": True} + assert os.environ["HERMES_VOICE"] == "0" + assert os.environ["HERMES_VOICE_TTS"] == "0" + assert calls["stop_continuous"] == 1 + assert ("voice.transcript", {"stop_phrase": True, "typed": True}) in emitted + + +def test_prompt_submit_typed_stop_passes_through_when_voice_off(monkeypatch): + """Outside a voice chat, typed "stop" is a normal message — the stop + matcher must not even be consulted (guard is on voice mode).""" + monkeypatch.setitem( + sys.modules, + "tools.voice_mode", + types.SimpleNamespace( + is_voice_stop_phrase=lambda t: (_ for _ in ()).throw( + AssertionError("stop matcher must not run when voice is off") + ) + ), + ) + monkeypatch.setenv("HERMES_VOICE", "0") + + resp = server.dispatch( + { + "id": "typed-stop-off", + "method": "prompt.submit", + "params": {"session_id": "missing-sid", "text": "stop"}, + } + ) + + # The submit proceeds into normal handling (here: unknown session error), + # NOT the voice_stopped consumption path. + assert resp.get("result") != {"voice_stopped": True} + + +def test_prompt_submit_longer_text_not_consumed_in_voice_mode(monkeypatch): + """"stop the build" while voice is on must reach the agent path.""" + monkeypatch.setitem( + sys.modules, + "tools.voice_mode", + types.SimpleNamespace( + is_voice_stop_phrase=lambda t: t.strip().lower().strip(".!?") == "stop" + ), + ) + monkeypatch.setenv("HERMES_VOICE", "1") + + resp = server.dispatch( + { + "id": "typed-long", + "method": "prompt.submit", + "params": {"session_id": "missing-sid", "text": "stop the build"}, + } + ) + + assert resp.get("result") != {"voice_stopped": True} + + def test_wake_owner_is_sticky_and_routes_detection_to_first_transport(monkeypatch): from tools import wake_word diff --git a/tests/tools/test_voice_cli_integration.py b/tests/tools/test_voice_cli_integration.py index 566d790e84e4..0fa4304b4876 100644 --- a/tests/tools/test_voice_cli_integration.py +++ b/tests/tools/test_voice_cli_integration.py @@ -1454,3 +1454,51 @@ class TestVoiceBargeCaptureSubmit: assert cli._pending_input.empty() assert not cli._voice_barge_capture.is_set() assert restarted.wait(2.0) # continuous mode resumes listening + + +# ============================================================================ +# Typed stop phrase — typing "stop" during a voice chat ends it +# ============================================================================ +class TestTypedVoiceStop: + """_typed_voice_stop: a TYPED bare stop phrase during an active voice chat + ends the chat (same as saying "stop"); outside voice mode it passes + through to the agent untouched.""" + + def _cli(self, **overrides): + cli = _make_voice_cli(**overrides) + cli._disable_calls = [] + cli._disable_voice_mode = lambda: cli._disable_calls.append(True) + return cli + + @pytest.fixture(autouse=True) + def _pin_stop_phrases(self, monkeypatch): + # Hermetic: don't let a dev machine's voice.stop_phrases config + # change which utterances count as a stop phrase. + monkeypatch.setattr( + "tools.voice_mode._load_voice_stop_phrases", lambda: ("stop",) + ) + + def test_typed_stop_ends_voice_chat_when_voice_on(self): + cli = self._cli(_voice_mode=True) + assert cli._typed_voice_stop("stop") is True + assert cli._disable_calls == [True] + + def test_typed_stop_during_continuous_mode(self): + cli = self._cli(_voice_mode=False, _voice_continuous=True) + assert cli._typed_voice_stop("Stop.") is True + assert cli._disable_calls == [True] + + def test_typed_stop_passes_through_when_voice_off(self): + cli = self._cli(_voice_mode=False, _voice_continuous=False) + assert cli._typed_voice_stop("stop") is False + assert cli._disable_calls == [] + + def test_longer_typed_message_passes_through_in_voice_mode(self): + cli = self._cli(_voice_mode=True) + assert cli._typed_voice_stop("stop the docker container") is False + assert cli._disable_calls == [] + + def test_non_string_input_passes_through(self): + cli = self._cli(_voice_mode=True) + assert cli._typed_voice_stop(("text", ["img.png"])) is False + assert cli._disable_calls == [] diff --git a/tests/tools/test_voice_stop_phrase.py b/tests/tools/test_voice_stop_phrase.py index b67b1e07b8ab..a6f0f48805b0 100644 --- a/tests/tools/test_voice_stop_phrase.py +++ b/tests/tools/test_voice_stop_phrase.py @@ -133,3 +133,173 @@ class TestContinuousLoopStopPhrase: delivered, silent_limit, _ = self._run_silence_cycle("stop the build and rerun") assert delivered == ["stop the build and rerun"] assert silent_limit == [] + + +class TestContinuousLoopStopPhraseSignal: + """The explicit on_stop_phrase signal: fired on a spoken stop phrase so + consumers (TUI, desktop) end the conversation as user intent, with + on_silent_limit as the legacy fallback when it isn't wired.""" + + class _FakeRecorder: + _peak_rms = 500 + + def stop(self): + return "/tmp/fake.wav" + + def cancel(self): + pass + + def start(self, on_silence_stop=None): + pass + + def _run_silence_cycle(self, transcript_text, on_stop_phrase): + import hermes_cli.voice as v + + delivered = [] + silent_limit_fired = [] + + fake_result = {"success": True, "transcript": transcript_text} + with patch.object(v, "_continuous_active", True), \ + patch.object(v, "_continuous_recorder", self._FakeRecorder()), \ + patch.object(v, "_continuous_on_transcript", delivered.append), \ + patch.object(v, "_continuous_on_status", None), \ + patch.object(v, "_continuous_on_silent_limit", + lambda: silent_limit_fired.append(True)), \ + patch.object(v, "_continuous_on_stop_phrase", on_stop_phrase), \ + patch.object(v, "_continuous_no_speech_count", 0), \ + patch.object(v, "transcribe_recording", return_value=fake_result), \ + patch.object(v, "_play_beep", lambda **kw: None), \ + patch.object(v.os.path, "isfile", return_value=False): + v._continuous_on_silence() + still_active = v._continuous_active + return delivered, silent_limit_fired, still_active + + def test_stop_phrase_fires_dedicated_signal_not_silent_limit(self): + stop_fired = [] + delivered, silent_limit, still_active = self._run_silence_cycle( + "Stop.", stop_fired.append + ) + assert stop_fired == ["Stop."] + assert silent_limit == [] + assert delivered == [] + assert still_active is False + + def test_normal_transcript_never_fires_stop_signal(self): + stop_fired = [] + delivered, silent_limit, _ = self._run_silence_cycle( + "stop the build and rerun", stop_fired.append + ) + assert stop_fired == [] + assert delivered == ["stop the build and rerun"] + assert silent_limit == [] + + def test_force_transcribe_stop_phrase_fires_signal(self): + """stop_continuous(force_transcribe=True) — the auto_restart=False + client-driven path (TUI/desktop voice.record stop) — must fire the + stop signal instead of silently discarding the transcript, or the + client re-arms the next capture and the conversation never ends.""" + import hermes_cli.voice as v + + delivered = [] + stop_fired = [] + threads = [] + + fake_result = {"success": True, "transcript": "stop"} + with patch.object(v, "_continuous_active", True), \ + patch.object(v, "_continuous_auto_restart", False), \ + patch.object(v, "_continuous_recorder", self._FakeRecorder()), \ + patch.object(v, "_continuous_on_transcript", delivered.append), \ + patch.object(v, "_continuous_on_status", None), \ + patch.object(v, "_continuous_on_silent_limit", None), \ + patch.object(v, "_continuous_on_stop_phrase", stop_fired.append), \ + patch.object(v, "_continuous_no_speech_count", 0), \ + patch.object(v, "transcribe_recording", return_value=fake_result), \ + patch.object(v, "_play_beep", lambda **kw: None), \ + patch.object(v.threading, "Thread", + side_effect=lambda target, daemon=None: threads.append(target) + or _ImmediateThread(target)), \ + patch.object(v.os.path, "isfile", return_value=False): + v.stop_continuous(force_transcribe=True) + + assert stop_fired == ["stop"] + assert delivered == [] + + def test_force_transcribe_stop_phrase_falls_back_to_silent_limit(self): + """Legacy consumers that never wired on_stop_phrase still get the + voice-off signal via on_silent_limit.""" + import hermes_cli.voice as v + + silent_limit_fired = [] + + fake_result = {"success": True, "transcript": "stop"} + with patch.object(v, "_continuous_active", True), \ + patch.object(v, "_continuous_auto_restart", False), \ + patch.object(v, "_continuous_recorder", self._FakeRecorder()), \ + patch.object(v, "_continuous_on_transcript", lambda t: None), \ + patch.object(v, "_continuous_on_status", None), \ + patch.object(v, "_continuous_on_silent_limit", + lambda: silent_limit_fired.append(True)), \ + patch.object(v, "_continuous_on_stop_phrase", None), \ + patch.object(v, "_continuous_no_speech_count", 0), \ + patch.object(v, "transcribe_recording", return_value=fake_result), \ + patch.object(v, "_play_beep", lambda **kw: None), \ + patch.object(v.threading, "Thread", + side_effect=lambda target, daemon=None: _ImmediateThread(target)), \ + patch.object(v.os.path, "isfile", return_value=False): + v.stop_continuous(force_transcribe=True) + + assert silent_limit_fired == [True] + + def test_start_continuous_accepts_on_stop_phrase_kwarg(self): + import inspect + + import hermes_cli.voice as v + + assert "on_stop_phrase" in inspect.signature(v.start_continuous).parameters + + +class _ImmediateThread: + """Thread stand-in that runs the target synchronously on start().""" + + def __init__(self, target): + self._target = target + + def start(self): + self._target() + + +class TestStopPhraseSurvivesHallucinationFilter: + """Ordering contract: a configured stop phrase must never be eaten by the + Whisper hallucination filter inside transcribe_recording. "bye" is BOTH a + known hallucination and a plausible stop phrase — when configured as a + stop phrase it must come through so the stop check can end the chat.""" + + def _transcribe(self, text, phrases): + import tools.voice_mode as vm + + with patch.object( + vm, "_load_voice_stop_phrases", return_value=tuple(phrases) + ), patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": text}, + ): + return vm.transcribe_recording("/tmp/fake.wav") + + def test_configured_stop_phrase_survives_blocklist(self): + result = self._transcribe("Bye.", ["stop", "bye"]) + assert result["success"] is True + assert result["transcript"] == "Bye." + assert not result.get("filtered") + + def test_unconfigured_hallucination_still_filtered(self): + result = self._transcribe("Bye.", ["stop"]) + assert result["success"] is True + assert result["transcript"] == "" + assert result.get("filtered") is True + + def test_default_stop_survives_repeat_regex_adjacent_phrases(self): + # "stop." is not in the blocklist/repeat regex today; this pins the + # contract so a future blocklist addition can't swallow it. + result = self._transcribe("Stop.", ["stop"]) + assert result["transcript"] == "Stop." + assert not result.get("filtered") diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 68eb59e7c96a..3016605c0bd6 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -1187,10 +1187,18 @@ def transcribe_recording(wav_path: str, model: Optional[str] = None) -> Dict[str if not result.get("success") and "File too large" in result.get("error", ""): result = _transcribe_wav_in_chunks(wav_path, model=model, max_file_size=MAX_FILE_SIZE) - # Filter out Whisper hallucinations (common on silent/near-silent audio) - if result.get("success") and is_whisper_hallucination(result.get("transcript", "")): - logger.info("Filtered Whisper hallucination: %r", result["transcript"]) - return {"success": True, "transcript": "", "filtered": True} + # Filter out Whisper hallucinations (common on silent/near-silent audio). + # A configured voice-chat stop phrase is checked FIRST and always survives: + # phrases like "bye" or "okay" overlap the hallucination blocklist/repeat + # regex, and swallowing them here would make saying "bye" (when configured + # as a stop phrase) silently fail to end the voice chat. + if result.get("success"): + raw_transcript = result.get("transcript", "") + if is_whisper_hallucination(raw_transcript) and not is_voice_stop_phrase( + raw_transcript + ): + logger.info("Filtered Whisper hallucination: %r", result["transcript"]) + return {"success": True, "transcript": "", "filtered": True} # Providers that flag no_speech (empty transcript) failed to hear words, # not to transcribe — treat like silence so the voice loop re-listens diff --git a/tui_gateway/server.py b/tui_gateway/server.py index dda53c8c856f..aa9b1caaebb6 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -11074,6 +11074,36 @@ def _(rid, params: dict) -> dict: sid = params.get("session_id", "") raw_text = params.get("text", "") text = sanitize_user_prompt_text(raw_text) if isinstance(raw_text, str) else raw_text + # Typed bare stop phrase while backend voice mode is active ends the + # voice chat instead of sending "stop" to the agent — the typed twin of + # the spoken stop phrase (PR #73106), applied at the ONE server-side + # choke point every TUI submit passes through. Guarded on voice mode + # being ON: typed "stop" outside a voice chat is a normal message. + # (The desktop's voice conversation is renderer-owned and never flips + # the backend flag, so it handles its own typed stop client-side.) + if isinstance(text, str) and _voice_mode_enabled(): + try: + from tools.voice_mode import is_voice_stop_phrase + + typed_stop = is_voice_stop_phrase(text) + except Exception: + typed_stop = False + if typed_stop: + os.environ["HERMES_VOICE"] = "0" + os.environ["HERMES_VOICE_TTS"] = "0" + try: + from hermes_cli.voice import stop_continuous + + stop_continuous() + except Exception: + pass + try: + _tts_stream_stop(user_barge=False) + except Exception: + pass + _voice_emit("voice.transcript", {"stop_phrase": True, "typed": True}) + logger.info("prompt.submit: typed stop phrase — voice chat ended") + return _ok(rid, {"voice_stopped": True}) truncate_user_ordinal = params.get("truncate_before_user_ordinal") if params.get("interrupted"): # Client-side barge-in (desktop VAD / typing over playback) — latch it @@ -17773,7 +17803,24 @@ def _tts_stream_barge_in_monitor(stop: threading.Event, done: threading.Event) - result = transcribe_recording(wav_path) text = (result.get("transcript") or "").strip() if result.get("success") else "" if text: - _voice_emit("voice.transcript", {"text": text}) + from tools.voice_mode import is_voice_stop_phrase + + if is_voice_stop_phrase(text): + # Barge-in with a bare stop phrase — the user talked over + # the agent's speech to END the voice chat. Same explicit + # stop signal as the continuous loop: flip mode off, halt + # any active capture, and tell clients it was user intent. + os.environ["HERMES_VOICE"] = "0" + os.environ["HERMES_VOICE_TTS"] = "0" + try: + from hermes_cli.voice import stop_continuous + + stop_continuous() + except Exception: + pass + _voice_emit("voice.transcript", {"stop_phrase": True, "text": text}) + else: + _voice_emit("voice.transcript", {"text": text}) finally: try: os.unlink(wav_path) @@ -18316,6 +18363,23 @@ def _(rid, params: dict) -> dict: _voice_emit("voice.transcript", {"no_speech_limit": True}) _resume_voice_wake() + def _on_stop_phrase(t): + # Explicit user intent: the user SAID a bare stop phrase + # ("stop"). End the voice chat exactly like a manual + # /voice off — flip the mode flags and silence any live + # streaming TTS — and emit a distinct signal so clients + # (TUI, desktop) end the conversation instead of treating + # it as a no-speech timeout. The continuous loop has + # already halted before this callback fires. + os.environ["HERMES_VOICE"] = "0" + os.environ["HERMES_VOICE_TTS"] = "0" + try: + _tts_stream_stop(user_barge=False) + except Exception: + pass + _voice_emit("voice.transcript", {"stop_phrase": True, "text": t}) + _resume_voice_wake() + def _on_status(state): _voice_emit("voice.status", {"state": state}) if state == "idle": @@ -18339,6 +18403,7 @@ def _(rid, params: dict) -> dict: silence_duration=safe_duration, auto_restart=False, max_recording_seconds=safe_max_rec, + on_stop_phrase=_on_stop_phrase, ) if started is False: _resume_voice_wake() diff --git a/ui-tui/src/__tests__/createGatewayEventHandler.test.ts b/ui-tui/src/__tests__/createGatewayEventHandler.test.ts index 451edcfc5917..06460a188289 100644 --- a/ui-tui/src/__tests__/createGatewayEventHandler.test.ts +++ b/ui-tui/src/__tests__/createGatewayEventHandler.test.ts @@ -859,6 +859,42 @@ describe('createGatewayEventHandler', () => { expect(ctx.gateway.rpc).toHaveBeenCalledWith('wake.start', { surface: 'tui' }) }) + it('ends voice mode on a stop-phrase transcript without submitting a turn', () => { + const ctx = buildCtx([]) + const onEvent = createGatewayEventHandler(ctx) + + onEvent({ payload: { stop_phrase: true, text: 'stop' }, type: 'voice.transcript' } as any) + + expect(ctx.voice.setVoiceEnabled).toHaveBeenCalledWith(false) + expect(ctx.voice.setRecording).toHaveBeenCalledWith(false) + expect(ctx.voice.setProcessing).toHaveBeenCalledWith(false) + expect(ctx.system.sys).toHaveBeenCalledWith('voice: stop phrase — voice chat ended') + // The stop phrase is user intent to END the chat — never a turn. + expect(ctx.submission.submitRef.current).not.toHaveBeenCalled() + }) + + it('ends voice mode on a typed stop phrase consumed server-side', () => { + const ctx = buildCtx([]) + const onEvent = createGatewayEventHandler(ctx) + + onEvent({ payload: { stop_phrase: true, typed: true }, type: 'voice.transcript' } as any) + + expect(ctx.voice.setVoiceEnabled).toHaveBeenCalledWith(false) + expect(ctx.submission.submitRef.current).not.toHaveBeenCalled() + }) + + it('still submits ordinary voice transcripts as turns', async () => { + const ctx = buildCtx([]) + const onEvent = createGatewayEventHandler(ctx) + + onEvent({ payload: { text: 'stop the docker container' }, type: 'voice.transcript' } as any) + + await vi.waitFor(() => + expect(ctx.submission.submitRef.current).toHaveBeenCalledWith('stop the docker container') + ) + expect(ctx.voice.setVoiceEnabled).not.toHaveBeenCalled() + }) + it('opens a fresh session before starting voice after wake detection', async () => { const ctx = buildCtx([]) ctx.session.newSession = vi.fn(async () => patchUiState({ sid: 'wake-session' })) diff --git a/ui-tui/src/app/createGatewayEventHandler.ts b/ui-tui/src/app/createGatewayEventHandler.ts index 338ea7ea498a..55a31f97ddd5 100644 --- a/ui-tui/src/app/createGatewayEventHandler.ts +++ b/ui-tui/src/app/createGatewayEventHandler.ts @@ -914,6 +914,19 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: } case 'voice.transcript': { + // Explicit user-intent stop: the user said (or typed) a bare stop + // phrase. The backend already halted the capture loop and flipped + // voice mode off — mirror it here like a manual /voice off, and say + // so (this is intent, not the no-speech timeout below). + if (ev.payload?.stop_phrase) { + setVoiceEnabled(false) + setVoiceRecording(false) + setVoiceProcessing(false) + sys('voice: stop phrase — voice chat ended') + + return + } + // CLI parity: the 3-strikes silence detector flipped off automatically. // Mirror that on the UI side and tell the user why the mode is off. if (ev.payload?.no_speech_limit) { diff --git a/ui-tui/src/app/submissionCore.ts b/ui-tui/src/app/submissionCore.ts index 98b416c44d96..c99eca09f2eb 100644 --- a/ui-tui/src/app/submissionCore.ts +++ b/ui-tui/src/app/submissionCore.ts @@ -81,6 +81,14 @@ export function submitPrompt( deps.gw .request('prompt.submit', { session_id: liveSid, text: submitText }) + .then(r => { + // The gateway consumed a typed voice stop phrase server-side (voice + // chat ended, no turn started) — release the busy latch; the + // voice.transcript {stop_phrase} event handles the mode flags + notice. + if (r?.voice_stopped) { + patchUiState({ busy: false, status: 'ready' }) + } + }) .catch((e: Error) => { // Defensive: prompt.submit no longer rejects a mid-turn send with // "session busy" (the gateway queues it and returns success), but keep diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index 4a8184dc3cda..607da742594f 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -320,6 +320,9 @@ export interface SessionSteerResponse { export interface PromptSubmitResponse { ok?: boolean + /** Set when the submitted text was a bare voice stop phrase consumed + * server-side to end the voice chat instead of starting a turn. */ + voice_stopped?: boolean } export interface BackgroundStartResponse { @@ -620,7 +623,11 @@ export type GatewayEvent = type: 'billing.step_up.verification' } | { payload?: { state?: 'idle' | 'listening' | 'transcribing' }; session_id?: string; type: 'voice.status' } - | { payload?: { no_speech_limit?: boolean; text?: string }; session_id?: string; type: 'voice.transcript' } + | { + payload?: { no_speech_limit?: boolean; stop_phrase?: boolean; text?: string; typed?: boolean } + session_id?: string + type: 'voice.transcript' + } | { payload?: { phrase?: string; profile?: null | string; start_new_session?: boolean } session_id?: string diff --git a/website/docs/user-guide/features/voice-mode.md b/website/docs/user-guide/features/voice-mode.md index fda828848bf0..8b31b23bf9b9 100644 --- a/website/docs/user-guide/features/voice-mode.md +++ b/website/docs/user-guide/features/voice-mode.md @@ -163,6 +163,8 @@ Both `silence_threshold` and `silence_duration` are configurable in `config.yaml Say **"stop"** — and nothing else — to end the voice conversation hands-free. The match is deliberately strict: the whole utterance (case-insensitive, surrounding punctuation ignored) must equal a configured phrase, so "stop doing that and try X instead" still reaches the agent normally. Customize the phrase list with `voice.stop_phrases` in `config.yaml` (e.g. `["stop", "goodbye hermes"]`), or set it to `[]` to disable. A voice chat also ends on its own after three consecutive silent cycles (no speech detected). +**Typing** a bare stop phrase while a voice chat is active works the same way on every surface (CLI, TUI, desktop): the message ends the voice chat instead of being sent to the agent. Outside a voice chat, typed "stop" is an ordinary message. + ### Streaming TTS When TTS is enabled, the agent speaks its reply **sentence-by-sentence** as it generates text — you don't wait for the full response. This works with **every TTS provider**: