diff --git a/cli.py b/cli.py index 462417d8158..5594ca32e68 100644 --- a/cli.py +++ b/cli.py @@ -12000,14 +12000,26 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): pass # Track consecutive no-speech cycles to avoid infinite restart loops. + # While the agent is mid-turn or TTS is speaking, the user is + # CORRECTLY silent (waiting/listening) — those cycles must not + # count, or a multi-minute tool run ends the voice chat under + # the user. The stop phrase and barge-in still work during the + # hold (they run on their own paths above). stop_continuous_restart = False + _activity_hold = bool( + getattr(self, "_agent_running", False) + or not self._voice_tts_done.is_set() + ) if not submitted: - self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1 - if self._no_speech_count >= 3: - self._voice_continuous = False - self._no_speech_count = 0 - _cprint(f"{_DIM}No speech detected 3 times, continuous mode stopped.{_RST}") - stop_continuous_restart = True + if _activity_hold: + pass # held: keep listening without counting the cycle + else: + self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1 + if self._no_speech_count >= 3: + self._voice_continuous = False + self._no_speech_count = 0 + _cprint(f"{_DIM}No speech detected 3 times, continuous mode stopped.{_RST}") + stop_continuous_restart = True else: self._no_speech_count = 0 diff --git a/hermes_cli/voice.py b/hermes_cli/voice.py index 83854a015fb..d72ab379bb2 100644 --- a/hermes_cli/voice.py +++ b/hermes_cli/voice.py @@ -301,6 +301,47 @@ _continuous_recorder: Any = None # leak into the mic. _tts_playing = threading.Event() _tts_playing.set() # initially "not playing" + +# ── Silence-count hold (agent busy) ────────────────────────────────── +# While the agent is mid-turn (thinking / tool-calling, possibly for +# minutes) or TTS is playing, the user is CORRECTLY silent — those cycles +# must not count toward the no-speech limit or a long tool run ends the +# voice chat under the user (#silence-must-not-end-the-chat). The host +# surface (tui_gateway) registers a probe that reports "agent busy"; +# TTS-playing is already tracked via _tts_playing above. +_voice_busy_probe: Optional[Callable[[], bool]] = None + + +def set_voice_busy_probe(probe: Optional[Callable[[], bool]]) -> None: + """Register a callable that returns True while the agent is mid-turn. + + Called by the hosting surface (tui_gateway registers one that checks + every session's ``running`` flag). ``None`` clears it. The probe must + be cheap and thread-safe — it runs on the silence-callback thread. + """ + global _voice_busy_probe + _voice_busy_probe = probe + + +def _voice_activity_held() -> bool: + """True while silent cycles must NOT count toward the no-speech limit. + + Held when TTS is playing (the user is listening) or when the + registered busy probe reports the agent mid-turn (the user is + waiting). Fail-open to "not held" so a broken probe can never make + the voice chat immortal. + """ + if not _tts_playing.is_set(): + return True + probe = _voice_busy_probe + if probe is None: + return False + try: + return bool(probe()) + except Exception: + return False + + _continuous_on_transcript: Optional[Callable[[str], None]] = None _continuous_on_status: Optional[Callable[[str], None]] = None _continuous_on_silent_limit: Optional[Callable[[], None]] = None @@ -574,9 +615,17 @@ def stop_continuous(force_transcribe: bool = False) -> None: logger.warning("on_transcript callback raised: %s", e) if track_no_speech: + held = _voice_activity_held() with _continuous_lock: if transcript or stop_phrase: _continuous_no_speech_count = 0 + elif held: + # Agent busy / TTS playing — the user is + # correctly silent; don't count the cycle. + _debug( + "stop_continuous: silent cycle ignored " + "(agent busy or TTS playing)" + ) else: _continuous_no_speech_count += 1 should_halt = ( @@ -712,6 +761,14 @@ def _continuous_on_silence() -> None: _debug(f"_continuous_on_silence: stop phrase {transcript!r} — ending loop") transcript = None + # Silent cycle while the agent is mid-turn or TTS is playing: the user + # is CORRECTLY quiet (waiting/listening), so the cycle must not count + # toward the no-speech limit — a multi-minute tool run would otherwise + # end the voice chat under the user. Checked outside the lock (probe + # may call into the host surface). + _silence_held = (transcript is None and not stop_phrase + and _voice_activity_held()) + with _continuous_lock: if not _continuous_active: # User stopped us while we were transcribing — discard. @@ -719,6 +776,11 @@ def _continuous_on_silence() -> None: return if transcript: _continuous_no_speech_count = 0 + elif _silence_held: + _debug( + "_continuous_on_silence: silent cycle ignored " + "(agent busy or TTS playing)" + ) elif not stop_phrase: _continuous_no_speech_count += 1 should_halt = stop_phrase or ( diff --git a/tests/hermes_cli/test_voice_wrapper.py b/tests/hermes_cli/test_voice_wrapper.py index 2d996969d45..a403d3c2892 100644 --- a/tests/hermes_cli/test_voice_wrapper.py +++ b/tests/hermes_cli/test_voice_wrapper.py @@ -424,6 +424,7 @@ class TestContinuousLoopSimulation: monkeypatch.setattr(voice, "_continuous_on_status", None) monkeypatch.setattr(voice, "_continuous_on_silent_limit", None) monkeypatch.setattr(voice, "_continuous_auto_restart", True, raising=False) + monkeypatch.setattr(voice, "_voice_busy_probe", None, raising=False) monkeypatch.setattr(voice, "_play_beep", lambda *_, **__: None) class FakeRecorder: @@ -724,6 +725,162 @@ class TestContinuousLoopSimulation: assert voice.is_continuous_active() is False assert fake_recorder.cancelled >= 1 + def test_silent_cycles_do_not_count_while_agent_busy(self, fake_recorder, monkeypatch): + """Agent mid-turn: silent cycles must NOT count toward the no-speech + limit — a multi-minute tool run would otherwise end the voice chat + while the user is correctly waiting quietly.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True) + + silent_limit_fired = [] + + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + ) + + # Way past the 3-strike limit while the agent is busy. + for _ in range(6): + fake_recorder.last_callback() + + assert silent_limit_fired == [] + assert voice.is_continuous_active() is True + assert voice._continuous_no_speech_count == 0 + + # Agent finishes → strikes count again, limit fires as before. + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: False) + for _ in range(3): + fake_recorder.last_callback() + assert silent_limit_fired == [True] + assert voice.is_continuous_active() is False + + def test_silent_cycles_do_not_count_while_tts_playing(self, fake_recorder, monkeypatch): + """TTS speaking: the user is listening, not ignoring the mic.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + # Keep the TTS-wait re-arm path from blocking: _tts_playing cleared + # means "playing"; use a tiny wait timeout via a fake event-like shim. + monkeypatch.setattr(voice, "_voice_busy_probe", None) + + class _FakePlaying: + def is_set(self): + return False + + def wait(self, timeout=None): + return True + + monkeypatch.setattr(voice, "_tts_playing", _FakePlaying()) + + silent_limit_fired = [] + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + ) + for _ in range(4): + fake_recorder.last_callback() + + assert silent_limit_fired == [] + assert voice._continuous_no_speech_count == 0 + voice.stop_continuous() + + def test_stop_phrase_still_ends_chat_during_busy_hold(self, fake_recorder, monkeypatch): + """The hold suppresses the silence counter only — a spoken stop + phrase must still end the voice chat instantly.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": "stop"}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + monkeypatch.setattr(voice, "is_voice_stop_phrase", lambda _t: True) + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True) + + stop_fired = [] + voice.start_continuous( + on_transcript=lambda _t: None, + on_stop_phrase=lambda t: stop_fired.append(t), + ) + fake_recorder.last_callback() + + assert stop_fired == ["stop"] + assert voice.is_continuous_active() is False + + def test_broken_busy_probe_fails_open(self, fake_recorder, monkeypatch): + """A raising probe must not make the voice chat immortal — silent + cycles count as if no probe were registered.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + + def _boom(): + raise RuntimeError("probe broken") + + monkeypatch.setattr(voice, "_voice_busy_probe", _boom) + + silent_limit_fired = [] + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + ) + for _ in range(3): + fake_recorder.last_callback() + + assert silent_limit_fired == [True] + assert voice.is_continuous_active() is False + + def test_force_transcribe_silent_cycle_held_while_busy(self, fake_recorder, monkeypatch): + """The single-shot (auto_restart=False, force_transcribe) strike path + honors the busy hold too — desktop/TUI clients drive that loop.""" + import hermes_cli.voice as voice + + class ImmediateThread: + def __init__(self, target, daemon=False): + self.target = target + + def start(self): + self.target() + + monkeypatch.setattr(voice.threading, "Thread", ImmediateThread) + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True) + + silent_limit_fired = [] + for _ in range(4): + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + auto_restart=False, + ) + voice.stop_continuous(force_transcribe=True) + + assert silent_limit_fired == [] + assert voice._continuous_no_speech_count == 0 + def test_stop_during_transcription_discards_restart(self, fake_recorder, monkeypatch): """User hits Ctrl+B mid-transcription: the in-flight transcript must still fire (it's a real utterance), but the loop must NOT restart.""" diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index 0286e9bea29..a39fcc0d6f6 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -14521,6 +14521,7 @@ def _fake_tts_modules(monkeypatch, *, requirements=True, playback_stops=None, li check_tts_requirements=lambda: requirements, stream_tts_to_speaker=fake_stream, _get_provider=lambda cfg: "edge", + _load_tts_config=lambda: {}, get_env_value=lambda key, default="": default, ), ) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index bc1cb1be384..0b49bab12a1 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -17701,6 +17701,21 @@ def _voice_tts_enabled() -> bool: return os.environ.get("HERMES_VOICE_TTS", "").strip() == "1" +def _any_session_running() -> bool: + """True while any session's agent turn is in flight. + + Registered as the voice busy-probe (``hermes_cli.voice.set_voice_busy_probe``) + so silent capture cycles during a long agent turn don't count toward the + no-speech limit — the user is correctly quiet while the agent works. + Voice is process-global (one microphone), so any running session holds. + """ + try: + with _sessions_lock: + return any(s.get("running") for s in _sessions.values()) + except Exception: + return False + + # ── Streaming TTS (one active pipeline per process — one speaker) ────────── # Token deltas from the running turn feed a sentence-buffering consumer # (tools.tts_tool.stream_tts_to_speaker) so speech starts on the first @@ -18354,6 +18369,18 @@ def _(rid, params: dict) -> dict: from hermes_cli.voice import start_continuous + # Register the agent-busy probe so the shared voice wrapper can + # hold the no-speech counter during long agent turns (item: + # silence must not end the chat while the agent works). Safe to + # re-register on every start; older wrappers without the setter + # are tolerated. + try: + from hermes_cli.voice import set_voice_busy_probe + + set_voice_busy_probe(_any_session_running) + except Exception: + pass + # Shape-safe lookups: malformed ``voice:`` YAML (bool/scalar/list) # must not crash /voice with a 5025 — fall back to VAD defaults. #