mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(voice): silent cycles never end the chat while the agent is busy or TTS is playing
The continuous-voice no-speech counter (3 strikes -> voice off) counted every silent capture cycle unconditionally. During a long agent turn (thinking/tool-calling for minutes) or while TTS is speaking, the user is CORRECTLY silent — those cycles ended the voice chat under them. - hermes_cli/voice.py: new set_voice_busy_probe() seam + _voice_activity_held() (TTS-playing via the existing _tts_playing Event, agent-busy via the registered probe). Both the continuous-loop strike path and the force-transcribe single-shot strike path skip counting while held. Fail-open: a broken probe counts cycles as before. - tui_gateway/server.py: registers _any_session_running() as the probe on voice.record start (voice is process-global; any running session holds). - cli.py: classic CLI strike path skips counting while _agent_running or TTS playback is in flight. Stop phrase and barge-in still work during the hold (own paths). Includes a fixture fix for the #71083 cherry-pick: the fake tools.tts_tool module needs _load_tts_config (main's tts_streaming imports it).
This commit is contained in:
Teknium
parent
be424703c4
commit
9e5f1b619c
5 changed files with 265 additions and 6 deletions
24
cli.py
24
cli.py
|
|
@ -12000,14 +12000,26 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
pass
|
||||
|
||||
# Track consecutive no-speech cycles to avoid infinite restart loops.
|
||||
# While the agent is mid-turn or TTS is speaking, the user is
|
||||
# CORRECTLY silent (waiting/listening) — those cycles must not
|
||||
# count, or a multi-minute tool run ends the voice chat under
|
||||
# the user. The stop phrase and barge-in still work during the
|
||||
# hold (they run on their own paths above).
|
||||
stop_continuous_restart = False
|
||||
_activity_hold = bool(
|
||||
getattr(self, "_agent_running", False)
|
||||
or not self._voice_tts_done.is_set()
|
||||
)
|
||||
if not submitted:
|
||||
self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1
|
||||
if self._no_speech_count >= 3:
|
||||
self._voice_continuous = False
|
||||
self._no_speech_count = 0
|
||||
_cprint(f"{_DIM}No speech detected 3 times, continuous mode stopped.{_RST}")
|
||||
stop_continuous_restart = True
|
||||
if _activity_hold:
|
||||
pass # held: keep listening without counting the cycle
|
||||
else:
|
||||
self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1
|
||||
if self._no_speech_count >= 3:
|
||||
self._voice_continuous = False
|
||||
self._no_speech_count = 0
|
||||
_cprint(f"{_DIM}No speech detected 3 times, continuous mode stopped.{_RST}")
|
||||
stop_continuous_restart = True
|
||||
else:
|
||||
self._no_speech_count = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -301,6 +301,47 @@ _continuous_recorder: Any = None
|
|||
# leak into the mic.
|
||||
_tts_playing = threading.Event()
|
||||
_tts_playing.set() # initially "not playing"
|
||||
|
||||
# ── Silence-count hold (agent busy) ──────────────────────────────────
|
||||
# While the agent is mid-turn (thinking / tool-calling, possibly for
|
||||
# minutes) or TTS is playing, the user is CORRECTLY silent — those cycles
|
||||
# must not count toward the no-speech limit or a long tool run ends the
|
||||
# voice chat under the user (#silence-must-not-end-the-chat). The host
|
||||
# surface (tui_gateway) registers a probe that reports "agent busy";
|
||||
# TTS-playing is already tracked via _tts_playing above.
|
||||
_voice_busy_probe: Optional[Callable[[], bool]] = None
|
||||
|
||||
|
||||
def set_voice_busy_probe(probe: Optional[Callable[[], bool]]) -> None:
|
||||
"""Register a callable that returns True while the agent is mid-turn.
|
||||
|
||||
Called by the hosting surface (tui_gateway registers one that checks
|
||||
every session's ``running`` flag). ``None`` clears it. The probe must
|
||||
be cheap and thread-safe — it runs on the silence-callback thread.
|
||||
"""
|
||||
global _voice_busy_probe
|
||||
_voice_busy_probe = probe
|
||||
|
||||
|
||||
def _voice_activity_held() -> bool:
|
||||
"""True while silent cycles must NOT count toward the no-speech limit.
|
||||
|
||||
Held when TTS is playing (the user is listening) or when the
|
||||
registered busy probe reports the agent mid-turn (the user is
|
||||
waiting). Fail-open to "not held" so a broken probe can never make
|
||||
the voice chat immortal.
|
||||
"""
|
||||
if not _tts_playing.is_set():
|
||||
return True
|
||||
probe = _voice_busy_probe
|
||||
if probe is None:
|
||||
return False
|
||||
try:
|
||||
return bool(probe())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
_continuous_on_transcript: Optional[Callable[[str], None]] = None
|
||||
_continuous_on_status: Optional[Callable[[str], None]] = None
|
||||
_continuous_on_silent_limit: Optional[Callable[[], None]] = None
|
||||
|
|
@ -574,9 +615,17 @@ def stop_continuous(force_transcribe: bool = False) -> None:
|
|||
logger.warning("on_transcript callback raised: %s", e)
|
||||
|
||||
if track_no_speech:
|
||||
held = _voice_activity_held()
|
||||
with _continuous_lock:
|
||||
if transcript or stop_phrase:
|
||||
_continuous_no_speech_count = 0
|
||||
elif held:
|
||||
# Agent busy / TTS playing — the user is
|
||||
# correctly silent; don't count the cycle.
|
||||
_debug(
|
||||
"stop_continuous: silent cycle ignored "
|
||||
"(agent busy or TTS playing)"
|
||||
)
|
||||
else:
|
||||
_continuous_no_speech_count += 1
|
||||
should_halt = (
|
||||
|
|
@ -712,6 +761,14 @@ def _continuous_on_silence() -> None:
|
|||
_debug(f"_continuous_on_silence: stop phrase {transcript!r} — ending loop")
|
||||
transcript = None
|
||||
|
||||
# Silent cycle while the agent is mid-turn or TTS is playing: the user
|
||||
# is CORRECTLY quiet (waiting/listening), so the cycle must not count
|
||||
# toward the no-speech limit — a multi-minute tool run would otherwise
|
||||
# end the voice chat under the user. Checked outside the lock (probe
|
||||
# may call into the host surface).
|
||||
_silence_held = (transcript is None and not stop_phrase
|
||||
and _voice_activity_held())
|
||||
|
||||
with _continuous_lock:
|
||||
if not _continuous_active:
|
||||
# User stopped us while we were transcribing — discard.
|
||||
|
|
@ -719,6 +776,11 @@ def _continuous_on_silence() -> None:
|
|||
return
|
||||
if transcript:
|
||||
_continuous_no_speech_count = 0
|
||||
elif _silence_held:
|
||||
_debug(
|
||||
"_continuous_on_silence: silent cycle ignored "
|
||||
"(agent busy or TTS playing)"
|
||||
)
|
||||
elif not stop_phrase:
|
||||
_continuous_no_speech_count += 1
|
||||
should_halt = stop_phrase or (
|
||||
|
|
|
|||
|
|
@ -424,6 +424,7 @@ class TestContinuousLoopSimulation:
|
|||
monkeypatch.setattr(voice, "_continuous_on_status", None)
|
||||
monkeypatch.setattr(voice, "_continuous_on_silent_limit", None)
|
||||
monkeypatch.setattr(voice, "_continuous_auto_restart", True, raising=False)
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", None, raising=False)
|
||||
monkeypatch.setattr(voice, "_play_beep", lambda *_, **__: None)
|
||||
|
||||
class FakeRecorder:
|
||||
|
|
@ -724,6 +725,162 @@ class TestContinuousLoopSimulation:
|
|||
assert voice.is_continuous_active() is False
|
||||
assert fake_recorder.cancelled >= 1
|
||||
|
||||
def test_silent_cycles_do_not_count_while_agent_busy(self, fake_recorder, monkeypatch):
|
||||
"""Agent mid-turn: silent cycles must NOT count toward the no-speech
|
||||
limit — a multi-minute tool run would otherwise end the voice chat
|
||||
while the user is correctly waiting quietly."""
|
||||
import hermes_cli.voice as voice
|
||||
|
||||
monkeypatch.setattr(
|
||||
voice,
|
||||
"transcribe_recording",
|
||||
lambda _p: {"success": True, "transcript": ""},
|
||||
)
|
||||
monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False)
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True)
|
||||
|
||||
silent_limit_fired = []
|
||||
|
||||
voice.start_continuous(
|
||||
on_transcript=lambda _t: None,
|
||||
on_silent_limit=lambda: silent_limit_fired.append(True),
|
||||
)
|
||||
|
||||
# Way past the 3-strike limit while the agent is busy.
|
||||
for _ in range(6):
|
||||
fake_recorder.last_callback()
|
||||
|
||||
assert silent_limit_fired == []
|
||||
assert voice.is_continuous_active() is True
|
||||
assert voice._continuous_no_speech_count == 0
|
||||
|
||||
# Agent finishes → strikes count again, limit fires as before.
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", lambda: False)
|
||||
for _ in range(3):
|
||||
fake_recorder.last_callback()
|
||||
assert silent_limit_fired == [True]
|
||||
assert voice.is_continuous_active() is False
|
||||
|
||||
def test_silent_cycles_do_not_count_while_tts_playing(self, fake_recorder, monkeypatch):
|
||||
"""TTS speaking: the user is listening, not ignoring the mic."""
|
||||
import hermes_cli.voice as voice
|
||||
|
||||
monkeypatch.setattr(
|
||||
voice,
|
||||
"transcribe_recording",
|
||||
lambda _p: {"success": True, "transcript": ""},
|
||||
)
|
||||
monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False)
|
||||
# Keep the TTS-wait re-arm path from blocking: _tts_playing cleared
|
||||
# means "playing"; use a tiny wait timeout via a fake event-like shim.
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", None)
|
||||
|
||||
class _FakePlaying:
|
||||
def is_set(self):
|
||||
return False
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(voice, "_tts_playing", _FakePlaying())
|
||||
|
||||
silent_limit_fired = []
|
||||
voice.start_continuous(
|
||||
on_transcript=lambda _t: None,
|
||||
on_silent_limit=lambda: silent_limit_fired.append(True),
|
||||
)
|
||||
for _ in range(4):
|
||||
fake_recorder.last_callback()
|
||||
|
||||
assert silent_limit_fired == []
|
||||
assert voice._continuous_no_speech_count == 0
|
||||
voice.stop_continuous()
|
||||
|
||||
def test_stop_phrase_still_ends_chat_during_busy_hold(self, fake_recorder, monkeypatch):
|
||||
"""The hold suppresses the silence counter only — a spoken stop
|
||||
phrase must still end the voice chat instantly."""
|
||||
import hermes_cli.voice as voice
|
||||
|
||||
monkeypatch.setattr(
|
||||
voice,
|
||||
"transcribe_recording",
|
||||
lambda _p: {"success": True, "transcript": "stop"},
|
||||
)
|
||||
monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False)
|
||||
monkeypatch.setattr(voice, "is_voice_stop_phrase", lambda _t: True)
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True)
|
||||
|
||||
stop_fired = []
|
||||
voice.start_continuous(
|
||||
on_transcript=lambda _t: None,
|
||||
on_stop_phrase=lambda t: stop_fired.append(t),
|
||||
)
|
||||
fake_recorder.last_callback()
|
||||
|
||||
assert stop_fired == ["stop"]
|
||||
assert voice.is_continuous_active() is False
|
||||
|
||||
def test_broken_busy_probe_fails_open(self, fake_recorder, monkeypatch):
|
||||
"""A raising probe must not make the voice chat immortal — silent
|
||||
cycles count as if no probe were registered."""
|
||||
import hermes_cli.voice as voice
|
||||
|
||||
monkeypatch.setattr(
|
||||
voice,
|
||||
"transcribe_recording",
|
||||
lambda _p: {"success": True, "transcript": ""},
|
||||
)
|
||||
monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False)
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("probe broken")
|
||||
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", _boom)
|
||||
|
||||
silent_limit_fired = []
|
||||
voice.start_continuous(
|
||||
on_transcript=lambda _t: None,
|
||||
on_silent_limit=lambda: silent_limit_fired.append(True),
|
||||
)
|
||||
for _ in range(3):
|
||||
fake_recorder.last_callback()
|
||||
|
||||
assert silent_limit_fired == [True]
|
||||
assert voice.is_continuous_active() is False
|
||||
|
||||
def test_force_transcribe_silent_cycle_held_while_busy(self, fake_recorder, monkeypatch):
|
||||
"""The single-shot (auto_restart=False, force_transcribe) strike path
|
||||
honors the busy hold too — desktop/TUI clients drive that loop."""
|
||||
import hermes_cli.voice as voice
|
||||
|
||||
class ImmediateThread:
|
||||
def __init__(self, target, daemon=False):
|
||||
self.target = target
|
||||
|
||||
def start(self):
|
||||
self.target()
|
||||
|
||||
monkeypatch.setattr(voice.threading, "Thread", ImmediateThread)
|
||||
monkeypatch.setattr(
|
||||
voice,
|
||||
"transcribe_recording",
|
||||
lambda _p: {"success": True, "transcript": ""},
|
||||
)
|
||||
monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False)
|
||||
monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True)
|
||||
|
||||
silent_limit_fired = []
|
||||
for _ in range(4):
|
||||
voice.start_continuous(
|
||||
on_transcript=lambda _t: None,
|
||||
on_silent_limit=lambda: silent_limit_fired.append(True),
|
||||
auto_restart=False,
|
||||
)
|
||||
voice.stop_continuous(force_transcribe=True)
|
||||
|
||||
assert silent_limit_fired == []
|
||||
assert voice._continuous_no_speech_count == 0
|
||||
|
||||
def test_stop_during_transcription_discards_restart(self, fake_recorder, monkeypatch):
|
||||
"""User hits Ctrl+B mid-transcription: the in-flight transcript must
|
||||
still fire (it's a real utterance), but the loop must NOT restart."""
|
||||
|
|
|
|||
|
|
@ -14521,6 +14521,7 @@ def _fake_tts_modules(monkeypatch, *, requirements=True, playback_stops=None, li
|
|||
check_tts_requirements=lambda: requirements,
|
||||
stream_tts_to_speaker=fake_stream,
|
||||
_get_provider=lambda cfg: "edge",
|
||||
_load_tts_config=lambda: {},
|
||||
get_env_value=lambda key, default="": default,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17701,6 +17701,21 @@ def _voice_tts_enabled() -> bool:
|
|||
return os.environ.get("HERMES_VOICE_TTS", "").strip() == "1"
|
||||
|
||||
|
||||
def _any_session_running() -> bool:
|
||||
"""True while any session's agent turn is in flight.
|
||||
|
||||
Registered as the voice busy-probe (``hermes_cli.voice.set_voice_busy_probe``)
|
||||
so silent capture cycles during a long agent turn don't count toward the
|
||||
no-speech limit — the user is correctly quiet while the agent works.
|
||||
Voice is process-global (one microphone), so any running session holds.
|
||||
"""
|
||||
try:
|
||||
with _sessions_lock:
|
||||
return any(s.get("running") for s in _sessions.values())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ── Streaming TTS (one active pipeline per process — one speaker) ──────────
|
||||
# Token deltas from the running turn feed a sentence-buffering consumer
|
||||
# (tools.tts_tool.stream_tts_to_speaker) so speech starts on the first
|
||||
|
|
@ -18354,6 +18369,18 @@ def _(rid, params: dict) -> dict:
|
|||
|
||||
from hermes_cli.voice import start_continuous
|
||||
|
||||
# Register the agent-busy probe so the shared voice wrapper can
|
||||
# hold the no-speech counter during long agent turns (item:
|
||||
# silence must not end the chat while the agent works). Safe to
|
||||
# re-register on every start; older wrappers without the setter
|
||||
# are tolerated.
|
||||
try:
|
||||
from hermes_cli.voice import set_voice_busy_probe
|
||||
|
||||
set_voice_busy_probe(_any_session_running)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Shape-safe lookups: malformed ``voice:`` YAML (bool/scalar/list)
|
||||
# must not crash /voice with a 5025 — fall back to VAD defaults.
|
||||
#
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue