From df093bf33cf1d491a19b8032c8d4624aaea32665 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:50:44 -0700 Subject: [PATCH] feat(voice): calm ambient "thinking" sound while the agent works in voice chat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Long thinking/tool stretches in a voice conversation are dead air — the user cannot tell whether the agent is alive. New: quiet, repeating soft bubble blips while the agent works and no speech audio is flowing. - tools/voice_mode.py: numpy-synthesized blips (no binary assets) — two alternating low pitches (G4/E4) with pitch glide + smooth attack/decay envelopes, ~0.8-1.2s randomized spacing, volume = voice.beep_volume * 0.5. start_thinking_sound(should_play=...) / stop_thinking_sound() daemon-loop lifecycle; macOS-TCC-safe (sounddevice output gated there → silent skip, no per-second afplay churn). New mark_audio_output_active()/ is_audio_output_active() ref-count wraps play_audio_file and the streaming OutputStream sentence writes so "audio is flowing" is accurate. - Config: voice.thinking_sound (default true) off-switch. - cli.py: starts when a voice-mode turn begins, per-blip gate skips while TTS speaks / mic records / barge capture owns the mic; stopped in the chat() finally on every exit path. - tui_gateway/server.py: same lifecycle around _run_prompt_submit turns (voice mode on), gated on is_audio_output_active + continuous capture. - Desktop: renderer owns voice-conversation audio, so a matching WebAudio implementation (src/lib/thinking-sound.ts, same envelope/pitches) runs while conversation status === "thinking"; honors voice.thinking_sound (via config store) and the shared sound-mute toggle; stops instantly on speaking/listening/end. --- .../composer/hooks/use-voice-conversation.ts | 17 ++ .../app/session/hooks/use-hermes-config.ts | 3 +- apps/desktop/src/lib/thinking-sound.test.ts | 118 +++++++++++ apps/desktop/src/lib/thinking-sound.ts | 108 ++++++++++ apps/desktop/src/store/voice-prefs.ts | 11 ++ apps/desktop/src/types/hermes.ts | 1 + cli.py | 30 +++ hermes_cli/config.py | 1 + tests/tools/test_voice_thinking_sound.py | 185 ++++++++++++++++++ tools/tts_tool.py | 22 ++- tools/voice_mode.py | 159 +++++++++++++++ tui_gateway/server.py | 40 ++++ 12 files changed, 690 insertions(+), 5 deletions(-) create mode 100644 apps/desktop/src/lib/thinking-sound.test.ts create mode 100644 apps/desktop/src/lib/thinking-sound.ts create mode 100644 tests/tools/test_voice_thinking_sound.py diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index b2e54da3f2e..9ac3c743298 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -1,6 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' +import { startThinkingSound, stopThinkingSound } from '@/lib/thinking-sound' import { monitorSpeechDuringPlayback } from '@/lib/voice-barge-in' import { markVoicePlaybackInterrupted, @@ -574,6 +575,22 @@ export function useVoiceConversation({ return () => window.removeEventListener('keydown', onKeyDown, { capture: true }) }, [enabled, stopTurn]) + // Ambient "thinking" sound: while the agent works (status 'thinking') no + // audio flows, which reads as dead air mid-conversation. Calm bubble blips + // fill the gap; they stop the INSTANT speech starts, the mic re-arms, or the + // conversation ends. Gated by voice.thinking_sound + the shared sound mute. + useEffect(() => { + if (enabled && !muted && status === 'thinking') { + startThinkingSound() + + return stopThinkingSound + } + + stopThinkingSound() + + return undefined + }, [enabled, muted, status]) + // Drive the loop: when a voice-submitted reply appears, open a live speech // session (which feeds itself from then on). Otherwise start listening when // idle between turns. diff --git a/apps/desktop/src/app/session/hooks/use-hermes-config.ts b/apps/desktop/src/app/session/hooks/use-hermes-config.ts index eeed199a529..22d6ae55246 100644 --- a/apps/desktop/src/app/session/hooks/use-hermes-config.ts +++ b/apps/desktop/src/app/session/hooks/use-hermes-config.ts @@ -14,7 +14,7 @@ import { setDefaultReasoningEffort, setIntroPersonality } from '@/store/session' -import { applyAutoSpeakFromConfig, applyVoiceStopPhraseFromConfig } from '@/store/voice-prefs' +import { applyAutoSpeakFromConfig, applyThinkingSoundFromConfig, applyVoiceStopPhraseFromConfig } from '@/store/voice-prefs' const DEFAULT_VOICE_SECONDS = 120 const FAST_TIERS = new Set(['fast', 'priority', 'on']) @@ -106,6 +106,7 @@ export function useHermesConfig({ activeSessionIdRef }: HermesConfigOptions) { setSttEnabled(config.stt?.enabled !== false) applyAutoSpeakFromConfig(config) applyVoiceStopPhraseFromConfig(config) + applyThinkingSoundFromConfig(config) } catch { // Config is nice-to-have; chat still works without it. } diff --git a/apps/desktop/src/lib/thinking-sound.test.ts b/apps/desktop/src/lib/thinking-sound.test.ts new file mode 100644 index 00000000000..536180cfa2d --- /dev/null +++ b/apps/desktop/src/lib/thinking-sound.test.ts @@ -0,0 +1,118 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/store/haptics', () => ({ $hapticsMuted: { get: vi.fn(() => false) } })) +vi.mock('@/hermes', () => ({ + getHermesConfigRecord: vi.fn(async () => ({})), + saveHermesConfig: vi.fn(async () => undefined) +})) + +import { $hapticsMuted } from '@/store/haptics' +import { $thinkingSoundEnabled } from '@/store/voice-prefs' + +import { isThinkingSoundActive, startThinkingSound, stopThinkingSound } from './thinking-sound' + +class FakeOscillator { + type = 'sine' + frequency = { exponentialRampToValueAtTime: vi.fn(), setValueAtTime: vi.fn() } + connect = vi.fn() + start = vi.fn() + stop = vi.fn() +} + +class FakeGain { + gain = { exponentialRampToValueAtTime: vi.fn(), setValueAtTime: vi.fn() } + connect = vi.fn() +} + +const started: FakeOscillator[] = [] + +class FakeAudioContext { + currentTime = 0 + destination = {} + state = 'running' + + createOscillator() { + const osc = new FakeOscillator() + + started.push(osc) + + return osc + } + + createGain() { + return new FakeGain() + } + + resume() { + return Promise.resolve() + } +} + +describe('thinking-sound', () => { + beforeEach(() => { + vi.useFakeTimers() + started.length = 0 + $thinkingSoundEnabled.set(true) + vi.mocked($hapticsMuted.get).mockReturnValue(false) + vi.stubGlobal('AudioContext', FakeAudioContext) + }) + + afterEach(() => { + stopThinkingSound() + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it('plays repeating blips while active and stops instantly', () => { + startThinkingSound() + expect(isThinkingSoundActive()).toBe(true) + + vi.advanceTimersByTime(5_000) + expect(started.length).toBeGreaterThanOrEqual(3) + + const count = started.length + + stopThinkingSound() + expect(isThinkingSoundActive()).toBe(false) + vi.advanceTimersByTime(5_000) + expect(started.length).toBe(count) // nothing after stop + }) + + it('respects the voice.thinking_sound config gate', () => { + $thinkingSoundEnabled.set(false) + startThinkingSound() + expect(isThinkingSoundActive()).toBe(false) + vi.advanceTimersByTime(3_000) + expect(started.length).toBe(0) + }) + + it('stays silent while sounds are muted but keeps the loop alive', () => { + vi.mocked($hapticsMuted.get).mockReturnValue(true) + startThinkingSound() + vi.advanceTimersByTime(3_000) + expect(started.length).toBe(0) + + // Unmute mid-loop → blips resume without a restart. + vi.mocked($hapticsMuted.get).mockReturnValue(false) + vi.advanceTimersByTime(3_000) + expect(started.length).toBeGreaterThan(0) + }) + + it('start is idempotent', () => { + startThinkingSound() + startThinkingSound() + vi.advanceTimersByTime(1_300) + + // One loop: at most ~2 blips in 1.3s (first at 400ms, next ≥800ms later). + expect(started.length).toBeLessThanOrEqual(2) + }) + + it('never throws when WebAudio is unavailable', () => { + vi.stubGlobal('AudioContext', undefined) + expect(() => { + startThinkingSound() + vi.advanceTimersByTime(2_000) + }).not.toThrow() + stopThinkingSound() + }) +}) diff --git a/apps/desktop/src/lib/thinking-sound.ts b/apps/desktop/src/lib/thinking-sound.ts new file mode 100644 index 00000000000..6a4d2bd6da8 --- /dev/null +++ b/apps/desktop/src/lib/thinking-sound.ts @@ -0,0 +1,108 @@ +// Ambient "thinking" sound for the desktop voice conversation. While the agent +// works (status === 'thinking') no audio flows, which reads as "it died" during +// long thinking/tool stretches. A calm, quiet, repeating pair of soft bubble +// blips fills the gap — same WebAudio oscillator synthesis approach as +// wake-sound.ts / completion-sound.ts (no asset to ship), mirroring the +// backend's numpy-synthesized blips in tools/voice_mode.py so CLI and desktop +// sound alike. +// +// Honours the shared sound-mute toggle ($hapticsMuted) and the +// voice.thinking_sound config gate ($thinkingSoundEnabled). Stops instantly on +// stopThinkingSound() — callers fire it the moment TTS starts, the mic re-arms, +// or the conversation ends. + +import { $hapticsMuted } from '@/store/haptics' +import { $thinkingSoundEnabled } from '@/store/voice-prefs' + +let ctx: AudioContext | null = null +let timer: number | null = null +let blipIndex = 0 + +function getCtx(): AudioContext | null { + if (typeof window === 'undefined') { + return null + } + + try { + if (!ctx) { + const Ctor = + window.AudioContext || (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext + + if (!Ctor) { + return null + } + + ctx = new Ctor() + } + + if (ctx.state === 'suspended') { + void ctx.resume().catch(() => undefined) + } + + return ctx + } catch { + return null + } +} + +// One soft "blub": short sine with a gentle downward pitch glide and a smooth +// attack into an exponential decay — no clicks, deliberately quiet. +function blub(ac: AudioContext, freq: number) { + const t0 = ac.currentTime + 0.01 + const dur = 0.16 + const osc = ac.createOscillator() + const env = ac.createGain() + + osc.type = 'sine' + osc.frequency.setValueAtTime(freq, t0) + osc.frequency.exponentialRampToValueAtTime(freq * 0.72, t0 + dur) + + env.gain.setValueAtTime(0.0001, t0) + env.gain.exponentialRampToValueAtTime(0.08, t0 + 0.02) + env.gain.exponentialRampToValueAtTime(0.0001, t0 + dur) + + osc.connect(env) + env.connect(ac.destination) + osc.start(t0) + osc.stop(t0 + dur + 0.02) +} + +export function isThinkingSoundActive(): boolean { + return timer !== null +} + +/** Start the repeating thinking blips (idempotent). Best-effort, never throws. */ +export function startThinkingSound(): void { + if (timer !== null || !$thinkingSoundEnabled.get()) { + return + } + + const tick = () => { + if ($hapticsMuted.get() === false) { + const ac = getCtx() + + if (ac) { + try { + // Alternate two calm pitches (G4 / E4), matching the backend blips. + blub(ac, blipIndex % 2 === 0 ? 392 : 329.6) + } catch { + // Audio backend unavailable — stay silent, keep the loop harmless. + } + } + } + + blipIndex += 1 + // ~0.8-1.2s spacing with slight randomization so it reads organic. + timer = window.setTimeout(tick, 800 + Math.random() * 400) + } + + timer = window.setTimeout(tick, 400) +} + +/** Stop the thinking blips instantly (idempotent). */ +export function stopThinkingSound(): void { + if (timer !== null) { + window.clearTimeout(timer) + timer = null + } +} diff --git a/apps/desktop/src/store/voice-prefs.ts b/apps/desktop/src/store/voice-prefs.ts index b291b118304..51a073c23b3 100644 --- a/apps/desktop/src/store/voice-prefs.ts +++ b/apps/desktop/src/store/voice-prefs.ts @@ -37,6 +37,17 @@ export function applyVoiceStopPhraseFromConfig( $voiceStopPhrase.set(first ?? null) } +// `voice.thinking_sound` — ambient bubble blips while the agent works during a +// voice conversation (default on, matching the backend default). +export const $thinkingSoundEnabled = atom(true) + +/** Seed the thinking-sound gate from a loaded config payload. */ +export function applyThinkingSoundFromConfig( + config: { voice?: { thinking_sound?: unknown } | null } | null | undefined +) { + $thinkingSoundEnabled.set(config?.voice?.thinking_sound !== false) +} + /** * Flip the preference and persist it. Optimistic — the atom updates instantly and * reverts if the config write fails. Read-modify-writes the whole record (the diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index c2d01f272fb..65979f5e20e 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -332,6 +332,7 @@ export interface HermesConfig { max_recording_seconds?: number auto_tts?: boolean stop_phrases?: unknown + thinking_sound?: unknown } } diff --git a/cli.py b/cli.py index 519f6158f9e..db22d1b4a4e 100644 --- a/cli.py +++ b/cli.py @@ -13350,6 +13350,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # chunks as they arrive, everything else synthesizes per sentence. use_streaming_tts = False _streaming_box_opened = False + _thinking_started = False text_queue = None tts_thread = None stream_callback = None @@ -13554,6 +13555,27 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): agent_thread = threading.Thread(target=run_agent, daemon=True) agent_thread.start() + # Ambient "thinking" sound: calm bubble blips while the agent + # works in voice mode with no audio flowing, so the user knows + # it's alive during long thinking/tool stretches. Skipped per-blip + # while TTS speaks, the mic records, or a barge capture is live; + # stopped outright as soon as the turn ends. voice.thinking_sound + # gates it (default on); macOS is handled inside (TCC-safe skip). + _thinking_started = False + if self._voice_mode: + try: + from tools.voice_mode import start_thinking_sound + + _thinking_started = start_thinking_sound( + should_play=lambda: ( + self._voice_tts_done.is_set() + and not self._voice_recording + and not self._voice_barge_capture.is_set() + ) + ) + except Exception: + _thinking_started = False + # Monitor the dedicated interrupt queue while the agent runs. # _interrupt_queue is separate from _pending_input, so process_loop # and chat() never compete for the same queue. @@ -13960,6 +13982,14 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): print(f"Error: {e}") return None finally: + # Stop the ambient thinking sound the moment the turn ends — + # every exit path (normal, error, interrupt) lands here. + if _thinking_started: + try: + from tools.voice_mode import stop_thinking_sound + stop_thinking_sound() + except Exception: + pass # Ensure streaming TTS resources are cleaned up even on error. # Normal path sends the sentinel at line ~3568; this is a safety # net for exception paths that skip it. Duplicate sentinels are diff --git a/hermes_cli/config.py b/hermes_cli/config.py index a7d8975ea39..fec977b0636 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2372,6 +2372,7 @@ DEFAULT_CONFIG = { "auto_tts": False, "beep_enabled": True, # Play record start/stop beeps in CLI voice mode "beep_volume": 0.3, # Beep amplitude multiplier (0.0-1.0, default keeps prior hardcoded value) + "thinking_sound": True, # Calm ambient bubble sound while the agent works in voice chat (volume follows beep_volume) "silence_threshold": 200, # RMS below this = silence (0-32767) "silence_duration": 3.0, # Seconds of silence before auto-stop "barge_in": True, # Stop TTS playback when the user starts talking diff --git a/tests/tools/test_voice_thinking_sound.py b/tests/tools/test_voice_thinking_sound.py new file mode 100644 index 00000000000..57d32418ab8 --- /dev/null +++ b/tests/tools/test_voice_thinking_sound.py @@ -0,0 +1,185 @@ +"""Tests for the ambient voice-chat "thinking" sound (tools/voice_mode.py). + +Contract: + - `voice.thinking_sound` config gates it (default True). + - `start_thinking_sound()` is idempotent, returns False when disabled. + - The loop synthesizes blips with numpy (no assets), scales volume by + `voice.beep_volume`, and NEVER plays through sounddevice on macOS + (_sounddevice_output_allowed → TCC-safe silent skip). + - `stop_thinking_sound()` stops the loop instantly and is idempotent. + - The should_play callback gates each blip (no blips while TTS speaks + or the mic captures). + - mark_audio_output_active / is_audio_output_active ref-count playback. +""" + +import threading +import time +from unittest.mock import patch + +import numpy as np + +import tools.voice_mode as vm + + +class _FakeSD: + def __init__(self): + self.played = [] + + def play(self, audio, samplerate=None): + self.played.append((audio, samplerate)) + + def stop(self): + pass + + +def _reset(): + vm.stop_thinking_sound() + # Drain any residual output ref-counts from prior tests. + with vm._audio_output_lock: + vm._audio_output_active_count = 0 + + +class TestConfigGate: + def test_default_enabled(self): + with patch("hermes_cli.config.load_config", return_value={"voice": {}}): + assert vm.thinking_sound_enabled() is True + + def test_disabled_via_config(self): + with patch( + "hermes_cli.config.load_config", + return_value={"voice": {"thinking_sound": False}}, + ): + assert vm.thinking_sound_enabled() is False + + def test_quoted_false_string(self): + with patch( + "hermes_cli.config.load_config", + return_value={"voice": {"thinking_sound": "false"}}, + ): + assert vm.thinking_sound_enabled() is False + + def test_start_refuses_when_disabled(self): + _reset() + with patch.object(vm, "thinking_sound_enabled", return_value=False): + assert vm.start_thinking_sound() is False + assert vm._thinking_stop is None + + +class TestBlipSynthesis: + def test_blip_is_int16_low_volume(self): + with patch.object(vm, "_get_beep_volume", return_value=0.3): + blip = vm._synth_thinking_blip(np, 392.0) + assert blip.dtype == np.int16 + assert len(blip) == int(vm.SAMPLE_RATE * 0.16) + # Quieter than the beeps: 0.3 * 0.5 * 32767 ≈ 4915 peak ceiling. + assert int(np.abs(blip).max()) <= int(0.3 * 0.5 * 32767) + 1 + assert int(np.abs(blip).max()) > 0 + + def test_blip_volume_follows_beep_volume(self): + with patch.object(vm, "_get_beep_volume", return_value=1.0): + loud = vm._synth_thinking_blip(np, 392.0) + with patch.object(vm, "_get_beep_volume", return_value=0.1): + quiet = vm._synth_thinking_blip(np, 392.0) + assert int(np.abs(loud).max()) > int(np.abs(quiet).max()) * 5 + + def test_no_click_smooth_attack(self): + blip = vm._synth_thinking_blip(np, 392.0) + # First sample near zero (enveloped attack, no click). + assert abs(int(blip[0])) < 200 + + +class TestLoopLifecycle: + def test_loop_plays_blips_and_stops_instantly(self): + _reset() + fake = _FakeSD() + stop = threading.Event() + with patch.object(vm, "_sounddevice_output_allowed", return_value=True), \ + patch.object(vm, "_import_audio", return_value=(fake, np)), \ + patch.object(vm, "_get_beep_volume", return_value=0.3): + t = threading.Thread( + target=vm._thinking_sound_loop, args=(stop, None), daemon=True + ) + t.start() + deadline = time.monotonic() + 3.0 + while not fake.played and time.monotonic() < deadline: + time.sleep(0.01) + stop.set() + t.join(timeout=3.0) + assert fake.played, "loop never played a blip" + assert not t.is_alive() + + def test_should_play_false_suppresses_blips(self): + _reset() + fake = _FakeSD() + stop = threading.Event() + with patch.object(vm, "_sounddevice_output_allowed", return_value=True), \ + patch.object(vm, "_import_audio", return_value=(fake, np)), \ + patch.object(vm, "_get_beep_volume", return_value=0.3): + t = threading.Thread( + target=vm._thinking_sound_loop, + args=(stop, lambda: False), + daemon=True, + ) + t.start() + time.sleep(0.3) + stop.set() + t.join(timeout=3.0) + assert fake.played == [] + + def test_macos_tcc_gate_skips_silently(self): + """sounddevice output is gated on macOS — the loop must exit without + importing/playing anything (per-second afplay churn is worse than + silence).""" + _reset() + stop = threading.Event() + + def _boom(): + raise AssertionError("must not import audio when output is gated") + + with patch.object(vm, "_sounddevice_output_allowed", return_value=False), \ + patch.object(vm, "_import_audio", _boom): + vm._thinking_sound_loop(stop, None) # returns immediately + + def test_start_is_idempotent_and_stop_clears(self): + _reset() + with patch.object(vm, "thinking_sound_enabled", return_value=True), \ + patch.object(vm, "_sounddevice_output_allowed", return_value=False): + assert vm.start_thinking_sound() is True + first_stop = vm._thinking_stop + assert vm.start_thinking_sound() is True + assert vm._thinking_stop is first_stop # no second loop + vm.stop_thinking_sound() + assert vm._thinking_stop is None + assert first_stop.is_set() + vm.stop_thinking_sound() # idempotent + + +class TestAudioOutputRefcount: + def test_refcount_tracks_nested_playback(self): + _reset() + assert vm.is_audio_output_active() is False + vm.mark_audio_output_active(True) + vm.mark_audio_output_active(True) + assert vm.is_audio_output_active() is True + vm.mark_audio_output_active(False) + assert vm.is_audio_output_active() is True + vm.mark_audio_output_active(False) + assert vm.is_audio_output_active() is False + # Never goes negative. + vm.mark_audio_output_active(False) + assert vm.is_audio_output_active() is False + + def test_play_audio_file_brackets_refcount(self, tmp_path): + """play_audio_file flags real speaker output for its whole duration, + so the thinking loop knows audio is flowing.""" + _reset() + seen = [] + + def fake_impl(path): + seen.append(vm.is_audio_output_active()) + return True + + with patch.object(vm, "_play_audio_file_impl", fake_impl): + vm.play_audio_file(str(tmp_path / "x.wav")) + assert seen == [True] + assert vm.is_audio_output_active() is False diff --git a/tools/tts_tool.py b/tools/tts_tool.py index c1febc274db..04abda135d3 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -3445,10 +3445,24 @@ def stream_tts_to_speaker( audio_iter = streamer.stream(cleaned) if output_stream is not None: import numpy as _np - for chunk in audio_iter: - if stop_event.is_set(): - break - output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1)) + + # Flag real speaker output for the duration of this + # sentence so ambient cues (thinking sound) stay quiet. + # Fail-open: stubbed/partial voice_mode modules (tests) + # must never break sentence playback. + try: + from tools.voice_mode import mark_audio_output_active + except Exception: + def mark_audio_output_active(_active): + return None + mark_audio_output_active(True) + try: + for chunk in audio_iter: + if stop_event.is_set(): + break + output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1)) + finally: + mark_audio_output_active(False) else: # No audio device: buffer chunks to a temp WAV and play it. _play_via_tempfile(audio_iter, stop_event, streamer.sample_rate) diff --git a/tools/voice_mode.py b/tools/voice_mode.py index b12c0672301..81f35883423 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -528,6 +528,155 @@ def play_beep(frequency: int = 880, duration: float = 0.12, count: int = 1) -> N logger.debug("Beep playback failed: %s", e) +# ============================================================================ +# Thinking sound — calm ambient "blub blub" while the agent works +# ============================================================================ +# During a voice conversation the agent can think / run tools for minutes with +# zero audio, which reads as "it died". A quiet, repeating pair of soft water- +# bubble blips fills that gap. Fully synthesized with numpy (no binary asset), +# volume-scaled by voice.beep_volume, gated by voice.thinking_sound (default +# on), and macOS-TCC-safe: sounddevice OUTPUT is gated there +# (_sounddevice_output_allowed), and spawning afplay every second would churn +# subprocesses, so on macOS the thinking sound is skipped silently. + +# The host's *should_play* callback decides when blips are allowed; the +# module-level output ref-count below tracks when real audio (TTS sentences, +# file playback) is actually flowing so hosts have an accurate signal. + +_audio_output_active_count = 0 +_audio_output_lock = threading.Lock() + + +def mark_audio_output_active(active: bool) -> None: + """Reference-count real audio output (TTS/file playback). + + Playback paths bracket their work with ``mark_audio_output_active(True)`` + / ``(False)`` so ``is_audio_output_active()`` reflects whether speech + audio is leaving the speakers RIGHT NOW — unlike the per-turn TTS-done + events, which stay 'busy' for a whole turn even while the pipeline is + silently waiting for text. + """ + global _audio_output_active_count + with _audio_output_lock: + _audio_output_active_count = max( + 0, _audio_output_active_count + (1 if active else -1) + ) + + +def is_audio_output_active() -> bool: + """True while TTS/file audio is actually playing on the speakers.""" + with _audio_output_lock: + return _audio_output_active_count > 0 + + +_thinking_lock = threading.Lock() +_thinking_stop: Optional[threading.Event] = None + + +def thinking_sound_enabled() -> bool: + """Config gate: ``voice.thinking_sound`` (default True).""" + try: + from hermes_cli.config import load_config + from utils import is_truthy_value + + voice_cfg = load_config().get("voice", {}) + if isinstance(voice_cfg, dict): + return is_truthy_value( + voice_cfg.get("thinking_sound", True), default=True + ) + except Exception: + pass + return True + + +def _synth_thinking_blip(np, frequency: float) -> "Any": + """One soft 'blub': short sine with a gentle downward pitch glide and a + smooth attack/decay envelope (no clicks), low-volume.""" + duration = 0.16 + n = int(SAMPLE_RATE * duration) + t = np.linspace(0, duration, n, endpoint=False) + # Downward glide (water-drop feel): freq → 0.72*freq over the blip. + glide = np.linspace(1.0, 0.72, n) + phase = 2 * np.pi * np.cumsum(frequency * glide) / SAMPLE_RATE + tone = np.sin(phase) + # Soften harmonics (cheap low-pass feel): add a quieter octave-down sine. + tone = 0.8 * tone + 0.2 * np.sin(phase / 2.0) + # Envelope: quick-but-smooth attack, long exponential-ish decay. + attack = int(0.02 * SAMPLE_RATE) + env = np.ones(n) + env[:attack] = np.linspace(0.0, 1.0, attack) + env *= np.exp(-t * 14.0) + volume = _get_beep_volume() * 0.5 # deliberately quieter than the beeps + return (tone * env * volume * 32767).astype(np.int16) + + +def _thinking_sound_loop(stop: threading.Event, should_play) -> None: + """Daemon loop: play alternating-pitch blips every ~0.8-1.2s until *stop*. + + Skips a blip (without stopping) whenever *should_play* returns False — + e.g. TTS audio started flowing or the mic re-armed. macOS: sounddevice + output is TCC-gated, and per-second afplay subprocess churn is worse + than silence, so the loop exits immediately there. + """ + if not _sounddevice_output_allowed(): + return + try: + sd, np = _import_audio() + except (ImportError, OSError): + return + + import random + + pitches = (392.0, 329.6) # G4 / E4 — calm, low, alternating + blips = [_synth_thinking_blip(np, p) for p in pitches] + i = 0 + while not stop.is_set(): + try: + if should_play is None or should_play(): + blip = blips[i % len(blips)] + sd.play(blip, samplerate=SAMPLE_RATE) + stop.wait(len(blip) / SAMPLE_RATE + 0.02) + sd.stop() + i += 1 + except Exception as e: + logger.debug("Thinking sound blip failed: %s", e) + return + stop.wait(0.8 + random.random() * 0.4) + + +def start_thinking_sound(should_play=None) -> bool: + """Start the ambient thinking sound (idempotent). + + *should_play* is polled before each blip; return False to skip while + speech audio flows or the mic is capturing. Returns True when the loop + was started (or already running), False when disabled/unavailable. + """ + global _thinking_stop + if not thinking_sound_enabled(): + return False + with _thinking_lock: + if _thinking_stop is not None and not _thinking_stop.is_set(): + return True # already running + stop = threading.Event() + _thinking_stop = stop + threading.Thread( + target=_thinking_sound_loop, + args=(stop, should_play), + daemon=True, + name="voice-thinking-sound", + ).start() + return True + + +def stop_thinking_sound() -> None: + """Stop the ambient thinking sound instantly (idempotent).""" + global _thinking_stop + with _thinking_lock: + stop, _thinking_stop = _thinking_stop, None + if stop is not None: + stop.set() + + # ============================================================================ # Termux Audio Recorder # ============================================================================ @@ -1413,6 +1562,16 @@ def play_audio_file(file_path: str) -> bool: Returns: ``True`` if playback succeeded, ``False`` otherwise. """ + # Ref-count real speaker output for the whole call so the thinking-sound + # loop (and any other ambient cue) knows audio is flowing right now. + mark_audio_output_active(True) + try: + return _play_audio_file_impl(file_path) + finally: + mark_audio_output_active(False) + + +def _play_audio_file_impl(file_path: str) -> bool: global _active_playback if not os.path.isfile(file_path): diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 5984952d0f7..5f8ba266c7d 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -11995,6 +11995,7 @@ def _run_prompt_submit( goal_followup = None # set by the post-turn goal hook below result = None # turn outcome; read after the finally for leftover /steer tts_queue = None # streaming-TTS feed for this turn (voice mode) + thinking_started = False # ambient thinking sound armed for this turn one_turn_restore = session.pop("one_turn_model_restore", None) # True once a failed turn's snapshot was retained for resume replay — # tells the finally below to skip the normal inflight clear. @@ -12143,6 +12144,36 @@ def _run_prompt_submit( # consume the latch below. tts_queue = _tts_stream_begin() + # Ambient "thinking" sound (voice mode only): calm bubble blips + # while the agent works with no audio flowing, so long + # thinking/tool stretches don't read as a dead session. Per-blip + # gate skips while real TTS audio flows or the mic is capturing; + # stopped in the finally the instant the turn ends. + # voice.thinking_sound config-gates it; macOS TCC handled inside. + thinking_started = False + if _voice_mode_enabled(): + try: + from tools.voice_mode import ( + is_audio_output_active, + start_thinking_sound, + ) + + def _thinking_should_play() -> bool: + if is_audio_output_active(): + return False + try: + from hermes_cli.voice import is_continuous_active + + return not is_continuous_active() + except Exception: + return True + + thinking_started = start_thinking_sound( + should_play=_thinking_should_play + ) + except Exception: + thinking_started = False + # Barged mid-speech? Tell the model (API-message note, same # enrichment channel as attached images) so it can react # ("rude!") instead of being oblivious to its own interruption. @@ -12551,6 +12582,15 @@ def _run_prompt_submit( ) _emit("error", sid, {"message": str(e)}) finally: + if thinking_started: + # Kill the ambient thinking sound the moment the turn ends — + # error and success paths both land here. + try: + from tools.voice_mode import stop_thinking_sound + + stop_thinking_sound() + except Exception: + pass if tts_queue is not None: tts_queue.put(None) # end-of-text sentinel — flush + finish speaking if one_turn_restore: