mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
feat(voice): calm ambient "thinking" sound while the agent works in voice chat
Long thinking/tool stretches in a voice conversation are dead air — the user cannot tell whether the agent is alive. New: quiet, repeating soft bubble blips while the agent works and no speech audio is flowing. - tools/voice_mode.py: numpy-synthesized blips (no binary assets) — two alternating low pitches (G4/E4) with pitch glide + smooth attack/decay envelopes, ~0.8-1.2s randomized spacing, volume = voice.beep_volume * 0.5. start_thinking_sound(should_play=...) / stop_thinking_sound() daemon-loop lifecycle; macOS-TCC-safe (sounddevice output gated there → silent skip, no per-second afplay churn). New mark_audio_output_active()/ is_audio_output_active() ref-count wraps play_audio_file and the streaming OutputStream sentence writes so "audio is flowing" is accurate. - Config: voice.thinking_sound (default true) off-switch. - cli.py: starts when a voice-mode turn begins, per-blip gate skips while TTS speaks / mic records / barge capture owns the mic; stopped in the chat() finally on every exit path. - tui_gateway/server.py: same lifecycle around _run_prompt_submit turns (voice mode on), gated on is_audio_output_active + continuous capture. - Desktop: renderer owns voice-conversation audio, so a matching WebAudio implementation (src/lib/thinking-sound.ts, same envelope/pitches) runs while conversation status === "thinking"; honors voice.thinking_sound (via config store) and the shared sound-mute toggle; stops instantly on speaking/listening/end.
This commit is contained in:
parent
6fdfdc1597
commit
df093bf33c
12 changed files with 690 additions and 5 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
|
||||
import { useI18n } from '@/i18n'
|
||||
import { startThinkingSound, stopThinkingSound } from '@/lib/thinking-sound'
|
||||
import { monitorSpeechDuringPlayback } from '@/lib/voice-barge-in'
|
||||
import {
|
||||
markVoicePlaybackInterrupted,
|
||||
|
|
@ -574,6 +575,22 @@ export function useVoiceConversation({
|
|||
return () => window.removeEventListener('keydown', onKeyDown, { capture: true })
|
||||
}, [enabled, stopTurn])
|
||||
|
||||
// Ambient "thinking" sound: while the agent works (status 'thinking') no
|
||||
// audio flows, which reads as dead air mid-conversation. Calm bubble blips
|
||||
// fill the gap; they stop the INSTANT speech starts, the mic re-arms, or the
|
||||
// conversation ends. Gated by voice.thinking_sound + the shared sound mute.
|
||||
useEffect(() => {
|
||||
if (enabled && !muted && status === 'thinking') {
|
||||
startThinkingSound()
|
||||
|
||||
return stopThinkingSound
|
||||
}
|
||||
|
||||
stopThinkingSound()
|
||||
|
||||
return undefined
|
||||
}, [enabled, muted, status])
|
||||
|
||||
// Drive the loop: when a voice-submitted reply appears, open a live speech
|
||||
// session (which feeds itself from then on). Otherwise start listening when
|
||||
// idle between turns.
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import {
|
|||
setDefaultReasoningEffort,
|
||||
setIntroPersonality
|
||||
} from '@/store/session'
|
||||
import { applyAutoSpeakFromConfig, applyVoiceStopPhraseFromConfig } from '@/store/voice-prefs'
|
||||
import { applyAutoSpeakFromConfig, applyThinkingSoundFromConfig, applyVoiceStopPhraseFromConfig } from '@/store/voice-prefs'
|
||||
|
||||
const DEFAULT_VOICE_SECONDS = 120
|
||||
const FAST_TIERS = new Set(['fast', 'priority', 'on'])
|
||||
|
|
@ -106,6 +106,7 @@ export function useHermesConfig({ activeSessionIdRef }: HermesConfigOptions) {
|
|||
setSttEnabled(config.stt?.enabled !== false)
|
||||
applyAutoSpeakFromConfig(config)
|
||||
applyVoiceStopPhraseFromConfig(config)
|
||||
applyThinkingSoundFromConfig(config)
|
||||
} catch {
|
||||
// Config is nice-to-have; chat still works without it.
|
||||
}
|
||||
|
|
|
|||
118
apps/desktop/src/lib/thinking-sound.test.ts
Normal file
118
apps/desktop/src/lib/thinking-sound.test.ts
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('@/store/haptics', () => ({ $hapticsMuted: { get: vi.fn(() => false) } }))
|
||||
vi.mock('@/hermes', () => ({
|
||||
getHermesConfigRecord: vi.fn(async () => ({})),
|
||||
saveHermesConfig: vi.fn(async () => undefined)
|
||||
}))
|
||||
|
||||
import { $hapticsMuted } from '@/store/haptics'
|
||||
import { $thinkingSoundEnabled } from '@/store/voice-prefs'
|
||||
|
||||
import { isThinkingSoundActive, startThinkingSound, stopThinkingSound } from './thinking-sound'
|
||||
|
||||
class FakeOscillator {
|
||||
type = 'sine'
|
||||
frequency = { exponentialRampToValueAtTime: vi.fn(), setValueAtTime: vi.fn() }
|
||||
connect = vi.fn()
|
||||
start = vi.fn()
|
||||
stop = vi.fn()
|
||||
}
|
||||
|
||||
class FakeGain {
|
||||
gain = { exponentialRampToValueAtTime: vi.fn(), setValueAtTime: vi.fn() }
|
||||
connect = vi.fn()
|
||||
}
|
||||
|
||||
const started: FakeOscillator[] = []
|
||||
|
||||
class FakeAudioContext {
|
||||
currentTime = 0
|
||||
destination = {}
|
||||
state = 'running'
|
||||
|
||||
createOscillator() {
|
||||
const osc = new FakeOscillator()
|
||||
|
||||
started.push(osc)
|
||||
|
||||
return osc
|
||||
}
|
||||
|
||||
createGain() {
|
||||
return new FakeGain()
|
||||
}
|
||||
|
||||
resume() {
|
||||
return Promise.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
describe('thinking-sound', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
started.length = 0
|
||||
$thinkingSoundEnabled.set(true)
|
||||
vi.mocked($hapticsMuted.get).mockReturnValue(false)
|
||||
vi.stubGlobal('AudioContext', FakeAudioContext)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
stopThinkingSound()
|
||||
vi.unstubAllGlobals()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('plays repeating blips while active and stops instantly', () => {
|
||||
startThinkingSound()
|
||||
expect(isThinkingSoundActive()).toBe(true)
|
||||
|
||||
vi.advanceTimersByTime(5_000)
|
||||
expect(started.length).toBeGreaterThanOrEqual(3)
|
||||
|
||||
const count = started.length
|
||||
|
||||
stopThinkingSound()
|
||||
expect(isThinkingSoundActive()).toBe(false)
|
||||
vi.advanceTimersByTime(5_000)
|
||||
expect(started.length).toBe(count) // nothing after stop
|
||||
})
|
||||
|
||||
it('respects the voice.thinking_sound config gate', () => {
|
||||
$thinkingSoundEnabled.set(false)
|
||||
startThinkingSound()
|
||||
expect(isThinkingSoundActive()).toBe(false)
|
||||
vi.advanceTimersByTime(3_000)
|
||||
expect(started.length).toBe(0)
|
||||
})
|
||||
|
||||
it('stays silent while sounds are muted but keeps the loop alive', () => {
|
||||
vi.mocked($hapticsMuted.get).mockReturnValue(true)
|
||||
startThinkingSound()
|
||||
vi.advanceTimersByTime(3_000)
|
||||
expect(started.length).toBe(0)
|
||||
|
||||
// Unmute mid-loop → blips resume without a restart.
|
||||
vi.mocked($hapticsMuted.get).mockReturnValue(false)
|
||||
vi.advanceTimersByTime(3_000)
|
||||
expect(started.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('start is idempotent', () => {
|
||||
startThinkingSound()
|
||||
startThinkingSound()
|
||||
vi.advanceTimersByTime(1_300)
|
||||
|
||||
// One loop: at most ~2 blips in 1.3s (first at 400ms, next ≥800ms later).
|
||||
expect(started.length).toBeLessThanOrEqual(2)
|
||||
})
|
||||
|
||||
it('never throws when WebAudio is unavailable', () => {
|
||||
vi.stubGlobal('AudioContext', undefined)
|
||||
expect(() => {
|
||||
startThinkingSound()
|
||||
vi.advanceTimersByTime(2_000)
|
||||
}).not.toThrow()
|
||||
stopThinkingSound()
|
||||
})
|
||||
})
|
||||
108
apps/desktop/src/lib/thinking-sound.ts
Normal file
108
apps/desktop/src/lib/thinking-sound.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
// Ambient "thinking" sound for the desktop voice conversation. While the agent
|
||||
// works (status === 'thinking') no audio flows, which reads as "it died" during
|
||||
// long thinking/tool stretches. A calm, quiet, repeating pair of soft bubble
|
||||
// blips fills the gap — same WebAudio oscillator synthesis approach as
|
||||
// wake-sound.ts / completion-sound.ts (no asset to ship), mirroring the
|
||||
// backend's numpy-synthesized blips in tools/voice_mode.py so CLI and desktop
|
||||
// sound alike.
|
||||
//
|
||||
// Honours the shared sound-mute toggle ($hapticsMuted) and the
|
||||
// voice.thinking_sound config gate ($thinkingSoundEnabled). Stops instantly on
|
||||
// stopThinkingSound() — callers fire it the moment TTS starts, the mic re-arms,
|
||||
// or the conversation ends.
|
||||
|
||||
import { $hapticsMuted } from '@/store/haptics'
|
||||
import { $thinkingSoundEnabled } from '@/store/voice-prefs'
|
||||
|
||||
let ctx: AudioContext | null = null
|
||||
let timer: number | null = null
|
||||
let blipIndex = 0
|
||||
|
||||
function getCtx(): AudioContext | null {
|
||||
if (typeof window === 'undefined') {
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
if (!ctx) {
|
||||
const Ctor =
|
||||
window.AudioContext || (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
|
||||
|
||||
if (!Ctor) {
|
||||
return null
|
||||
}
|
||||
|
||||
ctx = new Ctor()
|
||||
}
|
||||
|
||||
if (ctx.state === 'suspended') {
|
||||
void ctx.resume().catch(() => undefined)
|
||||
}
|
||||
|
||||
return ctx
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// One soft "blub": short sine with a gentle downward pitch glide and a smooth
|
||||
// attack into an exponential decay — no clicks, deliberately quiet.
|
||||
function blub(ac: AudioContext, freq: number) {
|
||||
const t0 = ac.currentTime + 0.01
|
||||
const dur = 0.16
|
||||
const osc = ac.createOscillator()
|
||||
const env = ac.createGain()
|
||||
|
||||
osc.type = 'sine'
|
||||
osc.frequency.setValueAtTime(freq, t0)
|
||||
osc.frequency.exponentialRampToValueAtTime(freq * 0.72, t0 + dur)
|
||||
|
||||
env.gain.setValueAtTime(0.0001, t0)
|
||||
env.gain.exponentialRampToValueAtTime(0.08, t0 + 0.02)
|
||||
env.gain.exponentialRampToValueAtTime(0.0001, t0 + dur)
|
||||
|
||||
osc.connect(env)
|
||||
env.connect(ac.destination)
|
||||
osc.start(t0)
|
||||
osc.stop(t0 + dur + 0.02)
|
||||
}
|
||||
|
||||
export function isThinkingSoundActive(): boolean {
|
||||
return timer !== null
|
||||
}
|
||||
|
||||
/** Start the repeating thinking blips (idempotent). Best-effort, never throws. */
|
||||
export function startThinkingSound(): void {
|
||||
if (timer !== null || !$thinkingSoundEnabled.get()) {
|
||||
return
|
||||
}
|
||||
|
||||
const tick = () => {
|
||||
if ($hapticsMuted.get() === false) {
|
||||
const ac = getCtx()
|
||||
|
||||
if (ac) {
|
||||
try {
|
||||
// Alternate two calm pitches (G4 / E4), matching the backend blips.
|
||||
blub(ac, blipIndex % 2 === 0 ? 392 : 329.6)
|
||||
} catch {
|
||||
// Audio backend unavailable — stay silent, keep the loop harmless.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blipIndex += 1
|
||||
// ~0.8-1.2s spacing with slight randomization so it reads organic.
|
||||
timer = window.setTimeout(tick, 800 + Math.random() * 400)
|
||||
}
|
||||
|
||||
timer = window.setTimeout(tick, 400)
|
||||
}
|
||||
|
||||
/** Stop the thinking blips instantly (idempotent). */
|
||||
export function stopThinkingSound(): void {
|
||||
if (timer !== null) {
|
||||
window.clearTimeout(timer)
|
||||
timer = null
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,17 @@ export function applyVoiceStopPhraseFromConfig(
|
|||
$voiceStopPhrase.set(first ?? null)
|
||||
}
|
||||
|
||||
// `voice.thinking_sound` — ambient bubble blips while the agent works during a
|
||||
// voice conversation (default on, matching the backend default).
|
||||
export const $thinkingSoundEnabled = atom<boolean>(true)
|
||||
|
||||
/** Seed the thinking-sound gate from a loaded config payload. */
|
||||
export function applyThinkingSoundFromConfig(
|
||||
config: { voice?: { thinking_sound?: unknown } | null } | null | undefined
|
||||
) {
|
||||
$thinkingSoundEnabled.set(config?.voice?.thinking_sound !== false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Flip the preference and persist it. Optimistic — the atom updates instantly and
|
||||
* reverts if the config write fails. Read-modify-writes the whole record (the
|
||||
|
|
|
|||
|
|
@ -332,6 +332,7 @@ export interface HermesConfig {
|
|||
max_recording_seconds?: number
|
||||
auto_tts?: boolean
|
||||
stop_phrases?: unknown
|
||||
thinking_sound?: unknown
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
30
cli.py
30
cli.py
|
|
@ -13350,6 +13350,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
# chunks as they arrive, everything else synthesizes per sentence.
|
||||
use_streaming_tts = False
|
||||
_streaming_box_opened = False
|
||||
_thinking_started = False
|
||||
text_queue = None
|
||||
tts_thread = None
|
||||
stream_callback = None
|
||||
|
|
@ -13554,6 +13555,27 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
agent_thread = threading.Thread(target=run_agent, daemon=True)
|
||||
agent_thread.start()
|
||||
|
||||
# Ambient "thinking" sound: calm bubble blips while the agent
|
||||
# works in voice mode with no audio flowing, so the user knows
|
||||
# it's alive during long thinking/tool stretches. Skipped per-blip
|
||||
# while TTS speaks, the mic records, or a barge capture is live;
|
||||
# stopped outright as soon as the turn ends. voice.thinking_sound
|
||||
# gates it (default on); macOS is handled inside (TCC-safe skip).
|
||||
_thinking_started = False
|
||||
if self._voice_mode:
|
||||
try:
|
||||
from tools.voice_mode import start_thinking_sound
|
||||
|
||||
_thinking_started = start_thinking_sound(
|
||||
should_play=lambda: (
|
||||
self._voice_tts_done.is_set()
|
||||
and not self._voice_recording
|
||||
and not self._voice_barge_capture.is_set()
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
_thinking_started = False
|
||||
|
||||
# Monitor the dedicated interrupt queue while the agent runs.
|
||||
# _interrupt_queue is separate from _pending_input, so process_loop
|
||||
# and chat() never compete for the same queue.
|
||||
|
|
@ -13960,6 +13982,14 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
print(f"Error: {e}")
|
||||
return None
|
||||
finally:
|
||||
# Stop the ambient thinking sound the moment the turn ends —
|
||||
# every exit path (normal, error, interrupt) lands here.
|
||||
if _thinking_started:
|
||||
try:
|
||||
from tools.voice_mode import stop_thinking_sound
|
||||
stop_thinking_sound()
|
||||
except Exception:
|
||||
pass
|
||||
# Ensure streaming TTS resources are cleaned up even on error.
|
||||
# Normal path sends the sentinel at line ~3568; this is a safety
|
||||
# net for exception paths that skip it. Duplicate sentinels are
|
||||
|
|
|
|||
|
|
@ -2372,6 +2372,7 @@ DEFAULT_CONFIG = {
|
|||
"auto_tts": False,
|
||||
"beep_enabled": True, # Play record start/stop beeps in CLI voice mode
|
||||
"beep_volume": 0.3, # Beep amplitude multiplier (0.0-1.0, default keeps prior hardcoded value)
|
||||
"thinking_sound": True, # Calm ambient bubble sound while the agent works in voice chat (volume follows beep_volume)
|
||||
"silence_threshold": 200, # RMS below this = silence (0-32767)
|
||||
"silence_duration": 3.0, # Seconds of silence before auto-stop
|
||||
"barge_in": True, # Stop TTS playback when the user starts talking
|
||||
|
|
|
|||
185
tests/tools/test_voice_thinking_sound.py
Normal file
185
tests/tools/test_voice_thinking_sound.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""Tests for the ambient voice-chat "thinking" sound (tools/voice_mode.py).
|
||||
|
||||
Contract:
|
||||
- `voice.thinking_sound` config gates it (default True).
|
||||
- `start_thinking_sound()` is idempotent, returns False when disabled.
|
||||
- The loop synthesizes blips with numpy (no assets), scales volume by
|
||||
`voice.beep_volume`, and NEVER plays through sounddevice on macOS
|
||||
(_sounddevice_output_allowed → TCC-safe silent skip).
|
||||
- `stop_thinking_sound()` stops the loop instantly and is idempotent.
|
||||
- The should_play callback gates each blip (no blips while TTS speaks
|
||||
or the mic captures).
|
||||
- mark_audio_output_active / is_audio_output_active ref-count playback.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
import tools.voice_mode as vm
|
||||
|
||||
|
||||
class _FakeSD:
|
||||
def __init__(self):
|
||||
self.played = []
|
||||
|
||||
def play(self, audio, samplerate=None):
|
||||
self.played.append((audio, samplerate))
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
def _reset():
|
||||
vm.stop_thinking_sound()
|
||||
# Drain any residual output ref-counts from prior tests.
|
||||
with vm._audio_output_lock:
|
||||
vm._audio_output_active_count = 0
|
||||
|
||||
|
||||
class TestConfigGate:
|
||||
def test_default_enabled(self):
|
||||
with patch("hermes_cli.config.load_config", return_value={"voice": {}}):
|
||||
assert vm.thinking_sound_enabled() is True
|
||||
|
||||
def test_disabled_via_config(self):
|
||||
with patch(
|
||||
"hermes_cli.config.load_config",
|
||||
return_value={"voice": {"thinking_sound": False}},
|
||||
):
|
||||
assert vm.thinking_sound_enabled() is False
|
||||
|
||||
def test_quoted_false_string(self):
|
||||
with patch(
|
||||
"hermes_cli.config.load_config",
|
||||
return_value={"voice": {"thinking_sound": "false"}},
|
||||
):
|
||||
assert vm.thinking_sound_enabled() is False
|
||||
|
||||
def test_start_refuses_when_disabled(self):
|
||||
_reset()
|
||||
with patch.object(vm, "thinking_sound_enabled", return_value=False):
|
||||
assert vm.start_thinking_sound() is False
|
||||
assert vm._thinking_stop is None
|
||||
|
||||
|
||||
class TestBlipSynthesis:
|
||||
def test_blip_is_int16_low_volume(self):
|
||||
with patch.object(vm, "_get_beep_volume", return_value=0.3):
|
||||
blip = vm._synth_thinking_blip(np, 392.0)
|
||||
assert blip.dtype == np.int16
|
||||
assert len(blip) == int(vm.SAMPLE_RATE * 0.16)
|
||||
# Quieter than the beeps: 0.3 * 0.5 * 32767 ≈ 4915 peak ceiling.
|
||||
assert int(np.abs(blip).max()) <= int(0.3 * 0.5 * 32767) + 1
|
||||
assert int(np.abs(blip).max()) > 0
|
||||
|
||||
def test_blip_volume_follows_beep_volume(self):
|
||||
with patch.object(vm, "_get_beep_volume", return_value=1.0):
|
||||
loud = vm._synth_thinking_blip(np, 392.0)
|
||||
with patch.object(vm, "_get_beep_volume", return_value=0.1):
|
||||
quiet = vm._synth_thinking_blip(np, 392.0)
|
||||
assert int(np.abs(loud).max()) > int(np.abs(quiet).max()) * 5
|
||||
|
||||
def test_no_click_smooth_attack(self):
|
||||
blip = vm._synth_thinking_blip(np, 392.0)
|
||||
# First sample near zero (enveloped attack, no click).
|
||||
assert abs(int(blip[0])) < 200
|
||||
|
||||
|
||||
class TestLoopLifecycle:
|
||||
def test_loop_plays_blips_and_stops_instantly(self):
|
||||
_reset()
|
||||
fake = _FakeSD()
|
||||
stop = threading.Event()
|
||||
with patch.object(vm, "_sounddevice_output_allowed", return_value=True), \
|
||||
patch.object(vm, "_import_audio", return_value=(fake, np)), \
|
||||
patch.object(vm, "_get_beep_volume", return_value=0.3):
|
||||
t = threading.Thread(
|
||||
target=vm._thinking_sound_loop, args=(stop, None), daemon=True
|
||||
)
|
||||
t.start()
|
||||
deadline = time.monotonic() + 3.0
|
||||
while not fake.played and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
stop.set()
|
||||
t.join(timeout=3.0)
|
||||
assert fake.played, "loop never played a blip"
|
||||
assert not t.is_alive()
|
||||
|
||||
def test_should_play_false_suppresses_blips(self):
|
||||
_reset()
|
||||
fake = _FakeSD()
|
||||
stop = threading.Event()
|
||||
with patch.object(vm, "_sounddevice_output_allowed", return_value=True), \
|
||||
patch.object(vm, "_import_audio", return_value=(fake, np)), \
|
||||
patch.object(vm, "_get_beep_volume", return_value=0.3):
|
||||
t = threading.Thread(
|
||||
target=vm._thinking_sound_loop,
|
||||
args=(stop, lambda: False),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
time.sleep(0.3)
|
||||
stop.set()
|
||||
t.join(timeout=3.0)
|
||||
assert fake.played == []
|
||||
|
||||
def test_macos_tcc_gate_skips_silently(self):
|
||||
"""sounddevice output is gated on macOS — the loop must exit without
|
||||
importing/playing anything (per-second afplay churn is worse than
|
||||
silence)."""
|
||||
_reset()
|
||||
stop = threading.Event()
|
||||
|
||||
def _boom():
|
||||
raise AssertionError("must not import audio when output is gated")
|
||||
|
||||
with patch.object(vm, "_sounddevice_output_allowed", return_value=False), \
|
||||
patch.object(vm, "_import_audio", _boom):
|
||||
vm._thinking_sound_loop(stop, None) # returns immediately
|
||||
|
||||
def test_start_is_idempotent_and_stop_clears(self):
|
||||
_reset()
|
||||
with patch.object(vm, "thinking_sound_enabled", return_value=True), \
|
||||
patch.object(vm, "_sounddevice_output_allowed", return_value=False):
|
||||
assert vm.start_thinking_sound() is True
|
||||
first_stop = vm._thinking_stop
|
||||
assert vm.start_thinking_sound() is True
|
||||
assert vm._thinking_stop is first_stop # no second loop
|
||||
vm.stop_thinking_sound()
|
||||
assert vm._thinking_stop is None
|
||||
assert first_stop.is_set()
|
||||
vm.stop_thinking_sound() # idempotent
|
||||
|
||||
|
||||
class TestAudioOutputRefcount:
|
||||
def test_refcount_tracks_nested_playback(self):
|
||||
_reset()
|
||||
assert vm.is_audio_output_active() is False
|
||||
vm.mark_audio_output_active(True)
|
||||
vm.mark_audio_output_active(True)
|
||||
assert vm.is_audio_output_active() is True
|
||||
vm.mark_audio_output_active(False)
|
||||
assert vm.is_audio_output_active() is True
|
||||
vm.mark_audio_output_active(False)
|
||||
assert vm.is_audio_output_active() is False
|
||||
# Never goes negative.
|
||||
vm.mark_audio_output_active(False)
|
||||
assert vm.is_audio_output_active() is False
|
||||
|
||||
def test_play_audio_file_brackets_refcount(self, tmp_path):
|
||||
"""play_audio_file flags real speaker output for its whole duration,
|
||||
so the thinking loop knows audio is flowing."""
|
||||
_reset()
|
||||
seen = []
|
||||
|
||||
def fake_impl(path):
|
||||
seen.append(vm.is_audio_output_active())
|
||||
return True
|
||||
|
||||
with patch.object(vm, "_play_audio_file_impl", fake_impl):
|
||||
vm.play_audio_file(str(tmp_path / "x.wav"))
|
||||
assert seen == [True]
|
||||
assert vm.is_audio_output_active() is False
|
||||
|
|
@ -3445,10 +3445,24 @@ def stream_tts_to_speaker(
|
|||
audio_iter = streamer.stream(cleaned)
|
||||
if output_stream is not None:
|
||||
import numpy as _np
|
||||
for chunk in audio_iter:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1))
|
||||
|
||||
# Flag real speaker output for the duration of this
|
||||
# sentence so ambient cues (thinking sound) stay quiet.
|
||||
# Fail-open: stubbed/partial voice_mode modules (tests)
|
||||
# must never break sentence playback.
|
||||
try:
|
||||
from tools.voice_mode import mark_audio_output_active
|
||||
except Exception:
|
||||
def mark_audio_output_active(_active):
|
||||
return None
|
||||
mark_audio_output_active(True)
|
||||
try:
|
||||
for chunk in audio_iter:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1))
|
||||
finally:
|
||||
mark_audio_output_active(False)
|
||||
else:
|
||||
# No audio device: buffer chunks to a temp WAV and play it.
|
||||
_play_via_tempfile(audio_iter, stop_event, streamer.sample_rate)
|
||||
|
|
|
|||
|
|
@ -528,6 +528,155 @@ def play_beep(frequency: int = 880, duration: float = 0.12, count: int = 1) -> N
|
|||
logger.debug("Beep playback failed: %s", e)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Thinking sound — calm ambient "blub blub" while the agent works
|
||||
# ============================================================================
|
||||
# During a voice conversation the agent can think / run tools for minutes with
|
||||
# zero audio, which reads as "it died". A quiet, repeating pair of soft water-
|
||||
# bubble blips fills that gap. Fully synthesized with numpy (no binary asset),
|
||||
# volume-scaled by voice.beep_volume, gated by voice.thinking_sound (default
|
||||
# on), and macOS-TCC-safe: sounddevice OUTPUT is gated there
|
||||
# (_sounddevice_output_allowed), and spawning afplay every second would churn
|
||||
# subprocesses, so on macOS the thinking sound is skipped silently.
|
||||
|
||||
# The host's *should_play* callback decides when blips are allowed; the
|
||||
# module-level output ref-count below tracks when real audio (TTS sentences,
|
||||
# file playback) is actually flowing so hosts have an accurate signal.
|
||||
|
||||
_audio_output_active_count = 0
|
||||
_audio_output_lock = threading.Lock()
|
||||
|
||||
|
||||
def mark_audio_output_active(active: bool) -> None:
|
||||
"""Reference-count real audio output (TTS/file playback).
|
||||
|
||||
Playback paths bracket their work with ``mark_audio_output_active(True)``
|
||||
/ ``(False)`` so ``is_audio_output_active()`` reflects whether speech
|
||||
audio is leaving the speakers RIGHT NOW — unlike the per-turn TTS-done
|
||||
events, which stay 'busy' for a whole turn even while the pipeline is
|
||||
silently waiting for text.
|
||||
"""
|
||||
global _audio_output_active_count
|
||||
with _audio_output_lock:
|
||||
_audio_output_active_count = max(
|
||||
0, _audio_output_active_count + (1 if active else -1)
|
||||
)
|
||||
|
||||
|
||||
def is_audio_output_active() -> bool:
|
||||
"""True while TTS/file audio is actually playing on the speakers."""
|
||||
with _audio_output_lock:
|
||||
return _audio_output_active_count > 0
|
||||
|
||||
|
||||
_thinking_lock = threading.Lock()
|
||||
_thinking_stop: Optional[threading.Event] = None
|
||||
|
||||
|
||||
def thinking_sound_enabled() -> bool:
|
||||
"""Config gate: ``voice.thinking_sound`` (default True)."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
from utils import is_truthy_value
|
||||
|
||||
voice_cfg = load_config().get("voice", {})
|
||||
if isinstance(voice_cfg, dict):
|
||||
return is_truthy_value(
|
||||
voice_cfg.get("thinking_sound", True), default=True
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _synth_thinking_blip(np, frequency: float) -> "Any":
|
||||
"""One soft 'blub': short sine with a gentle downward pitch glide and a
|
||||
smooth attack/decay envelope (no clicks), low-volume."""
|
||||
duration = 0.16
|
||||
n = int(SAMPLE_RATE * duration)
|
||||
t = np.linspace(0, duration, n, endpoint=False)
|
||||
# Downward glide (water-drop feel): freq → 0.72*freq over the blip.
|
||||
glide = np.linspace(1.0, 0.72, n)
|
||||
phase = 2 * np.pi * np.cumsum(frequency * glide) / SAMPLE_RATE
|
||||
tone = np.sin(phase)
|
||||
# Soften harmonics (cheap low-pass feel): add a quieter octave-down sine.
|
||||
tone = 0.8 * tone + 0.2 * np.sin(phase / 2.0)
|
||||
# Envelope: quick-but-smooth attack, long exponential-ish decay.
|
||||
attack = int(0.02 * SAMPLE_RATE)
|
||||
env = np.ones(n)
|
||||
env[:attack] = np.linspace(0.0, 1.0, attack)
|
||||
env *= np.exp(-t * 14.0)
|
||||
volume = _get_beep_volume() * 0.5 # deliberately quieter than the beeps
|
||||
return (tone * env * volume * 32767).astype(np.int16)
|
||||
|
||||
|
||||
def _thinking_sound_loop(stop: threading.Event, should_play) -> None:
|
||||
"""Daemon loop: play alternating-pitch blips every ~0.8-1.2s until *stop*.
|
||||
|
||||
Skips a blip (without stopping) whenever *should_play* returns False —
|
||||
e.g. TTS audio started flowing or the mic re-armed. macOS: sounddevice
|
||||
output is TCC-gated, and per-second afplay subprocess churn is worse
|
||||
than silence, so the loop exits immediately there.
|
||||
"""
|
||||
if not _sounddevice_output_allowed():
|
||||
return
|
||||
try:
|
||||
sd, np = _import_audio()
|
||||
except (ImportError, OSError):
|
||||
return
|
||||
|
||||
import random
|
||||
|
||||
pitches = (392.0, 329.6) # G4 / E4 — calm, low, alternating
|
||||
blips = [_synth_thinking_blip(np, p) for p in pitches]
|
||||
i = 0
|
||||
while not stop.is_set():
|
||||
try:
|
||||
if should_play is None or should_play():
|
||||
blip = blips[i % len(blips)]
|
||||
sd.play(blip, samplerate=SAMPLE_RATE)
|
||||
stop.wait(len(blip) / SAMPLE_RATE + 0.02)
|
||||
sd.stop()
|
||||
i += 1
|
||||
except Exception as e:
|
||||
logger.debug("Thinking sound blip failed: %s", e)
|
||||
return
|
||||
stop.wait(0.8 + random.random() * 0.4)
|
||||
|
||||
|
||||
def start_thinking_sound(should_play=None) -> bool:
|
||||
"""Start the ambient thinking sound (idempotent).
|
||||
|
||||
*should_play* is polled before each blip; return False to skip while
|
||||
speech audio flows or the mic is capturing. Returns True when the loop
|
||||
was started (or already running), False when disabled/unavailable.
|
||||
"""
|
||||
global _thinking_stop
|
||||
if not thinking_sound_enabled():
|
||||
return False
|
||||
with _thinking_lock:
|
||||
if _thinking_stop is not None and not _thinking_stop.is_set():
|
||||
return True # already running
|
||||
stop = threading.Event()
|
||||
_thinking_stop = stop
|
||||
threading.Thread(
|
||||
target=_thinking_sound_loop,
|
||||
args=(stop, should_play),
|
||||
daemon=True,
|
||||
name="voice-thinking-sound",
|
||||
).start()
|
||||
return True
|
||||
|
||||
|
||||
def stop_thinking_sound() -> None:
|
||||
"""Stop the ambient thinking sound instantly (idempotent)."""
|
||||
global _thinking_stop
|
||||
with _thinking_lock:
|
||||
stop, _thinking_stop = _thinking_stop, None
|
||||
if stop is not None:
|
||||
stop.set()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Termux Audio Recorder
|
||||
# ============================================================================
|
||||
|
|
@ -1413,6 +1562,16 @@ def play_audio_file(file_path: str) -> bool:
|
|||
Returns:
|
||||
``True`` if playback succeeded, ``False`` otherwise.
|
||||
"""
|
||||
# Ref-count real speaker output for the whole call so the thinking-sound
|
||||
# loop (and any other ambient cue) knows audio is flowing right now.
|
||||
mark_audio_output_active(True)
|
||||
try:
|
||||
return _play_audio_file_impl(file_path)
|
||||
finally:
|
||||
mark_audio_output_active(False)
|
||||
|
||||
|
||||
def _play_audio_file_impl(file_path: str) -> bool:
|
||||
global _active_playback
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
|
|
|
|||
|
|
@ -11995,6 +11995,7 @@ def _run_prompt_submit(
|
|||
goal_followup = None # set by the post-turn goal hook below
|
||||
result = None # turn outcome; read after the finally for leftover /steer
|
||||
tts_queue = None # streaming-TTS feed for this turn (voice mode)
|
||||
thinking_started = False # ambient thinking sound armed for this turn
|
||||
one_turn_restore = session.pop("one_turn_model_restore", None)
|
||||
# True once a failed turn's snapshot was retained for resume replay —
|
||||
# tells the finally below to skip the normal inflight clear.
|
||||
|
|
@ -12143,6 +12144,36 @@ def _run_prompt_submit(
|
|||
# consume the latch below.
|
||||
tts_queue = _tts_stream_begin()
|
||||
|
||||
# Ambient "thinking" sound (voice mode only): calm bubble blips
|
||||
# while the agent works with no audio flowing, so long
|
||||
# thinking/tool stretches don't read as a dead session. Per-blip
|
||||
# gate skips while real TTS audio flows or the mic is capturing;
|
||||
# stopped in the finally the instant the turn ends.
|
||||
# voice.thinking_sound config-gates it; macOS TCC handled inside.
|
||||
thinking_started = False
|
||||
if _voice_mode_enabled():
|
||||
try:
|
||||
from tools.voice_mode import (
|
||||
is_audio_output_active,
|
||||
start_thinking_sound,
|
||||
)
|
||||
|
||||
def _thinking_should_play() -> bool:
|
||||
if is_audio_output_active():
|
||||
return False
|
||||
try:
|
||||
from hermes_cli.voice import is_continuous_active
|
||||
|
||||
return not is_continuous_active()
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
thinking_started = start_thinking_sound(
|
||||
should_play=_thinking_should_play
|
||||
)
|
||||
except Exception:
|
||||
thinking_started = False
|
||||
|
||||
# Barged mid-speech? Tell the model (API-message note, same
|
||||
# enrichment channel as attached images) so it can react
|
||||
# ("rude!") instead of being oblivious to its own interruption.
|
||||
|
|
@ -12551,6 +12582,15 @@ def _run_prompt_submit(
|
|||
)
|
||||
_emit("error", sid, {"message": str(e)})
|
||||
finally:
|
||||
if thinking_started:
|
||||
# Kill the ambient thinking sound the moment the turn ends —
|
||||
# error and success paths both land here.
|
||||
try:
|
||||
from tools.voice_mode import stop_thinking_sound
|
||||
|
||||
stop_thinking_sound()
|
||||
except Exception:
|
||||
pass
|
||||
if tts_queue is not None:
|
||||
tts_queue.put(None) # end-of-text sentinel — flush + finish speaking
|
||||
if one_turn_restore:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue