mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-23 16:36:23 +00:00
Merge pull request #69511 from NousResearch/bb/voice-streaming
feat(voice): streaming, conversational TTS with barge-in across all surfaces
This commit is contained in:
brooklyn!
GitHub
commit
0ac07fdafd
20 changed files with 2399 additions and 411 deletions
|
|
@ -1,7 +1,13 @@
|
|||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
|
||||
import { useI18n } from '@/i18n'
|
||||
import { playSpeechText, stopVoicePlayback } from '@/lib/voice-playback'
|
||||
import { monitorSpeechDuringPlayback } from '@/lib/voice-barge-in'
|
||||
import {
|
||||
playSpeechText,
|
||||
type SpeechStreamSession,
|
||||
startSpeechStream,
|
||||
stopVoicePlayback
|
||||
} from '@/lib/voice-playback'
|
||||
import { notify, notifyError } from '@/store/notifications'
|
||||
|
||||
import { useMicRecorder } from './use-mic-recorder'
|
||||
|
|
@ -44,7 +50,9 @@ export function useVoiceConversation({
|
|||
const awaitingSpokenResponseRef = useRef(false)
|
||||
const responseIdRef = useRef<string | null>(null)
|
||||
const spokenSourceLengthRef = useRef(0)
|
||||
const speechBufferRef = useRef('')
|
||||
const speechSessionRef = useRef<null | SpeechStreamSession>(null)
|
||||
const stopBargeMonitorRef = useRef<(() => void) | null>(null)
|
||||
const bargeCapturePendingRef = useRef(false)
|
||||
const enabledRef = useRef(enabled)
|
||||
const mutedRef = useRef(muted)
|
||||
const busyRef = useRef(busy)
|
||||
|
|
@ -74,60 +82,13 @@ export function useVoiceConversation({
|
|||
}
|
||||
}
|
||||
|
||||
const resetSpeechBuffer = () => {
|
||||
const dropSpeechSession = () => {
|
||||
stopBargeMonitorRef.current?.()
|
||||
stopBargeMonitorRef.current = null
|
||||
bargeCapturePendingRef.current = false
|
||||
speechSessionRef.current = null
|
||||
responseIdRef.current = null
|
||||
spokenSourceLengthRef.current = 0
|
||||
speechBufferRef.current = ''
|
||||
}
|
||||
|
||||
const appendSpeechText = (text: string) => {
|
||||
if (!text) {
|
||||
return
|
||||
}
|
||||
|
||||
speechBufferRef.current = `${speechBufferRef.current}${text}`
|
||||
}
|
||||
|
||||
const takeSpeechChunk = (force = false): string | null => {
|
||||
const buffer = speechBufferRef.current.replace(/\s+/g, ' ').trim()
|
||||
|
||||
if (!buffer) {
|
||||
speechBufferRef.current = ''
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
const sentence = buffer.match(/^(.+?[.!?。！？])(?:\s+|$)/)
|
||||
|
||||
if (sentence?.[1] && (sentence[1].length >= 8 || force)) {
|
||||
const chunk = sentence[1].trim()
|
||||
speechBufferRef.current = buffer.slice(sentence[1].length).trim()
|
||||
|
||||
return chunk
|
||||
}
|
||||
|
||||
if (!force && buffer.length > 220) {
|
||||
const softBoundary = Math.max(
|
||||
buffer.lastIndexOf(', ', 180),
|
||||
buffer.lastIndexOf('; ', 180),
|
||||
buffer.lastIndexOf(': ', 180)
|
||||
)
|
||||
|
||||
if (softBoundary > 80) {
|
||||
const chunk = buffer.slice(0, softBoundary + 1).trim()
|
||||
speechBufferRef.current = buffer.slice(softBoundary + 1).trim()
|
||||
|
||||
return chunk
|
||||
}
|
||||
}
|
||||
|
||||
if (!force) {
|
||||
return null
|
||||
}
|
||||
|
||||
speechBufferRef.current = ''
|
||||
|
||||
return buffer
|
||||
}
|
||||
|
||||
const handleTurn = useCallback(
|
||||
|
|
@ -167,7 +128,7 @@ export function useVoiceConversation({
|
|||
}
|
||||
|
||||
awaitingSpokenResponseRef.current = true
|
||||
resetSpeechBuffer()
|
||||
dropSpeechSession()
|
||||
await onSubmit(transcript)
|
||||
setStatus('thinking')
|
||||
} catch (error) {
|
||||
|
|
@ -193,6 +154,10 @@ export function useVoiceConversation({
|
|||
return
|
||||
}
|
||||
|
||||
if (bargeCapturePendingRef.current) {
|
||||
return // the barge monitor is mid-capture and owns the mic
|
||||
}
|
||||
|
||||
if (statusRef.current !== 'idle') {
|
||||
return
|
||||
}
|
||||
|
|
@ -220,24 +185,237 @@ export function useVoiceConversation({
|
|||
}
|
||||
}, [handle, handleTurn, onFatalError, voiceCopy.couldNotStartSession, voiceCopy.microphoneFailed])
|
||||
|
||||
const speak = useCallback(
|
||||
async (text: string) => {
|
||||
setStatus('speaking')
|
||||
const settleAfterSpeech = useCallback(
|
||||
(barged: boolean) => {
|
||||
if (barged || !awaitingSpokenResponseRef.current) {
|
||||
awaitingSpokenResponseRef.current = false
|
||||
consumePendingResponse()
|
||||
}
|
||||
|
||||
if (bargeCapturePendingRef.current) {
|
||||
// The barge monitor is still capturing the user's interruption — it
|
||||
// owns the next turn. Keep it alive and don't re-open the mic; the
|
||||
// utterance callback transcribes and submits when they go quiet.
|
||||
speechSessionRef.current = null
|
||||
responseIdRef.current = null
|
||||
spokenSourceLengthRef.current = 0
|
||||
setStatus('listening')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
dropSpeechSession()
|
||||
|
||||
if (enabledRef.current) {
|
||||
pendingStartRef.current = true
|
||||
}
|
||||
|
||||
setStatus('idle')
|
||||
},
|
||||
[consumePendingResponse]
|
||||
)
|
||||
|
||||
/**
|
||||
* Submit the utterance the barge monitor captured — the user's interruption
|
||||
* from its first syllable, no re-listen round trip. Empty/failed captures
|
||||
* fall back to normal listening.
|
||||
*/
|
||||
const submitCapturedUtterance = useCallback(
|
||||
async (audio: Blob | null) => {
|
||||
const resumeListening = () => {
|
||||
if (enabledRef.current && !mutedRef.current) {
|
||||
pendingStartRef.current = true
|
||||
}
|
||||
|
||||
setStatus('idle')
|
||||
}
|
||||
|
||||
if (!audio || !onTranscribeAudio) {
|
||||
resumeListening()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
setStatus('transcribing')
|
||||
|
||||
try {
|
||||
await playSpeechText(text, { source: 'voice-conversation' })
|
||||
} catch (error) {
|
||||
notifyError(error, voiceCopy.playbackFailed)
|
||||
} finally {
|
||||
if (enabledRef.current) {
|
||||
pendingStartRef.current = true
|
||||
setStatus('idle')
|
||||
} else {
|
||||
setStatus('idle')
|
||||
const transcript = (await onTranscribeAudio(audio)).trim()
|
||||
|
||||
if (!transcript) {
|
||||
resumeListening()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
awaitingSpokenResponseRef.current = true
|
||||
dropSpeechSession()
|
||||
consumePendingResponse()
|
||||
await onSubmit(transcript)
|
||||
setStatus('thinking')
|
||||
} catch (error) {
|
||||
notifyError(error, voiceCopy.transcriptionFailed)
|
||||
resumeListening()
|
||||
}
|
||||
},
|
||||
[voiceCopy.playbackFailed]
|
||||
[consumePendingResponse, onSubmit, onTranscribeAudio, voiceCopy.transcriptionFailed]
|
||||
)
|
||||
|
||||
/** Barge-in monitor wiring shared by the live and fallback speech paths. */
|
||||
const openBargeMonitor = useCallback(
|
||||
(onBarge: () => void) =>
|
||||
monitorSpeechDuringPlayback({
|
||||
onSpeech: () => {
|
||||
bargeCapturePendingRef.current = true
|
||||
onBarge()
|
||||
stopVoicePlayback()
|
||||
},
|
||||
onUtterance: audio => {
|
||||
bargeCapturePendingRef.current = false
|
||||
stopBargeMonitorRef.current = null
|
||||
void submitCapturedUtterance(audio)
|
||||
}
|
||||
}),
|
||||
[submitCapturedUtterance]
|
||||
)
|
||||
|
||||
/** Push any new reply text into the live session; finish when complete. */
|
||||
const feedSpeechSession = useCallback(
|
||||
(responseId: string) => {
|
||||
const session = speechSessionRef.current
|
||||
|
||||
if (!session || responseIdRef.current !== responseId) {
|
||||
return
|
||||
}
|
||||
|
||||
const response = pendingResponse()
|
||||
|
||||
if (response && response.id === responseId) {
|
||||
if (response.text.length > spokenSourceLengthRef.current) {
|
||||
session.append(response.text.slice(spokenSourceLengthRef.current))
|
||||
spokenSourceLengthRef.current = response.text.length
|
||||
}
|
||||
|
||||
if (!response.pending && !busyRef.current) {
|
||||
session.finish()
|
||||
}
|
||||
} else if (!busyRef.current) {
|
||||
// Reply consumed/vanished while we were speaking — close out the turn.
|
||||
session.finish()
|
||||
}
|
||||
},
|
||||
[pendingResponse]
|
||||
)
|
||||
|
||||
/** Whole-text fallback: wait for the reply to complete, then speak it. */
|
||||
const awaitFallbackSpeech = useCallback(
|
||||
(responseId: string) => {
|
||||
const poll = () => {
|
||||
if (responseIdRef.current !== responseId) {
|
||||
return
|
||||
}
|
||||
|
||||
const response = pendingResponse()
|
||||
|
||||
if (!response || response.id !== responseId) {
|
||||
settleAfterSpeech(false)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if (response.pending || busyRef.current) {
|
||||
window.setTimeout(poll, 250)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
let barged = false
|
||||
|
||||
stopBargeMonitorRef.current?.()
|
||||
stopBargeMonitorRef.current = openBargeMonitor(() => {
|
||||
barged = true
|
||||
})
|
||||
|
||||
void playSpeechText(response.text, { source: 'voice-conversation' })
|
||||
.catch(error => notifyError(error, voiceCopy.playbackFailed))
|
||||
.finally(() => {
|
||||
if (responseIdRef.current === responseId) {
|
||||
awaitingSpokenResponseRef.current = false
|
||||
settleAfterSpeech(barged)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
poll()
|
||||
},
|
||||
[openBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed]
|
||||
)
|
||||
|
||||
/**
|
||||
* Live-speak the streaming reply: one speech session per response, fed
|
||||
* incremental text as the assistant generates it. Audio overlaps generation
|
||||
* — no wait for the full reply, no per-sentence gaps.
|
||||
*/
|
||||
const openLiveSpeech = useCallback(
|
||||
(responseId: string) => {
|
||||
responseIdRef.current = responseId
|
||||
spokenSourceLengthRef.current = 0
|
||||
setStatus('speaking')
|
||||
|
||||
let barged = false
|
||||
|
||||
// VAD barge-in: the user talking over the reply cuts playback, drops
|
||||
// the not-yet-spoken remainder, AND keeps capturing — the interruption
|
||||
// is transcribed from its first syllable instead of losing the opening
|
||||
// words to a mic re-open.
|
||||
stopBargeMonitorRef.current = openBargeMonitor(() => {
|
||||
barged = true
|
||||
})
|
||||
|
||||
void (async () => {
|
||||
const session = await startSpeechStream({ source: 'voice-conversation' })
|
||||
|
||||
// The session may resolve after the loop moved on (barge, disable).
|
||||
if (responseIdRef.current !== responseId) {
|
||||
if (session) {
|
||||
stopVoicePlayback()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if (!session) {
|
||||
// No streaming backend/provider: speak the whole reply once it lands.
|
||||
speechSessionRef.current = null
|
||||
awaitFallbackSpeech(responseId)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
speechSessionRef.current = session
|
||||
|
||||
// Timer-driven feed: reply text flows into the session at delta rate
|
||||
// regardless of React render cadence.
|
||||
const feedTimer = window.setInterval(() => feedSpeechSession(responseId), 150)
|
||||
feedSpeechSession(responseId)
|
||||
|
||||
const outcome = await session.done
|
||||
window.clearInterval(feedTimer)
|
||||
|
||||
if (responseIdRef.current !== responseId) {
|
||||
return
|
||||
}
|
||||
|
||||
if (outcome === 'fallback') {
|
||||
awaitFallbackSpeech(responseId)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
awaitingSpokenResponseRef.current = false
|
||||
settleAfterSpeech(barged)
|
||||
})()
|
||||
},
|
||||
[awaitFallbackSpeech, feedSpeechSession, openBargeMonitor, settleAfterSpeech]
|
||||
)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
|
|
@ -254,7 +432,7 @@ export function useVoiceConversation({
|
|||
|
||||
setMuted(false)
|
||||
awaitingSpokenResponseRef.current = false
|
||||
resetSpeechBuffer()
|
||||
dropSpeechSession()
|
||||
consumePendingResponse()
|
||||
pendingStartRef.current = true
|
||||
await startListening()
|
||||
|
|
@ -274,7 +452,7 @@ export function useVoiceConversation({
|
|||
handle.cancel()
|
||||
turnClosingRef.current = false
|
||||
awaitingSpokenResponseRef.current = false
|
||||
resetSpeechBuffer()
|
||||
dropSpeechSession()
|
||||
consumePendingResponse()
|
||||
setMuted(false)
|
||||
setStatus('idle')
|
||||
|
|
@ -325,8 +503,9 @@ export function useVoiceConversation({
|
|||
return () => window.removeEventListener('keydown', onKeyDown, { capture: true })
|
||||
}, [enabled, stopTurn])
|
||||
|
||||
// Drive the loop: after a voice-submitted turn, speak stable chunks as the
|
||||
// assistant stream grows. Otherwise start listening when idle between turns.
|
||||
// Drive the loop: when a voice-submitted reply appears, open a live speech
|
||||
// session (which feeds itself from then on). Otherwise start listening when
|
||||
// idle between turns.
|
||||
useEffect(() => {
|
||||
if (!enabled || muted) {
|
||||
return
|
||||
|
|
@ -336,38 +515,15 @@ export function useVoiceConversation({
|
|||
const response = pendingResponse()
|
||||
|
||||
if (response) {
|
||||
if (response.id !== responseIdRef.current) {
|
||||
resetSpeechBuffer()
|
||||
responseIdRef.current = response.id
|
||||
}
|
||||
openLiveSpeech(response.id)
|
||||
|
||||
if (response.text.length > spokenSourceLengthRef.current) {
|
||||
appendSpeechText(response.text.slice(spokenSourceLengthRef.current))
|
||||
spokenSourceLengthRef.current = response.text.length
|
||||
}
|
||||
|
||||
const chunk = takeSpeechChunk(!response.pending && !busy)
|
||||
|
||||
if (chunk) {
|
||||
void speak(chunk)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if (!response.pending && !busy) {
|
||||
awaitingSpokenResponseRef.current = false
|
||||
consumePendingResponse()
|
||||
resetSpeechBuffer()
|
||||
pendingStartRef.current = true
|
||||
setStatus('idle')
|
||||
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (!busy && status === 'thinking') {
|
||||
// Turn finished without any speakable reply (tool-only, error).
|
||||
awaitingSpokenResponseRef.current = false
|
||||
resetSpeechBuffer()
|
||||
dropSpeechSession()
|
||||
pendingStartRef.current = true
|
||||
setStatus('idle')
|
||||
|
||||
|
|
@ -382,7 +538,7 @@ export function useVoiceConversation({
|
|||
if (pendingStartRef.current) {
|
||||
void startListening()
|
||||
}
|
||||
}, [busy, consumePendingResponse, enabled, muted, pendingResponse, speak, startListening, status])
|
||||
}, [busy, enabled, muted, openLiveSpeech, pendingResponse, startListening, status])
|
||||
|
||||
useEffect(() => {
|
||||
if (enabled && !wasEnabledRef.current) {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { type ChatMessage, textPart } from '@/lib/chat-messages'
|
|||
import { optimisticAttachmentRef } from '@/lib/chat-runtime'
|
||||
import { sanitizeComposerInput } from '@/lib/composer-input-sanitize'
|
||||
import { setMutableRef } from '@/lib/mutable-ref'
|
||||
import { isVoicePlaybackActive, stopVoicePlayback } from '@/lib/voice-playback'
|
||||
import {
|
||||
$composerAttachments,
|
||||
clearComposerAttachments,
|
||||
|
|
@ -142,6 +143,11 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) {
|
|||
return false
|
||||
}
|
||||
|
||||
// Typing barge-in: a new send silences any in-flight spoken reply.
|
||||
if (isVoicePlaybackActive()) {
|
||||
stopVoicePlayback()
|
||||
}
|
||||
|
||||
// Queue drains carry their source session explicitly. A background drain
|
||||
// must never inherit the currently selected session after the user moves
|
||||
// to another chat.
|
||||
|
|
|
|||
238
apps/desktop/src/lib/voice-barge-in.ts
Normal file
238
apps/desktop/src/lib/voice-barge-in.ts
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
// VAD barge-in: watch the mic while TTS plays, fire the moment the user talks
|
||||
// over it, and CAPTURE what they say. Detection alone loses the first words —
|
||||
// by the time sustained speech trips the trigger and a fresh recorder spins
|
||||
// up, "stop, actually—" has become "actually—". So a MediaRecorder runs on
|
||||
// the monitor's stream the whole time (pre-roll), and once tripped it keeps
|
||||
// rolling until the user goes quiet, delivering the complete utterance.
|
||||
//
|
||||
// Echo cancellation strips the app's own speaker output from the capture, the
|
||||
// noise floor is calibrated while playback is already audible, and the
|
||||
// sustained window filters coughs/thumps — mirrors
|
||||
// tools/voice_mode.listen_for_speech on the Python surfaces.
|
||||
|
||||
const CALIBRATION_MS = 400
|
||||
const SUSTAINED_MS = 300
|
||||
const MIN_TRIGGER_LEVEL = 0.075 // matches the voice loop's silenceLevel
|
||||
const PRE_ROLL_RESTART_MS = 5_000 // cap pre-roll: restart the recorder while quiet
|
||||
const UTTERANCE_SILENCE_MS = 1_250 // matches the voice loop's silenceMs
|
||||
const UTTERANCE_MAX_MS = 30_000
|
||||
|
||||
export interface BargeMonitorCallbacks {
|
||||
/** Sustained speech detected — cut playback now. */
|
||||
onSpeech: () => void
|
||||
/**
|
||||
* The interrupting utterance, complete from its first syllable (pre-roll
|
||||
* included), delivered once the user goes quiet. `null` when capture was
|
||||
* unavailable — fall back to normal listening.
|
||||
*/
|
||||
onUtterance?: (audio: Blob | null) => void
|
||||
}
|
||||
|
||||
export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): () => void {
|
||||
let disposed = false
|
||||
let stream: MediaStream | null = null
|
||||
let context: AudioContext | null = null
|
||||
let frame: number | null = null
|
||||
let recorder: MediaRecorder | null = null
|
||||
let chunks: Blob[] = []
|
||||
let mimeType = ''
|
||||
|
||||
const cleanup = () => {
|
||||
disposed = true
|
||||
|
||||
if (frame !== null) {
|
||||
window.cancelAnimationFrame(frame)
|
||||
frame = null
|
||||
}
|
||||
|
||||
if (recorder && recorder.state !== 'inactive') {
|
||||
recorder.ondataavailable = null
|
||||
recorder.onstop = null
|
||||
|
||||
try {
|
||||
recorder.stop()
|
||||
} catch {
|
||||
// already stopped
|
||||
}
|
||||
}
|
||||
|
||||
recorder = null
|
||||
chunks = []
|
||||
void context?.close().catch(() => undefined)
|
||||
context = null
|
||||
stream?.getTracks().forEach(track => track.stop())
|
||||
stream = null
|
||||
}
|
||||
|
||||
const startSegment = () => {
|
||||
if (!stream || typeof MediaRecorder === 'undefined') {
|
||||
return
|
||||
}
|
||||
|
||||
mimeType =
|
||||
['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus'].find(type =>
|
||||
MediaRecorder.isTypeSupported(type)
|
||||
) ?? ''
|
||||
|
||||
try {
|
||||
recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined)
|
||||
} catch {
|
||||
recorder = null
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
chunks = []
|
||||
|
||||
recorder.ondataavailable = event => {
|
||||
if (event.data.size > 0) {
|
||||
chunks.push(event.data)
|
||||
}
|
||||
}
|
||||
|
||||
recorder.start(250)
|
||||
}
|
||||
|
||||
/** Restart the recorder to drop stale pre-roll — only valid while quiet. */
|
||||
const rotateSegment = () => {
|
||||
if (!recorder || recorder.state === 'inactive') {
|
||||
return
|
||||
}
|
||||
|
||||
recorder.ondataavailable = null
|
||||
recorder.onstop = null
|
||||
|
||||
try {
|
||||
recorder.stop()
|
||||
} catch {
|
||||
// already stopped
|
||||
}
|
||||
|
||||
startSegment()
|
||||
}
|
||||
|
||||
const finishCapture = () => {
|
||||
const active = recorder
|
||||
const type = active?.mimeType || mimeType || 'audio/webm'
|
||||
|
||||
if (!active || active.state === 'inactive') {
|
||||
cleanup()
|
||||
callbacks.onUtterance?.(chunks.length ? new Blob(chunks, { type }) : null)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
active.onstop = () => {
|
||||
const audio = chunks.length ? new Blob(chunks, { type }) : null
|
||||
|
||||
cleanup()
|
||||
callbacks.onUtterance?.(audio)
|
||||
}
|
||||
|
||||
active.stop()
|
||||
}
|
||||
void (async () => {
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true }
|
||||
})
|
||||
|
||||
if (disposed) {
|
||||
cleanup()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
startSegment()
|
||||
|
||||
context = new AudioContext()
|
||||
const analyser = context.createAnalyser()
|
||||
analyser.fftSize = 256
|
||||
context.createMediaStreamSource(stream).connect(analyser)
|
||||
|
||||
const data = new Uint8Array(analyser.fftSize)
|
||||
const startedAt = Date.now()
|
||||
const floorSamples: number[] = []
|
||||
let segmentStartedAt = Date.now()
|
||||
let speechStartedAt: number | null = null
|
||||
let tripped = false
|
||||
let trippedAt = 0
|
||||
let quietSince: number | null = null
|
||||
|
||||
const tick = () => {
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
|
||||
analyser.getByteTimeDomainData(data)
|
||||
|
||||
let sum = 0
|
||||
|
||||
for (const value of data) {
|
||||
const centered = value - 128
|
||||
sum += centered * centered
|
||||
}
|
||||
|
||||
const level = Math.min(1, Math.sqrt(sum / data.length) / 42)
|
||||
const now = Date.now()
|
||||
|
||||
if (!tripped && now - startedAt < CALIBRATION_MS) {
|
||||
floorSamples.push(level)
|
||||
} else if (!tripped) {
|
||||
const floor = floorSamples.length ? [...floorSamples].sort((a, b) => a - b)[floorSamples.length >> 1] : 0
|
||||
const trigger = Math.max(MIN_TRIGGER_LEVEL, floor * 3.5)
|
||||
|
||||
if (level >= trigger) {
|
||||
speechStartedAt ??= now
|
||||
|
||||
if (now - speechStartedAt >= SUSTAINED_MS) {
|
||||
tripped = true
|
||||
trippedAt = now
|
||||
quietSince = null
|
||||
callbacks.onSpeech()
|
||||
|
||||
if (!callbacks.onUtterance || !recorder) {
|
||||
cleanup()
|
||||
callbacks.onUtterance?.(null)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
speechStartedAt = null
|
||||
|
||||
// Bound the pre-roll while quiet so the utterance blob doesn't
|
||||
// accumulate the whole playback (rotating mid-speech would lose
|
||||
// the onset — the whole point).
|
||||
if (now - segmentStartedAt >= PRE_ROLL_RESTART_MS) {
|
||||
rotateSegment()
|
||||
segmentStartedAt = now
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Tripped: keep recording until the user goes quiet (endpoint).
|
||||
// Playback is already stopped, so plain silence-vs-speech works.
|
||||
if (level >= MIN_TRIGGER_LEVEL) {
|
||||
quietSince = null
|
||||
} else {
|
||||
quietSince ??= now
|
||||
}
|
||||
|
||||
if ((quietSince && now - quietSince >= UTTERANCE_SILENCE_MS) || now - trippedAt >= UTTERANCE_MAX_MS) {
|
||||
finishCapture()
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
frame = window.requestAnimationFrame(tick)
|
||||
}
|
||||
|
||||
tick()
|
||||
} catch {
|
||||
cleanup()
|
||||
}
|
||||
})()
|
||||
|
||||
return cleanup
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
import { resolveGatewayWsUrl } from '@hermes/shared'
|
||||
|
||||
import { speakText } from '@/hermes'
|
||||
import {
|
||||
$voicePlayback,
|
||||
|
|
@ -58,6 +60,321 @@ export function stopVoicePlayback() {
|
|||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Streaming path — /api/audio/speak-stream WebSocket, raw int16 PCM frames
|
||||
// scheduled through Web Audio. Speech starts on the provider's first chunk
|
||||
// instead of after full synthesis + base64 transfer.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function resolveSpeakStreamUrl(): Promise<null | string> {
|
||||
const desktop = window.hermesDesktop
|
||||
|
||||
if (!desktop?.getConnection) {
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
// Mint a fresh credential (single-use ticket in OAuth mode), then swap the
|
||||
// gateway endpoint for the PCM one — auth is shared across WS routes.
|
||||
const wsUrl = await resolveGatewayWsUrl(desktop, await desktop.getConnection())
|
||||
const url = new URL(wsUrl)
|
||||
|
||||
if (!url.pathname.endsWith('/api/ws')) {
|
||||
return null
|
||||
}
|
||||
|
||||
url.pathname = url.pathname.replace(/\/api\/ws$/, '/api/audio/speak-stream')
|
||||
|
||||
return url.toString()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export interface SpeechStreamSession {
|
||||
/** Feed more reply text as it streams in. Safe after `finish` (no-op). */
|
||||
append: (text: string) => void
|
||||
/** No more text coming — resolves `done` once the audio drains. */
|
||||
finish: () => void
|
||||
/**
|
||||
* 'done' — audio fully played (or barged via stopVoicePlayback)
|
||||
* 'fallback'— no audio ever produced; caller should speak the accumulated
|
||||
* text through `playSpeechText` instead.
|
||||
*/
|
||||
done: Promise<'done' | 'fallback'>
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a live speech session: one WebSocket + one AudioContext for a whole
|
||||
* reply. Text is appended as LLM deltas arrive; the server cuts sentences and
|
||||
* streams PCM back while generation continues, so speech overlaps the text
|
||||
* stream (ChatGPT-style) with no per-sentence connection or synthesis gaps.
|
||||
*/
|
||||
function openSpeechStream(wsUrl: string, options: VoicePlaybackOptions): SpeechStreamSession {
|
||||
const ws = new WebSocket(wsUrl)
|
||||
ws.binaryType = 'arraybuffer'
|
||||
|
||||
let context: AudioContext | null = null
|
||||
let streamRate = 24_000
|
||||
let nextStartAt = 0
|
||||
let carry: null | Uint8Array = null
|
||||
let started = false
|
||||
let settled = false
|
||||
let finished = false
|
||||
const pendingSends: string[] = []
|
||||
|
||||
let settle: (value: 'done' | 'fallback') => void = () => undefined
|
||||
|
||||
const done = new Promise<'done' | 'fallback'>(resolve => {
|
||||
settle = value => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
|
||||
settled = true
|
||||
currentStop = null
|
||||
|
||||
try {
|
||||
ws.close()
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
|
||||
void context?.close().catch(() => undefined)
|
||||
context = null
|
||||
resolve(value)
|
||||
}
|
||||
})
|
||||
|
||||
const send = (frame: object) => {
|
||||
const data = JSON.stringify(frame)
|
||||
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(data)
|
||||
} else if (ws.readyState === WebSocket.CONNECTING) {
|
||||
pendingSends.push(data)
|
||||
}
|
||||
}
|
||||
|
||||
// stopVoicePlayback() → immediate barge-in: kill the socket (the server
|
||||
// aborts synthesis on disconnect) and the audio context (cuts sound now).
|
||||
currentStop = () => settle('done')
|
||||
|
||||
const finishWhenDrained = () => {
|
||||
const remainingMs = context ? Math.max(0, nextStartAt - context.currentTime) * 1_000 : 0
|
||||
window.setTimeout(() => settle('done'), remainingMs + 100)
|
||||
}
|
||||
|
||||
const schedule = (data: ArrayBuffer) => {
|
||||
if (!context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Provider chunks are not sample-aligned — carry any odd byte over.
|
||||
let bytes = new Uint8Array(data)
|
||||
|
||||
if (carry) {
|
||||
const joined = new Uint8Array(carry.length + bytes.length)
|
||||
joined.set(carry)
|
||||
joined.set(bytes, carry.length)
|
||||
bytes = joined
|
||||
carry = null
|
||||
}
|
||||
|
||||
const usable = bytes.length - (bytes.length % 2)
|
||||
|
||||
if (bytes.length !== usable) {
|
||||
carry = bytes.slice(usable)
|
||||
}
|
||||
|
||||
if (!usable) {
|
||||
return
|
||||
}
|
||||
|
||||
const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, usable / 2)
|
||||
const buffer = context.createBuffer(1, pcm.length, streamRate)
|
||||
const channel = buffer.getChannelData(0)
|
||||
|
||||
for (let index = 0; index < pcm.length; index += 1) {
|
||||
channel[index] = pcm[index] / 32_768
|
||||
}
|
||||
|
||||
const source = context.createBufferSource()
|
||||
source.buffer = buffer
|
||||
source.connect(context.destination)
|
||||
|
||||
const startAt = Math.max(context.currentTime + 0.05, nextStartAt)
|
||||
source.start(startAt)
|
||||
nextStartAt = startAt + buffer.duration
|
||||
|
||||
if (!started) {
|
||||
started = true
|
||||
setVoicePlaybackState(currentState('speaking', options))
|
||||
}
|
||||
}
|
||||
|
||||
ws.onopen = () => {
|
||||
pendingSends.splice(0).forEach(data => ws.send(data))
|
||||
}
|
||||
|
||||
ws.onmessage = event => {
|
||||
if (typeof event.data !== 'string') {
|
||||
schedule(event.data as ArrayBuffer)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
let frame: { channels?: number; sample_rate?: number; type?: string }
|
||||
|
||||
try {
|
||||
frame = JSON.parse(event.data) as typeof frame
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
if (frame.type === 'start') {
|
||||
streamRate = frame.sample_rate || 24_000
|
||||
context = new AudioContext()
|
||||
nextStartAt = 0
|
||||
} else if (frame.type === 'end') {
|
||||
finishWhenDrained()
|
||||
} else if (frame.type === 'fallback') {
|
||||
settle(started ? 'done' : 'fallback')
|
||||
}
|
||||
}
|
||||
|
||||
// A drop before any audio means the endpoint is unavailable (old backend,
|
||||
// auth, network) → fall back. After audio started, replaying the whole
|
||||
// message via POST would stutter — treat what played as the playback.
|
||||
ws.onerror = () => settle(started ? 'done' : 'fallback')
|
||||
ws.onclose = () => (started ? finishWhenDrained() : settle('fallback'))
|
||||
|
||||
return {
|
||||
// Raw deltas — the server strips markdown/emoji per *sentence*, which is
|
||||
// the only safe granularity when constructs span delta boundaries.
|
||||
append: text => {
|
||||
if (text && !finished && !settled) {
|
||||
send({ text })
|
||||
}
|
||||
},
|
||||
finish: () => {
|
||||
if (!finished && !settled) {
|
||||
finished = true
|
||||
send({ done: true })
|
||||
}
|
||||
},
|
||||
done
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Live-speak an in-progress reply: open a session, then `append` deltas and
|
||||
* `finish` when generation completes. Resolves null when streaming is
|
||||
* unavailable (old backend / non-chunked provider) — the caller falls back to
|
||||
* whole-text `playSpeechText`.
|
||||
*/
|
||||
export async function startSpeechStream(options: VoicePlaybackOptions): Promise<null | SpeechStreamSession> {
|
||||
const wsUrl = await resolveSpeakStreamUrl()
|
||||
|
||||
if (!wsUrl) {
|
||||
return null
|
||||
}
|
||||
|
||||
stopVoicePlayback()
|
||||
setVoicePlaybackState(currentState('preparing', options))
|
||||
|
||||
const session = openSpeechStream(wsUrl, options)
|
||||
|
||||
void session.done.then(outcome => {
|
||||
if (outcome === 'done') {
|
||||
setVoicePlaybackState(currentState('idle'))
|
||||
}
|
||||
})
|
||||
|
||||
return session
|
||||
}
|
||||
|
||||
/** One-shot playback of complete text over the streaming WS. */
|
||||
function playSpeechStream(wsUrl: string, text: string, options: VoicePlaybackOptions): Promise<'fallback' | 'played'> {
|
||||
const session = openSpeechStream(wsUrl, options)
|
||||
session.append(text)
|
||||
session.finish()
|
||||
|
||||
return session.done.then(outcome => (outcome === 'done' ? 'played' : 'fallback'))
|
||||
}
|
||||
|
||||
async function playSpeechDataUrl(
|
||||
speakableText: string,
|
||||
options: VoicePlaybackOptions,
|
||||
isCurrent: () => boolean
|
||||
): Promise<boolean> {
|
||||
const response = await speakText(speakableText)
|
||||
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
}
|
||||
|
||||
const audio = new Audio(response.data_url)
|
||||
currentAudio = audio
|
||||
setVoicePlaybackState(currentState('speaking', options, audio))
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let stall: number | null = null
|
||||
|
||||
const cleanup = () => {
|
||||
if (stall !== null) {
|
||||
window.clearTimeout(stall)
|
||||
stall = null
|
||||
}
|
||||
|
||||
audio.removeEventListener('ended', onEnded)
|
||||
audio.removeEventListener('error', onError)
|
||||
audio.removeEventListener('timeupdate', armStall)
|
||||
currentStop = null
|
||||
}
|
||||
|
||||
const armStall = () => {
|
||||
if (stall !== null) {
|
||||
window.clearTimeout(stall)
|
||||
}
|
||||
|
||||
stall = window.setTimeout(() => {
|
||||
cleanup()
|
||||
reject(new Error('Playback stalled'))
|
||||
}, PLAYBACK_STALL_MS)
|
||||
}
|
||||
|
||||
const onEnded = () => {
|
||||
cleanup()
|
||||
resolve()
|
||||
}
|
||||
|
||||
const onError = () => {
|
||||
cleanup()
|
||||
reject(new Error('Playback failed'))
|
||||
}
|
||||
|
||||
currentStop = () => {
|
||||
cleanup()
|
||||
resolve()
|
||||
}
|
||||
|
||||
audio.addEventListener('ended', onEnded, { once: true })
|
||||
audio.addEventListener('error', onError, { once: true })
|
||||
audio.addEventListener('timeupdate', armStall)
|
||||
armStall()
|
||||
void audio.play().catch(onError)
|
||||
})
|
||||
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
}
|
||||
|
||||
currentAudio = null
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
export async function playSpeechText(text: string, options: VoicePlaybackOptions): Promise<boolean> {
|
||||
stopVoicePlayback()
|
||||
|
||||
|
|
@ -73,72 +390,35 @@ export async function playSpeechText(text: string, options: VoicePlaybackOptions
|
|||
setVoicePlaybackState(currentState('preparing', options))
|
||||
|
||||
try {
|
||||
const response = await speakText(speakableText)
|
||||
// Streaming first; the POST data-URL path is the fallback for backends
|
||||
// without the WS endpoint or providers without a chunked API.
|
||||
const streamUrl = await resolveSpeakStreamUrl()
|
||||
|
||||
if (streamUrl && isCurrent()) {
|
||||
const outcome = await playSpeechStream(streamUrl, speakableText, options)
|
||||
|
||||
if (outcome === 'played') {
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
}
|
||||
|
||||
setVoicePlaybackState(currentState('idle'))
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
}
|
||||
|
||||
const audio = new Audio(response.data_url)
|
||||
currentAudio = audio
|
||||
setVoicePlaybackState(currentState('speaking', options, audio))
|
||||
const played = await playSpeechDataUrl(speakableText, options, isCurrent)
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let stall: number | null = null
|
||||
|
||||
const cleanup = () => {
|
||||
if (stall !== null) {
|
||||
window.clearTimeout(stall)
|
||||
stall = null
|
||||
}
|
||||
|
||||
audio.removeEventListener('ended', onEnded)
|
||||
audio.removeEventListener('error', onError)
|
||||
audio.removeEventListener('timeupdate', armStall)
|
||||
currentStop = null
|
||||
}
|
||||
|
||||
const armStall = () => {
|
||||
if (stall !== null) {
|
||||
window.clearTimeout(stall)
|
||||
}
|
||||
|
||||
stall = window.setTimeout(() => {
|
||||
cleanup()
|
||||
reject(new Error('Playback stalled'))
|
||||
}, PLAYBACK_STALL_MS)
|
||||
}
|
||||
|
||||
const onEnded = () => {
|
||||
cleanup()
|
||||
resolve()
|
||||
}
|
||||
|
||||
const onError = () => {
|
||||
cleanup()
|
||||
reject(new Error('Playback failed'))
|
||||
}
|
||||
|
||||
currentStop = () => {
|
||||
cleanup()
|
||||
resolve()
|
||||
}
|
||||
|
||||
audio.addEventListener('ended', onEnded, { once: true })
|
||||
audio.addEventListener('error', onError, { once: true })
|
||||
audio.addEventListener('timeupdate', armStall)
|
||||
armStall()
|
||||
void audio.play().catch(onError)
|
||||
})
|
||||
|
||||
if (!isCurrent()) {
|
||||
return false
|
||||
if (played) {
|
||||
setVoicePlaybackState(currentState('idle'))
|
||||
}
|
||||
|
||||
currentAudio = null
|
||||
setVoicePlaybackState(currentState('idle'))
|
||||
|
||||
return true
|
||||
return played
|
||||
} catch (error) {
|
||||
if (isCurrent()) {
|
||||
currentStop = null
|
||||
|
|
|
|||
146
cli.py
146
cli.py
|
|
@ -4205,6 +4205,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
self._voice_continuous = False
|
||||
self._voice_tts_done = threading.Event()
|
||||
self._voice_tts_done.set()
|
||||
self._voice_tts_stop = None # active streaming pipeline's stop event
|
||||
self._voice_barge_capture = threading.Event() # barge monitor is capturing the interruption
|
||||
|
||||
# Status bar visibility (toggled via /statusbar)
|
||||
self._status_bar_visible = True
|
||||
|
|
@ -11099,6 +11101,26 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
time.sleep(0.15)
|
||||
threading.Thread(target=_refresh_level, daemon=True).start()
|
||||
|
||||
def _voice_stt_model(self) -> Optional[str]:
|
||||
"""STT model override from config, or None for the provider default."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
stt_config = load_config().get("stt", {})
|
||||
return stt_config.get("model") if isinstance(stt_config, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _voice_restart_recording_async(self) -> None:
|
||||
"""Restart continuous-mode recording off-thread (start() can block)."""
|
||||
def _restart_recording():
|
||||
try:
|
||||
self._voice_start_recording()
|
||||
if hasattr(self, '_app') and self._app:
|
||||
self._app.invalidate()
|
||||
except Exception as e:
|
||||
_cprint(f"{_DIM}Voice auto-restart failed: {e}{_RST}")
|
||||
threading.Thread(target=_restart_recording, daemon=True).start()
|
||||
|
||||
def _voice_stop_and_transcribe(self):
|
||||
"""Stop recording, transcribe via STT, and queue the transcript as input."""
|
||||
# Atomic guard: only one thread can enter stop-and-transcribe.
|
||||
|
|
@ -11136,17 +11158,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
self._app.invalidate()
|
||||
_cprint(f"{_DIM}Transcribing...{_RST}")
|
||||
|
||||
# Get STT model from config
|
||||
stt_model = None
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
stt_config = load_config().get("stt", {})
|
||||
stt_model = stt_config.get("model")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from tools.voice_mode import transcribe_recording
|
||||
result = transcribe_recording(wav_path, model=stt_model)
|
||||
result = transcribe_recording(wav_path, model=self._voice_stt_model())
|
||||
|
||||
if result.get("success") and result.get("transcript", "").strip():
|
||||
transcript = result["transcript"].strip()
|
||||
|
|
@ -11197,14 +11210,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
# (When transcript IS submitted, process_loop handles restart
|
||||
# after chat() completes.)
|
||||
if self._voice_continuous and not submitted and not self._voice_recording:
|
||||
def _restart_recording():
|
||||
try:
|
||||
self._voice_start_recording()
|
||||
if hasattr(self, '_app') and self._app:
|
||||
self._app.invalidate()
|
||||
except Exception as e:
|
||||
_cprint(f"{_DIM}Voice auto-restart failed: {e}{_RST}")
|
||||
threading.Thread(target=_restart_recording, daemon=True).start()
|
||||
self._voice_restart_recording_async()
|
||||
|
||||
def _voice_speak_response_async(self, text: str) -> None:
|
||||
"""Schedule TTS and mark it pending before continuous recording can restart."""
|
||||
|
|
@ -11270,6 +11276,68 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
self._voice_tts_done.set()
|
||||
|
||||
|
||||
def _voice_barge_in_monitor(self, stop_event: threading.Event) -> None:
|
||||
"""VAD barge-in: cut streaming TTS the moment the user starts talking.
|
||||
|
||||
Runs for one turn alongside the streaming pipeline (continuous voice
|
||||
mode only — the mic is otherwise idle during playback). On speech,
|
||||
playback is cut immediately while the monitor KEEPS capturing (with
|
||||
pre-roll, so the interruption is transcribed from its first syllable
|
||||
— restarting the recorder after detection would lose the opening
|
||||
words). ``_voice_barge_capture`` suppresses process_loop's auto-
|
||||
restart until the captured utterance has been submitted.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
voice_cfg = load_config().get("voice") or {}
|
||||
if not (isinstance(voice_cfg, dict) and voice_cfg.get("barge_in", True)):
|
||||
return
|
||||
from tools.voice_mode import listen_for_speech, stop_playback
|
||||
|
||||
def _cut_playback():
|
||||
if not self._voice_tts_done.is_set():
|
||||
self._voice_barge_capture.set()
|
||||
stop_event.set()
|
||||
stop_playback()
|
||||
|
||||
wav_path = listen_for_speech(
|
||||
lambda: stop_event.is_set() or self._voice_tts_done.is_set(),
|
||||
capture=True,
|
||||
on_trigger=_cut_playback,
|
||||
)
|
||||
if wav_path and self._voice_barge_capture.is_set():
|
||||
self._voice_submit_barge_utterance(wav_path)
|
||||
else:
|
||||
self._voice_barge_capture.clear()
|
||||
except Exception as e:
|
||||
self._voice_barge_capture.clear()
|
||||
logger.debug("Voice barge-in monitor failed: %s", e)
|
||||
|
||||
def _voice_submit_barge_utterance(self, wav_path: str) -> None:
|
||||
"""Transcribe a barge-captured interruption and queue it as the next turn."""
|
||||
submitted = False
|
||||
try:
|
||||
from tools.voice_mode import transcribe_recording
|
||||
result = transcribe_recording(wav_path, model=self._voice_stt_model())
|
||||
transcript = (result.get("transcript") or "").strip() if result.get("success") else ""
|
||||
if transcript:
|
||||
self._pending_input.put(transcript)
|
||||
submitted = True
|
||||
elif not result.get("success"):
|
||||
_cprint(f"\n{_DIM}Transcription failed: {result.get('error', 'Unknown error')}{_RST}")
|
||||
except Exception as e:
|
||||
_cprint(f"\n{_DIM}Voice processing error: {e}{_RST}")
|
||||
finally:
|
||||
try:
|
||||
if os.path.isfile(wav_path):
|
||||
os.unlink(wav_path)
|
||||
except OSError:
|
||||
pass
|
||||
self._voice_barge_capture.clear()
|
||||
# No usable transcript: hand the mic back to the normal loop.
|
||||
if not submitted and self._voice_mode and self._voice_continuous and not self._voice_recording:
|
||||
self._voice_restart_recording_async()
|
||||
|
||||
def _voice_beeps_enabled(self) -> bool:
|
||||
"""Return whether CLI voice mode should play record start/stop beeps."""
|
||||
try:
|
||||
|
|
@ -11363,8 +11431,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
threading.Thread(target=_bg_shutdown, daemon=True).start()
|
||||
self._voice_recorder = None
|
||||
|
||||
# Stop any active TTS playback
|
||||
# Stop any active TTS playback (file player + streaming pipeline)
|
||||
try:
|
||||
if self._voice_tts_stop is not None:
|
||||
self._voice_tts_stop.set()
|
||||
from tools.voice_mode import stop_playback
|
||||
stop_playback()
|
||||
except Exception:
|
||||
|
|
@ -12117,9 +12187,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
self._reasoning_shown_this_turn = False
|
||||
|
||||
# --- Streaming TTS setup ---
|
||||
# When ElevenLabs is the TTS provider and sounddevice is available,
|
||||
# we stream audio sentence-by-sentence as the agent generates tokens
|
||||
# instead of waiting for the full response.
|
||||
# Any working TTS provider streams sentence-by-sentence as the agent
|
||||
# generates tokens: PCM-streaming providers (ElevenLabs, OpenAI) play
|
||||
# chunks as they arrive, everything else synthesizes per sentence.
|
||||
use_streaming_tts = False
|
||||
_streaming_box_opened = False
|
||||
text_queue = None
|
||||
|
|
@ -12130,20 +12200,12 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
if self._voice_tts:
|
||||
try:
|
||||
from tools.tts_tool import (
|
||||
_load_tts_config as _load_tts_cfg,
|
||||
_get_provider as _get_prov,
|
||||
_import_elevenlabs,
|
||||
_import_sounddevice,
|
||||
check_tts_requirements,
|
||||
stream_tts_to_speaker,
|
||||
)
|
||||
_tts_cfg = _load_tts_cfg()
|
||||
if _get_prov(_tts_cfg) == "elevenlabs":
|
||||
# Verify both ElevenLabs SDK and audio output are available
|
||||
_import_elevenlabs()
|
||||
_import_sounddevice()
|
||||
use_streaming_tts = True
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
_import_sounddevice()
|
||||
use_streaming_tts = check_tts_requirements()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -12171,6 +12233,13 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
daemon=True,
|
||||
)
|
||||
tts_thread.start()
|
||||
# Expose the pipeline's stop event so barge-in paths (voice
|
||||
# key, VAD monitor) can cut playback from outside this turn.
|
||||
self._voice_tts_stop = stop_event
|
||||
if self._voice_continuous:
|
||||
threading.Thread(
|
||||
target=self._voice_barge_in_monitor, args=(stop_event,), daemon=True
|
||||
).start()
|
||||
|
||||
def stream_callback(delta: str):
|
||||
if text_queue is not None:
|
||||
|
|
@ -13316,6 +13385,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
self._voice_continuous = False # Whether to auto-restart after agent responds
|
||||
self._voice_tts_done = threading.Event() # Signals TTS playback finished
|
||||
self._voice_tts_done.set() # Initially "done" (no TTS pending)
|
||||
self._voice_tts_stop = None # active streaming pipeline's stop event
|
||||
self._voice_barge_capture = threading.Event() # barge monitor is capturing the interruption
|
||||
|
||||
if os.environ.get("HERMES_DEFER_AGENT_STARTUP") != "1":
|
||||
self._install_tool_callbacks()
|
||||
|
|
@ -14104,9 +14175,12 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
return
|
||||
|
||||
# Interrupt TTS if playing, so user can start talking.
|
||||
# stop_playback() is fast (just terminates a subprocess).
|
||||
# stop_playback() is fast (just terminates a subprocess);
|
||||
# the stop event drains the streaming pipeline if one is live.
|
||||
if not cli_ref._voice_tts_done.is_set():
|
||||
try:
|
||||
if cli_ref._voice_tts_stop is not None:
|
||||
cli_ref._voice_tts_stop.set()
|
||||
from tools.voice_mode import stop_playback
|
||||
stop_playback()
|
||||
cli_ref._voice_tts_done.set()
|
||||
|
|
@ -15328,6 +15402,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
if self._voice_tts:
|
||||
self._voice_tts_done.wait(timeout=60)
|
||||
time.sleep(0.3)
|
||||
# A barge-in capture already owns the mic and
|
||||
# will submit the interruption itself.
|
||||
if self._voice_barge_capture.is_set():
|
||||
return
|
||||
self._voice_start_recording()
|
||||
app.invalidate()
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -2254,6 +2254,7 @@ DEFAULT_CONFIG = {
|
|||
"beep_enabled": True, # Play record start/stop beeps in CLI voice mode
|
||||
"silence_threshold": 200, # RMS below this = silence (0-32767)
|
||||
"silence_duration": 3.0, # Seconds of silence before auto-stop
|
||||
"barge_in": True, # Stop TTS playback when the user starts talking
|
||||
},
|
||||
|
||||
"human_delay": {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ Usage:
|
|||
python -m hermes_cli.main web --port 8080
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from contextlib import asynccontextmanager, contextmanager
|
||||
|
||||
import asyncio
|
||||
|
|
@ -28,6 +29,7 @@ import json
|
|||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import secrets
|
||||
import shlex
|
||||
|
|
@ -4281,10 +4283,14 @@ async def transcribe_audio_upload(payload: AudioTranscriptionRequest):
|
|||
tmp.write(audio_bytes)
|
||||
temp_path = tmp.name
|
||||
|
||||
from tools.transcription_tools import transcribe_audio
|
||||
# transcribe_recording (not raw transcribe_audio): filters Whisper
|
||||
# hallucinations and maps provider "empty transcript" errors to a
|
||||
# successful empty result — the live voice loop treats "" as silence
|
||||
# and re-listens instead of surfacing a 400 on every quiet turn.
|
||||
from tools.voice_mode import transcribe_recording
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
result = await loop.run_in_executor(None, transcribe_audio, temp_path)
|
||||
result = await loop.run_in_executor(None, transcribe_recording, temp_path)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
|
@ -4476,6 +4482,151 @@ async def speak_text(payload: TTSSpeakRequest):
|
|||
}
|
||||
|
||||
|
||||
def _split_text_for_speak_stream(text: str, cap: int) -> list:
|
||||
"""Split *text* into provider-cap-sized pieces on sentence boundaries."""
|
||||
from tools.tts_streaming import SENTENCE_BOUNDARY_RE as _SENTENCE_BOUNDARY_RE
|
||||
|
||||
cap = cap if cap and cap > 0 else 4000
|
||||
pieces, buf = [], ""
|
||||
for sentence in filter(str.strip, _SENTENCE_BOUNDARY_RE.split(text)):
|
||||
while len(sentence) > cap:
|
||||
pieces.append(sentence[:cap])
|
||||
sentence = sentence[cap:]
|
||||
if buf and len(buf) + len(sentence) + 1 > cap:
|
||||
pieces.append(buf)
|
||||
buf = sentence
|
||||
else:
|
||||
buf = f"{buf} {sentence}" if buf else sentence
|
||||
if buf:
|
||||
pieces.append(buf)
|
||||
return pieces
|
||||
|
||||
|
||||
@app.websocket("/api/audio/speak-stream")
|
||||
async def speak_stream_ws(ws: "WebSocket") -> None:
|
||||
"""Streaming TTS for the desktop: text in, raw int16 PCM frames out.
|
||||
|
||||
The socket is a per-reply speech *session*: the client feeds text
|
||||
incrementally as LLM deltas arrive, the server cuts sentences
|
||||
(``SentenceChunker`` — same cutter as the CLI/TUI speaker pipeline) and
|
||||
streams each one's PCM the moment it's ready. Speech overlaps generation,
|
||||
exactly like the token→sentence→TTS pipelining the realtime-voice
|
||||
literature converges on.
|
||||
|
||||
Protocol:
|
||||
client → ``{"text": "..."}`` frames (incremental; may combine with done),
|
||||
``{"done": true}`` when the reply is complete,
|
||||
``{"stop": true}`` or disconnect = barge-in
|
||||
server → ``{"type": "start", "sample_rate": N, "channels": 1}``,
|
||||
binary PCM frames, then ``{"type": "end"}``
|
||||
server → ``{"type": "fallback"}`` when the configured provider has no
|
||||
chunked API — the client uses the POST endpoint instead.
|
||||
"""
|
||||
if not _ws_auth_ok(ws):
|
||||
await ws.close(code=4401)
|
||||
return
|
||||
if not _ws_request_is_allowed(ws):
|
||||
await ws.close(code=4403)
|
||||
return
|
||||
await ws.accept()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _resolve():
|
||||
from tools.tts_streaming import resolve_streaming_provider
|
||||
from tools.tts_tool import _get_provider, _load_tts_config, _resolve_max_text_length
|
||||
|
||||
cfg = _load_tts_config()
|
||||
streamer = resolve_streaming_provider(cfg)
|
||||
cap = _resolve_max_text_length(_get_provider(cfg), cfg) if streamer else 0
|
||||
return streamer, cap
|
||||
|
||||
try:
|
||||
streamer, cap = await loop.run_in_executor(None, _resolve)
|
||||
except Exception:
|
||||
_log.exception("speak-stream provider resolution failed")
|
||||
streamer, cap = None, 0
|
||||
if streamer is None:
|
||||
with contextlib.suppress(Exception):
|
||||
await ws.send_json({"type": "fallback"})
|
||||
await ws.close()
|
||||
return
|
||||
|
||||
await ws.send_json(
|
||||
{"type": "start", "sample_rate": streamer.sample_rate, "channels": streamer.channels}
|
||||
)
|
||||
|
||||
stop = threading.Event()
|
||||
text_q: queue.Queue = queue.Queue() # str deltas; None = end-of-text
|
||||
chunks: asyncio.Queue = asyncio.Queue() # PCM out; None = synthesis done
|
||||
|
||||
def _produce():
|
||||
from tools.tts_streaming import SentenceChunker
|
||||
from tools.tts_tool import _strip_markdown_for_tts
|
||||
|
||||
chunker = SentenceChunker()
|
||||
|
||||
def _sentences():
|
||||
while not stop.is_set():
|
||||
delta = text_q.get()
|
||||
if delta is None:
|
||||
yield from chunker.flush()
|
||||
return
|
||||
yield from chunker.feed(delta)
|
||||
|
||||
try:
|
||||
for sentence in _sentences():
|
||||
cleaned = _strip_markdown_for_tts(sentence)
|
||||
if not cleaned:
|
||||
continue
|
||||
for piece in _split_text_for_speak_stream(cleaned, cap):
|
||||
for chunk in streamer.stream(piece):
|
||||
if stop.is_set():
|
||||
return
|
||||
loop.call_soon_threadsafe(chunks.put_nowait, chunk)
|
||||
except Exception as exc:
|
||||
_log.warning("speak-stream synthesis failed: %s", exc)
|
||||
finally:
|
||||
loop.call_soon_threadsafe(chunks.put_nowait, None)
|
||||
|
||||
threading.Thread(target=_produce, daemon=True).start()
|
||||
|
||||
async def _pump_client():
|
||||
# Text frames feed synthesis; done ends the text; stop/disconnect
|
||||
# (or any unparseable frame) is barge-in.
|
||||
try:
|
||||
while True:
|
||||
frame = json.loads(await ws.receive_text())
|
||||
if frame.get("text"):
|
||||
text_q.put(str(frame["text"]))
|
||||
if frame.get("stop"):
|
||||
break
|
||||
if frame.get("done"):
|
||||
text_q.put(None)
|
||||
except Exception:
|
||||
pass
|
||||
stop.set()
|
||||
text_q.put(None) # unblock the producer
|
||||
|
||||
pump = asyncio.ensure_future(_pump_client())
|
||||
try:
|
||||
while True:
|
||||
chunk = await chunks.get()
|
||||
if chunk is None:
|
||||
break
|
||||
await ws.send_bytes(chunk)
|
||||
if not stop.is_set():
|
||||
await ws.send_json({"type": "end"})
|
||||
except (WebSocketDisconnect, RuntimeError):
|
||||
pass
|
||||
finally:
|
||||
stop.set()
|
||||
text_q.put(None)
|
||||
pump.cancel()
|
||||
with contextlib.suppress(Exception):
|
||||
await ws.close()
|
||||
|
||||
|
||||
@app.get("/api/actions/{name}/status")
|
||||
async def get_action_status(name: str, lines: int = 200):
|
||||
"""Tail an action log and report whether the process is still running."""
|
||||
|
|
|
|||
|
|
@ -2192,7 +2192,7 @@ class TestWebServerEndpoints:
|
|||
|
||||
captured = {}
|
||||
|
||||
def fake_transcribe_audio(path):
|
||||
def fake_transcribe_audio(path, model=None):
|
||||
captured["path"] = path
|
||||
return {
|
||||
"success": True,
|
||||
|
|
@ -2219,6 +2219,36 @@ class TestWebServerEndpoints:
|
|||
assert captured["path"].endswith(".webm")
|
||||
assert not Path(captured["path"]).exists()
|
||||
|
||||
def test_audio_transcription_no_speech_is_not_an_error(self, monkeypatch):
|
||||
"""A provider hearing silence (empty transcript) must return 200/"" —
|
||||
the live voice loop treats it as a quiet turn and re-listens, instead
|
||||
of surfacing a 400 toast on every pause (the ElevenLabs empty-
|
||||
transcript spam)."""
|
||||
import tools.transcription_tools as transcription_tools
|
||||
|
||||
monkeypatch.setattr(
|
||||
transcription_tools,
|
||||
"transcribe_audio",
|
||||
lambda path, model=None: {
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "ElevenLabs STT returned empty transcript",
|
||||
"no_speech": True,
|
||||
},
|
||||
)
|
||||
|
||||
resp = self.client.post(
|
||||
"/api/audio/transcribe",
|
||||
json={
|
||||
"data_url": "data:audio/webm;base64,aGVsbG8=",
|
||||
"mime_type": "audio/webm",
|
||||
},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["ok"] is True
|
||||
assert resp.json()["transcript"] == ""
|
||||
|
||||
def test_audio_transcription_rejects_invalid_base64(self):
|
||||
resp = self.client.post(
|
||||
"/api/audio/transcribe",
|
||||
|
|
|
|||
163
tests/hermes_cli/test_web_server_speak_stream.py
Normal file
163
tests/hermes_cli/test_web_server_speak_stream.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""/api/audio/speak-stream — desktop streaming TTS over WebSocket."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from hermes_cli import web_server
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stream_client(monkeypatch, _isolate_hermes_home):
|
||||
previous_auth_required = getattr(web_server.app.state, "auth_required", None)
|
||||
web_server.app.state.auth_required = False
|
||||
|
||||
client = TestClient(web_server.app)
|
||||
try:
|
||||
yield client
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if close is not None:
|
||||
close()
|
||||
if previous_auth_required is None:
|
||||
if hasattr(web_server.app.state, "auth_required"):
|
||||
delattr(web_server.app.state, "auth_required")
|
||||
else:
|
||||
web_server.app.state.auth_required = previous_auth_required
|
||||
|
||||
|
||||
def _url(token: str | None = None) -> str:
|
||||
return f"/api/audio/speak-stream?{urlencode({'token': token or web_server._SESSION_TOKEN})}"
|
||||
|
||||
|
||||
class _FakeStreamer:
|
||||
sample_rate = 24000
|
||||
channels = 1
|
||||
|
||||
def __init__(self, chunks):
|
||||
self.chunks = chunks
|
||||
self.requests: list[str] = []
|
||||
|
||||
def stream(self, text):
|
||||
self.requests.append(text)
|
||||
yield from self.chunks
|
||||
|
||||
|
||||
def _patch_provider(monkeypatch, streamer, cap=4000):
|
||||
monkeypatch.setattr("tools.tts_streaming.resolve_streaming_provider", lambda cfg: streamer)
|
||||
monkeypatch.setattr("tools.tts_tool._load_tts_config", lambda: {})
|
||||
monkeypatch.setattr("tools.tts_tool._get_provider", lambda cfg: "fake")
|
||||
monkeypatch.setattr("tools.tts_tool._resolve_max_text_length", lambda provider, cfg: cap)
|
||||
|
||||
|
||||
def test_rejects_bad_token(stream_client):
|
||||
with pytest.raises(WebSocketDisconnect) as exc:
|
||||
with stream_client.websocket_connect(_url(token="wrong")):
|
||||
pass
|
||||
assert exc.value.code == 4401
|
||||
|
||||
|
||||
def test_fallback_frame_when_no_streaming_provider(stream_client, monkeypatch):
|
||||
_patch_provider(monkeypatch, None)
|
||||
with stream_client.websocket_connect(_url()) as conn:
|
||||
assert conn.receive_json() == {"type": "fallback"}
|
||||
|
||||
|
||||
def test_streams_pcm_frames_then_end(stream_client, monkeypatch):
|
||||
streamer = _FakeStreamer([b"\x01\x02\x03\x04", b"\x05\x06"])
|
||||
_patch_provider(monkeypatch, streamer)
|
||||
|
||||
with stream_client.websocket_connect(_url()) as conn:
|
||||
start = conn.receive_json()
|
||||
assert start == {"type": "start", "sample_rate": 24000, "channels": 1}
|
||||
|
||||
conn.send_text(json.dumps({"text": "Hello there.", "done": True}))
|
||||
assert conn.receive_bytes() == b"\x01\x02\x03\x04"
|
||||
assert conn.receive_bytes() == b"\x05\x06"
|
||||
assert conn.receive_json() == {"type": "end"}
|
||||
|
||||
assert streamer.requests == ["Hello there."]
|
||||
|
||||
|
||||
def test_incremental_deltas_are_cut_into_sentences(stream_client, monkeypatch):
|
||||
"""Text fed across frames is chunked and synthesized while more arrives."""
|
||||
streamer = _FakeStreamer([b"\x00\x00"])
|
||||
_patch_provider(monkeypatch, streamer)
|
||||
|
||||
with stream_client.websocket_connect(_url()) as conn:
|
||||
assert conn.receive_json()["type"] == "start"
|
||||
conn.send_text(json.dumps({"text": "This is the first full"}))
|
||||
conn.send_text(json.dumps({"text": " sentence of the reply. And"}))
|
||||
# The first sentence is complete — PCM must arrive before `done`.
|
||||
assert conn.receive_bytes() == b"\x00\x00"
|
||||
conn.send_text(json.dumps({"text": " here is the second one.", "done": True}))
|
||||
assert conn.receive_bytes() == b"\x00\x00"
|
||||
assert conn.receive_json() == {"type": "end"}
|
||||
|
||||
assert streamer.requests == [
|
||||
"This is the first full sentence of the reply.",
|
||||
"And here is the second one.",
|
||||
]
|
||||
|
||||
|
||||
def test_stop_frame_cuts_synthesis(stream_client, monkeypatch):
|
||||
streamer = _FakeStreamer([b"\x00\x00"])
|
||||
_patch_provider(monkeypatch, streamer)
|
||||
|
||||
with stream_client.websocket_connect(_url()) as conn:
|
||||
assert conn.receive_json()["type"] == "start"
|
||||
conn.send_text(json.dumps({"stop": True}))
|
||||
# Socket closes without an "end" frame — barge-in, not completion.
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
conn.receive_text()
|
||||
assert streamer.requests == []
|
||||
|
||||
|
||||
def test_long_text_is_split_across_provider_requests(stream_client, monkeypatch):
|
||||
streamer = _FakeStreamer([b"\x00\x00"])
|
||||
_patch_provider(monkeypatch, streamer, cap=24)
|
||||
|
||||
with stream_client.websocket_connect(_url()) as conn:
|
||||
assert conn.receive_json()["type"] == "start"
|
||||
conn.send_text(
|
||||
json.dumps(
|
||||
{"text": "First sentence here. Second sentence here. Third one.", "done": True}
|
||||
)
|
||||
)
|
||||
# One PCM frame per split piece, then end.
|
||||
frames = 0
|
||||
while True:
|
||||
message = conn.receive()
|
||||
if message.get("bytes") is not None:
|
||||
frames += 1
|
||||
else:
|
||||
assert json.loads(message["text"]) == {"type": "end"}
|
||||
break
|
||||
|
||||
assert len(streamer.requests) > 1
|
||||
assert frames == len(streamer.requests)
|
||||
# Nothing lost in the split: every sentence reached the provider.
|
||||
joined = " ".join(streamer.requests)
|
||||
for fragment in ("First sentence here.", "Second sentence here.", "Third one."):
|
||||
assert fragment in joined
|
||||
|
||||
|
||||
def test_split_text_respects_cap_and_preserves_content():
|
||||
text = "Alpha beta. Gamma delta epsilon. Zeta eta theta iota kappa."
|
||||
pieces = web_server._split_text_for_speak_stream(text, 30)
|
||||
assert pieces
|
||||
assert all(len(piece) <= 30 for piece in pieces)
|
||||
joined = " ".join(pieces)
|
||||
for word in text.replace(".", "").split():
|
||||
assert word in joined
|
||||
|
||||
|
||||
def test_split_text_hard_splits_oversized_sentence():
|
||||
pieces = web_server._split_text_for_speak_stream("x" * 100, 30)
|
||||
assert all(len(piece) <= 30 for piece in pieces)
|
||||
assert sum(len(piece) for piece in pieces) == 100
|
||||
|
|
@ -1209,7 +1209,10 @@ def test_voice_toggle_tts_branch_also_carries_record_key(monkeypatch):
|
|||
),
|
||||
)
|
||||
monkeypatch.setenv("HERMES_VOICE", "1")
|
||||
monkeypatch.delenv("HERMES_VOICE_TTS", raising=False)
|
||||
# setenv (not delenv) — the handler writes HERMES_VOICE_TTS directly, and
|
||||
# delenv on an absent var registers no teardown, leaking TTS=1 into every
|
||||
# later test in the file (which now spins up the streaming TTS pipeline).
|
||||
monkeypatch.setenv("HERMES_VOICE_TTS", "0")
|
||||
|
||||
tts_resp = server.dispatch(
|
||||
{"id": "voice-tts", "method": "voice.toggle", "params": {"action": "tts"}}
|
||||
|
|
@ -11882,3 +11885,127 @@ def test_get_usage_clamps_post_compression_sentinel():
|
|||
usage = server._get_usage(agent)
|
||||
assert "context_used" not in usage
|
||||
assert "context_percent" not in usage
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming TTS — per-turn pipeline + barge-in
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _fake_tts_modules(monkeypatch, *, requirements=True, playback_stops=None, listen=None, transcribe=None):
|
||||
"""Install lightweight tools.tts_tool / tools.voice_mode fakes."""
|
||||
started = {}
|
||||
|
||||
def fake_stream(text_queue, stop, done, **_kw):
|
||||
started["queue"] = text_queue
|
||||
stop.wait(5)
|
||||
done.set()
|
||||
|
||||
def default_listen(should_stop, capture=False, on_trigger=None, **_kw):
|
||||
return None if capture else False
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"tools.tts_tool",
|
||||
types.SimpleNamespace(
|
||||
check_tts_requirements=lambda: requirements,
|
||||
stream_tts_to_speaker=fake_stream,
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"tools.voice_mode",
|
||||
types.SimpleNamespace(
|
||||
stop_playback=lambda: (playback_stops.append(True) if playback_stops is not None else None),
|
||||
listen_for_speech=listen or default_listen,
|
||||
transcribe_recording=transcribe or (lambda path, model=None: {"success": True, "transcript": ""}),
|
||||
),
|
||||
)
|
||||
return started
|
||||
|
||||
|
||||
def test_tts_stream_begin_requires_voice_tts(monkeypatch):
|
||||
monkeypatch.setenv("HERMES_VOICE_TTS", "0")
|
||||
assert server._tts_stream_begin() is None
|
||||
|
||||
|
||||
def test_tts_stream_begin_requires_working_provider(monkeypatch):
|
||||
monkeypatch.setenv("HERMES_VOICE_TTS", "1")
|
||||
_fake_tts_modules(monkeypatch, requirements=False)
|
||||
assert server._tts_stream_begin() is None
|
||||
|
||||
|
||||
def test_tts_stream_begin_and_stop_lifecycle(monkeypatch):
|
||||
"""begin() spawns the consumer; stop() cuts it and clears the slot."""
|
||||
monkeypatch.setenv("HERMES_VOICE_TTS", "1")
|
||||
monkeypatch.setenv("HERMES_VOICE", "0") # no barge-in monitor (no mic)
|
||||
playback_stops: list = []
|
||||
started = _fake_tts_modules(monkeypatch, playback_stops=playback_stops)
|
||||
|
||||
text_queue = server._tts_stream_begin()
|
||||
assert text_queue is not None
|
||||
assert started["queue"] is text_queue
|
||||
|
||||
with server._tts_stream_lock:
|
||||
state = server._tts_stream_state
|
||||
assert state is not None and not state["stop"].is_set()
|
||||
|
||||
server._tts_stream_stop()
|
||||
assert state["stop"].is_set()
|
||||
assert playback_stops == [True]
|
||||
with server._tts_stream_lock:
|
||||
assert server._tts_stream_state is None
|
||||
|
||||
|
||||
def test_tts_stream_begin_barges_in_on_previous_pipeline(monkeypatch):
|
||||
"""A new turn's pipeline stops the previous turn's speech (one speaker)."""
|
||||
monkeypatch.setenv("HERMES_VOICE_TTS", "1")
|
||||
monkeypatch.setenv("HERMES_VOICE", "0")
|
||||
_fake_tts_modules(monkeypatch)
|
||||
|
||||
server._tts_stream_begin()
|
||||
with server._tts_stream_lock:
|
||||
first = server._tts_stream_state
|
||||
server._tts_stream_begin()
|
||||
assert first is not None and first["stop"].is_set()
|
||||
server._tts_stream_stop()
|
||||
|
||||
|
||||
def test_tts_stream_vad_barge_in_cuts_pipeline_and_submits_capture(monkeypatch, tmp_path):
|
||||
"""User speech during playback cuts TTS at the moment of detection
|
||||
(voice.interrupted), then the captured interruption is transcribed and
|
||||
emitted as voice.transcript so the TUI submits it — complete from its
|
||||
first syllable, no re-record round trip."""
|
||||
monkeypatch.setenv("HERMES_VOICE_TTS", "1")
|
||||
monkeypatch.setenv("HERMES_VOICE", "1")
|
||||
monkeypatch.setattr(server, "_load_cfg", lambda: {"voice": {"barge_in": True}})
|
||||
events: list = []
|
||||
monkeypatch.setattr(
|
||||
server, "_voice_emit", lambda event, payload=None: events.append((event, payload))
|
||||
)
|
||||
|
||||
wav = tmp_path / "barge.wav"
|
||||
wav.write_bytes(b"RIFF")
|
||||
|
||||
def fake_listen(should_stop, capture=False, on_trigger=None, **_kw):
|
||||
assert capture is True
|
||||
on_trigger() # playback cut happens at detection, not after endpointing
|
||||
return str(wav)
|
||||
|
||||
_fake_tts_modules(
|
||||
monkeypatch,
|
||||
listen=fake_listen,
|
||||
transcribe=lambda path, model=None: {"success": True, "transcript": "stop, actually—"},
|
||||
)
|
||||
|
||||
server._tts_stream_begin()
|
||||
with server._tts_stream_lock:
|
||||
state = server._tts_stream_state
|
||||
assert state is not None
|
||||
assert state["stop"].wait(2.0)
|
||||
deadline = time.monotonic() + 2.0
|
||||
while time.monotonic() < deadline and wav.exists():
|
||||
time.sleep(0.01) # unlink (finally) runs after the transcript emit
|
||||
assert ("voice.interrupted", None) in events
|
||||
assert ("voice.transcript", {"text": "stop, actually—"}) in events
|
||||
assert not wav.exists() # capture temp file cleaned up
|
||||
server._tts_stream_stop()
|
||||
|
|
|
|||
|
|
@ -1209,6 +1209,7 @@ class TestTranscribeXAI:
|
|||
|
||||
assert result["success"] is False
|
||||
assert "empty transcript" in result["error"]
|
||||
assert result["no_speech"] is True # live voice loops treat this as silence
|
||||
|
||||
def test_permission_error(self, monkeypatch, sample_ogg, mock_xai_http_module):
|
||||
monkeypatch.setenv("XAI_API_KEY", "xai-test-key")
|
||||
|
|
@ -1453,6 +1454,7 @@ class TestTranscribeElevenLabs:
|
|||
|
||||
assert result["success"] is False
|
||||
assert "empty transcript" in result["error"]
|
||||
assert result["no_speech"] is True # live voice loops treat this as silence
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
|
|
|||
228
tests/tools/test_tts_streaming.py
Normal file
228
tests/tools/test_tts_streaming.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
"""Tests for the provider-agnostic streaming TTS backend (tools.tts_streaming)
|
||||
and its dispatch through tools.tts_tool.stream_tts_to_speaker.
|
||||
|
||||
No live audio or network: the ElevenLabs/OpenAI SDKs, sounddevice, and the sync
|
||||
synth path are all mocked. Covers the registry/resolver, provider availability,
|
||||
the chunked-streamer playback path, and the universal per-sentence sync fallback.
|
||||
"""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.tts_streaming as ts
|
||||
|
||||
pytest.importorskip("numpy")
|
||||
|
||||
|
||||
# ── SentenceChunker ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSentenceChunker:
|
||||
def test_cuts_sentence_the_moment_its_boundary_arrives(self):
|
||||
c = ts.SentenceChunker()
|
||||
assert c.feed("This is the first full") == []
|
||||
assert c.feed(" sentence of it all. And") == ["This is the first full sentence of it all. "]
|
||||
assert c.flush() == ["And"]
|
||||
|
||||
def test_short_fragment_rides_with_the_next_sentence(self):
|
||||
c = ts.SentenceChunker()
|
||||
# "Ha! " alone is under min_len — it must not become its own clip.
|
||||
assert c.feed("Ha! ") == []
|
||||
assert c.feed("That was a good one, honestly. ") == [
|
||||
"Ha! That was a good one, honestly. "
|
||||
]
|
||||
|
||||
def test_think_blocks_are_stripped_even_across_deltas(self):
|
||||
c = ts.SentenceChunker()
|
||||
assert c.feed("<think>secret reason") == []
|
||||
assert c.feed("ing</think>The actual spoken answer. ") == ["The actual spoken answer. "]
|
||||
|
||||
def test_flush_drains_the_tail(self):
|
||||
c = ts.SentenceChunker()
|
||||
c.feed("no boundary here")
|
||||
assert c.flush() == ["no boundary here"]
|
||||
assert c.flush() == []
|
||||
|
||||
def test_paragraph_break_is_a_boundary(self):
|
||||
c = ts.SentenceChunker()
|
||||
assert c.feed("A paragraph without punctuation\n\nnext one") == [
|
||||
"A paragraph without punctuation\n\n"
|
||||
]
|
||||
|
||||
|
||||
# ── Registry + resolver ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _register_fake(monkeypatch, name, available=True, chunks=(b"\x00\x00",)):
|
||||
class _Fake(ts.StreamingTTSProvider):
|
||||
sample_rate = 24000
|
||||
|
||||
@staticmethod
|
||||
def available():
|
||||
return available
|
||||
|
||||
def stream(self, text):
|
||||
yield from chunks
|
||||
|
||||
monkeypatch.setitem(ts._REGISTRY, name, _Fake)
|
||||
return _Fake
|
||||
|
||||
|
||||
def test_resolve_returns_configured_streamer(monkeypatch):
|
||||
_register_fake(monkeypatch, "faketts")
|
||||
prov = ts.resolve_streaming_provider({"provider": "faketts"})
|
||||
assert isinstance(prov, ts.StreamingTTSProvider)
|
||||
|
||||
|
||||
def test_resolve_none_for_unregistered_provider(monkeypatch):
|
||||
# edge is a sync provider — not registered — so the dispatcher keeps its voice.
|
||||
assert ts.resolve_streaming_provider({"provider": "edge"}) is None
|
||||
|
||||
|
||||
def test_resolve_none_when_provider_unavailable(monkeypatch):
|
||||
_register_fake(monkeypatch, "faketts", available=False)
|
||||
assert ts.resolve_streaming_provider({"provider": "faketts"}) is None
|
||||
|
||||
|
||||
def test_resolve_honors_preferred_override(monkeypatch):
|
||||
_register_fake(monkeypatch, "faketts")
|
||||
prov = ts.resolve_streaming_provider({"provider": "edge"}, preferred="faketts")
|
||||
assert isinstance(prov, ts.StreamingTTSProvider)
|
||||
|
||||
|
||||
def test_never_swaps_provider_for_streaming(monkeypatch):
|
||||
# A registered streamer must NOT be substituted when the user picked another
|
||||
# (non-streaming) provider — that would silently change their voice.
|
||||
_register_fake(monkeypatch, "elevenlabs")
|
||||
assert ts.resolve_streaming_provider({"provider": "edge"}) is None
|
||||
|
||||
|
||||
# ── Built-in provider availability ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_elevenlabs_available_reflects_key(monkeypatch):
|
||||
monkeypatch.setattr(ts, "get_env_value", lambda k, *a: "key" if k == "ELEVENLABS_API_KEY" else None)
|
||||
assert ts.ElevenLabsStreamer.available() is True
|
||||
monkeypatch.setattr(ts, "get_env_value", lambda k, *a: None)
|
||||
assert ts.ElevenLabsStreamer.available() is False
|
||||
|
||||
|
||||
def test_openai_available_reflects_key(monkeypatch):
|
||||
monkeypatch.setattr(ts, "get_env_value", lambda k, *a: "key" if k == "OPENAI_API_KEY" else None)
|
||||
assert ts.OpenAIStreamer.available() is True
|
||||
|
||||
|
||||
# ── Dispatch: chunked streamer path ──────────────────────────────────────
|
||||
|
||||
|
||||
def _drain_queue(sentences):
|
||||
q = queue.Queue()
|
||||
for s in sentences:
|
||||
q.put(s)
|
||||
q.put(None)
|
||||
return q
|
||||
|
||||
|
||||
def _sd_mock():
|
||||
sd = MagicMock()
|
||||
out = MagicMock()
|
||||
sd.OutputStream.return_value = out
|
||||
return sd, out
|
||||
|
||||
|
||||
def test_streamer_path_writes_pcm_to_output(monkeypatch):
|
||||
from tools import tts_tool
|
||||
|
||||
class _Fake(ts.StreamingTTSProvider):
|
||||
sample_rate = 24000
|
||||
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
def stream(self, text):
|
||||
yield b"\x01\x00" * 50
|
||||
yield b"\x02\x00" * 50
|
||||
|
||||
sd, out = _sd_mock()
|
||||
q = _drain_queue(["Hello there, this is a full sentence."])
|
||||
stop, done = threading.Event(), threading.Event()
|
||||
|
||||
with patch("tools.tts_streaming.resolve_streaming_provider", return_value=_Fake({}, {})), \
|
||||
patch.object(tts_tool, "_import_sounddevice", return_value=sd):
|
||||
tts_tool.stream_tts_to_speaker(q, stop, done)
|
||||
|
||||
assert out.write.called, "expected PCM chunks written to the output stream"
|
||||
assert done.is_set()
|
||||
|
||||
|
||||
def test_stop_event_aborts_streaming(monkeypatch):
|
||||
from tools import tts_tool
|
||||
|
||||
class _Fake(ts.StreamingTTSProvider):
|
||||
sample_rate = 24000
|
||||
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
def stream(self, text):
|
||||
for _ in range(1000):
|
||||
yield b"\x00\x00" * 50
|
||||
|
||||
sd, out = _sd_mock()
|
||||
stop, done = threading.Event(), threading.Event()
|
||||
stop.set() # pre-set: no audio should be written
|
||||
q = _drain_queue(["A complete sentence here."])
|
||||
|
||||
with patch("tools.tts_streaming.resolve_streaming_provider", return_value=_Fake({}, {})), \
|
||||
patch.object(tts_tool, "_import_sounddevice", return_value=sd):
|
||||
tts_tool.stream_tts_to_speaker(q, stop, done)
|
||||
|
||||
assert not out.write.called
|
||||
assert done.is_set()
|
||||
|
||||
|
||||
# ── Dispatch: universal per-sentence sync fallback ───────────────────────
|
||||
|
||||
|
||||
def test_sync_fallback_speaks_each_sentence(monkeypatch):
|
||||
from tools import tts_tool
|
||||
|
||||
spoken = []
|
||||
monkeypatch.setattr(tts_tool, "text_to_speech_tool",
|
||||
lambda text, output_path: spoken.append(text))
|
||||
played = []
|
||||
fake_vm = MagicMock()
|
||||
fake_vm.play_audio_file.side_effect = lambda p: played.append(p)
|
||||
monkeypatch.setitem(__import__("sys").modules, "tools.voice_mode", fake_vm)
|
||||
monkeypatch.setattr("os.path.getsize", lambda p: 100)
|
||||
monkeypatch.setattr("os.path.isfile", lambda p: True)
|
||||
|
||||
q = _drain_queue(["First full sentence here. ", "Second full sentence here. "])
|
||||
stop, done = threading.Event(), threading.Event()
|
||||
|
||||
with patch("tools.tts_streaming.resolve_streaming_provider", return_value=None):
|
||||
tts_tool.stream_tts_to_speaker(q, stop, done)
|
||||
|
||||
assert len(spoken) == 2, f"expected both sentences synthesized, got {spoken}"
|
||||
assert len(played) == 2
|
||||
assert done.is_set()
|
||||
|
||||
|
||||
def test_display_callback_fires_without_audio(monkeypatch):
|
||||
from tools import tts_tool
|
||||
|
||||
seen = []
|
||||
monkeypatch.setattr(tts_tool, "text_to_speech_tool", lambda text, output_path: None)
|
||||
q = _drain_queue(["A sentence to display aloud."])
|
||||
stop, done = threading.Event(), threading.Event()
|
||||
|
||||
with patch("tools.tts_streaming.resolve_streaming_provider", return_value=None):
|
||||
tts_tool.stream_tts_to_speaker(q, stop, done, display_callback=seen.append)
|
||||
|
||||
assert seen, "display_callback should fire even on the sync path"
|
||||
assert done.is_set()
|
||||
|
|
@ -29,6 +29,8 @@ def _make_voice_cli(**overrides):
|
|||
cli._voice_continuous = False
|
||||
cli._voice_tts_done = threading.Event()
|
||||
cli._voice_tts_done.set()
|
||||
cli._voice_tts_stop = None
|
||||
cli._voice_barge_capture = threading.Event()
|
||||
cli._pending_input = queue.Queue()
|
||||
cli._app = None
|
||||
cli._attached_images = []
|
||||
|
|
@ -176,114 +178,40 @@ class TestVoiceStateLock:
|
|||
# ============================================================================
|
||||
|
||||
class TestStreamingTTSActivation:
|
||||
"""Verify streaming TTS uses lazy imports to check availability."""
|
||||
"""The CLI streaming gate: sounddevice + a working provider, ANY provider.
|
||||
|
||||
def test_activates_when_elevenlabs_and_sounddevice_available(self):
|
||||
"""use_streaming_tts should be True when provider is elevenlabs
|
||||
and both lazy imports succeed."""
|
||||
use_streaming_tts = False
|
||||
Mirrors cli.py's gate exactly — streaming engages whenever audio output
|
||||
exists and check_tts_requirements() passes, regardless of which provider
|
||||
is configured (non-streamers get the per-sentence sync path downstream).
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _gate() -> bool:
|
||||
"""The cli.py streaming-TTS gate, verbatim."""
|
||||
try:
|
||||
from tools.tts_tool import (
|
||||
_load_tts_config as _load_tts_cfg,
|
||||
_get_provider as _get_prov,
|
||||
_import_elevenlabs,
|
||||
_import_sounddevice,
|
||||
)
|
||||
assert callable(_import_elevenlabs)
|
||||
assert callable(_import_sounddevice)
|
||||
except ImportError:
|
||||
pytest.skip("tools.tts_tool not available")
|
||||
from tools.tts_tool import _import_sounddevice, check_tts_requirements
|
||||
_import_sounddevice()
|
||||
return check_tts_requirements()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
with patch("tools.tts_tool._load_tts_config") as mock_cfg, \
|
||||
patch("tools.tts_tool._get_provider", return_value="elevenlabs"), \
|
||||
patch("tools.tts_tool._import_elevenlabs") as mock_el, \
|
||||
patch("tools.tts_tool._import_sounddevice") as mock_sd:
|
||||
mock_cfg.return_value = {"provider": "elevenlabs"}
|
||||
mock_el.return_value = MagicMock()
|
||||
mock_sd.return_value = MagicMock()
|
||||
def test_activates_for_any_working_provider(self):
|
||||
"""Any provider that passes check_tts_requirements engages streaming."""
|
||||
with patch("tools.tts_tool._import_sounddevice", return_value=MagicMock()), \
|
||||
patch("tools.tts_tool.check_tts_requirements", return_value=True):
|
||||
assert self._gate() is True
|
||||
|
||||
from tools.tts_tool import (
|
||||
_load_tts_config as load_cfg,
|
||||
_get_provider as get_prov,
|
||||
_import_elevenlabs as import_el,
|
||||
_import_sounddevice as import_sd,
|
||||
)
|
||||
cfg = load_cfg()
|
||||
if get_prov(cfg) == "elevenlabs":
|
||||
import_el()
|
||||
import_sd()
|
||||
use_streaming_tts = True
|
||||
|
||||
assert use_streaming_tts is True
|
||||
|
||||
def test_does_not_activate_when_elevenlabs_missing(self):
|
||||
"""use_streaming_tts stays False when elevenlabs import fails."""
|
||||
use_streaming_tts = False
|
||||
with patch("tools.tts_tool._load_tts_config", return_value={"provider": "elevenlabs"}), \
|
||||
patch("tools.tts_tool._get_provider", return_value="elevenlabs"), \
|
||||
patch("tools.tts_tool._import_elevenlabs", side_effect=ImportError("no elevenlabs")):
|
||||
try:
|
||||
from tools.tts_tool import (
|
||||
_load_tts_config as load_cfg,
|
||||
_get_provider as get_prov,
|
||||
_import_elevenlabs as import_el,
|
||||
_import_sounddevice as import_sd,
|
||||
)
|
||||
cfg = load_cfg()
|
||||
if get_prov(cfg) == "elevenlabs":
|
||||
import_el()
|
||||
import_sd()
|
||||
use_streaming_tts = True
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
assert use_streaming_tts is False
|
||||
def test_does_not_activate_when_provider_unavailable(self):
|
||||
"""No working TTS provider → no streaming pipeline."""
|
||||
with patch("tools.tts_tool._import_sounddevice", return_value=MagicMock()), \
|
||||
patch("tools.tts_tool.check_tts_requirements", return_value=False):
|
||||
assert self._gate() is False
|
||||
|
||||
def test_does_not_activate_when_sounddevice_missing(self):
|
||||
"""use_streaming_tts stays False when sounddevice import fails."""
|
||||
use_streaming_tts = False
|
||||
with patch("tools.tts_tool._load_tts_config", return_value={"provider": "elevenlabs"}), \
|
||||
patch("tools.tts_tool._get_provider", return_value="elevenlabs"), \
|
||||
patch("tools.tts_tool._import_elevenlabs", return_value=MagicMock()), \
|
||||
patch("tools.tts_tool._import_sounddevice", side_effect=OSError("no PortAudio")):
|
||||
try:
|
||||
from tools.tts_tool import (
|
||||
_load_tts_config as load_cfg,
|
||||
_get_provider as get_prov,
|
||||
_import_elevenlabs as import_el,
|
||||
_import_sounddevice as import_sd,
|
||||
)
|
||||
cfg = load_cfg()
|
||||
if get_prov(cfg) == "elevenlabs":
|
||||
import_el()
|
||||
import_sd()
|
||||
use_streaming_tts = True
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
assert use_streaming_tts is False
|
||||
|
||||
def test_does_not_activate_for_non_elevenlabs_provider(self):
|
||||
"""use_streaming_tts stays False when provider is not elevenlabs."""
|
||||
use_streaming_tts = False
|
||||
with patch("tools.tts_tool._load_tts_config", return_value={"provider": "edge"}), \
|
||||
patch("tools.tts_tool._get_provider", return_value="edge"):
|
||||
try:
|
||||
from tools.tts_tool import (
|
||||
_load_tts_config as load_cfg,
|
||||
_get_provider as get_prov,
|
||||
_import_elevenlabs as import_el,
|
||||
_import_sounddevice as import_sd,
|
||||
)
|
||||
cfg = load_cfg()
|
||||
if get_prov(cfg) == "elevenlabs":
|
||||
import_el()
|
||||
import_sd()
|
||||
use_streaming_tts = True
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
assert use_streaming_tts is False
|
||||
"""No audio output device → no streaming pipeline, even with a provider."""
|
||||
with patch("tools.tts_tool._import_sounddevice", side_effect=OSError("no PortAudio")), \
|
||||
patch("tools.tts_tool.check_tts_requirements", return_value=True):
|
||||
assert self._gate() is False
|
||||
|
||||
def test_stale_boolean_imports_no_longer_exist(self):
|
||||
"""Confirm _HAS_ELEVENLABS and _HAS_AUDIO are not in tts_tool module."""
|
||||
|
|
@ -1326,3 +1254,49 @@ class TestRefreshLevelLock:
|
|||
assert not t.is_alive()
|
||||
assert not t.is_alive(), "Refresh thread did not stop"
|
||||
assert iterations > 0, "Refresh thread never ran"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Barge-in capture — the interruption is transcribed and queued directly
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVoiceBargeCaptureSubmit:
|
||||
"""_voice_submit_barge_utterance: the barge monitor's captured WAV becomes
|
||||
the next turn without a re-record round trip."""
|
||||
|
||||
def test_transcript_is_queued_and_wav_removed(self, tmp_path, monkeypatch):
|
||||
cli = _make_voice_cli()
|
||||
cli._voice_barge_capture.set()
|
||||
wav = tmp_path / "barge.wav"
|
||||
wav.write_bytes(b"RIFF")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.voice_mode.transcribe_recording",
|
||||
lambda path, model=None: {"success": True, "transcript": "stop, do it differently"},
|
||||
)
|
||||
|
||||
cli._voice_submit_barge_utterance(str(wav))
|
||||
|
||||
assert cli._pending_input.get_nowait() == "stop, do it differently"
|
||||
assert not cli._voice_barge_capture.is_set()
|
||||
assert not wav.exists()
|
||||
|
||||
def test_no_speech_hands_mic_back_without_queueing(self, tmp_path, monkeypatch):
|
||||
cli = _make_voice_cli(_voice_mode=True, _voice_continuous=True)
|
||||
cli._voice_barge_capture.set()
|
||||
wav = tmp_path / "barge.wav"
|
||||
wav.write_bytes(b"RIFF")
|
||||
restarted = threading.Event()
|
||||
cli._voice_start_recording = lambda: restarted.set()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.voice_mode.transcribe_recording",
|
||||
lambda path, model=None: {"success": True, "transcript": "", "no_speech": True},
|
||||
)
|
||||
|
||||
cli._voice_submit_barge_utterance(str(wav))
|
||||
|
||||
assert cli._pending_input.empty()
|
||||
assert not cli._voice_barge_capture.is_set()
|
||||
assert restarted.wait(2.0) # continuous mode resumes listening
|
||||
|
|
|
|||
|
|
@ -776,6 +776,37 @@ class TestTranscribeRecording:
|
|||
assert result["transcript"] == ""
|
||||
assert result["filtered"] is True
|
||||
|
||||
def test_no_speech_failure_maps_to_silent_success(self):
|
||||
"""Provider "empty transcript" errors are silence, not failure — the
|
||||
voice loop should re-listen quietly instead of showing an error."""
|
||||
mock_transcribe = MagicMock(return_value={
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "ElevenLabs STT returned empty transcript",
|
||||
"no_speech": True,
|
||||
})
|
||||
|
||||
with patch("tools.transcription_tools.transcribe_audio", mock_transcribe):
|
||||
from tools.voice_mode import transcribe_recording
|
||||
result = transcribe_recording("/tmp/test.wav")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["transcript"] == ""
|
||||
assert result["no_speech"] is True
|
||||
|
||||
def test_real_failures_still_fail(self):
|
||||
mock_transcribe = MagicMock(return_value={
|
||||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "xAI STT API error (HTTP 500): boom",
|
||||
})
|
||||
|
||||
with patch("tools.transcription_tools.transcribe_audio", mock_transcribe):
|
||||
from tools.voice_mode import transcribe_recording
|
||||
result = transcribe_recording("/tmp/test.wav")
|
||||
|
||||
assert result["success"] is False
|
||||
|
||||
def test_does_not_filter_real_speech(self):
|
||||
mock_transcribe = MagicMock(return_value={
|
||||
"success": True,
|
||||
|
|
@ -1476,3 +1507,126 @@ class TestSilenceCallbackLock:
|
|||
recorder.cancel()
|
||||
with recorder._lock:
|
||||
assert recorder._on_silence_stop is None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# listen_for_speech — VAD barge-in monitor
|
||||
# ============================================================================
|
||||
|
||||
class _FakeInputStream:
|
||||
"""Context-manager InputStream serving a fixed sequence of RMS levels."""
|
||||
|
||||
def __init__(self, np, levels):
|
||||
self._np = np
|
||||
self._levels = list(levels)
|
||||
self.reads = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def read(self, frames):
|
||||
level = self._levels[min(self.reads, len(self._levels) - 1)]
|
||||
self.reads += 1
|
||||
return self._np.full((frames, 1), level, dtype=self._np.int16), False
|
||||
|
||||
|
||||
class TestListenForSpeech:
|
||||
"""listen_for_speech: calibration → sustained-speech trigger → barge-in."""
|
||||
|
||||
CALIB_BLOCKS = 14 # 400ms / 30ms
|
||||
TRIP_BLOCKS = 10 # 300ms / 30ms
|
||||
|
||||
def _run(self, mock_sd, levels, should_stop=None, **kwargs):
|
||||
np = pytest.importorskip("numpy")
|
||||
stream = _FakeInputStream(np, levels)
|
||||
mock_sd.InputStream.return_value = stream
|
||||
from tools.voice_mode import listen_for_speech
|
||||
stops = iter([False] * 200 + [True] * 10_000)
|
||||
return listen_for_speech(should_stop or (lambda: next(stops)), **kwargs), stream
|
||||
|
||||
def test_sustained_speech_triggers(self, mock_sd):
|
||||
levels = [0] * self.CALIB_BLOCKS + [5000] * 50
|
||||
heard, _ = self._run(mock_sd, levels)
|
||||
assert heard is True
|
||||
|
||||
def test_brief_spike_does_not_trigger(self, mock_sd):
|
||||
levels = [0] * self.CALIB_BLOCKS + [5000] * (self.TRIP_BLOCKS - 2) + [0] * 500
|
||||
heard, _ = self._run(mock_sd, levels)
|
||||
assert heard is False
|
||||
|
||||
def test_should_stop_wins_over_silence(self, mock_sd):
|
||||
"""TTS finishing (should_stop) ends the monitor without a trigger —
|
||||
the default _run stopper flips True after 200 silent reads."""
|
||||
heard, stream = self._run(mock_sd, [0] * 500)
|
||||
assert heard is False
|
||||
assert stream.reads <= 201
|
||||
|
||||
def test_returns_false_when_audio_unavailable(self, monkeypatch):
|
||||
monkeypatch.setattr("tools.voice_mode._import_audio", MagicMock(side_effect=OSError("no audio")))
|
||||
from tools.voice_mode import listen_for_speech
|
||||
assert listen_for_speech(lambda: False) is False
|
||||
|
||||
def test_loud_floor_raises_trigger(self, mock_sd):
|
||||
"""Speaker bleed during calibration bakes into the floor — playback-level
|
||||
audio after calibration must NOT trip (only louder speech does)."""
|
||||
levels = [2000] * self.CALIB_BLOCKS + [2000] * 100
|
||||
heard, _ = self._run(mock_sd, levels)
|
||||
assert heard is False
|
||||
|
||||
|
||||
class TestListenForSpeechCapture:
|
||||
"""capture=True: the barge monitor records the interruption with pre-roll,
|
||||
so the utterance is complete from its first syllable — nothing is lost
|
||||
between detection and a recorder restart."""
|
||||
|
||||
CALIB_BLOCKS = 14 # 400ms / 30ms
|
||||
LOUD_BLOCKS = 30 # speech: trips after 10, keeps talking
|
||||
BLOCK = 480 # 16000 * 0.03
|
||||
|
||||
def _run(self, mock_sd, monkeypatch, levels, should_stop=None, **kwargs):
|
||||
np = pytest.importorskip("numpy")
|
||||
stream = _FakeInputStream(np, levels)
|
||||
mock_sd.InputStream.return_value = stream
|
||||
written = {}
|
||||
monkeypatch.setattr(
|
||||
"tools.voice_mode.AudioRecorder._write_wav",
|
||||
staticmethod(lambda audio: written.update(audio=audio) or "/tmp/barge.wav"),
|
||||
)
|
||||
from tools.voice_mode import listen_for_speech
|
||||
stops = iter([False] * 200 + [True] * 10_000)
|
||||
path = listen_for_speech(
|
||||
should_stop or (lambda: next(stops)), capture=True, **kwargs
|
||||
)
|
||||
return path, written.get("audio"), stream
|
||||
|
||||
def test_captured_utterance_includes_speech_onset(self, mock_sd, monkeypatch):
|
||||
"""Every loud block — including the ones BEFORE detection tripped —
|
||||
must land in the WAV. That pre-roll is the whole point."""
|
||||
triggered = []
|
||||
levels = [0] * self.CALIB_BLOCKS + [5000] * self.LOUD_BLOCKS + [0] * 500
|
||||
path, audio, _ = self._run(
|
||||
mock_sd, monkeypatch, levels,
|
||||
should_stop=lambda: False,
|
||||
on_trigger=lambda: triggered.append(True),
|
||||
)
|
||||
assert path == "/tmp/barge.wav"
|
||||
assert triggered == [True]
|
||||
assert int((audio == 5000).sum()) == self.LOUD_BLOCKS * self.BLOCK
|
||||
|
||||
def test_no_trip_returns_none(self, mock_sd, monkeypatch):
|
||||
triggered = []
|
||||
path, audio, _ = self._run(
|
||||
mock_sd, monkeypatch, [0] * 500,
|
||||
on_trigger=lambda: triggered.append(True),
|
||||
)
|
||||
assert path is None
|
||||
assert audio is None
|
||||
assert triggered == []
|
||||
|
||||
def test_returns_none_when_audio_unavailable(self, monkeypatch):
|
||||
monkeypatch.setattr("tools.voice_mode._import_audio", MagicMock(side_effect=OSError("no audio")))
|
||||
from tools.voice_mode import listen_for_speech
|
||||
assert listen_for_speech(lambda: False, capture=True) is None
|
||||
|
|
|
|||
|
|
@ -1546,6 +1546,7 @@ def _transcribe_xai(file_path: str, model_name: str) -> Dict[str, Any]:
|
|||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "xAI STT returned empty transcript",
|
||||
"no_speech": True,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
|
|
@ -1633,6 +1634,7 @@ def _transcribe_elevenlabs(file_path: str, model_name: str) -> Dict[str, Any]:
|
|||
"success": False,
|
||||
"transcript": "",
|
||||
"error": "ElevenLabs STT returned empty transcript",
|
||||
"no_speech": True,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
|
|
|
|||
193
tools/tts_streaming.py
Normal file
193
tools/tts_streaming.py
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
"""Provider-agnostic streaming TTS: sentence text → int16 PCM chunk iterator.
|
||||
|
||||
The keystone of Hermes' conversational voice UX. `stream_tts_to_speaker`
|
||||
(``tools.tts_tool``) owns the sentence buffer, sounddevice output, and
|
||||
stop/queue protocol; this module owns the *provider* half — turning one
|
||||
sentence into audio the moment it's ready, so playback starts on sentence one
|
||||
instead of after the whole reply.
|
||||
|
||||
Two provider shapes, one contract (int16 mono PCM at ``sample_rate``):
|
||||
|
||||
* **True streamers** (`StreamingTTSProvider.stream`) — chunked APIs
|
||||
(ElevenLabs pcm_24000, OpenAI pcm, …) that yield audio as it synthesizes.
|
||||
Lowest time-to-first-audio.
|
||||
* **Everyone else** — providers with no chunked API still get per-*sentence*
|
||||
playback via the proven sync `text_to_speech_tool` path (handled by the
|
||||
dispatcher, not here), so edge (the default) is conversational too.
|
||||
|
||||
Adding a streamer is `@register("name")` on a `StreamingTTSProvider` subclass;
|
||||
the dispatcher, config gate (`tts.<name>.streaming`), and resolver come free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Callable, Dict, Iterator, List, Optional
|
||||
|
||||
from tools.tts_tool import _get_provider, get_env_value
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sentence boundary: after .!? followed by whitespace, or a blank line.
|
||||
SENTENCE_BOUNDARY_RE = re.compile(r"(?<=[.!?])(?:\s|\n)|(?:\n\n)")
|
||||
_THINK_BLOCK_RE = re.compile(r"<think[\s>].*?</think>", flags=re.DOTALL)
|
||||
|
||||
|
||||
class SentenceChunker:
|
||||
"""Incremental sentence cutter for LLM token deltas.
|
||||
|
||||
Shared by the speaker pipeline (`stream_tts_to_speaker`) and the
|
||||
speak-stream WebSocket so every surface cuts speech identically. Strips
|
||||
``<think>`` blocks (even split across deltas) and merges fragments shorter
|
||||
than *min_len* into the following sentence, so "Ha!" rides along with the
|
||||
sentence after it instead of stalling as a tiny clip.
|
||||
"""
|
||||
|
||||
def __init__(self, min_len: int = 20):
|
||||
self.min_len = min_len
|
||||
self.buf = ""
|
||||
|
||||
def feed(self, delta: str) -> List[str]:
|
||||
"""Absorb *delta*; return every complete sentence now ready to speak."""
|
||||
self.buf = _THINK_BLOCK_RE.sub("", self.buf + delta)
|
||||
if "<think" in self.buf and "</think>" not in self.buf:
|
||||
return [] # open think tag — the closing tag may arrive next delta
|
||||
out: List[str] = []
|
||||
start = 0 # skip boundaries that would leave the head too short
|
||||
while m := SENTENCE_BOUNDARY_RE.search(self.buf, start):
|
||||
head = self.buf[: m.end()]
|
||||
if len(head.strip()) < self.min_len:
|
||||
start = m.end()
|
||||
continue
|
||||
out.append(head)
|
||||
self.buf = self.buf[m.end():]
|
||||
start = 0
|
||||
return out
|
||||
|
||||
def flush(self) -> List[str]:
|
||||
"""Drain the tail (end-of-text or long-idle flush)."""
|
||||
tail = _THINK_BLOCK_RE.sub("", self.buf).strip()
|
||||
self.buf = ""
|
||||
return [tail] if tail else []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ABC + registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class StreamingTTSProvider(ABC):
|
||||
"""Yields raw int16, little-endian, mono PCM chunks at ``sample_rate``."""
|
||||
|
||||
sample_rate: int = 24000
|
||||
channels: int = 1
|
||||
sample_width: int = 2 # bytes/sample (int16)
|
||||
|
||||
def __init__(self, tts_config: Dict, section: Dict):
|
||||
self.tts_config = tts_config
|
||||
self.section = section
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def available() -> bool:
|
||||
"""True when this provider's credentials/SDK are usable right now."""
|
||||
|
||||
@abstractmethod
|
||||
def stream(self, text: str) -> Iterator[bytes]:
|
||||
"""Yield PCM chunks for ``text``. Raise on failure (caller logs)."""
|
||||
|
||||
|
||||
_REGISTRY: Dict[str, type[StreamingTTSProvider]] = {}
|
||||
|
||||
|
||||
def register(name: str) -> Callable[[type[StreamingTTSProvider]], type[StreamingTTSProvider]]:
|
||||
def _wrap(cls: type[StreamingTTSProvider]) -> type[StreamingTTSProvider]:
|
||||
_REGISTRY[name] = cls
|
||||
return cls
|
||||
|
||||
return _wrap
|
||||
|
||||
|
||||
def resolve_streaming_provider(
|
||||
tts_config: Dict,
|
||||
preferred: Optional[str] = None,
|
||||
) -> Optional[StreamingTTSProvider]:
|
||||
"""Return a ready streamer for the *configured* provider, else ``None``.
|
||||
|
||||
``None`` means "no chunked API for this provider" — the dispatcher then
|
||||
speaks per-sentence via the sync path, preserving the user's chosen voice.
|
||||
We never silently swap to a different provider just to get streaming.
|
||||
"""
|
||||
name = (preferred or _get_provider(tts_config)).lower().strip()
|
||||
cls = _REGISTRY.get(name)
|
||||
if cls is None or not cls.available():
|
||||
return None
|
||||
try:
|
||||
return cls(tts_config, tts_config.get(name) or {})
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("streaming provider %s init failed: %s", name, exc)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Providers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@register("elevenlabs")
|
||||
class ElevenLabsStreamer(StreamingTTSProvider):
|
||||
"""ElevenLabs chunked HTTP → pcm_24000 (the original reference path)."""
|
||||
|
||||
sample_rate = 24000
|
||||
|
||||
@staticmethod
|
||||
def available() -> bool:
|
||||
return bool(get_env_value("ELEVENLABS_API_KEY"))
|
||||
|
||||
def stream(self, text: str) -> Iterator[bytes]:
|
||||
from tools.tts_tool import (
|
||||
DEFAULT_ELEVENLABS_STREAMING_MODEL_ID,
|
||||
DEFAULT_ELEVENLABS_VOICE_ID,
|
||||
_import_elevenlabs,
|
||||
)
|
||||
|
||||
client = _import_elevenlabs()(api_key=get_env_value("ELEVENLABS_API_KEY"))
|
||||
voice_id = self.section.get("voice_id", DEFAULT_ELEVENLABS_VOICE_ID)
|
||||
model_id = self.section.get(
|
||||
"streaming_model_id",
|
||||
self.section.get("model_id", DEFAULT_ELEVENLABS_STREAMING_MODEL_ID),
|
||||
)
|
||||
yield from client.text_to_speech.convert(
|
||||
text=text,
|
||||
voice_id=voice_id,
|
||||
model_id=model_id,
|
||||
output_format="pcm_24000",
|
||||
)
|
||||
|
||||
|
||||
@register("openai")
|
||||
class OpenAIStreamer(StreamingTTSProvider):
|
||||
"""OpenAI speech with ``response_format=pcm`` (24 kHz mono int16)."""
|
||||
|
||||
sample_rate = 24000
|
||||
|
||||
@staticmethod
|
||||
def available() -> bool:
|
||||
return bool(get_env_value("OPENAI_API_KEY"))
|
||||
|
||||
def stream(self, text: str) -> Iterator[bytes]:
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key=get_env_value("OPENAI_API_KEY"),
|
||||
base_url=get_env_value("OPENAI_BASE_URL") or None,
|
||||
)
|
||||
model = self.section.get("model", "gpt-4o-mini-tts")
|
||||
voice = self.section.get("voice", "alloy")
|
||||
with client.audio.speech.with_streaming_response.create(
|
||||
model=model,
|
||||
voice=voice,
|
||||
input=text,
|
||||
response_format="pcm",
|
||||
) as response:
|
||||
yield from response.iter_bytes()
|
||||
|
|
@ -2716,11 +2716,8 @@ def _has_openai_audio_backend() -> bool:
|
|||
|
||||
|
||||
# ===========================================================================
|
||||
# Streaming TTS: sentence-by-sentence pipeline for ElevenLabs
|
||||
# Streaming TTS: sentence-by-sentence pipeline
|
||||
# ===========================================================================
|
||||
# Sentence boundary pattern: punctuation followed by space or newline
|
||||
_SENTENCE_BOUNDARY_RE = re.compile(r'(?<=[.!?])(?:\s|\n)|(?:\n\n)')
|
||||
|
||||
# Markdown stripping patterns (same as cli.py _voice_speak_response)
|
||||
_MD_CODE_BLOCK = re.compile(r'```[\s\S]*?```')
|
||||
_MD_LINK = re.compile(r'\[([^\]]+)\]\([^)]+\)')
|
||||
|
|
@ -2732,10 +2729,15 @@ _MD_HEADER = re.compile(r'^#+\s*', flags=re.MULTILINE)
|
|||
_MD_LIST_ITEM = re.compile(r'^\s*[-*]\s+', flags=re.MULTILINE)
|
||||
_MD_HR = re.compile(r'---+')
|
||||
_MD_EXCESS_NL = re.compile(r'\n{3,}')
|
||||
# Emoji + variation selectors/ZWJ — TTS providers render these as awkward
|
||||
# pauses or literal descriptions ("smiling face"), breaking the speech flow.
|
||||
_EMOJI = re.compile(
|
||||
'[\U0001F000-\U0001FAFF\u2600-\u27BF\uFE0F\u200D\U000E0020-\U000E007F]+'
|
||||
)
|
||||
|
||||
|
||||
def _strip_markdown_for_tts(text: str) -> str:
|
||||
"""Remove markdown formatting that shouldn't be spoken aloud."""
|
||||
"""Remove markdown formatting (and emoji) that shouldn't be spoken aloud."""
|
||||
text = _MD_CODE_BLOCK.sub(' ', text)
|
||||
text = _MD_LINK.sub(r'\1', text)
|
||||
text = _MD_URL.sub('', text)
|
||||
|
|
@ -2745,6 +2747,7 @@ def _strip_markdown_for_tts(text: str) -> str:
|
|||
text = _MD_HEADER.sub('', text)
|
||||
text = _MD_LIST_ITEM.sub('', text)
|
||||
text = _MD_HR.sub('', text)
|
||||
text = _EMOJI.sub(' ', text)
|
||||
text = _MD_EXCESS_NL.sub('\n\n', text)
|
||||
return text.strip()
|
||||
|
||||
|
|
@ -2754,75 +2757,62 @@ def stream_tts_to_speaker(
|
|||
stop_event: threading.Event,
|
||||
tts_done_event: threading.Event,
|
||||
display_callback: Optional[Callable[[str], None]] = None,
|
||||
provider: Optional[str] = None,
|
||||
):
|
||||
"""Consume text deltas from *text_queue*, buffer them into sentences,
|
||||
and stream each sentence through ElevenLabs TTS to the speaker in
|
||||
real-time.
|
||||
"""Consume text deltas from *text_queue*, buffer them into sentences, and
|
||||
speak each sentence the moment it's ready — the conversational path.
|
||||
|
||||
Provider-agnostic. A registered streaming provider (ElevenLabs, OpenAI, …)
|
||||
plays chunked PCM through one sounddevice stream for the lowest latency;
|
||||
every other provider (edge, the default) is spoken per-sentence via the sync
|
||||
``text_to_speech_tool`` path, so audio still starts on sentence one instead
|
||||
of after the whole reply.
|
||||
|
||||
Protocol:
|
||||
* The producer puts ``str`` deltas onto *text_queue*.
|
||||
* A ``None`` sentinel signals end-of-text (flush remaining buffer).
|
||||
* *stop_event* can be set to abort early (e.g. user interrupt).
|
||||
* *stop_event* can be set to abort early (barge-in / user interrupt).
|
||||
* *tts_done_event* is **set** in the ``finally`` block so callers
|
||||
waiting on it (continuous voice mode) know playback is finished.
|
||||
"""
|
||||
tts_done_event.clear()
|
||||
|
||||
try:
|
||||
# --- TTS client setup (optional -- display_callback works without it) ---
|
||||
client = None
|
||||
output_stream = None
|
||||
voice_id = DEFAULT_ELEVENLABS_VOICE_ID
|
||||
model_id = DEFAULT_ELEVENLABS_STREAMING_MODEL_ID
|
||||
|
||||
tts_config = _load_tts_config()
|
||||
el_config = tts_config.get("elevenlabs") or {}
|
||||
voice_id = el_config.get("voice_id", voice_id)
|
||||
model_id = el_config.get("streaming_model_id",
|
||||
el_config.get("model_id", model_id))
|
||||
# Per-sentence cap for the streaming path. Look up the cap against
|
||||
# the *streaming* model_id (defaults to eleven_flash_v2_5 = 40k chars),
|
||||
# not the sync model_id. A user override
|
||||
# (tts.elevenlabs.max_text_length) still wins.
|
||||
stream_max_len = _resolve_max_text_length(
|
||||
"elevenlabs",
|
||||
{**tts_config, "elevenlabs": {**el_config, "model_id": model_id}},
|
||||
)
|
||||
|
||||
api_key = (get_env_value("ELEVENLABS_API_KEY") or "")
|
||||
if not api_key:
|
||||
logger.warning("ELEVENLABS_API_KEY not set; streaming TTS audio disabled")
|
||||
else:
|
||||
# Prefer a chunked streamer for low time-to-first-audio; fall back to
|
||||
# per-sentence sync synthesis (universal — edge + every non-streamer).
|
||||
from tools.tts_streaming import SentenceChunker, resolve_streaming_provider
|
||||
streamer = resolve_streaming_provider(tts_config, preferred=provider)
|
||||
|
||||
stream_max_len = 0
|
||||
if streamer is not None:
|
||||
try:
|
||||
ElevenLabs = _import_elevenlabs()
|
||||
client = ElevenLabs(api_key=api_key)
|
||||
except ImportError:
|
||||
logger.warning("elevenlabs package not installed; streaming TTS disabled")
|
||||
stream_max_len = _resolve_max_text_length(
|
||||
provider or _get_provider(tts_config), tts_config
|
||||
)
|
||||
except Exception:
|
||||
stream_max_len = 0
|
||||
try:
|
||||
sd = _import_sounddevice()
|
||||
output_stream = sd.OutputStream(
|
||||
samplerate=streamer.sample_rate,
|
||||
channels=streamer.channels,
|
||||
dtype="int16",
|
||||
)
|
||||
output_stream.start()
|
||||
except (ImportError, OSError) as exc:
|
||||
logger.debug("sounddevice not available, streamer→tempfile: %s", exc)
|
||||
output_stream = None
|
||||
except Exception as exc:
|
||||
logger.warning("sounddevice OutputStream failed: %s", exc)
|
||||
output_stream = None
|
||||
|
||||
# Open a single sounddevice output stream for the lifetime of
|
||||
# this function. ElevenLabs pcm_24000 produces signed 16-bit
|
||||
# little-endian mono PCM at 24 kHz.
|
||||
if client is not None:
|
||||
try:
|
||||
sd = _import_sounddevice()
|
||||
output_stream = sd.OutputStream(
|
||||
samplerate=24000, channels=1, dtype="int16",
|
||||
)
|
||||
output_stream.start()
|
||||
except (ImportError, OSError) as exc:
|
||||
logger.debug("sounddevice not available: %s", exc)
|
||||
output_stream = None
|
||||
except Exception as exc:
|
||||
logger.warning("sounddevice OutputStream failed: %s", exc)
|
||||
output_stream = None
|
||||
|
||||
sentence_buf = ""
|
||||
min_sentence_len = 20
|
||||
chunker = SentenceChunker()
|
||||
long_flush_len = 100
|
||||
queue_timeout = 0.5
|
||||
_spoken_sentences: list[str] = [] # track spoken sentences to skip duplicates
|
||||
# Regex to strip complete <think>...</think> blocks from buffer
|
||||
_think_block_re = re.compile(r'<think[\s>].*?</think>', flags=re.DOTALL)
|
||||
|
||||
def _speak_sentence(sentence: str):
|
||||
"""Display sentence and optionally generate + play audio."""
|
||||
|
|
@ -2840,33 +2830,51 @@ def stream_tts_to_speaker(
|
|||
# Display raw sentence on screen before TTS processing
|
||||
if display_callback is not None:
|
||||
display_callback(sentence)
|
||||
# Skip audio generation if no TTS client available
|
||||
if client is None:
|
||||
# No chunked streamer → per-sentence sync synthesis (universal).
|
||||
if streamer is None:
|
||||
_speak_via_sync(cleaned)
|
||||
return
|
||||
# Truncate very long sentences (ElevenLabs streaming path)
|
||||
if len(cleaned) > stream_max_len:
|
||||
# Truncate very long sentences to the provider's per-request cap.
|
||||
if stream_max_len and len(cleaned) > stream_max_len:
|
||||
cleaned = cleaned[:stream_max_len]
|
||||
try:
|
||||
audio_iter = client.text_to_speech.convert(
|
||||
text=cleaned,
|
||||
voice_id=voice_id,
|
||||
model_id=model_id,
|
||||
output_format="pcm_24000",
|
||||
)
|
||||
audio_iter = streamer.stream(cleaned)
|
||||
if output_stream is not None:
|
||||
import numpy as _np
|
||||
for chunk in audio_iter:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
import numpy as _np
|
||||
audio_array = _np.frombuffer(chunk, dtype=_np.int16)
|
||||
output_stream.write(audio_array.reshape(-1, 1))
|
||||
output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1))
|
||||
else:
|
||||
# Fallback: write chunks to temp file and play via system player
|
||||
_play_via_tempfile(audio_iter, stop_event)
|
||||
# No audio device: buffer chunks to a temp WAV and play it.
|
||||
_play_via_tempfile(audio_iter, stop_event, streamer.sample_rate)
|
||||
except Exception as exc:
|
||||
logger.warning("Streaming TTS sentence failed: %s", exc)
|
||||
|
||||
def _play_via_tempfile(audio_iter, stop_evt):
|
||||
def _speak_via_sync(cleaned: str):
|
||||
"""Synthesize one sentence via the proven sync tool, then block on
|
||||
playback. No chunked API, but per-*sentence* granularity keeps the
|
||||
flow conversational for edge and every other non-streaming provider.
|
||||
"""
|
||||
tmp_path = None
|
||||
try:
|
||||
fd, tmp_path = tempfile.mkstemp(suffix=".mp3")
|
||||
os.close(fd)
|
||||
text_to_speech_tool(text=cleaned, output_path=tmp_path)
|
||||
if (not stop_event.is_set() and os.path.isfile(tmp_path)
|
||||
and os.path.getsize(tmp_path) > 0):
|
||||
from tools.voice_mode import play_audio_file
|
||||
play_audio_file(tmp_path)
|
||||
except Exception as exc:
|
||||
logger.warning("Sync per-sentence TTS failed: %s", exc)
|
||||
finally:
|
||||
if tmp_path:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _play_via_tempfile(audio_iter, stop_evt, sample_rate=24000):
|
||||
"""Write PCM chunks to a temp WAV file and play it."""
|
||||
tmp_path = None
|
||||
try:
|
||||
|
|
@ -2876,7 +2884,7 @@ def stream_tts_to_speaker(
|
|||
with wave.open(tmp, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2) # 16-bit
|
||||
wf.setframerate(24000)
|
||||
wf.setframerate(sample_rate)
|
||||
for chunk in audio_iter:
|
||||
if stop_evt.is_set():
|
||||
break
|
||||
|
|
@ -2897,43 +2905,19 @@ def stream_tts_to_speaker(
|
|||
try:
|
||||
delta = text_queue.get(timeout=queue_timeout)
|
||||
except queue.Empty:
|
||||
# Timeout: if we have accumulated a long buffer, flush it
|
||||
if len(sentence_buf) > long_flush_len:
|
||||
_speak_sentence(sentence_buf)
|
||||
sentence_buf = ""
|
||||
# Idle producer: flush a long buffer instead of sitting on it
|
||||
if len(chunker.buf) > long_flush_len:
|
||||
for sentence in chunker.flush():
|
||||
_speak_sentence(sentence)
|
||||
continue
|
||||
|
||||
if delta is None:
|
||||
# End-of-text sentinel: strip any remaining think blocks, flush
|
||||
sentence_buf = _think_block_re.sub('', sentence_buf)
|
||||
if sentence_buf.strip():
|
||||
_speak_sentence(sentence_buf)
|
||||
# End-of-text sentinel: flush whatever remains
|
||||
for sentence in chunker.flush():
|
||||
_speak_sentence(sentence)
|
||||
break
|
||||
|
||||
sentence_buf += delta
|
||||
|
||||
# --- Think block filtering ---
|
||||
# Strip complete <think>...</think> blocks from buffer.
|
||||
# Works correctly even when tags span multiple deltas.
|
||||
sentence_buf = _think_block_re.sub('', sentence_buf)
|
||||
|
||||
# If an incomplete <think tag is at the end, wait for more data
|
||||
# before extracting sentences (the closing tag may arrive next).
|
||||
if '<think' in sentence_buf and '</think>' not in sentence_buf:
|
||||
continue
|
||||
|
||||
# Check for sentence boundaries
|
||||
while True:
|
||||
m = _SENTENCE_BOUNDARY_RE.search(sentence_buf)
|
||||
if m is None:
|
||||
break
|
||||
end_pos = m.end()
|
||||
sentence = sentence_buf[:end_pos]
|
||||
sentence_buf = sentence_buf[end_pos:]
|
||||
# Merge short fragments into the next sentence
|
||||
if len(sentence.strip()) < min_sentence_len:
|
||||
sentence_buf = sentence + sentence_buf
|
||||
break
|
||||
for sentence in chunker.feed(delta):
|
||||
_speak_sentence(sentence)
|
||||
|
||||
# Drain any remaining items from the queue
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import tempfile
|
|||
import threading
|
||||
import time
|
||||
import wave
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -900,6 +900,12 @@ def transcribe_recording(wav_path: str, model: Optional[str] = None) -> Dict[str
|
|||
logger.info("Filtered Whisper hallucination: %r", result["transcript"])
|
||||
return {"success": True, "transcript": "", "filtered": True}
|
||||
|
||||
# Providers that flag no_speech (empty transcript) failed to hear words,
|
||||
# not to transcribe — treat like silence so the voice loop re-listens
|
||||
# quietly instead of surfacing "Transcription failed".
|
||||
if result.get("no_speech"):
|
||||
return {"success": True, "transcript": "", "no_speech": True}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
|
@ -1118,6 +1124,94 @@ def play_audio_file(file_path: str) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Barge-in — detect the user speaking over TTS playback
|
||||
# ============================================================================
|
||||
def listen_for_speech(
|
||||
should_stop: Callable[[], bool],
|
||||
threshold: Optional[int] = None,
|
||||
sustained_ms: int = 300,
|
||||
calibration_ms: int = 400,
|
||||
capture: bool = False,
|
||||
on_trigger: Optional[Callable[[], None]] = None,
|
||||
pre_roll_ms: int = 1200,
|
||||
endpoint_silence_ms: int = 1250,
|
||||
max_utterance_ms: int = 30_000,
|
||||
):
|
||||
"""Block until sustained speech is heard on the mic, or *should_stop*.
|
||||
|
||||
Barge-in monitor: run in a side thread while TTS is playing. Without
|
||||
*capture* it returns ``True`` when the user started talking (cut playback).
|
||||
With ``capture=True`` it ALSO records the interruption — a rolling
|
||||
*pre_roll_ms* buffer means the utterance is kept from its first syllable,
|
||||
not from the moment detection tripped — and keeps rolling until the user
|
||||
goes quiet for *endpoint_silence_ms*, then returns the WAV path (or
|
||||
``None`` if speech never tripped). *on_trigger* fires at the moment of
|
||||
detection so the caller can stop playback while capture continues.
|
||||
|
||||
The noise floor is calibrated from the first *calibration_ms* of input —
|
||||
playback is already audible then, so speaker bleed is baked into the
|
||||
floor and only louder-than-playback speech trips the trigger. Requiring
|
||||
*sustained_ms* of consecutive above-threshold blocks filters out coughs,
|
||||
keyboard thumps, and playback transients.
|
||||
"""
|
||||
try:
|
||||
sd, np = _import_audio()
|
||||
except (ImportError, OSError):
|
||||
return None if capture else False
|
||||
|
||||
from collections import deque
|
||||
|
||||
block = int(SAMPLE_RATE * 0.03) # 30ms blocks
|
||||
calib_blocks = max(1, calibration_ms // 30)
|
||||
trip_blocks = max(1, sustained_ms // 30)
|
||||
endpoint_blocks = max(1, endpoint_silence_ms // 30)
|
||||
max_blocks = max(1, max_utterance_ms // 30)
|
||||
floor_samples: List[float] = []
|
||||
pre_roll: deque = deque(maxlen=max(1, pre_roll_ms // 30))
|
||||
consecutive = 0
|
||||
|
||||
try:
|
||||
with sd.InputStream(samplerate=SAMPLE_RATE, channels=1, dtype="int16", blocksize=block) as stream:
|
||||
while not should_stop():
|
||||
data, _ = stream.read(block)
|
||||
rms = float(np.sqrt(np.mean(data.astype(np.float64) ** 2)))
|
||||
if capture:
|
||||
pre_roll.append(data.copy())
|
||||
if len(floor_samples) < calib_blocks:
|
||||
floor_samples.append(rms)
|
||||
continue
|
||||
trigger = max(float(threshold or SILENCE_RMS_THRESHOLD * 2), float(np.median(floor_samples)) * 3.5)
|
||||
consecutive = consecutive + 1 if rms >= trigger else 0
|
||||
if consecutive < trip_blocks:
|
||||
continue
|
||||
|
||||
# Tripped — the user is talking over playback.
|
||||
if on_trigger:
|
||||
try:
|
||||
on_trigger()
|
||||
except Exception as e:
|
||||
logger.debug("Barge-in trigger callback failed: %s", e)
|
||||
if not capture:
|
||||
return True
|
||||
|
||||
# Keep rolling until the user goes quiet. Playback is stopped
|
||||
# now, so plain silence endpointing (recorder threshold) works.
|
||||
frames: List[Any] = list(pre_roll)
|
||||
quiet = 0
|
||||
for _ in range(max_blocks):
|
||||
data, _ = stream.read(block)
|
||||
frames.append(data.copy())
|
||||
rms = float(np.sqrt(np.mean(data.astype(np.float64) ** 2)))
|
||||
quiet = quiet + 1 if rms < SILENCE_RMS_THRESHOLD else 0
|
||||
if quiet >= endpoint_blocks:
|
||||
break
|
||||
return AudioRecorder._write_wav(np.concatenate(frames, axis=0))
|
||||
except Exception as e:
|
||||
logger.debug("Barge-in listener failed: %s", e)
|
||||
return None if capture else False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Requirements check
|
||||
# ============================================================================
|
||||
|
|
|
|||
|
|
@ -9409,6 +9409,9 @@ def _(rid, params: dict) -> dict:
|
|||
|
||||
@method("session.interrupt")
|
||||
def _(rid, params: dict) -> dict:
|
||||
# Keypress barge-in: stopping the turn also silences its streaming TTS
|
||||
# (voice is process-global, so no per-session scoping is needed).
|
||||
_tts_stream_stop()
|
||||
session, err = _sess_nowait(params, rid)
|
||||
if err:
|
||||
return err
|
||||
|
|
@ -10339,6 +10342,7 @@ def _run_prompt_submit(rid, sid: str, session: dict, text: Any) -> None:
|
|||
session_tokens = []
|
||||
home_token = None # per-turn HERMES_HOME override for a resumed remote profile
|
||||
goal_followup = None # set by the post-turn goal hook below
|
||||
tts_queue = None # streaming-TTS feed for this turn (voice mode)
|
||||
one_turn_restore = session.pop("one_turn_model_restore", None)
|
||||
try:
|
||||
from tools.approval import (
|
||||
|
|
@ -10461,12 +10465,18 @@ def _run_prompt_submit(rid, sid: str, session: dict, text: Any) -> None:
|
|||
else:
|
||||
run_message = _enrich_with_attached_images(prompt, images)
|
||||
|
||||
# Streaming TTS: voice-mode replies are spoken sentence-by-sentence
|
||||
# as tokens arrive (CLI parity) instead of after the full turn.
|
||||
tts_queue = _tts_stream_begin()
|
||||
|
||||
def _stream(delta):
|
||||
with session["history_lock"]:
|
||||
_append_inflight_delta(session, delta)
|
||||
payload = {"text": delta}
|
||||
if streamer and (r := streamer.feed(delta)) is not None:
|
||||
payload["rendered"] = r
|
||||
if tts_queue is not None and isinstance(delta, str):
|
||||
tts_queue.put(delta)
|
||||
_emit("message.delta", sid, payload)
|
||||
|
||||
# Surface interim assistant text (commentary emitted alongside
|
||||
|
|
@ -10735,12 +10745,13 @@ def _run_prompt_submit(rid, sid: str, session: dict, text: Any) -> None:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# CLI parity: when voice-mode TTS is on, speak the agent reply
|
||||
# (cli.py:_voice_speak_response). Only the final text — tool
|
||||
# calls / reasoning already stream separately and would be
|
||||
# noisy to read aloud.
|
||||
# Voice TTS fallback: when the streaming pipeline couldn't start
|
||||
# (no provider / missing deps probed at turn start), speak the
|
||||
# final text whole (cli.py:_voice_speak_response parity). The
|
||||
# streaming path already spoke everything via tts_queue.
|
||||
if (
|
||||
status == "complete"
|
||||
and tts_queue is None
|
||||
and isinstance(raw, str)
|
||||
and raw.strip()
|
||||
and _voice_tts_enabled()
|
||||
|
|
@ -10775,6 +10786,8 @@ def _run_prompt_submit(rid, sid: str, session: dict, text: Any) -> None:
|
|||
)
|
||||
_emit("error", sid, {"message": str(e)})
|
||||
finally:
|
||||
if tts_queue is not None:
|
||||
tts_queue.put(None) # end-of-text sentinel — flush + finish speaking
|
||||
if one_turn_restore:
|
||||
try:
|
||||
_restore_agent_model_runtime(agent, one_turn_restore)
|
||||
|
|
@ -15549,6 +15562,107 @@ def _voice_tts_enabled() -> bool:
|
|||
return os.environ.get("HERMES_VOICE_TTS", "").strip() == "1"
|
||||
|
||||
|
||||
# ── Streaming TTS (one active pipeline per process — one speaker) ──────────
|
||||
# Token deltas from the running turn feed a sentence-buffering consumer
|
||||
# (tools.tts_tool.stream_tts_to_speaker) so speech starts on the first
|
||||
# sentence instead of after the full reply. Voice is process-global, so a
|
||||
# single slot suffices; starting a new turn's pipeline barges in on the
|
||||
# previous one.
|
||||
|
||||
_tts_stream_lock = threading.Lock()
|
||||
_tts_stream_state: Optional[dict] = None
|
||||
|
||||
|
||||
def _tts_stream_begin() -> Optional[queue.Queue]:
|
||||
"""Start a per-turn streaming TTS consumer; None when TTS can't stream."""
|
||||
if not _voice_tts_enabled():
|
||||
return None
|
||||
try:
|
||||
from tools.tts_tool import check_tts_requirements, stream_tts_to_speaker
|
||||
|
||||
if not check_tts_requirements():
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
_tts_stream_stop()
|
||||
text_queue: queue.Queue = queue.Queue()
|
||||
stop = threading.Event()
|
||||
done = threading.Event()
|
||||
threading.Thread(
|
||||
target=stream_tts_to_speaker, args=(text_queue, stop, done), daemon=True
|
||||
).start()
|
||||
|
||||
global _tts_stream_state
|
||||
with _tts_stream_lock:
|
||||
_tts_stream_state = {"stop": stop, "done": done}
|
||||
|
||||
if _voice_mode_enabled() and _voice_cfg_dict().get("barge_in", True):
|
||||
threading.Thread(
|
||||
target=_tts_stream_barge_in_monitor, args=(stop, done), daemon=True
|
||||
).start()
|
||||
|
||||
return text_queue
|
||||
|
||||
|
||||
def _tts_stream_stop() -> None:
|
||||
"""Barge-in: cut any in-flight streaming TTS (new turn, interrupt, /voice off)."""
|
||||
global _tts_stream_state
|
||||
with _tts_stream_lock:
|
||||
state, _tts_stream_state = _tts_stream_state, None
|
||||
if state is None:
|
||||
return
|
||||
state["stop"].set()
|
||||
try:
|
||||
from tools.voice_mode import stop_playback
|
||||
|
||||
stop_playback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _tts_stream_barge_in_monitor(stop: threading.Event, done: threading.Event) -> None:
|
||||
"""VAD barge-in: cut streaming TTS when the user starts talking.
|
||||
|
||||
Playback is cut at the moment of detection while the monitor keeps
|
||||
capturing (with pre-roll) until the user goes quiet — the interruption is
|
||||
then transcribed and emitted as ``voice.transcript``, which the TUI
|
||||
submits like any spoken turn. Without capture the opening words would be
|
||||
lost between detection and the next recording start.
|
||||
"""
|
||||
try:
|
||||
from tools.voice_mode import listen_for_speech, stop_playback, transcribe_recording
|
||||
|
||||
barged = threading.Event()
|
||||
|
||||
def _cut_playback():
|
||||
if not done.is_set():
|
||||
barged.set()
|
||||
stop.set()
|
||||
stop_playback()
|
||||
_voice_emit("voice.interrupted")
|
||||
|
||||
wav_path = listen_for_speech(
|
||||
lambda: stop.is_set() or done.is_set(),
|
||||
capture=True,
|
||||
on_trigger=_cut_playback,
|
||||
)
|
||||
if not (wav_path and barged.is_set()):
|
||||
return
|
||||
try:
|
||||
result = transcribe_recording(wav_path)
|
||||
text = (result.get("transcript") or "").strip() if result.get("success") else ""
|
||||
if text:
|
||||
_voice_emit("voice.transcript", {"text": text})
|
||||
finally:
|
||||
try:
|
||||
os.unlink(wav_path)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug("TTS barge-in monitor failed: %s", e)
|
||||
|
||||
|
||||
def _voice_cfg_dict() -> dict:
|
||||
"""Shape-safe accessor for the ``voice:`` block in config.yaml.
|
||||
|
||||
|
|
@ -15636,8 +15750,10 @@ def _(rid, params: dict) -> dict:
|
|||
except Exception as e:
|
||||
logger.warning("voice: stop_continuous failed during toggle off: %s", e)
|
||||
|
||||
# Clear TTS so it can be toggled independently after voice is off.
|
||||
# Clear TTS so it can be toggled independently after voice is off,
|
||||
# and silence any in-flight streaming speech.
|
||||
os.environ["HERMES_VOICE_TTS"] = "0"
|
||||
_tts_stream_stop()
|
||||
|
||||
return _ok(
|
||||
rid,
|
||||
|
|
@ -15654,6 +15770,8 @@ def _(rid, params: dict) -> dict:
|
|||
new_value = not _voice_tts_enabled()
|
||||
# Runtime-only flag (CLI parity) — see voice.toggle on/off above.
|
||||
os.environ["HERMES_VOICE_TTS"] = "1" if new_value else "0"
|
||||
if not new_value:
|
||||
_tts_stream_stop()
|
||||
# Include ``record_key`` on every branch so a /voice tts toggle
|
||||
# doesn't reset the TUI's cached shortcut to the default when a
|
||||
# user has a custom binding configured (Copilot review, round 2
|
||||
|
|
|
|||
|
|
@ -159,11 +159,20 @@ Both `silence_threshold` and `silence_duration` are configurable in `config.yaml
|
|||
|
||||
### Streaming TTS
|
||||
|
||||
When TTS is enabled, the agent speaks its reply **sentence-by-sentence** as it generates text — you don't wait for the full response:
|
||||
When TTS is enabled, the agent speaks its reply **sentence-by-sentence** as it generates text — you don't wait for the full response. This works with **every TTS provider**:
|
||||
|
||||
1. Buffers text deltas into complete sentences (min 20 chars)
|
||||
2. Strips markdown formatting and `<think>` blocks
|
||||
3. Generates and plays audio per sentence in real-time
|
||||
2. Strips markdown formatting, emoji, and `<think>` blocks
|
||||
3. Plays audio per sentence in real-time — providers with a chunked PCM API (ElevenLabs, OpenAI) stream raw audio for the lowest time-to-first-word; every other provider (including the default Edge) synthesizes and plays each sentence as it completes
|
||||
|
||||
The same pipeline runs in the classic CLI, the TUI, and the desktop app. In a desktop voice conversation the reply text is fed **live** into a per-reply speech WebSocket as the model generates it, so speech overlaps generation — one socket and one audio clock per reply, no per-sentence connection gaps.
|
||||
|
||||
### Barge-in
|
||||
|
||||
You can interrupt the agent mid-speech:
|
||||
|
||||
- **Talk over it** — in continuous voice mode, a voice-activity monitor listens while the agent speaks and cuts playback the moment you start talking, then goes straight back to recording. The detector calibrates its noise floor against the playback itself, so speaker bleed doesn't self-trigger. Disable with `voice.barge_in: false` in `config.yaml`.
|
||||
- **Type or press the record key** — sending a new message or hitting the push-to-talk key stops playback instantly on every surface.
|
||||
|
||||
### Hallucination Filter
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue