mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(desktop): full-duplex voice barge-in — interrupt during generation AND playback
The voice-interruption fix (5081551f0) covered the Python surfaces (CLI +
TUI gateway) but the desktop app has its own mic path in voice-barge-in.ts,
which still had both bugs on Windows:
- HALF-DUPLEX GAP: the barge monitor only opened when TTS playback started;
during LLM generation no mic was listening at all.
- PLAYBACK DEAFNESS: the monitor calibrated its noise floor while the
speakers were already playing TTS, baking bleed into the floor. On
Windows, Chromium echoCancellation does not reliably cancel same-app
output (measured live: quiet floor ~35-50 RMS vs playback bleed
600-1700 RMS), making the trigger unreachable.
Changes mirror tools/voice_mode.full_duplex_listen:
- voice-barge-in.ts: phase-aware full-duplex monitor — quiet-only
calibration (floor held through playback, never recalibrated against
bleed), playback min-trigger clamp + ceiling, 500ms grace on playback
onset only, windowed-majority detection.
- use-voice-conversation.ts: monitor arms at turn submit and spans
generation + playback (ensureBargeMonitor, idempotent). Mid-generation
speech fires the new onInterrupt callback; spoken stop-word during a
barge ends the conversation; submit waits for the interrupt to settle.
- use-composer-voice.ts + composer/index.tsx: plumb onInterrupt: haltRun
(same seam as the Stop button).
Tests: use-voice-conversation.test.tsx (6 tests) covering monitor lifecycle
across generation + playback, mid-generation interrupt, stop-word handling.
npm run typecheck + eslint clean; remaining desktop vitest failures
reproduce on a clean tree (pre-existing Windows env failures).
This commit is contained in:
Teknium
parent
70411a6152
commit
e0233f8fc5
5 changed files with 497 additions and 75 deletions
|
|
@ -27,6 +27,9 @@ interface UseComposerVoiceArgs {
|
|||
focusInput: () => void
|
||||
insertText: (text: string) => void
|
||||
maxRecordingSeconds: number
|
||||
/** Interrupt the in-flight agent turn (Stop-button seam) — fired when the
|
||||
* user speaks over the model while it is still generating. */
|
||||
onInterrupt?: () => Promise<void> | void
|
||||
onSubmit: ChatBarProps['onSubmit']
|
||||
onTranscribeAudio: ChatBarProps['onTranscribeAudio']
|
||||
sessionId: string | null | undefined
|
||||
|
|
@ -48,6 +51,7 @@ export function useComposerVoice({
|
|||
focusInput,
|
||||
insertText,
|
||||
maxRecordingSeconds,
|
||||
onInterrupt,
|
||||
onSubmit,
|
||||
onTranscribeAudio,
|
||||
sessionId,
|
||||
|
|
@ -129,6 +133,10 @@ export function useComposerVoice({
|
|||
consumePendingResponse,
|
||||
enabled: voiceConversationActive,
|
||||
onFatalError: () => setVoiceConversationActive(false),
|
||||
// Speaking over the model mid-generation interrupts the in-flight turn —
|
||||
// the same seam as the Stop button — so the interjection becomes the next
|
||||
// turn instead of waiting behind a reply the user already rejected.
|
||||
onInterrupt,
|
||||
// A spoken stop command ("stop", "never mind", "goodbye", …) ends the
|
||||
// hands-free conversation. Flipping the flag is the authoritative off
|
||||
// switch — the enabled=false prop + effect below drive conversation.end()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,266 @@
|
|||
import { act, cleanup, renderHook, waitFor } from '@testing-library/react'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import type { BargeMonitorCallbacks } from '@/lib/voice-barge-in'
|
||||
|
||||
import type { MicRecording } from './use-mic-recorder'
|
||||
import { useVoiceConversation } from './use-voice-conversation'
|
||||
|
||||
// The full-duplex contract: the barge monitor is live across the WHOLE agent
|
||||
// turn — generation (thinking) and playback (speaking) — so speaking over the
|
||||
// model interrupts it mid-generation instead of the mic being deaf until TTS
|
||||
// starts (the Windows report: interruption "never works" because the deaf
|
||||
// window covered generation, and playback bleed made the old monitor's
|
||||
// trigger unreachable).
|
||||
|
||||
const monitorCalls: BargeMonitorCallbacks[] = []
|
||||
const stopMonitor = vi.fn()
|
||||
|
||||
vi.mock('@/lib/voice-barge-in', () => ({
|
||||
monitorSpeechDuringPlayback: (callbacks: BargeMonitorCallbacks) => {
|
||||
monitorCalls.push(callbacks)
|
||||
|
||||
return stopMonitor
|
||||
}
|
||||
}))
|
||||
|
||||
const markVoicePlaybackInterrupted = vi.fn()
|
||||
const stopVoicePlayback = vi.fn()
|
||||
|
||||
vi.mock('@/lib/voice-playback', () => ({
|
||||
markVoicePlaybackInterrupted: () => markVoicePlaybackInterrupted(),
|
||||
playSpeechText: vi.fn(async () => true),
|
||||
startSpeechStream: vi.fn(async () => null),
|
||||
stopVoicePlayback: () => stopVoicePlayback()
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/thinking-sound', () => ({
|
||||
startThinkingSound: vi.fn(),
|
||||
stopThinkingSound: vi.fn()
|
||||
}))
|
||||
|
||||
const micHandle = {
|
||||
cancel: vi.fn(),
|
||||
start: vi.fn(async () => undefined),
|
||||
stop: vi.fn<() => Promise<MicRecording | null>>(async () => null)
|
||||
}
|
||||
|
||||
vi.mock('./use-mic-recorder', () => ({
|
||||
useMicRecorder: () => ({ handle: micHandle, level: 0, recording: false })
|
||||
}))
|
||||
|
||||
vi.mock('@/i18n', () => ({
|
||||
useI18n: () => ({
|
||||
t: {
|
||||
notifications: {
|
||||
voice: {
|
||||
configureSpeechToText: 'configure STT',
|
||||
couldNotStartSession: 'could not start',
|
||||
microphoneFailed: 'mic failed',
|
||||
playbackFailed: 'playback failed',
|
||||
transcriptionFailed: 'transcription failed',
|
||||
unavailable: 'unavailable'
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}))
|
||||
|
||||
vi.mock('@/store/notifications', () => ({
|
||||
notify: vi.fn(),
|
||||
notifyError: vi.fn()
|
||||
}))
|
||||
|
||||
interface HookProps {
|
||||
busy: boolean
|
||||
}
|
||||
|
||||
function renderConversation(overrides: { onInterrupt?: () => void; transcript?: string } = {}) {
|
||||
const onInterrupt = overrides.onInterrupt ?? vi.fn()
|
||||
|
||||
// Mirrors the real app: submitting a turn makes the agent busy.
|
||||
const onBusyChange: { current: (busy: boolean) => void } = { current: () => undefined }
|
||||
|
||||
const onSubmit = vi.fn(async () => {
|
||||
onBusyChange.current(true)
|
||||
})
|
||||
|
||||
const onStopWord = vi.fn()
|
||||
|
||||
// First transcription is the turn that starts the conversation; subsequent
|
||||
// ones are barge captures (the overridable transcript).
|
||||
let transcriptions = 0
|
||||
|
||||
const onTranscribeAudio = vi.fn(async () =>
|
||||
transcriptions++ === 0 ? 'kick off the task' : (overrides.transcript ?? 'and another thing')
|
||||
)
|
||||
|
||||
const hook = renderHook(
|
||||
({ busy }: HookProps) =>
|
||||
useVoiceConversation({
|
||||
busy,
|
||||
consumePendingResponse: vi.fn(),
|
||||
enabled: true,
|
||||
onInterrupt,
|
||||
onStopWord,
|
||||
onSubmit,
|
||||
onTranscribeAudio,
|
||||
pendingResponse: () => null
|
||||
}),
|
||||
{ initialProps: { busy: false } }
|
||||
)
|
||||
|
||||
onBusyChange.current = busy => hook.rerender({ busy })
|
||||
|
||||
return { hook, onInterrupt, onStopWord, onSubmit, onTranscribeAudio }
|
||||
}
|
||||
|
||||
/** Drive the hook into the generation phase (turn submitted, model working). */
|
||||
async function enterThinking(hook: ReturnType<typeof renderConversation>['hook']) {
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await waitFor(() => expect(hook.result.current.status).toBe('listening'))
|
||||
|
||||
micHandle.stop.mockResolvedValueOnce({
|
||||
audio: new Blob(['q'], { type: 'audio/webm' }),
|
||||
durationMs: 900,
|
||||
heardSpeech: true
|
||||
})
|
||||
|
||||
await act(async () => {
|
||||
hook.result.current.stopTurn()
|
||||
})
|
||||
await waitFor(() => expect(hook.result.current.status).toBe('thinking'))
|
||||
}
|
||||
|
||||
describe('useVoiceConversation full-duplex barge-in', () => {
|
||||
beforeEach(() => {
|
||||
monitorCalls.length = 0
|
||||
vi.clearAllMocks()
|
||||
micHandle.start.mockResolvedValue(undefined)
|
||||
micHandle.stop.mockResolvedValue(null)
|
||||
})
|
||||
|
||||
afterEach(cleanup)
|
||||
|
||||
it('arms the barge monitor during generation (before any reply audio exists)', async () => {
|
||||
const { hook } = renderConversation()
|
||||
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await enterThinking(hook)
|
||||
|
||||
await waitFor(() => expect(hook.result.current.status).toBe('thinking'))
|
||||
// busy=true + thinking → the full-duplex monitor must be live.
|
||||
await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0))
|
||||
})
|
||||
|
||||
it('interrupts the in-flight turn when speech trips mid-generation', async () => {
|
||||
const { hook, onInterrupt } = renderConversation()
|
||||
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await enterThinking(hook)
|
||||
await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0))
|
||||
|
||||
act(() => {
|
||||
monitorCalls.at(-1)?.onSpeech()
|
||||
})
|
||||
|
||||
expect(onInterrupt).toHaveBeenCalledTimes(1)
|
||||
expect(markVoicePlaybackInterrupted).toHaveBeenCalled()
|
||||
expect(stopVoicePlayback).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('submits the captured interruption once the interrupt settles (busy clears)', async () => {
|
||||
const { hook, onSubmit } = renderConversation({ transcript: 'no, do it differently' })
|
||||
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await enterThinking(hook)
|
||||
await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0))
|
||||
|
||||
const monitor = monitorCalls.at(-1)
|
||||
|
||||
act(() => {
|
||||
monitor?.onSpeech()
|
||||
})
|
||||
|
||||
// Interrupt lands → the turn ends → busy flips false.
|
||||
hook.rerender({ busy: false })
|
||||
|
||||
await act(async () => {
|
||||
monitor?.onUtterance?.(new Blob(['x'], { type: 'audio/webm' }))
|
||||
})
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('no, do it differently'))
|
||||
})
|
||||
|
||||
it('does not interrupt when speech trips during playback (turn already done)', async () => {
|
||||
const { hook, onInterrupt } = renderConversation()
|
||||
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await enterThinking(hook)
|
||||
await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0))
|
||||
|
||||
// Turn finished; playback phase.
|
||||
hook.rerender({ busy: false })
|
||||
|
||||
act(() => {
|
||||
monitorCalls.at(-1)?.onSpeech()
|
||||
})
|
||||
|
||||
expect(onInterrupt).not.toHaveBeenCalled()
|
||||
expect(stopVoicePlayback).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('a spoken stop command in the barge capture ends the conversation instead of submitting', async () => {
|
||||
const { hook, onStopWord, onSubmit } = renderConversation({ transcript: 'stop' })
|
||||
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await enterThinking(hook)
|
||||
await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0))
|
||||
|
||||
const monitor = monitorCalls.at(-1)
|
||||
|
||||
act(() => {
|
||||
monitor?.onSpeech()
|
||||
})
|
||||
hook.rerender({ busy: false })
|
||||
|
||||
await act(async () => {
|
||||
monitor?.onUtterance?.(new Blob(['s'], { type: 'audio/webm' }))
|
||||
})
|
||||
|
||||
await waitFor(() => expect(onStopWord).toHaveBeenCalledTimes(1))
|
||||
// Only the kickoff turn was submitted — the "stop" capture never was.
|
||||
expect(onSubmit).toHaveBeenCalledTimes(1)
|
||||
expect(onSubmit).not.toHaveBeenCalledWith('stop')
|
||||
})
|
||||
|
||||
it('re-arms a single monitor per turn (idempotent ensure)', async () => {
|
||||
const { hook } = renderConversation()
|
||||
|
||||
await act(async () => {
|
||||
await hook.result.current.start()
|
||||
})
|
||||
await enterThinking(hook)
|
||||
await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0))
|
||||
|
||||
const armed = monitorCalls.length
|
||||
|
||||
// Effect re-runs (busy toggles, status changes) must not open more mics.
|
||||
hook.rerender({ busy: true })
|
||||
hook.rerender({ busy: true })
|
||||
|
||||
expect(monitorCalls.length).toBe(armed)
|
||||
})
|
||||
})
|
||||
|
|
@ -28,6 +28,9 @@ interface VoiceConversationOptions {
|
|||
busy: boolean
|
||||
enabled: boolean
|
||||
onFatalError?: () => void
|
||||
/** Interrupt the in-flight agent turn (the same seam as the Stop button).
|
||||
* Fired when the user speaks while the model is still generating. */
|
||||
onInterrupt?: () => Promise<void> | void
|
||||
onStopWord?: () => void
|
||||
onSubmit: (text: string) => Promise<void> | void
|
||||
onTranscribeAudio?: (audio: Blob) => Promise<string>
|
||||
|
|
@ -38,10 +41,15 @@ interface VoiceConversationOptions {
|
|||
beforeMicOpen?: () => Promise<void> | void
|
||||
}
|
||||
|
||||
/** How long a barge-triggered interrupt may take to settle before we submit
|
||||
* the captured utterance anyway. */
|
||||
const INTERRUPT_SETTLE_TIMEOUT_MS = 5_000
|
||||
|
||||
export function useVoiceConversation({
|
||||
busy,
|
||||
enabled,
|
||||
onFatalError,
|
||||
onInterrupt,
|
||||
onStopWord,
|
||||
onSubmit,
|
||||
onTranscribeAudio,
|
||||
|
|
@ -63,6 +71,7 @@ export function useVoiceConversation({
|
|||
const speechSessionRef = useRef<null | SpeechStreamSession>(null)
|
||||
const stopBargeMonitorRef = useRef<(() => void) | null>(null)
|
||||
const bargeCapturePendingRef = useRef(false)
|
||||
const bargedRef = useRef(false)
|
||||
const speechStartSequenceRef = useRef(0)
|
||||
const enabledRef = useRef(enabled)
|
||||
const mutedRef = useRef(muted)
|
||||
|
|
@ -70,6 +79,12 @@ export function useVoiceConversation({
|
|||
const statusRef = useRef<ConversationStatus>('idle')
|
||||
const wasEnabledRef = useRef(enabled)
|
||||
const onStopWordRef = useRef(onStopWord)
|
||||
const onInterruptRef = useRef(onInterrupt)
|
||||
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
onInterruptRef.current = onInterrupt
|
||||
}, [onInterrupt])
|
||||
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
|
|
@ -114,6 +129,7 @@ export function useVoiceConversation({
|
|||
stopBargeMonitorRef.current?.()
|
||||
stopBargeMonitorRef.current = null
|
||||
bargeCapturePendingRef.current = false
|
||||
bargedRef.current = false
|
||||
speechSessionRef.current = null
|
||||
responseIdRef.current = null
|
||||
spokenSourceLengthRef.current = 0
|
||||
|
|
@ -315,6 +331,25 @@ export function useVoiceConversation({
|
|||
return
|
||||
}
|
||||
|
||||
// A spoken stop command while barging means "stop everything" — the
|
||||
// turn/playback was already cut at trip time; now end the conversation
|
||||
// instead of submitting "stop" as a new prompt.
|
||||
if (isVoiceStopCommand(transcript)) {
|
||||
dropSpeechSession()
|
||||
setStatus('idle')
|
||||
onStopWordRef.current?.()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// A generation-phase barge interrupted the in-flight turn; the submit
|
||||
// path refuses while `busy`, so wait for the interrupt to settle.
|
||||
const deadline = Date.now() + INTERRUPT_SETTLE_TIMEOUT_MS
|
||||
|
||||
while (busyRef.current && Date.now() < deadline) {
|
||||
await new Promise(resolve => window.setTimeout(resolve, 100))
|
||||
}
|
||||
|
||||
awaitingSpokenResponseRef.current = true
|
||||
dropSpeechSession()
|
||||
consumePendingResponse()
|
||||
|
|
@ -328,24 +363,46 @@ export function useVoiceConversation({
|
|||
[consumePendingResponse, onSubmit, onTranscribeAudio, voiceCopy.transcriptionFailed]
|
||||
)
|
||||
|
||||
/** Barge-in monitor wiring shared by the live and fallback speech paths. */
|
||||
const openBargeMonitor = useCallback(
|
||||
(onBarge: () => void) =>
|
||||
monitorSpeechDuringPlayback({
|
||||
onSpeech: () => {
|
||||
bargeCapturePendingRef.current = true
|
||||
onBarge()
|
||||
markVoicePlaybackInterrupted()
|
||||
stopVoicePlayback()
|
||||
},
|
||||
onUtterance: audio => {
|
||||
bargeCapturePendingRef.current = false
|
||||
stopBargeMonitorRef.current = null
|
||||
void submitCapturedUtterance(audio)
|
||||
/**
|
||||
* Full-duplex barge-in monitor for the WHOLE agent turn: armed at submit,
|
||||
* live through generation (thinking) AND playback (speaking).
|
||||
*
|
||||
* - generation phase (`busy`): speech interrupts the in-flight turn via
|
||||
* `onInterrupt` — the same seam as the Stop button — and cuts any TTS that
|
||||
* managed to start, so the stale reply never speaks.
|
||||
* - playback phase: speech cuts playback and the captured interruption is
|
||||
* transcribed and submitted as the next turn.
|
||||
*
|
||||
* Idempotent — one monitor owns the mic per turn; re-arming while one is
|
||||
* live is a no-op (the live/fallback speech paths and the turn-drive effect
|
||||
* all call this).
|
||||
*/
|
||||
const ensureBargeMonitor = useCallback(() => {
|
||||
if (stopBargeMonitorRef.current) {
|
||||
return
|
||||
}
|
||||
|
||||
stopBargeMonitorRef.current = monitorSpeechDuringPlayback({
|
||||
isPlaying: () => $voicePlayback.get().status === 'speaking',
|
||||
onSpeech: () => {
|
||||
bargeCapturePendingRef.current = true
|
||||
bargedRef.current = true
|
||||
markVoicePlaybackInterrupted()
|
||||
stopVoicePlayback()
|
||||
|
||||
if (busyRef.current) {
|
||||
// Mid-generation: stop the in-flight turn so the captured utterance
|
||||
// becomes the next one instead of queueing behind a stale reply.
|
||||
void onInterruptRef.current?.()
|
||||
}
|
||||
}),
|
||||
[submitCapturedUtterance]
|
||||
)
|
||||
},
|
||||
onUtterance: audio => {
|
||||
bargeCapturePendingRef.current = false
|
||||
stopBargeMonitorRef.current = null
|
||||
void submitCapturedUtterance(audio)
|
||||
}
|
||||
})
|
||||
}, [submitCapturedUtterance])
|
||||
|
||||
/** Push any new reply text into the live session; finish when complete. */
|
||||
const feedSpeechSession = useCallback(
|
||||
|
|
@ -397,12 +454,9 @@ export function useVoiceConversation({
|
|||
return
|
||||
}
|
||||
|
||||
let barged = false
|
||||
|
||||
stopBargeMonitorRef.current?.()
|
||||
stopBargeMonitorRef.current = openBargeMonitor(() => {
|
||||
barged = true
|
||||
})
|
||||
// The full-duplex monitor is normally already live (armed at submit);
|
||||
// this is a safety net for read-aloud-style entries into the loop.
|
||||
ensureBargeMonitor()
|
||||
|
||||
speechStartSequenceRef.current = $voicePlayback.get().sequence
|
||||
|
||||
|
|
@ -411,14 +465,14 @@ export function useVoiceConversation({
|
|||
.finally(() => {
|
||||
if (responseIdRef.current === responseId) {
|
||||
awaitingSpokenResponseRef.current = false
|
||||
settleAfterSpeech(barged)
|
||||
settleAfterSpeech(bargedRef.current)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
poll()
|
||||
},
|
||||
[openBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed]
|
||||
[ensureBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed]
|
||||
)
|
||||
|
||||
/**
|
||||
|
|
@ -433,15 +487,11 @@ export function useVoiceConversation({
|
|||
speechStartSequenceRef.current = $voicePlayback.get().sequence
|
||||
setStatus('speaking')
|
||||
|
||||
let barged = false
|
||||
|
||||
// VAD barge-in: the user talking over the reply cuts playback, drops
|
||||
// the not-yet-spoken remainder, AND keeps capturing — the interruption
|
||||
// is transcribed from its first syllable instead of losing the opening
|
||||
// words to a mic re-open.
|
||||
stopBargeMonitorRef.current = openBargeMonitor(() => {
|
||||
barged = true
|
||||
})
|
||||
// words to a mic re-open. Usually already live (armed at submit).
|
||||
ensureBargeMonitor()
|
||||
|
||||
void (async () => {
|
||||
const session = await startSpeechStream({ source: 'voice-conversation' })
|
||||
|
|
@ -484,10 +534,10 @@ export function useVoiceConversation({
|
|||
}
|
||||
|
||||
awaitingSpokenResponseRef.current = false
|
||||
settleAfterSpeech(barged)
|
||||
settleAfterSpeech(bargedRef.current)
|
||||
})()
|
||||
},
|
||||
[awaitFallbackSpeech, feedSpeechSession, openBargeMonitor, settleAfterSpeech]
|
||||
[awaitFallbackSpeech, ensureBargeMonitor, feedSpeechSession, settleAfterSpeech]
|
||||
)
|
||||
|
||||
const start = useCallback(async () => {
|
||||
|
|
@ -601,6 +651,13 @@ export function useVoiceConversation({
|
|||
}
|
||||
|
||||
if (awaitingSpokenResponseRef.current && status !== 'speaking') {
|
||||
// Generation phase: the turn is in flight but no reply audio exists
|
||||
// yet. Keep the mic live so speech can interrupt the model mid-
|
||||
// generation (full-duplex) instead of going deaf until playback.
|
||||
if (status === 'thinking' && (busy || bargeCapturePendingRef.current)) {
|
||||
ensureBargeMonitor()
|
||||
}
|
||||
|
||||
const response = pendingResponse()
|
||||
|
||||
if (response) {
|
||||
|
|
@ -609,8 +666,9 @@ export function useVoiceConversation({
|
|||
return
|
||||
}
|
||||
|
||||
if (!busy && status === 'thinking') {
|
||||
// Turn finished without any speakable reply (tool-only, error).
|
||||
if (!busy && status === 'thinking' && !bargeCapturePendingRef.current) {
|
||||
// Turn finished without any speakable reply (tool-only, error). A
|
||||
// live barge capture owns the loop instead — it submits or resumes.
|
||||
awaitingSpokenResponseRef.current = false
|
||||
dropSpeechSession()
|
||||
pendingStartRef.current = true
|
||||
|
|
@ -627,7 +685,7 @@ export function useVoiceConversation({
|
|||
if (pendingStartRef.current) {
|
||||
void startListening()
|
||||
}
|
||||
}, [busy, enabled, muted, openLiveSpeech, pendingResponse, startListening, status])
|
||||
}, [busy, enabled, muted, ensureBargeMonitor, openLiveSpeech, pendingResponse, startListening, status])
|
||||
|
||||
// eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -856,6 +856,8 @@ export function ChatBar({
|
|||
focusInput,
|
||||
insertText,
|
||||
maxRecordingSeconds,
|
||||
// Voice barge-in mid-generation halts the run like the Stop button.
|
||||
onInterrupt: haltRun,
|
||||
onSubmit,
|
||||
onTranscribeAudio,
|
||||
sessionId,
|
||||
|
|
|
|||
|
|
@ -1,24 +1,42 @@
|
|||
// VAD barge-in: watch the mic while TTS plays, fire the moment the user talks
|
||||
// over it, and CAPTURE what they say. Detection alone loses the first words —
|
||||
// by the time sustained speech trips the trigger and a fresh recorder spins
|
||||
// up, "stop, actually—" has become "actually—". So a MediaRecorder runs on
|
||||
// the monitor's stream the whole time (pre-roll), and once tripped it keeps
|
||||
// rolling until the user goes quiet, delivering the complete utterance.
|
||||
// Full-duplex VAD monitor: watch the mic across the agent turn — while the
|
||||
// model is generating (no audio yet) AND while TTS plays — fire the moment the
|
||||
// user talks over either phase, and CAPTURE what they say. Detection alone
|
||||
// loses the first words — by the time sustained speech trips the trigger and a
|
||||
// fresh recorder spins up, "stop, actually—" has become "actually—". So a
|
||||
// MediaRecorder runs on the monitor's stream the whole time (pre-roll), and
|
||||
// once tripped it keeps rolling until the user goes quiet, delivering the
|
||||
// complete utterance.
|
||||
//
|
||||
// Echo cancellation strips the app's own speaker output from the capture, the
|
||||
// noise floor is calibrated while playback is already audible, and the
|
||||
// sustained window filters coughs/thumps — mirrors
|
||||
// tools/voice_mode.listen_for_speech on the Python surfaces.
|
||||
// Phase-aware trigger (mirrors tools/voice_mode.full_duplex_listen on the
|
||||
// Python surfaces):
|
||||
// - The noise floor is calibrated from QUIET samples only — while no TTS audio
|
||||
// is flowing — and HELD through playback. Calibrating while the speaker is
|
||||
// audible bakes bleed into the floor and makes the trigger unreachable
|
||||
// (echoCancellation does not reliably cancel same-app playback on Windows).
|
||||
// - During playback the trigger is additionally clamped up to a minimum so
|
||||
// bleed alone can't trip it, and capped so speech always remains reachable.
|
||||
// - A short grace window after playback onset suppresses the start transient.
|
||||
// - Detection is a windowed majority (>=80% of the last SUSTAINED_MS above
|
||||
// trigger) so intra-word energy dips don't reset progress.
|
||||
|
||||
const CALIBRATION_MS = 400
|
||||
const SUSTAINED_MS = 300
|
||||
const SUSTAINED_MAJORITY = 0.8
|
||||
const MIN_TRIGGER_LEVEL = 0.075 // matches the voice loop's silenceLevel
|
||||
const FLOOR_MULTIPLIER = 3.5
|
||||
// Playback clamps, scaled from the Python constants (int16 RMS 1500 / 4000
|
||||
// ≈ byte-domain level 0.14 / 0.37 with the /42 normalization below).
|
||||
const PLAYBACK_MIN_TRIGGER_LEVEL = 0.14
|
||||
const TRIGGER_CEILING_LEVEL = 0.37
|
||||
const PLAYBACK_GRACE_MS = 500
|
||||
const PLAYBACK_GAP_FOR_GRACE_MS = 1_000
|
||||
const FLOOR_SAMPLE_CAP = 200 // ~3s of quiet-phase levels at rAF cadence
|
||||
const PRE_ROLL_RESTART_MS = 5_000 // cap pre-roll: restart the recorder while quiet
|
||||
const UTTERANCE_SILENCE_MS = 1_250 // matches the voice loop's silenceMs
|
||||
const UTTERANCE_MAX_MS = 30_000
|
||||
|
||||
export interface BargeMonitorCallbacks {
|
||||
/** Sustained speech detected — cut playback now. */
|
||||
/** Sustained speech detected — cut playback / interrupt the turn now. */
|
||||
onSpeech: () => void
|
||||
/**
|
||||
* The interrupting utterance, complete from its first syllable (pre-roll
|
||||
|
|
@ -26,6 +44,12 @@ export interface BargeMonitorCallbacks {
|
|||
* unavailable — fall back to normal listening.
|
||||
*/
|
||||
onUtterance?: (audio: Blob | null) => void
|
||||
/**
|
||||
* Is TTS audio flowing RIGHT NOW? Drives the phase-aware trigger. Omitted
|
||||
* (legacy playback-only callers) means "always playing", which preserves
|
||||
* the old behavior of a monitor opened at playback start.
|
||||
*/
|
||||
isPlaying?: () => boolean
|
||||
}
|
||||
|
||||
export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): () => void {
|
||||
|
|
@ -151,14 +175,30 @@ export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): (
|
|||
context.createMediaStreamSource(stream).connect(analyser)
|
||||
|
||||
const data = new Uint8Array(analyser.fftSize)
|
||||
const startedAt = Date.now()
|
||||
const floorSamples: number[] = []
|
||||
const recentAbove: { above: boolean; at: number }[] = []
|
||||
let calibratedSince: number | null = null
|
||||
let floorLocked = false
|
||||
let quietFloor = 0
|
||||
let segmentStartedAt = Date.now()
|
||||
let speechStartedAt: number | null = null
|
||||
let wasPlaying = false
|
||||
let playbackSeen = false
|
||||
let lastPlayingAt = 0
|
||||
let graceUntil = 0
|
||||
let tripped = false
|
||||
let trippedAt = 0
|
||||
let quietSince: number | null = null
|
||||
|
||||
const pushFloorSample = (level: number) => {
|
||||
floorSamples.push(level)
|
||||
|
||||
if (floorSamples.length > FLOOR_SAMPLE_CAP) {
|
||||
floorSamples.shift()
|
||||
}
|
||||
|
||||
quietFloor = [...floorSamples].sort((a, b) => a - b)[floorSamples.length >> 1] ?? 0
|
||||
}
|
||||
|
||||
const tick = () => {
|
||||
if (disposed) {
|
||||
return
|
||||
|
|
@ -175,35 +215,83 @@ export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): (
|
|||
|
||||
const level = Math.min(1, Math.sqrt(sum / data.length) / 42)
|
||||
const now = Date.now()
|
||||
const playing = callbacks.isPlaying ? callbacks.isPlaying() : true
|
||||
|
||||
if (!tripped && now - startedAt < CALIBRATION_MS) {
|
||||
floorSamples.push(level)
|
||||
} else if (!tripped) {
|
||||
const floor = floorSamples.length ? [...floorSamples].sort((a, b) => a - b)[floorSamples.length >> 1] : 0
|
||||
const trigger = Math.max(MIN_TRIGGER_LEVEL, floor * 3.5)
|
||||
|
||||
if (level >= trigger) {
|
||||
speechStartedAt ??= now
|
||||
|
||||
if (now - speechStartedAt >= SUSTAINED_MS) {
|
||||
tripped = true
|
||||
trippedAt = now
|
||||
quietSince = null
|
||||
callbacks.onSpeech()
|
||||
|
||||
if (!callbacks.onUtterance || !recorder) {
|
||||
cleanup()
|
||||
callbacks.onUtterance?.(null)
|
||||
|
||||
return
|
||||
}
|
||||
if (!tripped) {
|
||||
// Quiet-floor calibration: quiet-phase samples only. The floor is
|
||||
// HELD while audio plays — never recalibrated against speaker bleed.
|
||||
if (!floorLocked) {
|
||||
if (!playing) {
|
||||
calibratedSince ??= now
|
||||
pushFloorSample(level)
|
||||
}
|
||||
} else {
|
||||
speechStartedAt = null
|
||||
|
||||
if (playing || (calibratedSince !== null && now - calibratedSince >= CALIBRATION_MS)) {
|
||||
floorLocked = true
|
||||
}
|
||||
}
|
||||
|
||||
// Grace only when playback starts after a real gap, so flapping of
|
||||
// the playing flag between sentences can't chain grace windows.
|
||||
if (playing && !wasPlaying) {
|
||||
if (!playbackSeen || now - lastPlayingAt >= PLAYBACK_GAP_FOR_GRACE_MS) {
|
||||
graceUntil = now + PLAYBACK_GRACE_MS
|
||||
}
|
||||
|
||||
playbackSeen = true
|
||||
}
|
||||
|
||||
wasPlaying = playing
|
||||
|
||||
if (playing) {
|
||||
lastPlayingAt = now
|
||||
}
|
||||
|
||||
// Phase-aware trigger: quiet baseline x multiplier; playback clamps
|
||||
// it up (bleed alone can't trip) but a ceiling keeps speech
|
||||
// reachable even over loud playback.
|
||||
let trigger = Math.max(MIN_TRIGGER_LEVEL, quietFloor * FLOOR_MULTIPLIER)
|
||||
|
||||
if (playing) {
|
||||
trigger = Math.min(Math.max(trigger, PLAYBACK_MIN_TRIGGER_LEVEL), TRIGGER_CEILING_LEVEL)
|
||||
}
|
||||
|
||||
// Track ambient drift while quiet and below trigger.
|
||||
if (floorLocked && !playing && level < trigger) {
|
||||
pushFloorSample(level)
|
||||
}
|
||||
|
||||
const above = floorLocked && level >= trigger && now >= graceUntil
|
||||
|
||||
recentAbove.push({ above, at: now })
|
||||
|
||||
while (recentAbove.length && now - recentAbove[0].at > SUSTAINED_MS) {
|
||||
recentAbove.shift()
|
||||
}
|
||||
|
||||
const aboveCount = recentAbove.reduce((count, sample) => count + (sample.above ? 1 : 0), 0)
|
||||
const spanMs = recentAbove.length ? now - recentAbove[0].at : 0
|
||||
|
||||
if (
|
||||
above &&
|
||||
spanMs >= SUSTAINED_MS * SUSTAINED_MAJORITY &&
|
||||
aboveCount >= recentAbove.length * SUSTAINED_MAJORITY
|
||||
) {
|
||||
tripped = true
|
||||
trippedAt = now
|
||||
quietSince = null
|
||||
callbacks.onSpeech()
|
||||
|
||||
if (!callbacks.onUtterance || !recorder) {
|
||||
cleanup()
|
||||
callbacks.onUtterance?.(null)
|
||||
|
||||
return
|
||||
}
|
||||
} else if (!above) {
|
||||
// Bound the pre-roll while quiet so the utterance blob doesn't
|
||||
// accumulate the whole playback (rotating mid-speech would lose
|
||||
// the onset — the whole point).
|
||||
// accumulate the whole turn (rotating mid-speech would lose the
|
||||
// onset — the whole point).
|
||||
if (now - segmentStartedAt >= PRE_ROLL_RESTART_MS) {
|
||||
rotateSegment()
|
||||
segmentStartedAt = now
|
||||
|
|
@ -211,7 +299,7 @@ export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): (
|
|||
}
|
||||
} else {
|
||||
// Tripped: keep recording until the user goes quiet (endpoint).
|
||||
// Playback is already stopped, so plain silence-vs-speech works.
|
||||
// Playback/generation was already cut, so silence-vs-speech works.
|
||||
if (level >= MIN_TRIGGER_LEVEL) {
|
||||
quietSince = null
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue