From e0233f8fc592328590832dfdc795ac2ec6f08736 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:20:11 -0700 Subject: [PATCH] =?UTF-8?q?fix(desktop):=20full-duplex=20voice=20barge-in?= =?UTF-8?q?=20=E2=80=94=20interrupt=20during=20generation=20AND=20playback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The voice-interruption fix (5081551f0) covered the Python surfaces (CLI + TUI gateway) but the desktop app has its own mic path in voice-barge-in.ts, which still had both bugs on Windows: - HALF-DUPLEX GAP: the barge monitor only opened when TTS playback started; during LLM generation no mic was listening at all. - PLAYBACK DEAFNESS: the monitor calibrated its noise floor while the speakers were already playing TTS, baking bleed into the floor. On Windows, Chromium echoCancellation does not reliably cancel same-app output (measured live: quiet floor ~35-50 RMS vs playback bleed 600-1700 RMS), making the trigger unreachable. Changes mirror tools/voice_mode.full_duplex_listen: - voice-barge-in.ts: phase-aware full-duplex monitor — quiet-only calibration (floor held through playback, never recalibrated against bleed), playback min-trigger clamp + ceiling, 500ms grace on playback onset only, windowed-majority detection. - use-voice-conversation.ts: monitor arms at turn submit and spans generation + playback (ensureBargeMonitor, idempotent). Mid-generation speech fires the new onInterrupt callback; spoken stop-word during a barge ends the conversation; submit waits for the interrupt to settle. - use-composer-voice.ts + composer/index.tsx: plumb onInterrupt: haltRun (same seam as the Stop button). Tests: use-voice-conversation.test.tsx (6 tests) covering monitor lifecycle across generation + playback, mid-generation interrupt, stop-word handling. npm run typecheck + eslint clean; remaining desktop vitest failures reproduce on a clean tree (pre-existing Windows env failures). --- .../chat/composer/hooks/use-composer-voice.ts | 8 + .../hooks/use-voice-conversation.test.tsx | 266 ++++++++++++++++++ .../composer/hooks/use-voice-conversation.ts | 130 ++++++--- apps/desktop/src/app/chat/composer/index.tsx | 2 + apps/desktop/src/lib/voice-barge-in.ts | 166 ++++++++--- 5 files changed, 497 insertions(+), 75 deletions(-) create mode 100644 apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index 3321f909148..49e0ba5ce7f 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -27,6 +27,9 @@ interface UseComposerVoiceArgs { focusInput: () => void insertText: (text: string) => void maxRecordingSeconds: number + /** Interrupt the in-flight agent turn (Stop-button seam) — fired when the + * user speaks over the model while it is still generating. */ + onInterrupt?: () => Promise | void onSubmit: ChatBarProps['onSubmit'] onTranscribeAudio: ChatBarProps['onTranscribeAudio'] sessionId: string | null | undefined @@ -48,6 +51,7 @@ export function useComposerVoice({ focusInput, insertText, maxRecordingSeconds, + onInterrupt, onSubmit, onTranscribeAudio, sessionId, @@ -129,6 +133,10 @@ export function useComposerVoice({ consumePendingResponse, enabled: voiceConversationActive, onFatalError: () => setVoiceConversationActive(false), + // Speaking over the model mid-generation interrupts the in-flight turn — + // the same seam as the Stop button — so the interjection becomes the next + // turn instead of waiting behind a reply the user already rejected. + onInterrupt, // A spoken stop command ("stop", "never mind", "goodbye", …) ends the // hands-free conversation. Flipping the flag is the authoritative off // switch — the enabled=false prop + effect below drive conversation.end() diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx new file mode 100644 index 00000000000..1e43ef8b186 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx @@ -0,0 +1,266 @@ +import { act, cleanup, renderHook, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { BargeMonitorCallbacks } from '@/lib/voice-barge-in' + +import type { MicRecording } from './use-mic-recorder' +import { useVoiceConversation } from './use-voice-conversation' + +// The full-duplex contract: the barge monitor is live across the WHOLE agent +// turn — generation (thinking) and playback (speaking) — so speaking over the +// model interrupts it mid-generation instead of the mic being deaf until TTS +// starts (the Windows report: interruption "never works" because the deaf +// window covered generation, and playback bleed made the old monitor's +// trigger unreachable). + +const monitorCalls: BargeMonitorCallbacks[] = [] +const stopMonitor = vi.fn() + +vi.mock('@/lib/voice-barge-in', () => ({ + monitorSpeechDuringPlayback: (callbacks: BargeMonitorCallbacks) => { + monitorCalls.push(callbacks) + + return stopMonitor + } +})) + +const markVoicePlaybackInterrupted = vi.fn() +const stopVoicePlayback = vi.fn() + +vi.mock('@/lib/voice-playback', () => ({ + markVoicePlaybackInterrupted: () => markVoicePlaybackInterrupted(), + playSpeechText: vi.fn(async () => true), + startSpeechStream: vi.fn(async () => null), + stopVoicePlayback: () => stopVoicePlayback() +})) + +vi.mock('@/lib/thinking-sound', () => ({ + startThinkingSound: vi.fn(), + stopThinkingSound: vi.fn() +})) + +const micHandle = { + cancel: vi.fn(), + start: vi.fn(async () => undefined), + stop: vi.fn<() => Promise>(async () => null) +} + +vi.mock('./use-mic-recorder', () => ({ + useMicRecorder: () => ({ handle: micHandle, level: 0, recording: false }) +})) + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + notifications: { + voice: { + configureSpeechToText: 'configure STT', + couldNotStartSession: 'could not start', + microphoneFailed: 'mic failed', + playbackFailed: 'playback failed', + transcriptionFailed: 'transcription failed', + unavailable: 'unavailable' + } + } + } + }) +})) + +vi.mock('@/store/notifications', () => ({ + notify: vi.fn(), + notifyError: vi.fn() +})) + +interface HookProps { + busy: boolean +} + +function renderConversation(overrides: { onInterrupt?: () => void; transcript?: string } = {}) { + const onInterrupt = overrides.onInterrupt ?? vi.fn() + + // Mirrors the real app: submitting a turn makes the agent busy. + const onBusyChange: { current: (busy: boolean) => void } = { current: () => undefined } + + const onSubmit = vi.fn(async () => { + onBusyChange.current(true) + }) + + const onStopWord = vi.fn() + + // First transcription is the turn that starts the conversation; subsequent + // ones are barge captures (the overridable transcript). + let transcriptions = 0 + + const onTranscribeAudio = vi.fn(async () => + transcriptions++ === 0 ? 'kick off the task' : (overrides.transcript ?? 'and another thing') + ) + + const hook = renderHook( + ({ busy }: HookProps) => + useVoiceConversation({ + busy, + consumePendingResponse: vi.fn(), + enabled: true, + onInterrupt, + onStopWord, + onSubmit, + onTranscribeAudio, + pendingResponse: () => null + }), + { initialProps: { busy: false } } + ) + + onBusyChange.current = busy => hook.rerender({ busy }) + + return { hook, onInterrupt, onStopWord, onSubmit, onTranscribeAudio } +} + +/** Drive the hook into the generation phase (turn submitted, model working). */ +async function enterThinking(hook: ReturnType['hook']) { + await act(async () => { + await hook.result.current.start() + }) + await waitFor(() => expect(hook.result.current.status).toBe('listening')) + + micHandle.stop.mockResolvedValueOnce({ + audio: new Blob(['q'], { type: 'audio/webm' }), + durationMs: 900, + heardSpeech: true + }) + + await act(async () => { + hook.result.current.stopTurn() + }) + await waitFor(() => expect(hook.result.current.status).toBe('thinking')) +} + +describe('useVoiceConversation full-duplex barge-in', () => { + beforeEach(() => { + monitorCalls.length = 0 + vi.clearAllMocks() + micHandle.start.mockResolvedValue(undefined) + micHandle.stop.mockResolvedValue(null) + }) + + afterEach(cleanup) + + it('arms the barge monitor during generation (before any reply audio exists)', async () => { + const { hook } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + + await waitFor(() => expect(hook.result.current.status).toBe('thinking')) + // busy=true + thinking → the full-duplex monitor must be live. + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + }) + + it('interrupts the in-flight turn when speech trips mid-generation', async () => { + const { hook, onInterrupt } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + act(() => { + monitorCalls.at(-1)?.onSpeech() + }) + + expect(onInterrupt).toHaveBeenCalledTimes(1) + expect(markVoicePlaybackInterrupted).toHaveBeenCalled() + expect(stopVoicePlayback).toHaveBeenCalled() + }) + + it('submits the captured interruption once the interrupt settles (busy clears)', async () => { + const { hook, onSubmit } = renderConversation({ transcript: 'no, do it differently' }) + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + const monitor = monitorCalls.at(-1) + + act(() => { + monitor?.onSpeech() + }) + + // Interrupt lands → the turn ends → busy flips false. + hook.rerender({ busy: false }) + + await act(async () => { + monitor?.onUtterance?.(new Blob(['x'], { type: 'audio/webm' })) + }) + + await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('no, do it differently')) + }) + + it('does not interrupt when speech trips during playback (turn already done)', async () => { + const { hook, onInterrupt } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + // Turn finished; playback phase. + hook.rerender({ busy: false }) + + act(() => { + monitorCalls.at(-1)?.onSpeech() + }) + + expect(onInterrupt).not.toHaveBeenCalled() + expect(stopVoicePlayback).toHaveBeenCalled() + }) + + it('a spoken stop command in the barge capture ends the conversation instead of submitting', async () => { + const { hook, onStopWord, onSubmit } = renderConversation({ transcript: 'stop' }) + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + const monitor = monitorCalls.at(-1) + + act(() => { + monitor?.onSpeech() + }) + hook.rerender({ busy: false }) + + await act(async () => { + monitor?.onUtterance?.(new Blob(['s'], { type: 'audio/webm' })) + }) + + await waitFor(() => expect(onStopWord).toHaveBeenCalledTimes(1)) + // Only the kickoff turn was submitted — the "stop" capture never was. + expect(onSubmit).toHaveBeenCalledTimes(1) + expect(onSubmit).not.toHaveBeenCalledWith('stop') + }) + + it('re-arms a single monitor per turn (idempotent ensure)', async () => { + const { hook } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + const armed = monitorCalls.length + + // Effect re-runs (busy toggles, status changes) must not open more mics. + hook.rerender({ busy: true }) + hook.rerender({ busy: true }) + + expect(monitorCalls.length).toBe(armed) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index 9ac3c743298..94fb912ff1c 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -28,6 +28,9 @@ interface VoiceConversationOptions { busy: boolean enabled: boolean onFatalError?: () => void + /** Interrupt the in-flight agent turn (the same seam as the Stop button). + * Fired when the user speaks while the model is still generating. */ + onInterrupt?: () => Promise | void onStopWord?: () => void onSubmit: (text: string) => Promise | void onTranscribeAudio?: (audio: Blob) => Promise @@ -38,10 +41,15 @@ interface VoiceConversationOptions { beforeMicOpen?: () => Promise | void } +/** How long a barge-triggered interrupt may take to settle before we submit + * the captured utterance anyway. */ +const INTERRUPT_SETTLE_TIMEOUT_MS = 5_000 + export function useVoiceConversation({ busy, enabled, onFatalError, + onInterrupt, onStopWord, onSubmit, onTranscribeAudio, @@ -63,6 +71,7 @@ export function useVoiceConversation({ const speechSessionRef = useRef(null) const stopBargeMonitorRef = useRef<(() => void) | null>(null) const bargeCapturePendingRef = useRef(false) + const bargedRef = useRef(false) const speechStartSequenceRef = useRef(0) const enabledRef = useRef(enabled) const mutedRef = useRef(muted) @@ -70,6 +79,12 @@ export function useVoiceConversation({ const statusRef = useRef('idle') const wasEnabledRef = useRef(enabled) const onStopWordRef = useRef(onStopWord) + const onInterruptRef = useRef(onInterrupt) + + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) + useEffect(() => { + onInterruptRef.current = onInterrupt + }, [onInterrupt]) // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { @@ -114,6 +129,7 @@ export function useVoiceConversation({ stopBargeMonitorRef.current?.() stopBargeMonitorRef.current = null bargeCapturePendingRef.current = false + bargedRef.current = false speechSessionRef.current = null responseIdRef.current = null spokenSourceLengthRef.current = 0 @@ -315,6 +331,25 @@ export function useVoiceConversation({ return } + // A spoken stop command while barging means "stop everything" — the + // turn/playback was already cut at trip time; now end the conversation + // instead of submitting "stop" as a new prompt. + if (isVoiceStopCommand(transcript)) { + dropSpeechSession() + setStatus('idle') + onStopWordRef.current?.() + + return + } + + // A generation-phase barge interrupted the in-flight turn; the submit + // path refuses while `busy`, so wait for the interrupt to settle. + const deadline = Date.now() + INTERRUPT_SETTLE_TIMEOUT_MS + + while (busyRef.current && Date.now() < deadline) { + await new Promise(resolve => window.setTimeout(resolve, 100)) + } + awaitingSpokenResponseRef.current = true dropSpeechSession() consumePendingResponse() @@ -328,24 +363,46 @@ export function useVoiceConversation({ [consumePendingResponse, onSubmit, onTranscribeAudio, voiceCopy.transcriptionFailed] ) - /** Barge-in monitor wiring shared by the live and fallback speech paths. */ - const openBargeMonitor = useCallback( - (onBarge: () => void) => - monitorSpeechDuringPlayback({ - onSpeech: () => { - bargeCapturePendingRef.current = true - onBarge() - markVoicePlaybackInterrupted() - stopVoicePlayback() - }, - onUtterance: audio => { - bargeCapturePendingRef.current = false - stopBargeMonitorRef.current = null - void submitCapturedUtterance(audio) + /** + * Full-duplex barge-in monitor for the WHOLE agent turn: armed at submit, + * live through generation (thinking) AND playback (speaking). + * + * - generation phase (`busy`): speech interrupts the in-flight turn via + * `onInterrupt` — the same seam as the Stop button — and cuts any TTS that + * managed to start, so the stale reply never speaks. + * - playback phase: speech cuts playback and the captured interruption is + * transcribed and submitted as the next turn. + * + * Idempotent — one monitor owns the mic per turn; re-arming while one is + * live is a no-op (the live/fallback speech paths and the turn-drive effect + * all call this). + */ + const ensureBargeMonitor = useCallback(() => { + if (stopBargeMonitorRef.current) { + return + } + + stopBargeMonitorRef.current = monitorSpeechDuringPlayback({ + isPlaying: () => $voicePlayback.get().status === 'speaking', + onSpeech: () => { + bargeCapturePendingRef.current = true + bargedRef.current = true + markVoicePlaybackInterrupted() + stopVoicePlayback() + + if (busyRef.current) { + // Mid-generation: stop the in-flight turn so the captured utterance + // becomes the next one instead of queueing behind a stale reply. + void onInterruptRef.current?.() } - }), - [submitCapturedUtterance] - ) + }, + onUtterance: audio => { + bargeCapturePendingRef.current = false + stopBargeMonitorRef.current = null + void submitCapturedUtterance(audio) + } + }) + }, [submitCapturedUtterance]) /** Push any new reply text into the live session; finish when complete. */ const feedSpeechSession = useCallback( @@ -397,12 +454,9 @@ export function useVoiceConversation({ return } - let barged = false - - stopBargeMonitorRef.current?.() - stopBargeMonitorRef.current = openBargeMonitor(() => { - barged = true - }) + // The full-duplex monitor is normally already live (armed at submit); + // this is a safety net for read-aloud-style entries into the loop. + ensureBargeMonitor() speechStartSequenceRef.current = $voicePlayback.get().sequence @@ -411,14 +465,14 @@ export function useVoiceConversation({ .finally(() => { if (responseIdRef.current === responseId) { awaitingSpokenResponseRef.current = false - settleAfterSpeech(barged) + settleAfterSpeech(bargedRef.current) } }) } poll() }, - [openBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed] + [ensureBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed] ) /** @@ -433,15 +487,11 @@ export function useVoiceConversation({ speechStartSequenceRef.current = $voicePlayback.get().sequence setStatus('speaking') - let barged = false - // VAD barge-in: the user talking over the reply cuts playback, drops // the not-yet-spoken remainder, AND keeps capturing — the interruption // is transcribed from its first syllable instead of losing the opening - // words to a mic re-open. - stopBargeMonitorRef.current = openBargeMonitor(() => { - barged = true - }) + // words to a mic re-open. Usually already live (armed at submit). + ensureBargeMonitor() void (async () => { const session = await startSpeechStream({ source: 'voice-conversation' }) @@ -484,10 +534,10 @@ export function useVoiceConversation({ } awaitingSpokenResponseRef.current = false - settleAfterSpeech(barged) + settleAfterSpeech(bargedRef.current) })() }, - [awaitFallbackSpeech, feedSpeechSession, openBargeMonitor, settleAfterSpeech] + [awaitFallbackSpeech, ensureBargeMonitor, feedSpeechSession, settleAfterSpeech] ) const start = useCallback(async () => { @@ -601,6 +651,13 @@ export function useVoiceConversation({ } if (awaitingSpokenResponseRef.current && status !== 'speaking') { + // Generation phase: the turn is in flight but no reply audio exists + // yet. Keep the mic live so speech can interrupt the model mid- + // generation (full-duplex) instead of going deaf until playback. + if (status === 'thinking' && (busy || bargeCapturePendingRef.current)) { + ensureBargeMonitor() + } + const response = pendingResponse() if (response) { @@ -609,8 +666,9 @@ export function useVoiceConversation({ return } - if (!busy && status === 'thinking') { - // Turn finished without any speakable reply (tool-only, error). + if (!busy && status === 'thinking' && !bargeCapturePendingRef.current) { + // Turn finished without any speakable reply (tool-only, error). A + // live barge capture owns the loop instead — it submits or resumes. awaitingSpokenResponseRef.current = false dropSpeechSession() pendingStartRef.current = true @@ -627,7 +685,7 @@ export function useVoiceConversation({ if (pendingStartRef.current) { void startListening() } - }, [busy, enabled, muted, openLiveSpeech, pendingResponse, startListening, status]) + }, [busy, enabled, muted, ensureBargeMonitor, openLiveSpeech, pendingResponse, startListening, status]) // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { diff --git a/apps/desktop/src/app/chat/composer/index.tsx b/apps/desktop/src/app/chat/composer/index.tsx index 567159662aa..cfb5e751889 100644 --- a/apps/desktop/src/app/chat/composer/index.tsx +++ b/apps/desktop/src/app/chat/composer/index.tsx @@ -856,6 +856,8 @@ export function ChatBar({ focusInput, insertText, maxRecordingSeconds, + // Voice barge-in mid-generation halts the run like the Stop button. + onInterrupt: haltRun, onSubmit, onTranscribeAudio, sessionId, diff --git a/apps/desktop/src/lib/voice-barge-in.ts b/apps/desktop/src/lib/voice-barge-in.ts index c9170e7d986..4d521f33d1b 100644 --- a/apps/desktop/src/lib/voice-barge-in.ts +++ b/apps/desktop/src/lib/voice-barge-in.ts @@ -1,24 +1,42 @@ -// VAD barge-in: watch the mic while TTS plays, fire the moment the user talks -// over it, and CAPTURE what they say. Detection alone loses the first words — -// by the time sustained speech trips the trigger and a fresh recorder spins -// up, "stop, actually—" has become "actually—". So a MediaRecorder runs on -// the monitor's stream the whole time (pre-roll), and once tripped it keeps -// rolling until the user goes quiet, delivering the complete utterance. +// Full-duplex VAD monitor: watch the mic across the agent turn — while the +// model is generating (no audio yet) AND while TTS plays — fire the moment the +// user talks over either phase, and CAPTURE what they say. Detection alone +// loses the first words — by the time sustained speech trips the trigger and a +// fresh recorder spins up, "stop, actually—" has become "actually—". So a +// MediaRecorder runs on the monitor's stream the whole time (pre-roll), and +// once tripped it keeps rolling until the user goes quiet, delivering the +// complete utterance. // -// Echo cancellation strips the app's own speaker output from the capture, the -// noise floor is calibrated while playback is already audible, and the -// sustained window filters coughs/thumps — mirrors -// tools/voice_mode.listen_for_speech on the Python surfaces. +// Phase-aware trigger (mirrors tools/voice_mode.full_duplex_listen on the +// Python surfaces): +// - The noise floor is calibrated from QUIET samples only — while no TTS audio +// is flowing — and HELD through playback. Calibrating while the speaker is +// audible bakes bleed into the floor and makes the trigger unreachable +// (echoCancellation does not reliably cancel same-app playback on Windows). +// - During playback the trigger is additionally clamped up to a minimum so +// bleed alone can't trip it, and capped so speech always remains reachable. +// - A short grace window after playback onset suppresses the start transient. +// - Detection is a windowed majority (>=80% of the last SUSTAINED_MS above +// trigger) so intra-word energy dips don't reset progress. const CALIBRATION_MS = 400 const SUSTAINED_MS = 300 +const SUSTAINED_MAJORITY = 0.8 const MIN_TRIGGER_LEVEL = 0.075 // matches the voice loop's silenceLevel +const FLOOR_MULTIPLIER = 3.5 +// Playback clamps, scaled from the Python constants (int16 RMS 1500 / 4000 +// ≈ byte-domain level 0.14 / 0.37 with the /42 normalization below). +const PLAYBACK_MIN_TRIGGER_LEVEL = 0.14 +const TRIGGER_CEILING_LEVEL = 0.37 +const PLAYBACK_GRACE_MS = 500 +const PLAYBACK_GAP_FOR_GRACE_MS = 1_000 +const FLOOR_SAMPLE_CAP = 200 // ~3s of quiet-phase levels at rAF cadence const PRE_ROLL_RESTART_MS = 5_000 // cap pre-roll: restart the recorder while quiet const UTTERANCE_SILENCE_MS = 1_250 // matches the voice loop's silenceMs const UTTERANCE_MAX_MS = 30_000 export interface BargeMonitorCallbacks { - /** Sustained speech detected — cut playback now. */ + /** Sustained speech detected — cut playback / interrupt the turn now. */ onSpeech: () => void /** * The interrupting utterance, complete from its first syllable (pre-roll @@ -26,6 +44,12 @@ export interface BargeMonitorCallbacks { * unavailable — fall back to normal listening. */ onUtterance?: (audio: Blob | null) => void + /** + * Is TTS audio flowing RIGHT NOW? Drives the phase-aware trigger. Omitted + * (legacy playback-only callers) means "always playing", which preserves + * the old behavior of a monitor opened at playback start. + */ + isPlaying?: () => boolean } export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): () => void { @@ -151,14 +175,30 @@ export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): ( context.createMediaStreamSource(stream).connect(analyser) const data = new Uint8Array(analyser.fftSize) - const startedAt = Date.now() const floorSamples: number[] = [] + const recentAbove: { above: boolean; at: number }[] = [] + let calibratedSince: number | null = null + let floorLocked = false + let quietFloor = 0 let segmentStartedAt = Date.now() - let speechStartedAt: number | null = null + let wasPlaying = false + let playbackSeen = false + let lastPlayingAt = 0 + let graceUntil = 0 let tripped = false let trippedAt = 0 let quietSince: number | null = null + const pushFloorSample = (level: number) => { + floorSamples.push(level) + + if (floorSamples.length > FLOOR_SAMPLE_CAP) { + floorSamples.shift() + } + + quietFloor = [...floorSamples].sort((a, b) => a - b)[floorSamples.length >> 1] ?? 0 + } + const tick = () => { if (disposed) { return @@ -175,35 +215,83 @@ export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): ( const level = Math.min(1, Math.sqrt(sum / data.length) / 42) const now = Date.now() + const playing = callbacks.isPlaying ? callbacks.isPlaying() : true - if (!tripped && now - startedAt < CALIBRATION_MS) { - floorSamples.push(level) - } else if (!tripped) { - const floor = floorSamples.length ? [...floorSamples].sort((a, b) => a - b)[floorSamples.length >> 1] : 0 - const trigger = Math.max(MIN_TRIGGER_LEVEL, floor * 3.5) - - if (level >= trigger) { - speechStartedAt ??= now - - if (now - speechStartedAt >= SUSTAINED_MS) { - tripped = true - trippedAt = now - quietSince = null - callbacks.onSpeech() - - if (!callbacks.onUtterance || !recorder) { - cleanup() - callbacks.onUtterance?.(null) - - return - } + if (!tripped) { + // Quiet-floor calibration: quiet-phase samples only. The floor is + // HELD while audio plays — never recalibrated against speaker bleed. + if (!floorLocked) { + if (!playing) { + calibratedSince ??= now + pushFloorSample(level) } - } else { - speechStartedAt = null + if (playing || (calibratedSince !== null && now - calibratedSince >= CALIBRATION_MS)) { + floorLocked = true + } + } + + // Grace only when playback starts after a real gap, so flapping of + // the playing flag between sentences can't chain grace windows. + if (playing && !wasPlaying) { + if (!playbackSeen || now - lastPlayingAt >= PLAYBACK_GAP_FOR_GRACE_MS) { + graceUntil = now + PLAYBACK_GRACE_MS + } + + playbackSeen = true + } + + wasPlaying = playing + + if (playing) { + lastPlayingAt = now + } + + // Phase-aware trigger: quiet baseline x multiplier; playback clamps + // it up (bleed alone can't trip) but a ceiling keeps speech + // reachable even over loud playback. + let trigger = Math.max(MIN_TRIGGER_LEVEL, quietFloor * FLOOR_MULTIPLIER) + + if (playing) { + trigger = Math.min(Math.max(trigger, PLAYBACK_MIN_TRIGGER_LEVEL), TRIGGER_CEILING_LEVEL) + } + + // Track ambient drift while quiet and below trigger. + if (floorLocked && !playing && level < trigger) { + pushFloorSample(level) + } + + const above = floorLocked && level >= trigger && now >= graceUntil + + recentAbove.push({ above, at: now }) + + while (recentAbove.length && now - recentAbove[0].at > SUSTAINED_MS) { + recentAbove.shift() + } + + const aboveCount = recentAbove.reduce((count, sample) => count + (sample.above ? 1 : 0), 0) + const spanMs = recentAbove.length ? now - recentAbove[0].at : 0 + + if ( + above && + spanMs >= SUSTAINED_MS * SUSTAINED_MAJORITY && + aboveCount >= recentAbove.length * SUSTAINED_MAJORITY + ) { + tripped = true + trippedAt = now + quietSince = null + callbacks.onSpeech() + + if (!callbacks.onUtterance || !recorder) { + cleanup() + callbacks.onUtterance?.(null) + + return + } + } else if (!above) { // Bound the pre-roll while quiet so the utterance blob doesn't - // accumulate the whole playback (rotating mid-speech would lose - // the onset — the whole point). + // accumulate the whole turn (rotating mid-speech would lose the + // onset — the whole point). if (now - segmentStartedAt >= PRE_ROLL_RESTART_MS) { rotateSegment() segmentStartedAt = now @@ -211,7 +299,7 @@ export function monitorSpeechDuringPlayback(callbacks: BargeMonitorCallbacks): ( } } else { // Tripped: keep recording until the user goes quiet (endpoint). - // Playback is already stopped, so plain silence-vs-speech works. + // Playback/generation was already cut, so silence-vs-speech works. if (level >= MIN_TRIGGER_LEVEL) { quietSince = null } else {