From 177a57f70cfcc65624337ea61a91a6cc6e9d63be Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Wed, 22 Jul 2026 17:53:06 -0500 Subject: [PATCH] feat(voice): desktop flags interrupted submits markVoicePlaybackInterrupted() / takeVoicePlaybackInterrupted() mirror the backend latch in the renderer (the barge happens client-side, where the audio plays). VAD barges and typing over playback mark it; the next prompt.submit carries interrupted:true, which the TUI gateway latches into the model note. --- .../composer/hooks/use-voice-conversation.ts | 2 + .../hooks/use-prompt-actions/index.test.tsx | 38 +++++++++++++++++++ .../hooks/use-prompt-actions/submit.ts | 22 +++++++++-- apps/desktop/src/lib/voice-playback.ts | 21 ++++++++++ .../docs/user-guide/features/voice-mode.md | 2 + 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index dc8e7dcdf138..2ec43c83c0b8 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -3,6 +3,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' import { monitorSpeechDuringPlayback } from '@/lib/voice-barge-in' import { + markVoicePlaybackInterrupted, playSpeechText, type SpeechStreamSession, startSpeechStream, @@ -267,6 +268,7 @@ export function useVoiceConversation({ onSpeech: () => { bargeCapturePendingRef.current = true onBarge() + markVoicePlaybackInterrupted() stopVoicePlayback() }, onUtterance: audio => { diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx index f83aa1235725..030ed8751fbf 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx @@ -536,6 +536,44 @@ describe('usePromptActions submit / queue drain semantics', () => { ) }) + it('flags prompt.submit with interrupted:true after a voice-playback barge', async () => { + const { markVoicePlaybackInterrupted } = await import('@/lib/voice-playback') + const requestGateway = vi.fn(async () => ({}) as never) + + let handle: HarnessHandle | null = null + await actRender( + (handle = h)} + refreshSessions={async () => undefined} + requestGateway={requestGateway} + /> + ) + + markVoicePlaybackInterrupted() + await handle!.submitText('stop! rude interruption') + + // The latch is one-shot: the flag rides this submit, the next is clean. + expect(requestGateway).toHaveBeenCalledWith( + 'prompt.submit', + { + session_id: RUNTIME_SESSION_ID, + text: 'stop! rude interruption', + interrupted: true + }, + 1_800_000 + ) + + await handle!.submitText('follow-up without a barge') + expect(requestGateway).toHaveBeenLastCalledWith( + 'prompt.submit', + { + session_id: RUNTIME_SESSION_ID, + text: 'follow-up without a barge' + }, + 1_800_000 + ) + }) + it('a fromQueue drain sends even when busyRef is still true on the settle edge', async () => { // busyRef lags $busy by one effect tick on the busy→false settle edge, so a // drained queue send would otherwise hit the busy guard and silently no-op. diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts index 9493227b9da9..d50647a68942 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/submit.ts @@ -6,7 +6,12 @@ import { type ChatMessage, textPart } from '@/lib/chat-messages' import { optimisticAttachmentRef } from '@/lib/chat-runtime' import { sanitizeComposerInput } from '@/lib/composer-input-sanitize' import { setMutableRef } from '@/lib/mutable-ref' -import { isVoicePlaybackActive, stopVoicePlayback } from '@/lib/voice-playback' +import { + isVoicePlaybackActive, + markVoicePlaybackInterrupted, + stopVoicePlayback, + takeVoicePlaybackInterrupted +} from '@/lib/voice-playback' import { $composerAttachments, clearComposerAttachments, @@ -144,9 +149,14 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) { // Typing barge-in: a new send silences any in-flight spoken reply. if (isVoicePlaybackActive()) { + markVoicePlaybackInterrupted() stopVoicePlayback() } + // Barged mid-speech (here or via the voice loop's VAD)? Flag the submit + // so the backend notes the interruption to the model. + const interrupted = takeVoicePlaybackInterrupted() + // Queue drains carry their source session explicitly. A background drain // must never inherit the currently selected session after the user moves // to another chat. @@ -456,6 +466,12 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) { rewriteOptimistic(sessionId) const text = buildContextText(syncedAttachments) + const submitParams = (targetId: string) => ({ + session_id: targetId, + text, + ...(interrupted && { interrupted }) + }) + // On sleep/wake the gateway's in-memory session may have been cleared // while the desktop app still holds the old session ID. Detect this, // resume the stored session to re-register it, and retry once. @@ -463,7 +479,7 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) { try { await withSessionBusyRetry(() => - requestGateway('prompt.submit', { session_id: sessionId, text }, PROMPT_SUBMIT_REQUEST_TIMEOUT_MS) + requestGateway('prompt.submit', submitParams(sessionId), PROMPT_SUBMIT_REQUEST_TIMEOUT_MS) ) } catch (firstErr) { const recoverStoredSessionId = targetStoredSessionId ?? selectedStoredSessionIdRef.current @@ -491,7 +507,7 @@ export function useSubmitPrompt(deps: SubmitPromptDeps) { } await withSessionBusyRetry(() => - requestGateway('prompt.submit', { session_id: recoveredId, text }, PROMPT_SUBMIT_REQUEST_TIMEOUT_MS) + requestGateway('prompt.submit', submitParams(recoveredId), PROMPT_SUBMIT_REQUEST_TIMEOUT_MS) ) } else { submitErr = firstErr diff --git a/apps/desktop/src/lib/voice-playback.ts b/apps/desktop/src/lib/voice-playback.ts index 142423d5d5bd..80d9a3f22a2e 100644 --- a/apps/desktop/src/lib/voice-playback.ts +++ b/apps/desktop/src/lib/voice-playback.ts @@ -433,3 +433,24 @@ export async function playSpeechText(text: string, options: VoicePlaybackOptions export function isVoicePlaybackActive() { return $voicePlayback.get().status !== 'idle' } + +// --------------------------------------------------------------------------- +// Interruption latch — the next prompt.submit carries `interrupted: true` so +// the model knows its spoken reply was cut off (it can react: "rude!"). +// Marked by the barge-in paths (VAD, typing over playback); TTL'd so a stale +// barge never annotates an unrelated message minutes later. +// --------------------------------------------------------------------------- + +const INTERRUPT_TTL_MS = 120_000 +let interruptedAt: null | number = null + +export function markVoicePlaybackInterrupted() { + interruptedAt = Date.now() +} + +export function takeVoicePlaybackInterrupted(): boolean { + const at = interruptedAt + interruptedAt = null + + return at !== null && Date.now() - at < INTERRUPT_TTL_MS +} diff --git a/website/docs/user-guide/features/voice-mode.md b/website/docs/user-guide/features/voice-mode.md index d0945478dac6..7c3222b6438a 100644 --- a/website/docs/user-guide/features/voice-mode.md +++ b/website/docs/user-guide/features/voice-mode.md @@ -174,6 +174,8 @@ You can interrupt the agent mid-speech: - **Talk over it** — in continuous voice mode, a voice-activity monitor listens while the agent speaks and cuts playback the moment you start talking, then goes straight back to recording. The detector calibrates its noise floor against the playback itself, so speaker bleed doesn't self-trigger. Disable with `voice.barge_in: false` in `config.yaml`. - **Type or press the record key** — sending a new message or hitting the push-to-talk key stops playback instantly on every surface. +The agent **knows** it was interrupted: the next message carries a short note telling the model its spoken reply was cut off, so it can react naturally ("rude!") or pick up where it left off instead of being oblivious. + ### Hallucination Filter Whisper sometimes generates phantom text from silence or background noise ("Thank you for watching", "Subscribe", etc.). The agent filters these out using a set of 26 known hallucination phrases across multiple languages, plus a regex pattern that catches repetitive variations.