From a56275771759af1b05620d07a3dc439d2f8044de Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:17:04 -0700 Subject: [PATCH] fix(wake): reconcile the listener back to config after a voice turn ends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ending a voice conversation left the wake word silently off even with wake_word.enabled: true — the desktop fired one wake.resume and hoped; if the mic was still held by the just-released WebRTC capture (or the resume raced teardown), the listener stayed dead until the user re-toggled it. The wake word is a persistent setting: on is on until the user explicitly turns it off. - Desktop: resumeWakeAfterVoice() replaces the fire-and-forget resume — resume, then verify against wake.status (config 'enabled' is the authority) and re-arm via wake.start, with spaced retries to ride out mic-release latency. Passive path: never passes persist, never writes config; respects an explicit off and another surface's mic lease. - Backend: wake.status now reports 'enabled' (config truth) so clients reconcile against the setting, not runtime listener state. - Backend: _wake_resume_if_owner self-heals — a resume that throws (mic still busy) retries in a background thread for up to 15s. A False return (lease gone/moved) is final, never retried, so the retry can't steal another surface's mic. Covers the TUI/gateway voice.record path which had no recovery at all (CLI has its idle watchdog; the gateway had nothing). - 6 new vitest cases: re-arm on enabled+down, no persist on the passive path, resume-alone success, disabled stays off, owned lease yields, older-backend no-op. --- .../chat/composer/hooks/use-composer-voice.ts | 9 +- apps/desktop/src/store/wake-word.test.ts | 119 ++++++++++++++++++ apps/desktop/src/store/wake-word.ts | 56 +++++++++ tui_gateway/server.py | 60 ++++++++- 4 files changed, 235 insertions(+), 9 deletions(-) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index 0fb5c288751..414676e3c9b 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -9,6 +9,7 @@ import { resetBrowseState } from '@/store/composer-input-history' import { $gateway } from '@/store/gateway' import { notifyError } from '@/store/notifications' import { $autoSpeakReplies, setAutoSpeakReplies } from '@/store/voice-prefs' +import { resumeWakeAfterVoice } from '@/store/wake-word' import type { ComposerTarget } from '../focus' import { onComposerVoiceToggleRequest } from '../focus' @@ -160,10 +161,10 @@ export function useComposerVoice({ } wakePausedRef.current = false - void $gateway - .get() - ?.request('wake.resume', {}) - .catch(() => undefined) + // Reconcile, don't just resume: the wake word is a persistent setting, so + // ending a voice chat must re-arm the listener whenever config says + // enabled — including when the raw resume loses the mic-release race. + void resumeWakeAfterVoice() }, []) // The ref is a request token (did WE issue wake.pause?), not an atom mirror — diff --git a/apps/desktop/src/store/wake-word.test.ts b/apps/desktop/src/store/wake-word.test.ts index 0236f150e7a..f969b4cf2d4 100644 --- a/apps/desktop/src/store/wake-word.test.ts +++ b/apps/desktop/src/store/wake-word.test.ts @@ -7,6 +7,7 @@ import { applyWakeStopResult, armWakeWord, resetWakeWordState, + resumeWakeAfterVoice, toggleWakeWord, type WakeRequester } from './wake-word' @@ -238,3 +239,121 @@ describe('applyWakeStartResult', () => { expect($wakeWord.get()).toMatchObject({ available: true, listening: true, phrase: 'computer' }) }) }) + +describe('resumeWakeAfterVoice (post-voice reconcile)', () => { + it('re-arms when config says enabled but the listener is down', async () => { + const calls: string[] = [] + + const request = requester(method => { + calls.push(method) + + if (method === 'wake.resume') { + return { reason: 'not_owner', resumed: false } + } + + if (method === 'wake.status') { + return { available: true, enabled: true, listening: false, phrase: 'hey hermes' } + } + + return { phrase: 'hey hermes', started: true } + }) + + await resumeWakeAfterVoice(request) + + expect(calls).toEqual(['wake.resume', 'wake.status', 'wake.start']) + expect($wakeWord.get()).toMatchObject({ listening: true }) + }) + + it('re-arm start never passes persist (passive path must not write config)', async () => { + const startParams: Array | undefined> = [] + + const request = vi.fn(async (method: string, params?: Record) => { + if (method === 'wake.resume') { + return { resumed: false } + } + + if (method === 'wake.status') { + return { available: true, enabled: true, listening: false } + } + + startParams.push(params) + + return { started: true } + }) as unknown as WakeRequester + + await resumeWakeAfterVoice(request) + + expect(startParams).toEqual([{ surface: 'gui' }]) + }) + + it('stops after the resume alone brings the listener back', async () => { + const calls: string[] = [] + + const request = requester(method => { + calls.push(method) + + if (method === 'wake.resume') { + return { resumed: true } + } + + return { available: true, enabled: true, listening: true, owned_by_caller: true } + }) + + await resumeWakeAfterVoice(request) + + expect(calls).toEqual(['wake.resume', 'wake.status']) + expect($wakeWord.get()).toMatchObject({ listening: true }) + }) + + it('leaves the listener off when config says disabled', async () => { + const calls: string[] = [] + + const request = requester(method => { + calls.push(method) + + if (method === 'wake.resume') { + return { resumed: false } + } + + return { available: true, enabled: false, listening: false } + }) + + await resumeWakeAfterVoice(request) + + expect(calls).toEqual(['wake.resume', 'wake.status']) + expect($wakeWord.get().listening).toBe(false) + }) + + it('yields when another surface owns the mic lease', async () => { + const calls: string[] = [] + + const request = requester(method => { + calls.push(method) + + if (method === 'wake.resume') { + return { resumed: false } + } + + if (method === 'wake.status') { + return { available: true, enabled: true, listening: false, owner_surface: 'tui' } + } + + return { owner_surface: 'tui', reason: 'owned', started: false } + }) + + await resumeWakeAfterVoice(request) + + expect(calls).toEqual(['wake.resume', 'wake.status', 'wake.start']) + expect($wakeWord.get().listening).toBe(false) + }) + + it('is a no-op against older backends without wake.* methods', async () => { + const request = requester(() => { + throw new Error('Unknown method: wake.resume') + }) + + await resumeWakeAfterVoice(request) + + expect($wakeWord.get()).toMatchObject({ available: false, listening: false }) + }) +}) diff --git a/apps/desktop/src/store/wake-word.ts b/apps/desktop/src/store/wake-word.ts index 69175d19a6d..9b9e61a42ae 100644 --- a/apps/desktop/src/store/wake-word.ts +++ b/apps/desktop/src/store/wake-word.ts @@ -32,6 +32,8 @@ export const $wakeWord = atom(INITIAL_WAKE_WORD_STATE) export interface WakeStatusResponse { available?: boolean + /** Config truth (wake_word.enabled) — drives post-voice re-arm. */ + enabled?: boolean hint?: string listening?: boolean owned_by_caller?: boolean @@ -202,6 +204,60 @@ export async function toggleWakeWord(request: WakeRequester = gatewayRequester): } } +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * Post-voice-turn reconcile: the wake word is a persistent setting, so ending a + * voice conversation must land the listener back where config says it belongs. + * `wake.resume` alone isn't enough — the mic can still be held by the just-torn + * -down WebRTC capture, and a fire-and-forget resume that loses that race left + * the ear silently off until the user re-toggled. Resume, then verify against + * `wake.status` (config `enabled` is the authority) and re-arm, with a couple + * of spaced retries to ride out mic-release latency. Never passes `persist` — + * this is a passive path and must not flip config. + */ +export async function resumeWakeAfterVoice(request: WakeRequester = gatewayRequester): Promise { + try { + await request('wake.resume', {}) + } catch { + // Older backend without wake.* — nothing to reconcile. + return + } + + for (let attempt = 0; attempt < 3; attempt++) { + try { + const status = await request('wake.status', {}) + applyWakeStatus(status) + + // Config says off (or the feature can't run) — off is the correct rest + // state. A user /wake off during the voice turn stays respected. + if (!status?.enabled || !status.available) { + return + } + + if (status.listening) { + return + } + + const started = await request('wake.start', { surface: 'gui' }) + applyWakeStartResult(started) + + if (started?.started) { + return + } + + // Another surface holds the mic lease — theirs to keep. + if (started?.reason === 'owned') { + return + } + } catch { + // Transient (mic still releasing) — fall through to the next attempt. + } + + await sleep(1500) + } +} + /** Test-only reset. */ export function resetWakeWordState(): void { $wakeWord.set(INITIAL_WAKE_WORD_STATE) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 8c978967da6..4eb2229af11 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -17638,14 +17638,61 @@ def _release_gateway_wake_owner() -> bool: return owner is not None and _release_wake_for_transport(owner) -def _wake_resume_if_owner(owner: "Transport") -> bool: - try: - from tools.wake_word import resume_listening +_wake_resume_retry_lock = threading.Lock() +_wake_resume_retry_active = False + +def _wake_resume_if_owner(owner: "Transport", *, retry_seconds: float = 15.0, + retry_interval: float = 1.0) -> bool: + """Resume the wake detector for ``owner``; self-heal a busy microphone. + + Reopening the mic right after a voice turn can fail while the capture + device is still being released (browser WebRTC tracks release async). + The CLI covers this with its idle watchdog; the gateway had nothing, so + one failed resume left the listener silently dead until the user toggled + it by hand — despite ``wake_word.enabled: true``. On an exception (mic + open failure) we retry in a background thread until it sticks, the lease + changes hands, or ``retry_seconds`` elapses. ``False`` from + ``resume_listening`` (lease gone / different owner) is final — never + retried, so this can't steal another surface's mic. + """ + from tools.wake_word import resume_listening + + try: return resume_listening(owner=owner) except Exception as e: - logger.debug("wake resume failed: %s", e) - return False + logger.debug("wake resume failed (will retry): %s", e) + + global _wake_resume_retry_active + with _wake_resume_retry_lock: + if _wake_resume_retry_active: + return False + _wake_resume_retry_active = True + + def _retry() -> None: + global _wake_resume_retry_active + deadline = time.monotonic() + retry_seconds + try: + while time.monotonic() < deadline: + time.sleep(retry_interval) + try: + if resume_listening(owner=owner): + logger.info("wake: detector resumed after retry") + return + except Exception: + continue + # False — detector gone or lease moved: stop, don't fight it. + return + logger.warning( + "wake: could not resume detector after voice turn " + "(microphone still busy?) — toggle the wake word to re-arm" + ) + finally: + with _wake_resume_retry_lock: + _wake_resume_retry_active = False + + threading.Thread(target=_retry, daemon=True, name="wake-resume-retry").start() + return False def _persist_wake_enabled(enabled: bool) -> bool: @@ -17866,6 +17913,9 @@ def _(rid, params: dict) -> dict: "provider": reqs["provider"], "available": reqs["available"], "hint": reqs.get("hint", ""), + # Config truth: clients use this to re-arm after a voice turn + # ("permanent on") without guessing from runtime listener state. + "enabled": bool(cfg.get("enabled")), }) except Exception as e: return _err(rid, 5026, str(e))