From 64686e6ec51f79ba325768a9e9d08f87b8ef199e Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Thu, 23 Jul 2026 01:48:22 -0500 Subject: [PATCH] feat(desktop,tui): arm the wake word over the gateway and start hands-free voice The TUI and desktop GUI share the Python tui_gateway, which runs the detector server-side and exposes wake.start/stop/pause/resume/status plus a wake.detected event routed back over the same transport that armed it. Clients arm it on connect; on wake the desktop opens a fresh session, starts voice, and hands the mic between the detector and its browser voice loop. An empty STT transcript (silence) is treated as a quiet re-listen rather than a "transcription failed" toast. --- .../chat/composer/hooks/use-composer-voice.ts | 54 +++++ apps/desktop/src/app/contrib/wiring.tsx | 30 ++- apps/desktop/src/store/composer.ts | 23 +++ hermes_cli/web_server.py | 13 +- tui_gateway/server.py | 192 +++++++++++++++++- ui-tui/src/app/createGatewayEventHandler.ts | 23 +++ ui-tui/src/gatewayTypes.ts | 1 + 7 files changed, 327 insertions(+), 9 deletions(-) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index f8b75183f30c..c2c84a442ea3 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -1,9 +1,12 @@ +import { useStore } from '@nanostores/react' import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages' import { triggerHaptic } from '@/lib/haptics' +import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' +import { $gateway } from '@/store/gateway' import { notifyError } from '@/store/notifications' import { $messages } from '@/store/session' import { $autoSpeakReplies, setAutoSpeakReplies } from '@/store/voice-prefs' @@ -140,6 +143,57 @@ export function useComposerVoice({ [target, toggleVoiceConversation] ) + // "Hey Hermes" wake word: a latched start request (nanostore) the composer + // claims once it's mounted and the gateway is ready. Survives the fresh-session + // remount the wake handler triggers, and waits out a transient `disabled`. + const voiceStartReq = useStore($voiceConversationStartRequest) + useEffect(() => { + if (disabled) { + return // not ready — re-runs when `disabled` flips false + } + + if (!takeVoiceConversationStart(voiceStartReq)) { + return + } + + if (!voiceConversationActive) { + setVoiceConversationActive(true) + } + }, [voiceStartReq, disabled, voiceConversationActive]) + + // Hand the mic between the server-side wake detector and the browser's voice + // loop: pause the detector while a conversation is live, resume it after + // (no-ops server-side when the wake word isn't armed). wakePausedRef tracks + // whether WE paused, so resume always runs once — including on unmount, where + // ending voice can tear the composer down before the `false` render lands and + // would otherwise leave the detector paused forever. + const wakePausedRef = useRef(false) + + const wakeRpc = useCallback( + (method: string) => void $gateway.get()?.request(method, {}).catch(() => undefined), + [] + ) + + const resumeWakeIfPaused = useCallback(() => { + if (!wakePausedRef.current) { + return + } + + wakePausedRef.current = false + wakeRpc('wake.resume') + }, [wakeRpc]) + + useEffect(() => { + if (voiceConversationActive) { + wakePausedRef.current = true + wakeRpc('wake.pause') + } else { + resumeWakeIfPaused() + } + }, [voiceConversationActive, wakeRpc, resumeWakeIfPaused]) + + useEffect(() => resumeWakeIfPaused, [resumeWakeIfPaused]) + // Explicit start/end for the on-screen conversation controls (the hotkey uses // the gated toggle above). const startConversation = useCallback(() => setVoiceConversationActive(true), []) diff --git a/apps/desktop/src/app/contrib/wiring.tsx b/apps/desktop/src/app/contrib/wiring.tsx index 978d328e228f..bd650ad1e04b 100644 --- a/apps/desktop/src/app/contrib/wiring.tsx +++ b/apps/desktop/src/app/contrib/wiring.tsx @@ -29,6 +29,7 @@ import { sessionMessagesSignature } from '@/lib/session-signatures' import { isMessagingSource } from '@/lib/session-source' import { latestSessionTodos } from '@/lib/todos' import { $billingSettingsRequest } from '@/store/billing-block' +import { requestVoiceConversationStart } from '@/store/composer' import { setCronFocusJobId } from '@/store/cron' import { $pinnedSessionIds, pinSession, restoreWorktree, unpinSession } from '@/store/layout' import { $filePreviewTarget, $previewTarget } from '@/store/preview' @@ -645,9 +646,25 @@ export function ContribWiring({ children }: { children: ReactNode }) { const handleGatewayEventWithPlugins = useCallback( (event: Parameters[0]) => { emitGatewayEvent(event) + + // "Hey Hermes": handle the wake event on the canonical onEvent pipeline — + // open a fresh session (unless the config keeps the current one) and begin + // back-and-forth voice via the composer's latched start request. + if (event.type === 'wake.detected') { + const payload = event.payload as { start_new_session?: boolean } | undefined + + if (payload?.start_new_session !== false) { + startFreshSessionDraft() + } + + requestVoiceConversationStart() + + return + } + handleDesktopGatewayEvent(event) }, - [handleDesktopGatewayEvent] + [handleDesktopGatewayEvent, startFreshSessionDraft] ) useGatewayBoot({ @@ -668,6 +685,17 @@ export function ContribWiring({ children }: { children: ReactNode }) { refreshSessions }) + // "Hey Hermes" wake word: arm the server-side detector for this surface + // (gated on config server-side). Detection arrives as a wake.detected event + // handled in handleGatewayEventWithPlugins. Idempotent, so reconnects are safe. + useEffect(() => { + if (gatewayState !== 'open') { + return + } + + void requestGateway('wake.start', { surface: 'gui' }).catch(() => undefined) + }, [gatewayState, requestGateway]) + // Only the open messaging transcript needs its own poll — local chats are // live over the websocket already. const activeIsMessaging = diff --git a/apps/desktop/src/store/composer.ts b/apps/desktop/src/store/composer.ts index b984fe8bfc3b..50e199d028fe 100644 --- a/apps/desktop/src/store/composer.ts +++ b/apps/desktop/src/store/composer.ts @@ -93,6 +93,29 @@ export function createComposerAttachmentScope($attachments = atom + $voiceConversationStartRequest.set(Date.now()) + +/** Returns true exactly once per new request id (claims it). */ +export const takeVoiceConversationStart = (current: number): boolean => { + if (current > _voiceStartHandled) { + _voiceStartHandled = current + + return true + } + + return false +} + // Per-thread draft stash for the decoupled composer. Session lifecycle never // touches this — only ChatBar's scope swap reads/writes it. Text mirrors to // localStorage; attachments are memory-only (blobs, upload state). diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index f73a77bb655d..16070191b4d1 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -4304,10 +4304,15 @@ async def transcribe_audio_upload(payload: AudioTranscriptionRequest): pass if not result.get("success"): - raise HTTPException( - status_code=400, - detail=result.get("error") or "Transcription failed", - ) + err = result.get("error") or "Transcription failed" + # An empty transcript means no speech was detected — a normal outcome + # for VAD/continuous voice loops (e.g. a wake-word conversation + # re-listening on silence), not an error. Return an empty transcript so + # the client quietly re-listens instead of surfacing a "transcription + # failed" toast on every silent gap. + if "empty transcript" in err.lower(): + return {"ok": True, "transcript": "", "provider": result.get("provider")} + raise HTTPException(status_code=400, detail=err) return { "ok": True, diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 6cc0c8c22d74..eb73919cd1d0 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -905,6 +905,11 @@ def _close_sessions_for_transport( def _shutdown_sessions() -> None: + try: + from tools.wake_word import stop_listening as _stop_wake + _stop_wake() + except Exception: + pass with _sessions_lock: sids = list(_sessions) for sid in sids: @@ -15814,6 +15819,168 @@ def _voice_record_key() -> str: return str(record_key) if isinstance(record_key, str) and record_key else "ctrl+b" +# ── Wake word ("Hey Hermes") ────────────────────────────────────────────── +# The detector is process-global (one mic), like voice. It runs server-side so +# both the TUI and desktop GUI share it; clients pass their surface identity to +# wake.start and the shared gate (wake_surface_enabled) decides whether to arm. +# On detection we emit wake.detected; the client opens a new session and starts +# its own voice capture. The detector yields the mic to gateway voice.record +# (pause/resume below) and to the desktop's browser mic (wake.pause/resume RPCs). +_wake_lock = threading.Lock() +_wake_active = False +_wake_event_sid = "" +# Transport captured at wake.start time. The detector callback fires on a +# background thread where the request-scoped transport ContextVar is unset, so +# write_json would fall back to stdio and the event would never cross the +# desktop's websocket (#wake-detected-not-delivered). We pin the arming +# request's transport here and bind it for the emit. +_wake_transport: "Optional[Transport]" = None + + +def _wake_is_active() -> bool: + with _wake_lock: + return _wake_active + + +def _wake_resume_if_active() -> None: + if not _wake_is_active(): + return + try: + from tools.wake_word import resume_listening + resume_listening() + except Exception as e: + logger.debug("wake resume failed: %s", e) + + +def _wake_on_detect() -> None: + """Detector-thread callback: tell the client to open a fresh voice session.""" + with _wake_lock: + sid = _wake_event_sid + transport = _wake_transport + phrase, new_session = "", True + try: + from tools.wake_word import load_wake_word_config, wake_phrase + cfg = load_wake_word_config() + phrase = wake_phrase(cfg) + new_session = bool(cfg.get("start_new_session", True)) + except Exception: + pass + logger.info("wake.detected: emitting to sid=%r (transport=%s)", + sid, type(transport).__name__ if transport else None) + # Bind the arming request's transport so write_json reaches the right peer + # (WS for desktop/dashboard) instead of falling back to stdio on this + # background thread. Carry start_new_session so every surface honors it. + token = bind_transport(transport) if transport is not None else None + try: + _emit("wake.detected", sid, {"phrase": phrase, "start_new_session": new_session}) + finally: + if token is not None: + reset_transport(token) + + +@method("wake.start") +def _(rid, params: dict) -> dict: + """Arm the wake-word listener for the calling surface ("tui" | "gui"). + + Idempotent and gated: returns ``{started: False, reason}`` when the wake + word is disabled, scoped to another surface, or its deps/mic aren't ready. + """ + global _wake_active, _wake_event_sid, _wake_transport + surface = str(params.get("surface") or "auto").strip().lower() + try: + from tools.wake_word import ( + check_wake_word_requirements, + load_wake_word_config, + start_listening, + wake_surface_enabled, + ) + except Exception as e: + return _err(rid, 5026, f"wake module unavailable: {e}") + + cfg = load_wake_word_config() + if not wake_surface_enabled(surface, cfg): + logger.info("wake.start(%s): disabled for surface (enabled=%s, surface=%s)", + surface, cfg.get("enabled"), cfg.get("surface")) + return _ok(rid, {"started": False, "reason": "disabled_for_surface"}) + reqs = check_wake_word_requirements(cfg) + if not reqs["available"]: + logger.warning("wake.start(%s): not available — %s", surface, reqs.get("hint")) + return _ok(rid, {"started": False, "reason": reqs.get("hint") or "unavailable"}) + + with _wake_lock: + _wake_event_sid = params.get("session_id") or _wake_event_sid + # Capture the live transport (WS for desktop) so the background detector + # thread can route wake.detected back to this client, not stdio. + _wake_transport = current_transport() or _wake_transport + try: + start_listening(_wake_on_detect, config=cfg) + except Exception as e: + logger.warning("wake.start(%s): failed to start listener: %s", surface, e) + return _err(rid, 5026, str(e)) + with _wake_lock: + _wake_active = True + logger.info("wake.start(%s): listening for %r (%s)", surface, reqs["phrase"], reqs["provider"]) + return _ok(rid, {"started": True, "phrase": reqs["phrase"], "provider": reqs["provider"]}) + + +@method("wake.stop") +def _(rid, params: dict) -> dict: + global _wake_active + with _wake_lock: + _wake_active = False + try: + from tools.wake_word import stop_listening + stop_listening() + except Exception: + pass + return _ok(rid, {"stopped": True}) + + +@method("wake.pause") +def _(rid, params: dict) -> dict: + """Release the mic (e.g. while the desktop's browser captures audio).""" + try: + from tools.wake_word import pause_listening + pause_listening() + logger.info("wake.pause: detector paused") + except Exception as e: + logger.debug("wake.pause failed: %s", e) + return _ok(rid, {"paused": True}) + + +@method("wake.resume") +def _(rid, params: dict) -> dict: + """Reclaim the mic after a pause; no-op if the listener isn't armed.""" + active = _wake_is_active() + if active: + _wake_resume_if_active() + logger.info("wake.resume: detector resumed") + else: + logger.info("wake.resume: ignored (listener not armed)") + return _ok(rid, {"resumed": active}) + + +@method("wake.status") +def _(rid, params: dict) -> dict: + try: + from tools.wake_word import ( + check_wake_word_requirements, + is_listening, + load_wake_word_config, + ) + cfg = load_wake_word_config() + reqs = check_wake_word_requirements(cfg) + return _ok(rid, { + "listening": _wake_is_active() and is_listening(), + "phrase": reqs["phrase"], + "provider": reqs["provider"], + "available": reqs["available"], + "hint": reqs.get("hint", ""), + }) + except Exception as e: + return _err(rid, 5026, str(e)) + + @method("voice.toggle") def _(rid, params: dict) -> dict: """CLI parity for the ``/voice`` slash command. @@ -15962,12 +16129,28 @@ def _(rid, params: dict) -> dict: if isinstance(duration, (int, float)) and not isinstance(duration, bool) else 3.0 ) + # Hand the mic to STT if the wake-word detector holds it; resume + # once a terminal capture event fires (one-shot transcript / silence + # limit), so wake-triggered and manual captures both coexist. + if _wake_is_active(): + try: + from tools.wake_word import pause_listening + pause_listening() + except Exception: + pass + + def _on_transcript(t): + _voice_emit("voice.transcript", {"text": t}) + _wake_resume_if_active() + + def _on_silent(): + _voice_emit("voice.transcript", {"no_speech_limit": True}) + _wake_resume_if_active() + started = start_continuous( - on_transcript=lambda t: _voice_emit("voice.transcript", {"text": t}), + on_transcript=_on_transcript, on_status=lambda s: _voice_emit("voice.status", {"state": s}), - on_silent_limit=lambda: _voice_emit( - "voice.transcript", {"no_speech_limit": True} - ), + on_silent_limit=_on_silent, silence_threshold=safe_threshold, silence_duration=safe_duration, auto_restart=False, @@ -15983,6 +16166,7 @@ def _(rid, params: dict) -> dict: from hermes_cli.voice import stop_continuous stop_continuous(force_transcribe=True) + _wake_resume_if_active() return _ok(rid, {"status": "stopped"}) except ImportError: return _err( diff --git a/ui-tui/src/app/createGatewayEventHandler.ts b/ui-tui/src/app/createGatewayEventHandler.ts index 8527e083a580..ebb2409a44b3 100644 --- a/ui-tui/src/app/createGatewayEventHandler.ts +++ b/ui-tui/src/app/createGatewayEventHandler.ts @@ -619,6 +619,10 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: // "too many re-renders" guard in embedded dashboard PTYs. ensureAgentsNudgeConfig() + // Arm "Hey Hermes" if this surface owns it (server gates on config). + // Fire-and-forget + idempotent server-side, so reconnects are harmless. + void rpc('wake.start', { surface: 'tui' }) + rpc('commands.catalog', {}) .then(r => { if (!r?.pairs) { @@ -934,6 +938,25 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: return } + case 'wake.detected': { + // "Hey Hermes": optionally open a fresh session (start_new_session), + // then arm voice capture so the user can speak hands-free. Mirrors CLI. + void (async () => { + if (ev.payload?.start_new_session !== false) { + await newSession() + } + const sid = getUiState().sid + if (!sid) { + return + } + setVoiceEnabled(true) + await rpc('voice.toggle', { action: 'on' }) + await rpc('voice.record', { action: 'start', session_id: sid }) + })() + + return + } + case 'gateway.start_timeout': { const { cwd, python, stderr_tail: stderrTail } = ev.payload ?? {} const trace = python || cwd ? ` · ${String(python || '')} ${String(cwd || '')}`.trim() : '' diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index b9cc8f999cb8..ce7bb8fec29b 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -582,6 +582,7 @@ export type GatewayEvent = } | { payload?: { state?: 'idle' | 'listening' | 'transcribing' }; session_id?: string; type: 'voice.status' } | { payload?: { no_speech_limit?: boolean; text?: string }; session_id?: string; type: 'voice.transcript' } + | { payload?: { phrase?: string; start_new_session?: boolean }; session_id?: string; type: 'wake.detected' } | { payload?: { reason?: string }; session_id?: string; type: 'dashboard.new_session_requested' } | { payload: { line: string }; session_id?: string; type: 'gateway.stderr' } | {