feat(desktop,tui): arm the wake word over the gateway and start hands-free voice

The TUI and desktop GUI share the Python tui_gateway, which runs the detector
server-side and exposes wake.start/stop/pause/resume/status plus a wake.detected
event routed back over the same transport that armed it. Clients arm it on
connect; on wake the desktop opens a fresh session, starts voice, and hands the
mic between the detector and its browser voice loop. An empty STT transcript
(silence) is treated as a quiet re-listen rather than a "transcription failed"
toast.

apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts

@@ -1,9 +1,12 @@
+import { useStore } from '@nanostores/react'
 import { useCallback, useEffect, useRef, useState } from 'react'
 
 import { useI18n } from '@/i18n'
 import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages'
 import { triggerHaptic } from '@/lib/haptics'
+import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer'
 import { resetBrowseState } from '@/store/composer-input-history'
+import { $gateway } from '@/store/gateway'
 import { notifyError } from '@/store/notifications'
 import { $messages } from '@/store/session'
 import { $autoSpeakReplies, setAutoSpeakReplies } from '@/store/voice-prefs'
@@ -140,6 +143,57 @@ export function useComposerVoice({
     [target, toggleVoiceConversation]
   )
 
+  // "Hey Hermes" wake word: a latched start request (nanostore) the composer
+  // claims once it's mounted and the gateway is ready. Survives the fresh-session
+  // remount the wake handler triggers, and waits out a transient `disabled`.
+  const voiceStartReq = useStore($voiceConversationStartRequest)
+  useEffect(() => {
+    if (disabled) {
+      return // not ready — re-runs when `disabled` flips false
+    }
+
+    if (!takeVoiceConversationStart(voiceStartReq)) {
+      return
+    }
+
+    if (!voiceConversationActive) {
+      setVoiceConversationActive(true)
+    }
+  }, [voiceStartReq, disabled, voiceConversationActive])
+
+  // Hand the mic between the server-side wake detector and the browser's voice
+  // loop: pause the detector while a conversation is live, resume it after
+  // (no-ops server-side when the wake word isn't armed). wakePausedRef tracks
+  // whether WE paused, so resume always runs once — including on unmount, where
+  // ending voice can tear the composer down before the `false` render lands and
+  // would otherwise leave the detector paused forever.
+  const wakePausedRef = useRef(false)
+
+  const wakeRpc = useCallback(
+    (method: string) => void $gateway.get()?.request(method, {}).catch(() => undefined),
+    []
+  )
+
+  const resumeWakeIfPaused = useCallback(() => {
+    if (!wakePausedRef.current) {
+      return
+    }
+
+    wakePausedRef.current = false
+    wakeRpc('wake.resume')
+  }, [wakeRpc])
+
+  useEffect(() => {
+    if (voiceConversationActive) {
+      wakePausedRef.current = true
+      wakeRpc('wake.pause')
+    } else {
+      resumeWakeIfPaused()
+    }
+  }, [voiceConversationActive, wakeRpc, resumeWakeIfPaused])
+
+  useEffect(() => resumeWakeIfPaused, [resumeWakeIfPaused])
+
   // Explicit start/end for the on-screen conversation controls (the hotkey uses
   // the gated toggle above).
   const startConversation = useCallback(() => setVoiceConversationActive(true), [])

apps/desktop/src/app/contrib/wiring.tsx

@@ -29,6 +29,7 @@ import { sessionMessagesSignature } from '@/lib/session-signatures'
 import { isMessagingSource } from '@/lib/session-source'
 import { latestSessionTodos } from '@/lib/todos'
 import { $billingSettingsRequest } from '@/store/billing-block'
+import { requestVoiceConversationStart } from '@/store/composer'
 import { setCronFocusJobId } from '@/store/cron'
 import { $pinnedSessionIds, pinSession, restoreWorktree, unpinSession } from '@/store/layout'
 import { $filePreviewTarget, $previewTarget } from '@/store/preview'
@@ -645,9 +646,25 @@ export function ContribWiring({ children }: { children: ReactNode }) {
   const handleGatewayEventWithPlugins = useCallback(
     (event: Parameters<typeof handleDesktopGatewayEvent>[0]) => {
       emitGatewayEvent(event)
+
+      // "Hey Hermes": handle the wake event on the canonical onEvent pipeline —
+      // open a fresh session (unless the config keeps the current one) and begin
+      // back-and-forth voice via the composer's latched start request.
+      if (event.type === 'wake.detected') {
+        const payload = event.payload as { start_new_session?: boolean } | undefined
+
+        if (payload?.start_new_session !== false) {
+          startFreshSessionDraft()
+        }
+
+        requestVoiceConversationStart()
+
+        return
+      }
+
       handleDesktopGatewayEvent(event)
     },
-    [handleDesktopGatewayEvent]
+    [handleDesktopGatewayEvent, startFreshSessionDraft]
   )
 
   useGatewayBoot({
@@ -668,6 +685,17 @@ export function ContribWiring({ children }: { children: ReactNode }) {
     refreshSessions
   })
 
+  // "Hey Hermes" wake word: arm the server-side detector for this surface
+  // (gated on config server-side). Detection arrives as a wake.detected event
+  // handled in handleGatewayEventWithPlugins. Idempotent, so reconnects are safe.
+  useEffect(() => {
+    if (gatewayState !== 'open') {
+      return
+    }
+
+    void requestGateway('wake.start', { surface: 'gui' }).catch(() => undefined)
+  }, [gatewayState, requestGateway])
+
   // Only the open messaging transcript needs its own poll — local chats are
   // live over the websocket already.
   const activeIsMessaging =

apps/desktop/src/store/composer.ts

@@ -93,6 +93,29 @@ export function createComposerAttachmentScope($attachments = atom<ComposerAttach
  *  `$composerAttachments` reader/writer IS this scope. */
 export const mainComposerScope = createComposerAttachmentScope($composerAttachments)
 
+// Latched "start a voice conversation" request (the "Hey Hermes" wake word).
+// A nanostore, not a fire-once window event, because the wake handler opens a
+// fresh session first — the composer may remount before it can observe the
+// intent. The composer reads the current value on (re)mount and consumes it
+// once it's ready (gateway open). Module-level _voiceStartHandled tracks the
+// last consumed id so a remount doesn't re-trigger or miss a just-set request.
+export const $voiceConversationStartRequest = atom(0)
+let _voiceStartHandled = 0
+
+export const requestVoiceConversationStart = (): void =>
+  $voiceConversationStartRequest.set(Date.now())
+
+/** Returns true exactly once per new request id (claims it). */
+export const takeVoiceConversationStart = (current: number): boolean => {
+  if (current > _voiceStartHandled) {
+    _voiceStartHandled = current
+
+    return true
+  }
+
+  return false
+}
+
 // Per-thread draft stash for the decoupled composer. Session lifecycle never
 // touches this — only ChatBar's scope swap reads/writes it. Text mirrors to
 // localStorage; attachments are memory-only (blobs, upload state).

hermes_cli/web_server.py

@@ -4304,10 +4304,15 @@ async def transcribe_audio_upload(payload: AudioTranscriptionRequest):
                 pass
 
     if not result.get("success"):
-        raise HTTPException(
-            status_code=400,
-            detail=result.get("error") or "Transcription failed",
-        )
+        err = result.get("error") or "Transcription failed"
+        # An empty transcript means no speech was detected — a normal outcome
+        # for VAD/continuous voice loops (e.g. a wake-word conversation
+        # re-listening on silence), not an error. Return an empty transcript so
+        # the client quietly re-listens instead of surfacing a "transcription
+        # failed" toast on every silent gap.
+        if "empty transcript" in err.lower():
+            return {"ok": True, "transcript": "", "provider": result.get("provider")}
+        raise HTTPException(status_code=400, detail=err)
 
     return {
         "ok": True,

tui_gateway/server.py

@@ -905,6 +905,11 @@ def _close_sessions_for_transport(
 
 
 def _shutdown_sessions() -> None:
+    try:
+        from tools.wake_word import stop_listening as _stop_wake
+        _stop_wake()
+    except Exception:
+        pass
     with _sessions_lock:
         sids = list(_sessions)
     for sid in sids:
@@ -15814,6 +15819,168 @@ def _voice_record_key() -> str:
     return str(record_key) if isinstance(record_key, str) and record_key else "ctrl+b"
 
 
+# ── Wake word ("Hey Hermes") ──────────────────────────────────────────────
+# The detector is process-global (one mic), like voice. It runs server-side so
+# both the TUI and desktop GUI share it; clients pass their surface identity to
+# wake.start and the shared gate (wake_surface_enabled) decides whether to arm.
+# On detection we emit wake.detected; the client opens a new session and starts
+# its own voice capture. The detector yields the mic to gateway voice.record
+# (pause/resume below) and to the desktop's browser mic (wake.pause/resume RPCs).
+_wake_lock = threading.Lock()
+_wake_active = False
+_wake_event_sid = ""
+# Transport captured at wake.start time. The detector callback fires on a
+# background thread where the request-scoped transport ContextVar is unset, so
+# write_json would fall back to stdio and the event would never cross the
+# desktop's websocket (#wake-detected-not-delivered). We pin the arming
+# request's transport here and bind it for the emit.
+_wake_transport: "Optional[Transport]" = None
+
+
+def _wake_is_active() -> bool:
+    with _wake_lock:
+        return _wake_active
+
+
+def _wake_resume_if_active() -> None:
+    if not _wake_is_active():
+        return
+    try:
+        from tools.wake_word import resume_listening
+        resume_listening()
+    except Exception as e:
+        logger.debug("wake resume failed: %s", e)
+
+
+def _wake_on_detect() -> None:
+    """Detector-thread callback: tell the client to open a fresh voice session."""
+    with _wake_lock:
+        sid = _wake_event_sid
+        transport = _wake_transport
+    phrase, new_session = "", True
+    try:
+        from tools.wake_word import load_wake_word_config, wake_phrase
+        cfg = load_wake_word_config()
+        phrase = wake_phrase(cfg)
+        new_session = bool(cfg.get("start_new_session", True))
+    except Exception:
+        pass
+    logger.info("wake.detected: emitting to sid=%r (transport=%s)",
+                sid, type(transport).__name__ if transport else None)
+    # Bind the arming request's transport so write_json reaches the right peer
+    # (WS for desktop/dashboard) instead of falling back to stdio on this
+    # background thread. Carry start_new_session so every surface honors it.
+    token = bind_transport(transport) if transport is not None else None
+    try:
+        _emit("wake.detected", sid, {"phrase": phrase, "start_new_session": new_session})
+    finally:
+        if token is not None:
+            reset_transport(token)
+
+
+@method("wake.start")
+def _(rid, params: dict) -> dict:
+    """Arm the wake-word listener for the calling surface ("tui" | "gui").
+
+    Idempotent and gated: returns ``{started: False, reason}`` when the wake
+    word is disabled, scoped to another surface, or its deps/mic aren't ready.
+    """
+    global _wake_active, _wake_event_sid, _wake_transport
+    surface = str(params.get("surface") or "auto").strip().lower()
+    try:
+        from tools.wake_word import (
+            check_wake_word_requirements,
+            load_wake_word_config,
+            start_listening,
+            wake_surface_enabled,
+        )
+    except Exception as e:
+        return _err(rid, 5026, f"wake module unavailable: {e}")
+
+    cfg = load_wake_word_config()
+    if not wake_surface_enabled(surface, cfg):
+        logger.info("wake.start(%s): disabled for surface (enabled=%s, surface=%s)",
+                    surface, cfg.get("enabled"), cfg.get("surface"))
+        return _ok(rid, {"started": False, "reason": "disabled_for_surface"})
+    reqs = check_wake_word_requirements(cfg)
+    if not reqs["available"]:
+        logger.warning("wake.start(%s): not available — %s", surface, reqs.get("hint"))
+        return _ok(rid, {"started": False, "reason": reqs.get("hint") or "unavailable"})
+
+    with _wake_lock:
+        _wake_event_sid = params.get("session_id") or _wake_event_sid
+        # Capture the live transport (WS for desktop) so the background detector
+        # thread can route wake.detected back to this client, not stdio.
+        _wake_transport = current_transport() or _wake_transport
+    try:
+        start_listening(_wake_on_detect, config=cfg)
+    except Exception as e:
+        logger.warning("wake.start(%s): failed to start listener: %s", surface, e)
+        return _err(rid, 5026, str(e))
+    with _wake_lock:
+        _wake_active = True
+    logger.info("wake.start(%s): listening for %r (%s)", surface, reqs["phrase"], reqs["provider"])
+    return _ok(rid, {"started": True, "phrase": reqs["phrase"], "provider": reqs["provider"]})
+
+
+@method("wake.stop")
+def _(rid, params: dict) -> dict:
+    global _wake_active
+    with _wake_lock:
+        _wake_active = False
+    try:
+        from tools.wake_word import stop_listening
+        stop_listening()
+    except Exception:
+        pass
+    return _ok(rid, {"stopped": True})
+
+
+@method("wake.pause")
+def _(rid, params: dict) -> dict:
+    """Release the mic (e.g. while the desktop's browser captures audio)."""
+    try:
+        from tools.wake_word import pause_listening
+        pause_listening()
+        logger.info("wake.pause: detector paused")
+    except Exception as e:
+        logger.debug("wake.pause failed: %s", e)
+    return _ok(rid, {"paused": True})
+
+
+@method("wake.resume")
+def _(rid, params: dict) -> dict:
+    """Reclaim the mic after a pause; no-op if the listener isn't armed."""
+    active = _wake_is_active()
+    if active:
+        _wake_resume_if_active()
+        logger.info("wake.resume: detector resumed")
+    else:
+        logger.info("wake.resume: ignored (listener not armed)")
+    return _ok(rid, {"resumed": active})
+
+
+@method("wake.status")
+def _(rid, params: dict) -> dict:
+    try:
+        from tools.wake_word import (
+            check_wake_word_requirements,
+            is_listening,
+            load_wake_word_config,
+        )
+        cfg = load_wake_word_config()
+        reqs = check_wake_word_requirements(cfg)
+        return _ok(rid, {
+            "listening": _wake_is_active() and is_listening(),
+            "phrase": reqs["phrase"],
+            "provider": reqs["provider"],
+            "available": reqs["available"],
+            "hint": reqs.get("hint", ""),
+        })
+    except Exception as e:
+        return _err(rid, 5026, str(e))
+
+
 @method("voice.toggle")
 def _(rid, params: dict) -> dict:
     """CLI parity for the ``/voice`` slash command.
@@ -15962,12 +16129,28 @@ def _(rid, params: dict) -> dict:
                 if isinstance(duration, (int, float)) and not isinstance(duration, bool)
                 else 3.0
             )
+            # Hand the mic to STT if the wake-word detector holds it; resume
+            # once a terminal capture event fires (one-shot transcript / silence
+            # limit), so wake-triggered and manual captures both coexist.
+            if _wake_is_active():
+                try:
+                    from tools.wake_word import pause_listening
+                    pause_listening()
+                except Exception:
+                    pass
+
+            def _on_transcript(t):
+                _voice_emit("voice.transcript", {"text": t})
+                _wake_resume_if_active()
+
+            def _on_silent():
+                _voice_emit("voice.transcript", {"no_speech_limit": True})
+                _wake_resume_if_active()
+
             started = start_continuous(
-                on_transcript=lambda t: _voice_emit("voice.transcript", {"text": t}),
+                on_transcript=_on_transcript,
                 on_status=lambda s: _voice_emit("voice.status", {"state": s}),
-                on_silent_limit=lambda: _voice_emit(
-                    "voice.transcript", {"no_speech_limit": True}
-                ),
+                on_silent_limit=_on_silent,
                 silence_threshold=safe_threshold,
                 silence_duration=safe_duration,
                 auto_restart=False,
@@ -15983,6 +16166,7 @@ def _(rid, params: dict) -> dict:
         from hermes_cli.voice import stop_continuous
 
         stop_continuous(force_transcribe=True)
+        _wake_resume_if_active()
         return _ok(rid, {"status": "stopped"})
     except ImportError:
         return _err(

ui-tui/src/app/createGatewayEventHandler.ts

@@ -619,6 +619,10 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev:
     // "too many re-renders" guard in embedded dashboard PTYs.
     ensureAgentsNudgeConfig()
 
+    // Arm "Hey Hermes" if this surface owns it (server gates on config).
+    // Fire-and-forget + idempotent server-side, so reconnects are harmless.
+    void rpc('wake.start', { surface: 'tui' })
+
     rpc<CommandsCatalogResponse>('commands.catalog', {})
       .then(r => {
         if (!r?.pairs) {
@@ -934,6 +938,25 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev:
         return
       }
 
+      case 'wake.detected': {
+        // "Hey Hermes": optionally open a fresh session (start_new_session),
+        // then arm voice capture so the user can speak hands-free. Mirrors CLI.
+        void (async () => {
+          if (ev.payload?.start_new_session !== false) {
+            await newSession()
+          }
+          const sid = getUiState().sid
+          if (!sid) {
+            return
+          }
+          setVoiceEnabled(true)
+          await rpc('voice.toggle', { action: 'on' })
+          await rpc('voice.record', { action: 'start', session_id: sid })
+        })()
+
+        return
+      }
+
       case 'gateway.start_timeout': {
         const { cwd, python, stderr_tail: stderrTail } = ev.payload ?? {}
         const trace = python || cwd ? ` · ${String(python || '')} ${String(cwd || '')}`.trim() : ''

ui-tui/src/gatewayTypes.ts

@@ -582,6 +582,7 @@ export type GatewayEvent =
     }
   | { payload?: { state?: 'idle' | 'listening' | 'transcribing' }; session_id?: string; type: 'voice.status' }
   | { payload?: { no_speech_limit?: boolean; text?: string }; session_id?: string; type: 'voice.transcript' }
+  | { payload?: { phrase?: string; start_new_session?: boolean }; session_id?: string; type: 'wake.detected' }
   | { payload?: { reason?: string }; session_id?: string; type: 'dashboard.new_session_requested' }
   | { payload: { line: string }; session_id?: string; type: 'gateway.stderr' }
   | {