From 514dd59cad8fc8d65137d45a34b76db6be38e3f5 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:18:24 -0700 Subject: [PATCH] fix(wake): detect dead-mic streams, stop the ear freezing during first-use install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal testing (macOS): clicking the ear froze the button for ~30s, then it went blue but 'hey hermes' never fired even though STT worked. Two distinct bugs: 1. Frozen-then-timeout button: first-use wake.start lazy-installs the detection engine (onnxruntime is a large wheel), which blows the desktop's default 30s WS request timeout — the RPC 'failed' client- side while the backend kept installing and armed later on its own. wake.start now gets a 180s budget and the pending state says 'arming — first use may take a minute while the engine installs' instead of a silent disabled button. 2. Armed but deaf: macOS grants mic permission per PROCESS. The renderer having mic access (STT working) does not grant the Python backend anything — CoreAudio hands an unentitled process a 'working' stream that delivers zeros forever, so the listener looks healthy and can never hear the phrase. The detector now tracks frame peaks: 10s of consecutive near-zero frames sets audio_silent, logged with the exact macOS Settings path, cleared automatically when audio appears. Surfaced everywhere: wake.status (audio_silent + hint), desktop ear tooltip (kept visible while listening), /wake status on TUI (⚠ line) and classic CLI, plus a docs troubleshooting section. Tests: detector silent-flag set/recover cycle (fake silent/loud streams), desktop tooltip keeps the dead-mic hint while listening. 44 wake Python tests, 22 desktop + 14 TUI vitest green. --- apps/desktop/src/store/wake-word.test.ts | 14 +++++ apps/desktop/src/store/wake-word.ts | 24 +++++++- cli.py | 5 ++ tests/tools/test_wake_word.py | 58 +++++++++++++++++++ tools/wake_word.py | 48 +++++++++++++++ tui_gateway/server.py | 14 ++++- ui-tui/src/app/slash/commands/wake.ts | 6 ++ ui-tui/src/gatewayTypes.ts | 4 ++ website/docs/user-guide/features/wake-word.md | 12 ++++ 9 files changed, 180 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/store/wake-word.test.ts b/apps/desktop/src/store/wake-word.test.ts index f969b4cf2d4..2234d468a7d 100644 --- a/apps/desktop/src/store/wake-word.test.ts +++ b/apps/desktop/src/store/wake-word.test.ts @@ -47,6 +47,20 @@ describe('applyWakeStatus', () => { expect(state.listening).toBe(false) expect(state.notice).toBe('pip install openwakeword') }) + + it('keeps the dead-mic hint visible while listening (audio_silent)', () => { + applyWakeStatus({ + audio_silent: true, + available: true, + hint: 'Microphone delivers only silence — grant mic access', + listening: true, + phrase: 'hey hermes' + }) + + const state = $wakeWord.get() + expect(state.listening).toBe(true) + expect(state.notice).toBe('Microphone delivers only silence — grant mic access') + }) }) describe('toggleWakeWord', () => { diff --git a/apps/desktop/src/store/wake-word.ts b/apps/desktop/src/store/wake-word.ts index 9b9e61a42ae..3e64285a703 100644 --- a/apps/desktop/src/store/wake-word.ts +++ b/apps/desktop/src/store/wake-word.ts @@ -31,6 +31,8 @@ const INITIAL_WAKE_WORD_STATE: WakeWordState = { export const $wakeWord = atom(INITIAL_WAKE_WORD_STATE) export interface WakeStatusResponse { + /** Armed but the mic delivers only silence (macOS backend-permission gap). */ + audio_silent?: boolean available?: boolean /** Config truth (wake_word.enabled) — drives post-voice re-arm. */ enabled?: boolean @@ -62,6 +64,11 @@ export interface WakeStopResponse { * `requestGateway` and the `$gateway` instance wrapper below. */ export type WakeRequester = (method: string, params?: Record) => Promise +// First-use wake.start lazy-installs the detection engine (onnxruntime is a +// large wheel) — that legitimately takes minutes. The default 30s WS timeout +// fired mid-install, leaving a dead button that went blue on its own later. +const WAKE_START_TIMEOUT_MS = 180_000 + const gatewayRequester: WakeRequester = async (method: string, params: Record = {}) => { const gateway = $gateway.get() @@ -69,7 +76,9 @@ const gatewayRequester: WakeRequester = async (method: string, params: Record throw new Error('Hermes gateway unavailable') } - return gateway.request(method, params) + return method === 'wake.start' + ? gateway.request(method, params, WAKE_START_TIMEOUT_MS) + : gateway.request(method, params) } // Friendly text for the gateway's wake refusal codes (mirrors the TUI's @@ -99,12 +108,15 @@ const noticeFrom = (result: { hint?: string; reason?: string | null } | null | u export function applyWakeStatus(status: WakeStatusResponse | null | undefined): void { const current = $wakeWord.get() const listening = Boolean(status?.listening) + // "Armed but deaf" (macOS backend without mic permission) keeps its hint + // visible in the tooltip even though the toggle shows listening. + const silent = Boolean(status?.audio_silent) $wakeWord.set({ ...current, available: Boolean(status?.available), listening, - notice: listening ? '' : noticeFrom(status), + notice: listening && !silent ? '' : noticeFrom(status), phrase: status?.phrase?.trim() || current.phrase }) } @@ -182,7 +194,13 @@ export async function toggleWakeWord(request: WakeRequester = gatewayRequester): return } - $wakeWord.set({ ...state, pending: true }) + $wakeWord.set({ + ...state, + // First arm may lazy-install the detection engine — say so instead of + // freezing a silent disabled button for the duration. + notice: state.listening ? '' : 'arming — first use may take a minute while the engine installs', + pending: true + }) try { if (state.listening) { diff --git a/cli.py b/cli.py index 04528f20933..1d22d893d05 100644 --- a/cli.py +++ b/cli.py @@ -12367,6 +12367,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): def _show_wake_word_status(self): """Show current wake-word listener status.""" from tools.wake_word import ( + audio_is_silent, check_wake_word_requirements, is_listening, load_wake_word_config, @@ -12384,6 +12385,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): _cprint(f" Provider: {reqs['provider']}") _cprint(f" Surface: {cfg.get('surface', 'auto')}") _cprint(f" New session: {'yes' if cfg.get('start_new_session', True) else 'no'}") + if state == "LISTENING" and audio_is_silent(): + _cprint(f" {_ACCENT}⚠ Microphone delivers only silence — the listener can't hear anything.{_RST}") + _cprint(f" {_DIM}On macOS: System Settings > Privacy & Security > Microphone — allow your" + f" terminal/Hermes, then /wake off + /wake on.{_RST}") if not reqs["available"] and reqs.get("hint"): _cprint(f" {_DIM}{reqs['hint']}{_RST}") if not owned: diff --git a/tests/tools/test_wake_word.py b/tests/tools/test_wake_word.py index 45e87b36eec..0bcab048607 100644 --- a/tests/tools/test_wake_word.py +++ b/tests/tools/test_wake_word.py @@ -508,6 +508,64 @@ def _fake_audio(monkeypatch): monkeypatch.setattr(ww, "_import_audio", lambda: (fake_sd, None)) +class _Frame(list): + """List with numpy-ish abs()/max() so the silence probe sees real peaks.""" + + def __abs__(self): + return _Frame(abs(x) for x in self) + + def max(self): + return max(self) if self else 0 + + +class _SilentStream(_FakeStream): + """Stream that always yields near-zero frames (dead macOS mic).""" + + def read(self, n): + time.sleep(0.005) + return _Frame([0] * n), False + + +class _LoudStream(_FakeStream): + """Stream that yields audible frames.""" + + def read(self, n): + time.sleep(0.005) + return _Frame([500] * n), False + + +def test_detector_flags_silent_stream_and_recovers(monkeypatch): + """A stream of zeros sets audio_silent (macOS no-permission mode); audio clears it.""" + monkeypatch.setattr(ww, "_SILENCE_ALERT_SECONDS", 0.001) # trip on the first frame + stream_cls = {"cls": _SilentStream} + fake_sd = types.SimpleNamespace(InputStream=lambda **kw: stream_cls["cls"](**kw)) + monkeypatch.setattr(ww, "_import_audio", lambda: (fake_sd, None)) + + det = ww.WakeWordDetector(_FakeEngine(fire=False), lambda: None) + det.start() + try: + deadline = time.monotonic() + 2.0 + while not det.audio_silent and time.monotonic() < deadline: + time.sleep(0.01) + assert det.audio_silent is True + assert ww.audio_is_silent() is False # module accessor needs the singleton + + monkeypatch.setattr(ww, "_detector", det) + assert ww.audio_is_silent() is True + + # Audio returns (permission granted / real mic) — flag clears. + det.pause() + stream_cls["cls"] = _LoudStream + det.resume() + deadline = time.monotonic() + 2.0 + while det.audio_silent and time.monotonic() < deadline: + time.sleep(0.01) + assert det.audio_silent is False + finally: + monkeypatch.setattr(ww, "_detector", None) + det.stop() + + def test_detector_fires_once_under_cooldown(monkeypatch): _fake_audio(monkeypatch) calls = [] diff --git a/tools/wake_word.py b/tools/wake_word.py index e1211a61811..7f33a8895c9 100644 --- a/tools/wake_word.py +++ b/tools/wake_word.py @@ -48,6 +48,14 @@ SAMPLE_RATE = 16000 _FIRE_COOLDOWN_SECONDS = 2.0 _START_TIMEOUT_SECONDS = 5.0 +# Dead-mic detection: an int16 stream whose peak stays at/below this for this +# many consecutive seconds is flagged as silent. macOS grants the *app* mic +# permission per-process — a backend spawned without the entitlement gets a +# "working" CoreAudio stream that delivers zeros forever, so the listener +# looks armed but can never hear the phrase. +_SILENCE_PEAK = 10 +_SILENCE_ALERT_SECONDS = 10 + class WakeWordInUse(RuntimeError): """Raised when another surface or process owns the wake-word listener.""" @@ -561,6 +569,12 @@ class WakeWordDetector: self._callback_inflight = threading.Event() self._last_fire = 0.0 self._lock = threading.Lock() + # True when the stream is open but every frame is (near-)silence — the + # classic macOS symptom of a backend process without mic permission: + # CoreAudio "succeeds" and delivers zeros forever. Surfaced via + # wake.status / /wake status so users can tell "armed" from "deaf". + self.audio_silent = False + self._silent_frames = 0 @property def running(self) -> bool: @@ -653,6 +667,9 @@ class WakeWordDetector: logger.info("wake word: listening (frame=%d, rate=%d)", frame_length, SAMPLE_RATE) ready.set() failed = False + # ~seconds of consecutive near-zero frames before we flag the stream + # as silent (macOS no-permission streams deliver zeros forever). + silent_alert_frames = max(1, int(_SILENCE_ALERT_SECONDS * SAMPLE_RATE / max(1, frame_length))) try: while not self._stop.is_set(): try: @@ -662,6 +679,25 @@ class WakeWordDetector: failed = not self._stop.is_set() break frame = data[:, 0] if getattr(data, "ndim", 1) == 2 else data + try: + peak = int(abs(frame).max()) if len(frame) else 0 + except Exception: + peak = _SILENCE_PEAK + 1 + if peak <= _SILENCE_PEAK: + self._silent_frames += 1 + if self._silent_frames == silent_alert_frames: + self.audio_silent = True + logger.warning( + "wake word: mic delivers only silence (peak<=%d for %ds) — " + "on macOS check System Settings > Privacy & Security > " + "Microphone for the Hermes backend process", + _SILENCE_PEAK, _SILENCE_ALERT_SECONDS, + ) + elif self._silent_frames: + if self.audio_silent: + logger.info("wake word: mic audio detected — stream healthy") + self._silent_frames = 0 + self.audio_silent = False try: fired = self.engine.process(frame) except Exception as e: @@ -861,6 +897,18 @@ def is_listening() -> bool: return det is not None and det.running +def audio_is_silent() -> bool: + """True when the armed stream has delivered only silence (dead mic). + + The macOS no-permission failure mode: the stream opens fine but every + frame is zeros, so detection can never fire. Lets status surfaces show + "listening but the microphone appears silent" instead of a healthy state. + """ + with _detector_lock: + det = _detector + return det is not None and det.audio_silent + + def get_last_match() -> Optional[tuple[str, str]]: """(matched phrase, profile) of the most recent wake fire, if the engine reports per-phrase matches (sherpa multi-profile routing). None otherwise.""" diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 4eb2229af11..28da1643615 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -17895,6 +17895,7 @@ def _(rid, params: dict) -> dict: def _(rid, params: dict) -> dict: try: from tools.wake_word import ( + audio_is_silent, check_wake_word_requirements, is_listening, load_wake_word_config, @@ -17905,17 +17906,26 @@ def _(rid, params: dict) -> dict: transport = current_transport() or _stdio_transport owner, owner_surface = _wake_owner_snapshot() owned_by_caller = owns_listener(transport) + listening = owned_by_caller and is_listening() + silent = listening and audio_is_silent() + hint = reqs.get("hint", "") + if silent and not hint: + hint = ("Microphone delivers only silence — on macOS grant the " + "Hermes backend mic access (System Settings > Privacy & " + "Security > Microphone), then toggle the wake word.") return _ok(rid, { - "listening": owned_by_caller and is_listening(), + "listening": listening, "owned_by_caller": owned_by_caller, "owner_surface": owner_surface if owner is not None else None, "phrase": reqs["phrase"], "provider": reqs["provider"], "available": reqs["available"], - "hint": reqs.get("hint", ""), + "hint": hint, # Config truth: clients use this to re-arm after a voice turn # ("permanent on") without guessing from runtime listener state. "enabled": bool(cfg.get("enabled")), + # Armed but deaf (macOS permission failure mode) — see hint. + "audio_silent": silent, }) except Exception as e: return _err(rid, 5026, str(e)) diff --git a/ui-tui/src/app/slash/commands/wake.ts b/ui-tui/src/app/slash/commands/wake.ts index 1984678f9db..8011b853809 100644 --- a/ui-tui/src/app/slash/commands/wake.ts +++ b/ui-tui/src/app/slash/commands/wake.ts @@ -32,6 +32,12 @@ const statusLine = (r: WakeStatusResponse): string => { const provider = r.provider ? ` · ${r.provider}` : '' if (r.listening) { + if (r.audio_silent) { + const hint = r.hint?.trim() ? ` — ${r.hint.trim()}` : '' + + return `wake: listening${phrase}${provider} · ⚠ mic delivers only silence${hint}` + } + return `wake: listening${phrase}${provider}` } diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index a47b904e3ab..954190d028a 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -419,7 +419,11 @@ export interface WakeStopResponse { } export interface WakeStatusResponse { + /** Armed but the mic delivers only silence (macOS backend-permission gap). */ + audio_silent?: boolean available?: boolean + /** Config truth (wake_word.enabled). */ + enabled?: boolean hint?: string listening?: boolean owned_by_caller?: boolean diff --git a/website/docs/user-guide/features/wake-word.md b/website/docs/user-guide/features/wake-word.md index 86dbe967801..25673fbc451 100644 --- a/website/docs/user-guide/features/wake-word.md +++ b/website/docs/user-guide/features/wake-word.md @@ -208,6 +208,18 @@ PORCUPINE_ACCESS_KEY=your-key-here `/wake status` reports exactly what's missing if the listener won't start. +### "Listening" but never wakes (macOS) + +macOS grants microphone access per **process**. STT working in the desktop app +proves the *renderer* has mic access — the wake listener runs in the Python +*backend*, which needs its own grant. Without it, CoreAudio hands the backend a +"working" stream that only ever delivers silence, so the ear shows listening +but the phrase never fires. Hermes detects this (`/wake status` shows +"mic delivers only silence"; the desktop ear tooltip carries the same hint). +Fix: System Settings → Privacy & Security → Microphone → enable the Hermes +backend (it may appear as your terminal, `python`, or Hermes), then toggle the +wake word off and on. + ## Notes & limits - **Local surfaces only.** The wake word runs in the CLI, TUI, and desktop GUI —