mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(wake): detect dead-mic streams, stop the ear freezing during first-use install
Internal testing (macOS): clicking the ear froze the button for ~30s, then it went blue but 'hey hermes' never fired even though STT worked. Two distinct bugs: 1. Frozen-then-timeout button: first-use wake.start lazy-installs the detection engine (onnxruntime is a large wheel), which blows the desktop's default 30s WS request timeout — the RPC 'failed' client- side while the backend kept installing and armed later on its own. wake.start now gets a 180s budget and the pending state says 'arming — first use may take a minute while the engine installs' instead of a silent disabled button. 2. Armed but deaf: macOS grants mic permission per PROCESS. The renderer having mic access (STT working) does not grant the Python backend anything — CoreAudio hands an unentitled process a 'working' stream that delivers zeros forever, so the listener looks healthy and can never hear the phrase. The detector now tracks frame peaks: 10s of consecutive near-zero frames sets audio_silent, logged with the exact macOS Settings path, cleared automatically when audio appears. Surfaced everywhere: wake.status (audio_silent + hint), desktop ear tooltip (kept visible while listening), /wake status on TUI (⚠ line) and classic CLI, plus a docs troubleshooting section. Tests: detector silent-flag set/recover cycle (fake silent/loud streams), desktop tooltip keeps the dead-mic hint while listening. 44 wake Python tests, 22 desktop + 14 TUI vitest green.
This commit is contained in:
parent
a562757717
commit
514dd59cad
9 changed files with 180 additions and 5 deletions
|
|
@ -47,6 +47,20 @@ describe('applyWakeStatus', () => {
|
|||
expect(state.listening).toBe(false)
|
||||
expect(state.notice).toBe('pip install openwakeword')
|
||||
})
|
||||
|
||||
it('keeps the dead-mic hint visible while listening (audio_silent)', () => {
|
||||
applyWakeStatus({
|
||||
audio_silent: true,
|
||||
available: true,
|
||||
hint: 'Microphone delivers only silence — grant mic access',
|
||||
listening: true,
|
||||
phrase: 'hey hermes'
|
||||
})
|
||||
|
||||
const state = $wakeWord.get()
|
||||
expect(state.listening).toBe(true)
|
||||
expect(state.notice).toBe('Microphone delivers only silence — grant mic access')
|
||||
})
|
||||
})
|
||||
|
||||
describe('toggleWakeWord', () => {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ const INITIAL_WAKE_WORD_STATE: WakeWordState = {
|
|||
export const $wakeWord = atom<WakeWordState>(INITIAL_WAKE_WORD_STATE)
|
||||
|
||||
export interface WakeStatusResponse {
|
||||
/** Armed but the mic delivers only silence (macOS backend-permission gap). */
|
||||
audio_silent?: boolean
|
||||
available?: boolean
|
||||
/** Config truth (wake_word.enabled) — drives post-voice re-arm. */
|
||||
enabled?: boolean
|
||||
|
|
@ -62,6 +64,11 @@ export interface WakeStopResponse {
|
|||
* `requestGateway` and the `$gateway` instance wrapper below. */
|
||||
export type WakeRequester = <T>(method: string, params?: Record<string, unknown>) => Promise<T>
|
||||
|
||||
// First-use wake.start lazy-installs the detection engine (onnxruntime is a
|
||||
// large wheel) — that legitimately takes minutes. The default 30s WS timeout
|
||||
// fired mid-install, leaving a dead button that went blue on its own later.
|
||||
const WAKE_START_TIMEOUT_MS = 180_000
|
||||
|
||||
const gatewayRequester: WakeRequester = async <T>(method: string, params: Record<string, unknown> = {}) => {
|
||||
const gateway = $gateway.get()
|
||||
|
||||
|
|
@ -69,7 +76,9 @@ const gatewayRequester: WakeRequester = async <T>(method: string, params: Record
|
|||
throw new Error('Hermes gateway unavailable')
|
||||
}
|
||||
|
||||
return gateway.request<T>(method, params)
|
||||
return method === 'wake.start'
|
||||
? gateway.request<T>(method, params, WAKE_START_TIMEOUT_MS)
|
||||
: gateway.request<T>(method, params)
|
||||
}
|
||||
|
||||
// Friendly text for the gateway's wake refusal codes (mirrors the TUI's
|
||||
|
|
@ -99,12 +108,15 @@ const noticeFrom = (result: { hint?: string; reason?: string | null } | null | u
|
|||
export function applyWakeStatus(status: WakeStatusResponse | null | undefined): void {
|
||||
const current = $wakeWord.get()
|
||||
const listening = Boolean(status?.listening)
|
||||
// "Armed but deaf" (macOS backend without mic permission) keeps its hint
|
||||
// visible in the tooltip even though the toggle shows listening.
|
||||
const silent = Boolean(status?.audio_silent)
|
||||
|
||||
$wakeWord.set({
|
||||
...current,
|
||||
available: Boolean(status?.available),
|
||||
listening,
|
||||
notice: listening ? '' : noticeFrom(status),
|
||||
notice: listening && !silent ? '' : noticeFrom(status),
|
||||
phrase: status?.phrase?.trim() || current.phrase
|
||||
})
|
||||
}
|
||||
|
|
@ -182,7 +194,13 @@ export async function toggleWakeWord(request: WakeRequester = gatewayRequester):
|
|||
return
|
||||
}
|
||||
|
||||
$wakeWord.set({ ...state, pending: true })
|
||||
$wakeWord.set({
|
||||
...state,
|
||||
// First arm may lazy-install the detection engine — say so instead of
|
||||
// freezing a silent disabled button for the duration.
|
||||
notice: state.listening ? '' : 'arming — first use may take a minute while the engine installs',
|
||||
pending: true
|
||||
})
|
||||
|
||||
try {
|
||||
if (state.listening) {
|
||||
|
|
|
|||
5
cli.py
5
cli.py
|
|
@ -12367,6 +12367,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
def _show_wake_word_status(self):
|
||||
"""Show current wake-word listener status."""
|
||||
from tools.wake_word import (
|
||||
audio_is_silent,
|
||||
check_wake_word_requirements,
|
||||
is_listening,
|
||||
load_wake_word_config,
|
||||
|
|
@ -12384,6 +12385,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
_cprint(f" Provider: {reqs['provider']}")
|
||||
_cprint(f" Surface: {cfg.get('surface', 'auto')}")
|
||||
_cprint(f" New session: {'yes' if cfg.get('start_new_session', True) else 'no'}")
|
||||
if state == "LISTENING" and audio_is_silent():
|
||||
_cprint(f" {_ACCENT}⚠ Microphone delivers only silence — the listener can't hear anything.{_RST}")
|
||||
_cprint(f" {_DIM}On macOS: System Settings > Privacy & Security > Microphone — allow your"
|
||||
f" terminal/Hermes, then /wake off + /wake on.{_RST}")
|
||||
if not reqs["available"] and reqs.get("hint"):
|
||||
_cprint(f" {_DIM}{reqs['hint']}{_RST}")
|
||||
if not owned:
|
||||
|
|
|
|||
|
|
@ -508,6 +508,64 @@ def _fake_audio(monkeypatch):
|
|||
monkeypatch.setattr(ww, "_import_audio", lambda: (fake_sd, None))
|
||||
|
||||
|
||||
class _Frame(list):
|
||||
"""List with numpy-ish abs()/max() so the silence probe sees real peaks."""
|
||||
|
||||
def __abs__(self):
|
||||
return _Frame(abs(x) for x in self)
|
||||
|
||||
def max(self):
|
||||
return max(self) if self else 0
|
||||
|
||||
|
||||
class _SilentStream(_FakeStream):
|
||||
"""Stream that always yields near-zero frames (dead macOS mic)."""
|
||||
|
||||
def read(self, n):
|
||||
time.sleep(0.005)
|
||||
return _Frame([0] * n), False
|
||||
|
||||
|
||||
class _LoudStream(_FakeStream):
|
||||
"""Stream that yields audible frames."""
|
||||
|
||||
def read(self, n):
|
||||
time.sleep(0.005)
|
||||
return _Frame([500] * n), False
|
||||
|
||||
|
||||
def test_detector_flags_silent_stream_and_recovers(monkeypatch):
|
||||
"""A stream of zeros sets audio_silent (macOS no-permission mode); audio clears it."""
|
||||
monkeypatch.setattr(ww, "_SILENCE_ALERT_SECONDS", 0.001) # trip on the first frame
|
||||
stream_cls = {"cls": _SilentStream}
|
||||
fake_sd = types.SimpleNamespace(InputStream=lambda **kw: stream_cls["cls"](**kw))
|
||||
monkeypatch.setattr(ww, "_import_audio", lambda: (fake_sd, None))
|
||||
|
||||
det = ww.WakeWordDetector(_FakeEngine(fire=False), lambda: None)
|
||||
det.start()
|
||||
try:
|
||||
deadline = time.monotonic() + 2.0
|
||||
while not det.audio_silent and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert det.audio_silent is True
|
||||
assert ww.audio_is_silent() is False # module accessor needs the singleton
|
||||
|
||||
monkeypatch.setattr(ww, "_detector", det)
|
||||
assert ww.audio_is_silent() is True
|
||||
|
||||
# Audio returns (permission granted / real mic) — flag clears.
|
||||
det.pause()
|
||||
stream_cls["cls"] = _LoudStream
|
||||
det.resume()
|
||||
deadline = time.monotonic() + 2.0
|
||||
while det.audio_silent and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert det.audio_silent is False
|
||||
finally:
|
||||
monkeypatch.setattr(ww, "_detector", None)
|
||||
det.stop()
|
||||
|
||||
|
||||
def test_detector_fires_once_under_cooldown(monkeypatch):
|
||||
_fake_audio(monkeypatch)
|
||||
calls = []
|
||||
|
|
|
|||
|
|
@ -48,6 +48,14 @@ SAMPLE_RATE = 16000
|
|||
_FIRE_COOLDOWN_SECONDS = 2.0
|
||||
_START_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
# Dead-mic detection: an int16 stream whose peak stays at/below this for this
|
||||
# many consecutive seconds is flagged as silent. macOS grants the *app* mic
|
||||
# permission per-process — a backend spawned without the entitlement gets a
|
||||
# "working" CoreAudio stream that delivers zeros forever, so the listener
|
||||
# looks armed but can never hear the phrase.
|
||||
_SILENCE_PEAK = 10
|
||||
_SILENCE_ALERT_SECONDS = 10
|
||||
|
||||
|
||||
class WakeWordInUse(RuntimeError):
|
||||
"""Raised when another surface or process owns the wake-word listener."""
|
||||
|
|
@ -561,6 +569,12 @@ class WakeWordDetector:
|
|||
self._callback_inflight = threading.Event()
|
||||
self._last_fire = 0.0
|
||||
self._lock = threading.Lock()
|
||||
# True when the stream is open but every frame is (near-)silence — the
|
||||
# classic macOS symptom of a backend process without mic permission:
|
||||
# CoreAudio "succeeds" and delivers zeros forever. Surfaced via
|
||||
# wake.status / /wake status so users can tell "armed" from "deaf".
|
||||
self.audio_silent = False
|
||||
self._silent_frames = 0
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
|
|
@ -653,6 +667,9 @@ class WakeWordDetector:
|
|||
logger.info("wake word: listening (frame=%d, rate=%d)", frame_length, SAMPLE_RATE)
|
||||
ready.set()
|
||||
failed = False
|
||||
# ~seconds of consecutive near-zero frames before we flag the stream
|
||||
# as silent (macOS no-permission streams deliver zeros forever).
|
||||
silent_alert_frames = max(1, int(_SILENCE_ALERT_SECONDS * SAMPLE_RATE / max(1, frame_length)))
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
|
|
@ -662,6 +679,25 @@ class WakeWordDetector:
|
|||
failed = not self._stop.is_set()
|
||||
break
|
||||
frame = data[:, 0] if getattr(data, "ndim", 1) == 2 else data
|
||||
try:
|
||||
peak = int(abs(frame).max()) if len(frame) else 0
|
||||
except Exception:
|
||||
peak = _SILENCE_PEAK + 1
|
||||
if peak <= _SILENCE_PEAK:
|
||||
self._silent_frames += 1
|
||||
if self._silent_frames == silent_alert_frames:
|
||||
self.audio_silent = True
|
||||
logger.warning(
|
||||
"wake word: mic delivers only silence (peak<=%d for %ds) — "
|
||||
"on macOS check System Settings > Privacy & Security > "
|
||||
"Microphone for the Hermes backend process",
|
||||
_SILENCE_PEAK, _SILENCE_ALERT_SECONDS,
|
||||
)
|
||||
elif self._silent_frames:
|
||||
if self.audio_silent:
|
||||
logger.info("wake word: mic audio detected — stream healthy")
|
||||
self._silent_frames = 0
|
||||
self.audio_silent = False
|
||||
try:
|
||||
fired = self.engine.process(frame)
|
||||
except Exception as e:
|
||||
|
|
@ -861,6 +897,18 @@ def is_listening() -> bool:
|
|||
return det is not None and det.running
|
||||
|
||||
|
||||
def audio_is_silent() -> bool:
|
||||
"""True when the armed stream has delivered only silence (dead mic).
|
||||
|
||||
The macOS no-permission failure mode: the stream opens fine but every
|
||||
frame is zeros, so detection can never fire. Lets status surfaces show
|
||||
"listening but the microphone appears silent" instead of a healthy state.
|
||||
"""
|
||||
with _detector_lock:
|
||||
det = _detector
|
||||
return det is not None and det.audio_silent
|
||||
|
||||
|
||||
def get_last_match() -> Optional[tuple[str, str]]:
|
||||
"""(matched phrase, profile) of the most recent wake fire, if the engine
|
||||
reports per-phrase matches (sherpa multi-profile routing). None otherwise."""
|
||||
|
|
|
|||
|
|
@ -17895,6 +17895,7 @@ def _(rid, params: dict) -> dict:
|
|||
def _(rid, params: dict) -> dict:
|
||||
try:
|
||||
from tools.wake_word import (
|
||||
audio_is_silent,
|
||||
check_wake_word_requirements,
|
||||
is_listening,
|
||||
load_wake_word_config,
|
||||
|
|
@ -17905,17 +17906,26 @@ def _(rid, params: dict) -> dict:
|
|||
transport = current_transport() or _stdio_transport
|
||||
owner, owner_surface = _wake_owner_snapshot()
|
||||
owned_by_caller = owns_listener(transport)
|
||||
listening = owned_by_caller and is_listening()
|
||||
silent = listening and audio_is_silent()
|
||||
hint = reqs.get("hint", "")
|
||||
if silent and not hint:
|
||||
hint = ("Microphone delivers only silence — on macOS grant the "
|
||||
"Hermes backend mic access (System Settings > Privacy & "
|
||||
"Security > Microphone), then toggle the wake word.")
|
||||
return _ok(rid, {
|
||||
"listening": owned_by_caller and is_listening(),
|
||||
"listening": listening,
|
||||
"owned_by_caller": owned_by_caller,
|
||||
"owner_surface": owner_surface if owner is not None else None,
|
||||
"phrase": reqs["phrase"],
|
||||
"provider": reqs["provider"],
|
||||
"available": reqs["available"],
|
||||
"hint": reqs.get("hint", ""),
|
||||
"hint": hint,
|
||||
# Config truth: clients use this to re-arm after a voice turn
|
||||
# ("permanent on") without guessing from runtime listener state.
|
||||
"enabled": bool(cfg.get("enabled")),
|
||||
# Armed but deaf (macOS permission failure mode) — see hint.
|
||||
"audio_silent": silent,
|
||||
})
|
||||
except Exception as e:
|
||||
return _err(rid, 5026, str(e))
|
||||
|
|
|
|||
|
|
@ -32,6 +32,12 @@ const statusLine = (r: WakeStatusResponse): string => {
|
|||
const provider = r.provider ? ` · ${r.provider}` : ''
|
||||
|
||||
if (r.listening) {
|
||||
if (r.audio_silent) {
|
||||
const hint = r.hint?.trim() ? ` — ${r.hint.trim()}` : ''
|
||||
|
||||
return `wake: listening${phrase}${provider} · ⚠ mic delivers only silence${hint}`
|
||||
}
|
||||
|
||||
return `wake: listening${phrase}${provider}`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -419,7 +419,11 @@ export interface WakeStopResponse {
|
|||
}
|
||||
|
||||
export interface WakeStatusResponse {
|
||||
/** Armed but the mic delivers only silence (macOS backend-permission gap). */
|
||||
audio_silent?: boolean
|
||||
available?: boolean
|
||||
/** Config truth (wake_word.enabled). */
|
||||
enabled?: boolean
|
||||
hint?: string
|
||||
listening?: boolean
|
||||
owned_by_caller?: boolean
|
||||
|
|
|
|||
|
|
@ -208,6 +208,18 @@ PORCUPINE_ACCESS_KEY=your-key-here
|
|||
|
||||
`/wake status` reports exactly what's missing if the listener won't start.
|
||||
|
||||
### "Listening" but never wakes (macOS)
|
||||
|
||||
macOS grants microphone access per **process**. STT working in the desktop app
|
||||
proves the *renderer* has mic access — the wake listener runs in the Python
|
||||
*backend*, which needs its own grant. Without it, CoreAudio hands the backend a
|
||||
"working" stream that only ever delivers silence, so the ear shows listening
|
||||
but the phrase never fires. Hermes detects this (`/wake status` shows
|
||||
"mic delivers only silence"; the desktop ear tooltip carries the same hint).
|
||||
Fix: System Settings → Privacy & Security → Microphone → enable the Hermes
|
||||
backend (it may appear as your terminal, `python`, or Hermes), then toggle the
|
||||
wake word off and on.
|
||||
|
||||
## Notes & limits
|
||||
|
||||
- **Local surfaces only.** The wake word runs in the CLI, TUI, and desktop GUI —
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue