mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(wake): raise default sensitivity to 0.6 and fix inverted Porcupine direction
'hey hor' triggered the wake word: the default sensitivity was 0.5, which for openWakeWord IS the raw per-frame score threshold — openWakeWord's own permissive baseline that near-misses clear. Raised the default to 0.6 so phonetic near-misses fall short while real 'hey hermes' (typically 0.9+) still fires easily. Also fixed a real cross-engine inconsistency found while checking: the sensitivity knob is documented 'higher = stricter' and behaves that way for openWakeWord (threshold = sensitivity) and sherpa (0.05 + 0.4*s), but Porcupine's own 'sensitivities' param runs the opposite way (higher = MORE false alarms, per Picovoice). Turning sensitivity up made Porcupine looser — backwards. Now inverted (1 - sensitivity) so 'higher = stricter' holds for every engine. - tools/wake_word.py: default 0.6; _sensitivity fallback uses _DEFAULTS; Porcupine sensitivity inverted with rationale - hermes_cli/config.py + docs: default + consistent-direction note - tests: Porcupine inversion, default>=0.6 regression, fallback-to-default
This commit is contained in:
parent
a8bc64a418
commit
f106e0ebc2
4 changed files with 65 additions and 9 deletions
|
|
@ -2382,7 +2382,7 @@ DEFAULT_CONFIG = {
|
|||
"surface": "auto", # eligible surface: "auto" (first claimant) | "cli" | "tui" | "gui"
|
||||
"provider": "openwakeword", # "openwakeword" (free, local) | "sherpa" (free, ANY phrase, no training) | "porcupine" (premium; needs PORCUPINE_ACCESS_KEY)
|
||||
"phrase": "hey hermes", # for "sherpa" this IS the detected phrase (any text works); for other engines it's a cosmetic label — detection is keyed by the model/keyword below
|
||||
"sensitivity": 0.5, # 0.0-1.0 detection threshold (higher = stricter)
|
||||
"sensitivity": 0.6, # 0.0-1.0 detection threshold, consistent across engines (higher = stricter, fewer false triggers)
|
||||
"confirmation_frames": 3, # openWakeWord only: consecutive over-threshold frames required to fire (higher = fewer false triggers on ambient speech, slightly more latency; 1 = old single-frame behavior)
|
||||
"start_new_session": True, # start a fresh session on wake vs. continue the current one
|
||||
"profile_routing": True, # sherpa only: also listen for every wake-enabled profile's phrase and route the wake to the matching profile
|
||||
|
|
|
|||
|
|
@ -27,7 +27,9 @@ def test_config_defaults_and_clamping():
|
|||
assert ww._provider({"provider": "Porcupine"}) == "porcupine"
|
||||
assert ww._sensitivity({"sensitivity": 5}) == 1.0
|
||||
assert ww._sensitivity({"sensitivity": -1}) == 0.0
|
||||
assert ww._sensitivity({"sensitivity": "nope"}) == 0.5
|
||||
# Invalid input falls back to the configured default, not a hardcoded 0.5.
|
||||
assert ww._sensitivity({"sensitivity": "nope"}) == ww._DEFAULTS["sensitivity"]
|
||||
assert ww._sensitivity({}) == ww._DEFAULTS["sensitivity"]
|
||||
assert ww.wake_phrase({"phrase": "hey hermes"}) == "hey hermes"
|
||||
assert ww.wake_phrase({}) == "hey hermes"
|
||||
|
||||
|
|
@ -428,6 +430,44 @@ def test_confirmation_frames_config_clamped(monkeypatch):
|
|||
assert ww._confirmation_frames({}) == ww._DEFAULT_CONFIRMATION_FRAMES
|
||||
|
||||
|
||||
def test_porcupine_sensitivity_is_inverted_to_match_shared_contract(monkeypatch):
|
||||
# Our config contract is "higher sensitivity = stricter" for every engine.
|
||||
# Porcupine's own `sensitivities` param means the OPPOSITE (higher = looser,
|
||||
# more false alarms), so the engine must pass 1 - sensitivity.
|
||||
captured = {}
|
||||
|
||||
class _FakePorcupine:
|
||||
frame_length = 512
|
||||
|
||||
def process(self, frame):
|
||||
return -1
|
||||
|
||||
def _create(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _FakePorcupine()
|
||||
|
||||
pv = types.ModuleType("pvporcupine")
|
||||
pv.create = _create
|
||||
monkeypatch.setitem(sys.modules, "pvporcupine", pv)
|
||||
monkeypatch.setattr("tools.lazy_deps.ensure", lambda *a, **k: None)
|
||||
monkeypatch.setenv("PORCUPINE_ACCESS_KEY", "test-key")
|
||||
|
||||
# Strict (0.9) → Porcupine gets a low 0.1 (few false alarms).
|
||||
ww._PorcupineEngine({"provider": "porcupine", "sensitivity": 0.9})
|
||||
assert captured["sensitivities"] == [pytest.approx(0.1)]
|
||||
|
||||
# Loose (0.2) → Porcupine gets a high 0.8.
|
||||
ww._PorcupineEngine({"provider": "porcupine", "sensitivity": 0.2})
|
||||
assert captured["sensitivities"] == [pytest.approx(0.8)]
|
||||
|
||||
|
||||
def test_default_sensitivity_is_stricter_than_openwakeword_baseline():
|
||||
# Regression: the 0.5 default let near-misses ("hey hor") through. The
|
||||
# default must sit above openWakeWord's permissive 0.5 baseline.
|
||||
assert ww._DEFAULTS["sensitivity"] >= 0.6
|
||||
assert ww._sensitivity({}) >= 0.6
|
||||
|
||||
|
||||
def test_macos_tflite_refuses_silent_onnx_downgrade(monkeypatch):
|
||||
# openWakeWord silently falls back to onnx when no tflite runtime imports.
|
||||
# On macOS ARM64 that lands on the broken backend, so we must raise instead
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ _DEFAULTS: Dict[str, Any] = {
|
|||
"surface": "auto",
|
||||
"provider": "openwakeword",
|
||||
"phrase": "hey hermes",
|
||||
"sensitivity": 0.5,
|
||||
"sensitivity": 0.6,
|
||||
"confirmation_frames": _DEFAULT_CONFIRMATION_FRAMES,
|
||||
"start_new_session": True,
|
||||
}
|
||||
|
|
@ -170,7 +170,7 @@ def _sensitivity(cfg: Dict[str, Any]) -> float:
|
|||
try:
|
||||
s = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
s = 0.5
|
||||
s = float(_DEFAULTS["sensitivity"])
|
||||
return min(max(s, 0.0), 1.0)
|
||||
|
||||
|
||||
|
|
@ -327,6 +327,10 @@ class _OpenWakeWordEngine(_Engine):
|
|||
framework = str(sub.get("inference_framework") or "").strip().lower()
|
||||
if not framework:
|
||||
framework = default_inference_framework()
|
||||
# openWakeWord returns a 0..1 score per frame; sensitivity IS the raw
|
||||
# threshold a score must clear. Higher = stricter (fewer false fires).
|
||||
# Default 0.6 sits above openWakeWord's permissive 0.5 baseline, which
|
||||
# let near-misses like "hey hor" through.
|
||||
self._threshold = _sensitivity(cfg)
|
||||
self._confirm_needed = _confirmation_frames(cfg)
|
||||
self._confirm_streak = 0
|
||||
|
|
@ -575,9 +579,13 @@ class _PorcupineEngine(_Engine):
|
|||
|
||||
sub = cfg.get("porcupine") if isinstance(cfg.get("porcupine"), dict) else {}
|
||||
keyword = str(sub.get("keyword") or "jarvis").strip()
|
||||
sensitivity = _sensitivity(cfg)
|
||||
# Porcupine's `sensitivities` runs the OPPOSITE way to our shared knob:
|
||||
# per Picovoice, higher = more true positives AND more false alarms
|
||||
# (looser). Our config contract is "higher = stricter" everywhere, so
|
||||
# invert it here to keep one consistent meaning across all engines.
|
||||
porcupine_sensitivity = 1.0 - _sensitivity(cfg)
|
||||
|
||||
kwargs: Dict[str, Any] = {"access_key": access_key, "sensitivities": [sensitivity]}
|
||||
kwargs: Dict[str, Any] = {"access_key": access_key, "sensitivities": [porcupine_sensitivity]}
|
||||
if _looks_like_path(keyword):
|
||||
kwargs["keyword_paths"] = [keyword]
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ wake_word:
|
|||
surface: auto # eligible surface: "auto" | "cli" | "tui" | "gui"
|
||||
provider: openwakeword # "openwakeword" (free, local) | "porcupine"
|
||||
phrase: "hey hermes" # cosmetic label only — detection is keyed by the model/keyword below
|
||||
sensitivity: 0.5 # 0.0-1.0 — raise to reduce false triggers
|
||||
sensitivity: 0.6 # 0.0-1.0 — higher = stricter (fewer false triggers), consistent across all engines
|
||||
confirmation_frames: 3 # openWakeWord only — consecutive over-threshold frames required to fire
|
||||
start_new_session: true # start a fresh session on wake vs. continue the current one
|
||||
openwakeword:
|
||||
|
|
@ -101,10 +101,18 @@ threshold and fire the wake word unintentionally. Two knobs control this:
|
|||
just one. Raise it (e.g. `4`–`5`) if you still get false triggers in a noisy
|
||||
room; the cost is a few tens of milliseconds of extra latency. `1` restores
|
||||
the old fire-on-first-frame behavior.
|
||||
- **`sensitivity`** — raise toward `1.0` to demand a higher score per frame.
|
||||
- **`sensitivity`** (default `0.6`) — the detection threshold, `0.0`–`1.0`.
|
||||
Higher is stricter (fewer false triggers). This direction is consistent across
|
||||
**all** engines — for openWakeWord it's the raw per-frame score threshold, for
|
||||
sherpa it maps onto the keyword threshold, and for Porcupine it's inverted
|
||||
internally so "higher = stricter" holds there too. The `0.6` default sits
|
||||
above openWakeWord's permissive `0.5` baseline, which let near-misses like
|
||||
"hey hor" through; raise toward `0.8` if you still get false fires, lower it
|
||||
if real "hey hermes" utterances are missed.
|
||||
|
||||
The `sherpa` and `porcupine` engines decode the whole phrase internally, so they
|
||||
don't have the single-frame-spike problem and ignore `confirmation_frames`.
|
||||
don't have the single-frame-spike problem and ignore `confirmation_frames`
|
||||
(but they still honor `sensitivity`).
|
||||
|
||||
`inference_framework` picks the openWakeWord backend. Leave it empty (the
|
||||
default) to let Hermes choose per platform: **tflite on Apple Silicon**, onnx
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue