diff --git a/hermes_cli/config.py b/hermes_cli/config.py index a6f6a14fb70..92b79a968b4 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2371,7 +2371,11 @@ DEFAULT_CONFIG = { # ...) OR a path to a custom .onnx/.tflite model for another phrase. # See the wake-word docs for the custom-model training guide. "model": "hey_hermes", - "inference_framework": "onnx", # "onnx" | "tflite" + # "" (auto — tflite on macOS ARM64, onnx elsewhere) | "onnx" | "tflite". + # openWakeWord's onnx backend scores near-zero on macOS ARM64 + # (dscripka/openWakeWord#336), so auto avoids a listener that arms + # but never fires. Set explicitly only to override that choice. + "inference_framework": "", }, "sherpa": { # Optional path to a sherpa-onnx KWS model directory. Empty = diff --git a/pyproject.toml b/pyproject.toml index ac2318ea6dc..7aeb812dd9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -188,6 +188,11 @@ wake = [ "pvporcupine==4.0.3", "sounddevice==0.5.5", "numpy==2.4.3", + # openWakeWord's onnx embedding model scores near-zero on macOS ARM64 + # (dscripka/openWakeWord#336), so the wake word runs on tflite there. + # Upstream declares tflite-runtime for Linux only; ai-edge-litert is the + # macOS equivalent, bridged in tools/wake_word.py. + "ai-edge-litert==2.1.6; platform_system == 'Darwin'", ] honcho = ["honcho-ai==2.2.0"] # Cloud memory providers — opt-in, lazy-installed via tools/lazy_deps.py diff --git a/tests/tools/test_wake_word.py b/tests/tools/test_wake_word.py index 45e87b36eec..35c0db4f3af 100644 --- a/tests/tools/test_wake_word.py +++ b/tests/tools/test_wake_word.py @@ -224,11 +224,13 @@ def test_bundled_hey_hermes_model_ships_on_disk(): def test_openwakeword_default_resolves_to_bundled_model(monkeypatch, model_value): # The default (and any "hey_hermes" alias) must load the bundled file, not be # passed through as a bogus built-in name that openWakeWord can't resolve. + # Which artifact is bundled follows the platform's default backend (tflite on + # macOS ARM64, where openWakeWord's onnx path scores near-zero). calls = _install_fake_openwakeword(monkeypatch) sub = {} if model_value is None else {"model": model_value} ww._OpenWakeWordEngine({"provider": "openwakeword", "openwakeword": sub}) (downloaded,) = calls["download"] - assert downloaded == [ww._bundled_wakeword_path("onnx")] + assert downloaded == [ww._bundled_wakeword_path(ww.default_inference_framework())] def test_openwakeword_bundled_model_matches_framework(monkeypatch): @@ -238,7 +240,78 @@ def test_openwakeword_bundled_model_matches_framework(monkeypatch): ) (downloaded,) = calls["download"] assert downloaded == [ww._bundled_wakeword_path("tflite")] - assert downloaded[0].endswith(".tflite") + + +# ── platform-aware backend selection (openWakeWord onnx is broken on macOS ARM64, +# upstream dscripka/openWakeWord#336) ──────────────────────────────────────── + +def test_default_framework_is_tflite_on_macos_arm64(monkeypatch): + monkeypatch.setattr(ww.sys, "platform", "darwin") + monkeypatch.setattr("platform.machine", lambda: "arm64") + assert ww.default_inference_framework() == "tflite" + + +@pytest.mark.parametrize( + "plat,machine", + [("linux", "x86_64"), ("linux", "aarch64"), ("win32", "AMD64"), ("darwin", "x86_64")], +) +def test_default_framework_is_onnx_elsewhere(monkeypatch, plat, machine): + # Only the broken platform changes behaviour; everyone else keeps onnx. + monkeypatch.setattr(ww.sys, "platform", plat) + monkeypatch.setattr("platform.machine", lambda: machine) + assert ww.default_inference_framework() == "onnx" + + +def test_explicit_framework_overrides_platform_default(monkeypatch): + # An operator who pins a backend keeps it, even on macOS ARM64. + calls = _install_fake_openwakeword(monkeypatch) + monkeypatch.setattr(ww.sys, "platform", "darwin") + monkeypatch.setattr("platform.machine", lambda: "arm64") + ww._OpenWakeWordEngine( + {"provider": "openwakeword", "openwakeword": {"inference_framework": "onnx"}} + ) + (downloaded,) = calls["download"] + assert downloaded == [ww._bundled_wakeword_path("onnx")] + + +def test_macos_tflite_refuses_silent_onnx_downgrade(monkeypatch): + # openWakeWord silently falls back to onnx when no tflite runtime imports. + # On macOS ARM64 that lands on the broken backend, so we must raise instead + # of arming a listener that can never fire. + _install_fake_openwakeword(monkeypatch) + monkeypatch.setattr(ww.sys, "platform", "darwin") + monkeypatch.setattr("platform.machine", lambda: "arm64") + monkeypatch.setattr(ww, "ensure_tflite_runtime", lambda: False) + with pytest.raises(RuntimeError, match="ai-edge-litert"): + ww._OpenWakeWordEngine({"provider": "openwakeword", "openwakeword": {}}) + + +def test_non_macos_tflite_falls_back_to_onnx(monkeypatch): + # Off macOS the onnx backend works, so a missing tflite runtime is a + # downgrade, not a hard failure. + calls = _install_fake_openwakeword(monkeypatch) + monkeypatch.setattr(ww.sys, "platform", "linux") + monkeypatch.setattr("platform.machine", lambda: "x86_64") + monkeypatch.setattr(ww, "ensure_tflite_runtime", lambda: False) + ww._OpenWakeWordEngine( + {"provider": "openwakeword", "openwakeword": {"inference_framework": "tflite"}} + ) + (downloaded,) = calls["download"] + assert downloaded == [ww._bundled_wakeword_path("onnx")] + + +def test_requirements_report_missing_tflite_runtime(monkeypatch): + # A missing runtime must surface as unavailable + an actionable hint rather + # than an armed-but-deaf detector. + monkeypatch.setattr(ww.sys, "platform", "darwin") + monkeypatch.setattr("platform.machine", lambda: "arm64") + monkeypatch.setattr(ww, "ensure_tflite_runtime", lambda: False) + monkeypatch.setattr("tools.lazy_deps.is_available", lambda feature: feature != "wake.openwakeword.tflite") + monkeypatch.setattr("tools.lazy_deps._allow_lazy_installs", lambda: False) + monkeypatch.setattr(ww, "_audio_available", lambda: True) + reqs = ww.check_wake_word_requirements({"provider": "openwakeword", "openwakeword": {}}) + assert reqs["available"] is False + assert "ai-edge-litert" in reqs["hint"] # ── sherpa-onnx open-vocabulary engine ─────────────────────────────────── diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 67c7833dfa8..55b1fe5e418 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -140,6 +140,16 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Wake word ("Hey Hermes") engines ────────────────────────────────── # Keep in sync with the `wake` extra in pyproject.toml. openWakeWord is the # free, local default (ONNX runtime); Porcupine is the premium engine. + # openWakeWord's ONNX embedding model returns near-zero scores on macOS + # ARM64 (dscripka/openWakeWord#336), so the wake word runs on the tflite + # backend there. Upstream declares tflite-runtime for Linux only; + # ai-edge-litert is the macOS equivalent, bridged in tools/wake_word.py. + # It lives in its own feature because lazy-dep specs cannot carry PEP 508 + # environment markers (_spec_is_safe rejects ";"), so the platform gate is + # applied by the caller instead. + "wake.openwakeword.tflite": ( + "ai-edge-litert==2.1.6", + ), "wake.openwakeword": ( "openwakeword==0.6.0", "onnxruntime==1.27.0", diff --git a/tools/wake_word.py b/tools/wake_word.py index e1211a61811..eb926cb14ae 100644 --- a/tools/wake_word.py +++ b/tools/wake_word.py @@ -33,6 +33,7 @@ from __future__ import annotations import logging import os +import sys import threading import time from pathlib import Path @@ -78,6 +79,56 @@ def _bundled_wakeword_path(framework: str = "onnx") -> str: return os.path.join(os.path.dirname(__file__), "wakewords", f"{_BUNDLED_MODEL_NAME}.{ext}") +def _is_macos_arm64() -> bool: + import platform + + return sys.platform == "darwin" and platform.machine() == "arm64" + + +def default_inference_framework() -> str: + """The openWakeWord backend to use on this platform. + + openWakeWord's ONNX backend produces near-zero scores on macOS ARM64 — its + shared *embedding* model is the broken stage (the melspectrogram front-end + and the wake classifier both match tflite exactly). The detector arms, the + microphone works, and no phrase can ever cross the threshold. Prefer the + tflite backend there; ONNX stays the default everywhere else. + + Upstream: https://github.com/dscripka/openWakeWord/issues/336 + """ + return "tflite" if _is_macos_arm64() else "onnx" + + +def ensure_tflite_runtime() -> bool: + """Make ``import tflite_runtime.interpreter`` resolve, returning success. + + openWakeWord hardcodes that import but only declares ``tflite-runtime`` for + ``platform_system == "Linux"``; on macOS the equivalent wheel is + ``ai-edge-litert``. Alias the module so the upstream import succeeds. The + alias is process-local — nothing is written to site-packages. + """ + try: + import tflite_runtime.interpreter # noqa: F401 + + return True + except ImportError: + pass + + try: + from ai_edge_litert import interpreter as _litert # type: ignore[import-not-found] + except ImportError: + return False + + import types + + pkg = types.ModuleType("tflite_runtime") + pkg.__path__ = [] # type: ignore[attr-defined] # mark as package + sys.modules.setdefault("tflite_runtime", pkg) + sys.modules["tflite_runtime.interpreter"] = _litert + logger.debug("wake word: bridged tflite_runtime -> ai_edge_litert") + return True + + def load_wake_word_config() -> Dict[str, Any]: """Return the ``wake_word`` config section, shape-guarded to a dict.""" try: @@ -242,9 +293,32 @@ class _OpenWakeWordEngine(_Engine): sub = cfg.get("openwakeword") if isinstance(cfg.get("openwakeword"), dict) else {} model_ref = str(sub.get("model") or _BUNDLED_MODEL_NAME).strip() - framework = str(sub.get("inference_framework") or "onnx").strip().lower() + framework = str(sub.get("inference_framework") or "").strip().lower() + if not framework: + framework = default_inference_framework() self._threshold = _sensitivity(cfg) + # openWakeWord silently downgrades tflite -> onnx when no tflite runtime + # imports (model.py). On macOS ARM64 that lands on the backend whose + # embedding model is broken, so the listener would arm and never fire. + # Install + bridge the runtime first, and refuse the downgrade rather + # than ship a dead ear. + if framework == "tflite" and not ensure_tflite_runtime(): + # Same lazy-install contract as every other backend; the platform + # gate lives here because dep specs can't carry PEP 508 markers. + try: + lazy_deps.ensure("wake.openwakeword.tflite", prompt=False) + except Exception as e: + logger.debug("wake word: tflite runtime install failed: %s", e) + if not ensure_tflite_runtime(): + if _is_macos_arm64(): + raise RuntimeError( + "The wake word needs the tflite backend on this Mac, but its " + "runtime is missing. Install it with: pip install ai-edge-litert" + ) + logger.warning("wake word: no tflite runtime available — falling back to onnx") + framework = "onnx" + # Default (or explicit "hey_hermes") → the bundled model; a built-in name # or custom path is used as-is. if model_ref.lower() in _BUNDLED_MODEL_ALIASES: @@ -519,16 +593,27 @@ def check_wake_word_requirements(cfg: Optional[Dict[str, Any]] = None) -> Dict[s key_ok = True hint = "" + # The tflite backend needs a runtime openWakeWord doesn't declare off Linux. + # Report it as a real remediation instead of arming a detector that can't fire. + tflite_ok = True + if provider not in ("porcupine", "sherpa", "sherpa-onnx", "kws", "open"): + sub = cfg.get("openwakeword") if isinstance(cfg.get("openwakeword"), dict) else {} + framework = str(sub.get("inference_framework") or "").strip().lower() or default_inference_framework() + if framework == "tflite": + tflite_ok = ensure_tflite_runtime() or lazy_deps.is_available("wake.openwakeword.tflite") or lazy_ok + if provider == "porcupine" and not (os.getenv("PORCUPINE_ACCESS_KEY") or "").strip(): key_ok = False hint = "Set PORCUPINE_ACCESS_KEY (free key at https://console.picovoice.ai)." elif not deps_ok and not lazy_ok: hint = lazy_deps.feature_install_command(feature) or "" + elif not tflite_ok: + hint = "The wake word needs the tflite runtime on this Mac: pip install ai-edge-litert" elif deps_ok and not audio_ok: hint = "Microphone capture needs sounddevice + numpy and a working audio device." return { - "available": key_ok and ((deps_ok and audio_ok) or (not deps_ok and lazy_ok)), + "available": key_ok and tflite_ok and ((deps_ok and audio_ok) or (not deps_ok and lazy_ok)), "provider": provider, "deps_available": deps_ok, "audio_available": audio_ok, diff --git a/uv.lock b/uv.lock index 2f6856b93d8..d9796e09648 100644 --- a/uv.lock +++ b/uv.lock @@ -19,6 +19,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/ed/c284543c08aa443a4ef2c8bd120be51da8433dd174c01749b5d87c333f22/agent_client_protocol-0.9.0-py3-none-any.whl", hash = "sha256:06911500b51d8cb69112544e2be01fc5e7db39ef88fecbc3848c5c6f194798ee", size = 56850, upload-time = "2026-03-26T01:20:59.252Z" }, ] +[[package]] +name = "ai-edge-litert" +version = "2.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-strenum" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "protobuf" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/3d/41a85023e1c6cc76d895f3ba6ac7c22b0785db7e08d0827ebeb8a403eefc/ai_edge_litert-2.1.6-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:edf598814004e594b40c888f52cae59e950dbeffd821e83ba45d28db0a0aa3f5", size = 10031164, upload-time = "2026-07-01T21:42:50.986Z" }, + { url = "https://files.pythonhosted.org/packages/52/d5/164aaf69f60f72b7076900ef1cc6153bf50d82cd15202bdf1239c0dbfb1c/ai_edge_litert-2.1.6-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5adf0c9afde6151dc7f2989d039c800f3060d98d40bb5dfc95e426ad4eb3680b", size = 10033170, upload-time = "2026-07-01T21:42:53.208Z" }, + { url = "https://files.pythonhosted.org/packages/ad/92/578b31c4c05afa9081a664cd86afe33c304fecd70de1c2a4d3a3b9ca51c9/ai_edge_litert-2.1.6-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:2e8a3f92fa407690189533bea8b64d49bb1b8a9e96f707ba27e1a64a9c3cc8cf", size = 10032950, upload-time = "2026-07-01T21:42:55.102Z" }, +] + [[package]] name = "aiofiles" version = "24.1.0" @@ -472,6 +490,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, ] +[[package]] +name = "backports-strenum" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/c7/2ed54c32fed313591ffb21edbd48db71e68827d43a61938e5a0bc2b6ec91/backports_strenum-1.3.1.tar.gz", hash = "sha256:77c52407342898497714f0596e86188bb7084f89063226f4ba66863482f42414", size = 7257, upload-time = "2023-12-09T14:36:40.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/50/56cf20e2ee5127b603b81d5a69580a1a325083e2b921aa8f067da83927c0/backports_strenum-1.3.1-py3-none-any.whl", hash = "sha256:cdcfe36dc897e2615dc793b7d3097f54d359918fc448754a517e6f23044ccf83", size = 8304, upload-time = "2023-12-09T14:36:39.905Z" }, +] + [[package]] name = "base58" version = "2.1.1" @@ -1721,6 +1748,7 @@ voice = [ { name = "sounddevice" }, ] wake = [ + { name = "ai-edge-litert", marker = "sys_platform == 'darwin'" }, { name = "numpy" }, { name = "onnxruntime" }, { name = "openwakeword" }, @@ -1745,6 +1773,7 @@ youtube = [ [package.metadata] requires-dist = [ { name = "agent-client-protocol", marker = "extra == 'acp'", specifier = "==0.9.0" }, + { name = "ai-edge-litert", marker = "sys_platform == 'darwin' and extra == 'wake'", specifier = "==2.1.6" }, { name = "aiohttp", marker = "extra == 'homeassistant'", specifier = "==3.14.1" }, { name = "aiohttp", marker = "extra == 'matrix'", specifier = "==3.14.1" }, { name = "aiohttp", marker = "extra == 'messaging'", specifier = "==3.14.1" }, @@ -3942,7 +3971,7 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.12'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3997,7 +4026,7 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ diff --git a/website/docs/user-guide/features/wake-word.md b/website/docs/user-guide/features/wake-word.md index 86dbe967801..eb6573d026c 100644 --- a/website/docs/user-guide/features/wake-word.md +++ b/website/docs/user-guide/features/wake-word.md @@ -79,7 +79,7 @@ wake_word: start_new_session: true # start a fresh session on wake vs. continue the current one openwakeword: model: hey_hermes # bundled default; OR a built-in name OR a path to a custom .onnx/.tflite - inference_framework: onnx # "onnx" | "tflite" + inference_framework: "" # "" (auto) | "onnx" | "tflite" porcupine: keyword: jarvis # built-in keyword OR path to a custom .ppn ``` @@ -87,6 +87,14 @@ wake_word: `sensitivity`, `phrase`, and `start_new_session` apply to both engines. The `openwakeword` and `porcupine` blocks select the actual detection model. +`inference_framework` picks the openWakeWord backend. Leave it empty (the +default) to let Hermes choose per platform: **tflite on Apple Silicon**, onnx +everywhere else. openWakeWord's onnx backend returns near-zero scores on macOS +ARM64 ([openWakeWord#336](https://github.com/dscripka/openWakeWord/issues/336)), +so a listener pinned to `onnx` there will arm, show as listening, and never +fire. The tflite backend needs `ai-edge-litert` on macOS, which Hermes installs +on demand alongside the other wake-word deps. + ### Surfaces (CLI, TUI, GUI) The wake word works in all three Hermes surfaces, and `surface` picks which one