mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-29 18:46:59 +00:00
feat(voice): route wake phrases to their profile — "hey <profile>" wakes that profile
One sherpa listener now enrolls every wake-enabled profile's phrase (defaulting to "hey <profile name>") and reports WHICH phrase fired. wake.detected gains a profile field; the desktop live-switches to the matching profile (same path as the profile rail), opens a fresh session there, and starts hands-free voice. The single-profile CLI/TUI print the hermes -p switch command for foreign-profile phrases instead of answering as the wrong profile. Opt out per listener with wake_word.profile_routing: false.
This commit is contained in:
Hermes Agent
Teknium
parent
567f47f01f
commit
2a35c8f0b8
10 changed files with 274 additions and 16 deletions
|
|
@ -34,7 +34,7 @@ import { requestVoiceConversationStart } from '@/store/composer'
|
|||
import { setCronFocusJobId } from '@/store/cron'
|
||||
import { $pinnedSessionIds, pinSession, restoreWorktree, unpinSession } from '@/store/layout'
|
||||
import { $previewTarget } from '@/store/preview'
|
||||
import { $activeGatewayProfile, $freshSessionRequest, $profileScope, refreshActiveProfile } from '@/store/profile'
|
||||
import { $activeGatewayProfile, $freshSessionRequest, $profileScope, ensureGatewayProfile, newSessionInProfile, normalizeProfileKey, refreshActiveProfile } from '@/store/profile'
|
||||
import { $startWorkSessionRequest, followActiveSessionCwd } from '@/store/projects'
|
||||
import {
|
||||
$activeSessionId,
|
||||
|
|
@ -666,9 +666,24 @@ export function ContribWiring({ children }: { children: ReactNode }) {
|
|||
emitGatewayEvent(event)
|
||||
|
||||
if (event.type === 'wake.detected') {
|
||||
const payload = event.payload as { start_new_session?: boolean } | undefined
|
||||
const payload = event.payload as
|
||||
| { profile?: null | string; start_new_session?: boolean }
|
||||
| undefined
|
||||
|
||||
if (payload?.start_new_session !== false) {
|
||||
// Multi-profile routing: a wake phrase enrolled by another profile
|
||||
// re-homes the gateway to that profile first (live swap — same path
|
||||
// as clicking it in the profile rail), then opens the fresh session
|
||||
// and starts voice there.
|
||||
const targetProfile = payload?.profile?.trim()
|
||||
const activeProfile = normalizeProfileKey($activeGatewayProfile.get())
|
||||
|
||||
if (targetProfile && normalizeProfileKey(targetProfile) !== activeProfile) {
|
||||
if (payload?.start_new_session !== false) {
|
||||
newSessionInProfile(targetProfile)
|
||||
} else {
|
||||
void ensureGatewayProfile(normalizeProfileKey(targetProfile))
|
||||
}
|
||||
} else if (payload?.start_new_session !== false) {
|
||||
startFreshSessionDraft()
|
||||
}
|
||||
|
||||
|
|
|
|||
16
cli.py
16
cli.py
|
|
@ -12278,6 +12278,22 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
return
|
||||
self._wake_suspended = True
|
||||
|
||||
# Multi-profile routing: the CLI is a single-profile process, so a
|
||||
# phrase enrolled by ANOTHER profile can't be routed here — print the
|
||||
# switch command and re-arm rather than answering as the wrong profile.
|
||||
try:
|
||||
from tools.wake_word import get_last_match
|
||||
_match = get_last_match()
|
||||
except Exception:
|
||||
_match = None
|
||||
if _match and _match[1]:
|
||||
from tools.wake_word import _active_profile_name
|
||||
if _match[1] != _active_profile_name():
|
||||
_cprint(f"\n{_DIM}Wake phrase for profile '{_match[1]}' — "
|
||||
f"run: hermes -p {_match[1]}{_RST}")
|
||||
self._wake_suspended = True # watchdog resumes the listener
|
||||
return
|
||||
|
||||
_cprint(f"\n{_ACCENT}✦ Wake word detected — listening...{_RST}")
|
||||
if getattr(self, "_app", None):
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -2384,6 +2384,7 @@ DEFAULT_CONFIG = {
|
|||
"phrase": "hey hermes", # for "sherpa" this IS the detected phrase (any text works); for other engines it's a cosmetic label — detection is keyed by the model/keyword below
|
||||
"sensitivity": 0.5, # 0.0-1.0 detection threshold (higher = stricter)
|
||||
"start_new_session": True, # start a fresh session on wake vs. continue the current one
|
||||
"profile_routing": True, # sherpa only: also listen for every wake-enabled profile's phrase and route the wake to the matching profile
|
||||
"openwakeword": {
|
||||
# "hey_hermes" (the bundled, works-out-of-the-box default) OR a
|
||||
# built-in openWakeWord name ("hey_jarvis", "alexa", "hey_mycroft",
|
||||
|
|
|
|||
|
|
@ -1426,7 +1426,7 @@ def test_wake_owner_is_sticky_and_routes_detection_to_first_transport(monkeypatc
|
|||
assert emitted == [(
|
||||
"wake.detected",
|
||||
"first-session",
|
||||
{"phrase": "hey hermes", "start_new_session": True},
|
||||
{"phrase": "hey hermes", "profile": None, "start_new_session": True},
|
||||
first,
|
||||
)]
|
||||
assert state["paused"] is True
|
||||
|
|
@ -1459,7 +1459,7 @@ def test_wake_owner_is_sticky_and_routes_detection_to_first_transport(monkeypatc
|
|||
assert emitted[-1] == (
|
||||
"wake.detected",
|
||||
"second-session",
|
||||
{"phrase": "hey hermes", "start_new_session": True},
|
||||
{"phrase": "hey hermes", "profile": None, "start_new_session": True},
|
||||
second,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -302,6 +302,106 @@ def test_sherpa_requirements_probe_uses_sherpa_feature(monkeypatch):
|
|||
assert r["phrase"] == "anything at all"
|
||||
|
||||
|
||||
# ── Multi-profile phrase routing ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_sherpa_engine_enrolls_all_profile_phrases(monkeypatch, tmp_path):
|
||||
calls, model_dir = _install_fake_sherpa(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(ww, "_active_profile_name", lambda: "default")
|
||||
monkeypatch.setattr(
|
||||
ww, "enrolled_profile_phrases",
|
||||
lambda: {"coder": "hey coder", "trader": "hey trader"},
|
||||
)
|
||||
eng = ww._SherpaKwsEngine({
|
||||
"provider": "sherpa", "phrase": "hey hermes",
|
||||
"sherpa": {"model_dir": str(model_dir)},
|
||||
})
|
||||
with open(eng._keywords_file, encoding="utf-8") as f:
|
||||
lines = f.read().strip().splitlines()
|
||||
assert len(lines) == 3
|
||||
assert eng._display_to_profile == {
|
||||
"HEY_HERMES": "default",
|
||||
"HEY_CODER": "coder",
|
||||
"HEY_TRADER": "trader",
|
||||
}
|
||||
eng.close()
|
||||
|
||||
|
||||
def test_sherpa_engine_profile_routing_can_be_disabled(monkeypatch, tmp_path):
|
||||
calls, model_dir = _install_fake_sherpa(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(ww, "_active_profile_name", lambda: "default")
|
||||
monkeypatch.setattr(
|
||||
ww, "enrolled_profile_phrases", lambda: {"coder": "hey coder"}
|
||||
)
|
||||
eng = ww._SherpaKwsEngine({
|
||||
"provider": "sherpa", "phrase": "hey hermes", "profile_routing": False,
|
||||
"sherpa": {"model_dir": str(model_dir)},
|
||||
})
|
||||
assert eng._display_to_profile == {"HEY_HERMES": "default"}
|
||||
eng.close()
|
||||
|
||||
|
||||
def test_sherpa_engine_match_maps_back_to_profile(monkeypatch, tmp_path):
|
||||
calls, model_dir = _install_fake_sherpa(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(ww, "_active_profile_name", lambda: "default")
|
||||
monkeypatch.setattr(
|
||||
ww, "enrolled_profile_phrases", lambda: {"coder": "hey coder"}
|
||||
)
|
||||
eng = ww._SherpaKwsEngine({
|
||||
"provider": "sherpa", "phrase": "hey hermes",
|
||||
"sherpa": {"model_dir": str(model_dir)},
|
||||
})
|
||||
frame = [0] * eng.frame_length
|
||||
calls["results"].append("HEY_CODER")
|
||||
assert eng.process(frame) is True
|
||||
assert eng.last_match == ("hey coder", "coder")
|
||||
calls["results"].append("HEY_HERMES")
|
||||
assert eng.process(frame) is True
|
||||
assert eng.last_match == ("hey hermes", "default")
|
||||
|
||||
|
||||
def test_enrolled_profile_phrases_reads_profile_configs(monkeypatch, tmp_path):
|
||||
profiles_root = tmp_path / "profiles"
|
||||
for name, body in (
|
||||
("coder", "wake_word:\n enabled: true\n phrase: hey coder\n"),
|
||||
("trader", "wake_word:\n enabled: true\n"), # phrase defaults
|
||||
("quiet", "wake_word:\n enabled: false\n"), # not enrolled
|
||||
("empty", ""), # no wake_word at all
|
||||
):
|
||||
d = profiles_root / name
|
||||
d.mkdir(parents=True)
|
||||
(d / "config.yaml").write_text(body, encoding="utf-8")
|
||||
|
||||
class _Info:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
import types as _types
|
||||
fake_profiles = _types.ModuleType("hermes_cli.profiles")
|
||||
fake_profiles.list_profiles = lambda: [
|
||||
_Info(p.name) for p in sorted(profiles_root.iterdir())
|
||||
]
|
||||
fake_profiles.get_profile_dir = lambda name: str(profiles_root / name)
|
||||
fake_profiles.get_active_profile_name = lambda: "default"
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli.profiles", fake_profiles)
|
||||
|
||||
phrases = ww.enrolled_profile_phrases()
|
||||
assert phrases == {"coder": "hey coder", "trader": "hey trader"}
|
||||
|
||||
|
||||
def test_get_last_match_reads_detector_engine(monkeypatch):
|
||||
class _Eng:
|
||||
last_match = ("hey coder", "coder")
|
||||
|
||||
class _Det:
|
||||
engine = _Eng()
|
||||
|
||||
monkeypatch.setattr(ww, "_detector", _Det())
|
||||
assert ww.get_last_match() == ("hey coder", "coder")
|
||||
monkeypatch.setattr(ww, "_detector", None)
|
||||
assert ww.get_last_match() is None
|
||||
|
||||
|
||||
# ── Detector loop ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -127,6 +127,52 @@ def wake_surface_enabled(surface: str, cfg: Optional[Dict[str, Any]] = None) ->
|
|||
return want == "auto" or want == surface.strip().lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-profile phrase enrollment (open-vocabulary routing)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _active_profile_name() -> str:
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
|
||||
return get_active_profile_name() or "default"
|
||||
except Exception:
|
||||
return "default"
|
||||
|
||||
|
||||
def enrolled_profile_phrases() -> Dict[str, str]:
|
||||
"""Map ``profile name -> wake phrase`` for every wake-enabled profile.
|
||||
|
||||
Reads each profile's own ``config.yaml`` raw (cheap, no full config merge).
|
||||
A profile is enrolled when its ``wake_word.enabled`` is truthy; its phrase
|
||||
defaults to ``"hey <profile>"`` when unset. Used by the sherpa engine to
|
||||
listen for every enrolled profile's phrase at once and route the wake to
|
||||
the matching profile. Best-effort: unreadable profiles are skipped.
|
||||
"""
|
||||
phrases: Dict[str, str] = {}
|
||||
try:
|
||||
import yaml
|
||||
|
||||
from hermes_cli.profiles import get_profile_dir, list_profiles
|
||||
|
||||
for info in list_profiles():
|
||||
name = getattr(info, "name", None) or str(info)
|
||||
try:
|
||||
cfg_path = Path(get_profile_dir(name)) / "config.yaml"
|
||||
raw = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
|
||||
wc = raw.get("wake_word") or {}
|
||||
if not isinstance(wc, dict) or not wc.get("enabled"):
|
||||
continue
|
||||
phrase = str(wc.get("phrase") or f"hey {name}").strip()
|
||||
if phrase:
|
||||
phrases[name] = phrase
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
return phrases
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio capture (lazy — never import sounddevice at module load)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -155,6 +201,12 @@ class _Engine:
|
|||
|
||||
frame_length: int = 1280 # 80 ms at 16 kHz
|
||||
|
||||
#: Optional (matched phrase, profile name) of the most recent fire.
|
||||
#: Multi-phrase engines (sherpa) set this for profile routing; the
|
||||
#: single-phrase engines leave it None (callers fall back to the
|
||||
#: configured phrase / active profile).
|
||||
last_match: Optional[tuple[str, str]] = None
|
||||
|
||||
def process(self, frame) -> bool: # frame: 1-D int16 ndarray
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -292,24 +344,41 @@ class _SherpaKwsEngine(_Engine):
|
|||
if not (d / "tokens.txt").exists():
|
||||
raise RuntimeError(f"sherpa KWS model not found at {d}")
|
||||
|
||||
# Phrase set: this profile's own phrase, plus — when profile routing is
|
||||
# on — every other wake-enabled profile's phrase, so ONE listener can
|
||||
# wake any profile ("hey hermes" / "hey coder" / ...). display-name →
|
||||
# profile is kept for routing the match back.
|
||||
phrase = str(_get(cfg, "phrase") or "hey hermes").strip()
|
||||
# Runtime tokenization of the arbitrary phrase — the open-vocab core.
|
||||
# sherpa keyword entries reject spaces in the @display-name; underscore it.
|
||||
own_profile = _active_profile_name()
|
||||
phrase_map: Dict[str, str] = {phrase: own_profile}
|
||||
if bool(cfg.get("profile_routing", True)):
|
||||
for prof, p in enrolled_profile_phrases().items():
|
||||
phrase_map.setdefault(p.strip(), prof)
|
||||
|
||||
phrases = list(phrase_map)
|
||||
# Runtime tokenization of the arbitrary phrases — the open-vocab core.
|
||||
tokens = text2token(
|
||||
[phrase.upper()],
|
||||
[p.upper() for p in phrases],
|
||||
tokens=str(d / "tokens.txt"),
|
||||
tokens_type="bpe",
|
||||
bpe_model=str(d / "bpe.model"),
|
||||
)
|
||||
import tempfile
|
||||
|
||||
# sherpa keyword entries reject spaces in the @display-name; underscore
|
||||
# them and map display → profile for match routing.
|
||||
self._display_to_profile: Dict[str, str] = {}
|
||||
kw = tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".txt", prefix="hermes-kws-", delete=False, encoding="utf-8"
|
||||
)
|
||||
display = phrase.upper().replace(" ", "_")
|
||||
kw.write(" ".join(tokens[0]) + f" @{display}\n")
|
||||
for p, toks in zip(phrases, tokens):
|
||||
display = p.upper().replace(" ", "_")
|
||||
self._display_to_profile[display] = phrase_map[p]
|
||||
kw.write(" ".join(toks) + f" @{display}\n")
|
||||
kw.close()
|
||||
self._keywords_file = kw.name
|
||||
#: (phrase display name, profile) of the most recent fire, for routing.
|
||||
self.last_match: Optional[tuple[str, str]] = None
|
||||
|
||||
# Map the shared 0..1 sensitivity onto sherpa's keywords_threshold
|
||||
# (posterior probability; its default is 0.25).
|
||||
|
|
@ -340,8 +409,14 @@ class _SherpaKwsEngine(_Engine):
|
|||
fired = False
|
||||
while self._spotter.is_ready(self._stream):
|
||||
self._spotter.decode_stream(self._stream)
|
||||
if self._spotter.get_result(self._stream):
|
||||
result = self._spotter.get_result(self._stream)
|
||||
if result:
|
||||
fired = True
|
||||
display = str(result)
|
||||
self.last_match = (
|
||||
display.replace("_", " ").lower(),
|
||||
self._display_to_profile.get(display, ""),
|
||||
)
|
||||
# Reset decoder state so one utterance can't fire repeatedly.
|
||||
self._spotter.reset_stream(self._stream)
|
||||
return fired
|
||||
|
|
@ -774,3 +849,13 @@ def is_listening() -> bool:
|
|||
with _detector_lock:
|
||||
det = _detector
|
||||
return det is not None and det.running
|
||||
|
||||
|
||||
def get_last_match() -> Optional[tuple[str, str]]:
|
||||
"""(matched phrase, profile) of the most recent wake fire, if the engine
|
||||
reports per-phrase matches (sherpa multi-profile routing). None otherwise."""
|
||||
with _detector_lock:
|
||||
det = _detector
|
||||
if det is None:
|
||||
return None
|
||||
return getattr(det.engine, "last_match", None)
|
||||
|
|
|
|||
|
|
@ -17703,7 +17703,7 @@ def _(rid, params: dict) -> dict:
|
|||
new_session = bool(cfg.get("start_new_session", True))
|
||||
|
||||
def _on_detect() -> None:
|
||||
from tools.wake_word import owns_listener, pause_listening
|
||||
from tools.wake_word import get_last_match, owns_listener, pause_listening
|
||||
|
||||
if not pause_listening(owner=transport):
|
||||
return
|
||||
|
|
@ -17712,12 +17712,18 @@ def _(rid, params: dict) -> dict:
|
|||
if _transport_is_dead(transport):
|
||||
_release_wake_for_transport(transport)
|
||||
return
|
||||
logger.info("wake.detected: emitting to sid=%r (transport=%s)",
|
||||
sid, type(transport).__name__)
|
||||
# Multi-phrase engines report WHICH phrase fired and the profile it
|
||||
# belongs to, so one listener can wake any enrolled profile. Falls
|
||||
# back to the owner's configured phrase / no profile for
|
||||
# single-phrase engines.
|
||||
matched_phrase, matched_profile = get_last_match() or (phrase, "")
|
||||
logger.info("wake.detected: emitting to sid=%r (transport=%s, profile=%r)",
|
||||
sid, type(transport).__name__, matched_profile)
|
||||
token = bind_transport(transport)
|
||||
try:
|
||||
_emit("wake.detected", sid, {
|
||||
"phrase": phrase,
|
||||
"phrase": matched_phrase or phrase,
|
||||
"profile": matched_profile or None,
|
||||
"start_new_session": new_session,
|
||||
})
|
||||
finally:
|
||||
|
|
|
|||
|
|
@ -949,6 +949,19 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev:
|
|||
// "Hey Hermes": optionally open a fresh session (start_new_session),
|
||||
// then arm voice capture so the user can speak hands-free. Mirrors CLI.
|
||||
void (async () => {
|
||||
// Multi-profile routing: the TUI is a single-profile process, so a
|
||||
// phrase enrolled by ANOTHER profile can't be routed here — surface
|
||||
// the switch command instead of starting voice on the wrong profile.
|
||||
const wakeProfile = ev.payload?.profile?.trim()
|
||||
const ownProfile = getUiState().info?.profile_name || 'default'
|
||||
|
||||
if (wakeProfile && wakeProfile !== ownProfile) {
|
||||
sys(`wake phrase for profile '${wakeProfile}' — run: hermes -p ${wakeProfile} --tui`)
|
||||
await rpc('wake.resume', {}).catch(() => undefined)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if (ev.payload?.start_new_session !== false) {
|
||||
await newSession()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -615,7 +615,7 @@ export type GatewayEvent =
|
|||
}
|
||||
| { payload?: { state?: 'idle' | 'listening' | 'transcribing' }; session_id?: string; type: 'voice.status' }
|
||||
| { payload?: { no_speech_limit?: boolean; text?: string }; session_id?: string; type: 'voice.transcript' }
|
||||
| { payload?: { phrase?: string; start_new_session?: boolean }; session_id?: string; type: 'wake.detected' }
|
||||
| { payload?: { phrase?: string; profile?: null | string; start_new_session?: boolean }; session_id?: string; type: 'wake.detected' }
|
||||
| { payload?: { reason?: string }; session_id?: string; type: 'dashboard.new_session_requested' }
|
||||
| { payload: { line: string }; session_id?: string; type: 'gateway.stderr' }
|
||||
| {
|
||||
|
|
|
|||
|
|
@ -125,6 +125,28 @@ wake_word:
|
|||
The small English KWS model (~13 MB) downloads once on first use. Each
|
||||
profile can set its own phrase — "hey \<profile\>" for every profile you run.
|
||||
|
||||
### Waking a specific profile (desktop)
|
||||
|
||||
With the sherpa engine, ONE listener can wake ANY profile. Every profile
|
||||
whose config has `wake_word.enabled: true` is enrolled automatically; its
|
||||
phrase defaults to `hey <profile name>` when unset. Say a profile's phrase
|
||||
and the desktop app live-switches to that profile, opens a fresh session
|
||||
there, and starts hands-free voice:
|
||||
|
||||
- "hey hermes" → default profile
|
||||
- "hey coder" → the `coder` profile
|
||||
- "hey trader" → the `trader` profile
|
||||
|
||||
Set `wake_word.profile_routing: false` on the listener's profile to opt out
|
||||
and listen only for its own phrase. The CLI and TUI are single-profile
|
||||
processes: a wake phrase belonging to another profile prints the switch
|
||||
command (`hermes -p <profile>`) instead of routing.
|
||||
|
||||
Names are matched acoustically by their English subword sounds: two-word
|
||||
phrases with distinct, 2+ syllable names work best. Very short names, heavy
|
||||
non-English phonology, or two profiles with similar-sounding names will
|
||||
degrade accuracy — tune per-profile `sensitivity` if needed.
|
||||
|
||||
### Option B — openWakeWord (free, trained model)
|
||||
|
||||
Name a built-in model (`hey_jarvis`, `alexa`, `hey_mycroft`, …), or train a
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue