diff --git a/apps/desktop/src/app/desktop-controller.tsx b/apps/desktop/src/app/desktop-controller.tsx new file mode 100644 index 000000000000..6623636f1876 --- /dev/null +++ b/apps/desktop/src/app/desktop-controller.tsx @@ -0,0 +1,1435 @@ +import { useStore } from '@nanostores/react' +import { useQueryClient } from '@tanstack/react-query' +import { lazy, Suspense, useCallback, useEffect, useMemo, useRef } from 'react' +import { Navigate, Route, Routes, useLocation, useNavigate, useParams } from 'react-router-dom' + +import { BootFailureOverlay } from '@/components/boot-failure-overlay' +import { DesktopInstallOverlay } from '@/components/desktop-install-overlay' +import { DesktopOnboardingOverlay } from '@/components/desktop-onboarding-overlay' +import { GatewayConnectingOverlay } from '@/components/gateway-connecting-overlay' +import { Pane, PaneMain } from '@/components/pane-shell' +import { RemoteDisplayBanner } from '@/components/remote-display-banner' +import { useMediaQuery } from '@/hooks/use-media-query' +import { cn } from '@/lib/utils' +import { useSkinCommand } from '@/themes/use-skin-command' + +import { formatRefValue } from '../components/assistant-ui/directive-text' +import { getCronJobs, getSessionMessages, listAllProfileSessions, type SessionInfo, triggerCronJob } from '../hermes' +import { type ChatMessage, chatMessageText, preserveLocalAssistantErrors, toChatMessages } from '../lib/chat-messages' +import { storedSessionIdForNotification } from '../lib/session-ids' +import { + isMessagingSource, + LOCAL_SESSION_SOURCE_IDS, + MESSAGING_SESSION_SOURCE_IDS, + normalizeSessionSource +} from '../lib/session-source' +import { latestSessionTodos } from '../lib/todos' +import { setCronFocusJobId, setCronJobs } from '../store/cron' +import { + $fileBrowserOpen, + $panesFlipped, + $pinnedSessionIds, + $sessionsLimit, + bumpSessionsLimit, + FILE_BROWSER_DEFAULT_WIDTH, + FILE_BROWSER_MAX_WIDTH, + FILE_BROWSER_MIN_WIDTH, + pinSession, + PREVIEW_PANE_ID, + restoreWorktree, + setSidebarOverlayMounted, + SIDEBAR_DEFAULT_WIDTH, + SIDEBAR_MAX_WIDTH, + SIDEBAR_SESSIONS_PAGE_SIZE, + unpinSession +} from '../store/layout' +import { respondToApprovalAction } from '../store/native-notifications' +import { $paneOpen } from '../store/panes' +import { setPetActivity } from '../store/pet' +import { setPetScale } from '../store/pet-gallery' +import { + setPetOverlayOpenAppHandler, + setPetOverlayScaleHandler, + setPetOverlaySubmitHandler +} from '../store/pet-overlay' +import { $filePreviewTarget, $previewTarget, closeActiveRightRailTab } from '../store/preview' +import { + $activeGatewayProfile, + $freshSessionRequest, + $profileScope, + ALL_PROFILES, + normalizeProfileKey, + refreshActiveProfile +} from '../store/profile' +import { $startWorkSessionRequest, followActiveSessionCwd, resolveNewSessionCwd } from '../store/projects' +import { $reviewOpen, REVIEW_PANE_ID } from '../store/review' +import { + $activeSessionId, + $attentionSessionIds, + $currentCwd, + $freshDraftReady, + $gatewayState, + $messages, + $messagingSessions, + $resumeExhaustedSessionId, + $resumeFailedSessionId, + $selectedStoredSessionId, + $sessions, + $workingSessionIds, + CRON_SECTION_LIMIT, + getRecentlySettledSessionIds, + getRememberedSessionId, + mergeSessionPage, + MESSAGING_SECTION_LIMIT, + sessionPinId, + setAwaitingResponse, + setBusy, + setCronSessions, + setCurrentBranch, + setCurrentCwd, + setCurrentModel, + setCurrentProvider, + setMessages, + setMessagingPlatformTotals, + setMessagingSessions, + setMessagingTruncated, + setRememberedSessionId, + setSessionProfileTotals, + setSessions, + setSessionsLoading, + setSessionsTotal +} from '../store/session' +import { onSessionsChanged } from '../store/session-sync' +import { clearSessionTodos, setSessionTodos, todoListActive } from '../store/todos' +import { openUpdatesWindow, startUpdatePoller, stopUpdatePoller } from '../store/updates' +import { isSecondaryWindow } from '../store/windows' + +import { ChatView } from './chat' +import { requestComposerFocus, requestComposerInsert } from './chat/composer/focus' +import { useComposerActions } from './chat/hooks/use-composer-actions' +import { + ChatPreviewRail, + PREVIEW_RAIL_MAX_WIDTH, + PREVIEW_RAIL_MIN_WIDTH, + PREVIEW_RAIL_PANE_WIDTH +} from './chat/right-rail' +import { ChatSidebar } from './chat/sidebar' +import { CommandPalette } from './command-palette' +import { useGatewayBoot } from './gateway/hooks/use-gateway-boot' +import { useGatewayRequest } from './gateway/hooks/use-gateway-request' +import { useKeybinds } from './hooks/use-keybinds' +import { SIDEBAR_COLLAPSE_MEDIA_QUERY } from './layout-constants' +import { ModelPickerOverlay } from './model-picker-overlay' +import { ModelVisibilityOverlay } from './model-visibility-overlay' +import { PetGenerateOverlay } from './pet-generate/pet-generate-overlay' +import { RightSidebarPane } from './right-sidebar' +import { FileActionDialogs } from './right-sidebar/file-actions' +import { ReviewPane } from './right-sidebar/review' +import { $terminalTakeover } from './right-sidebar/store' +import { PersistentTerminal, TerminalSlot } from './right-sidebar/terminal/persistent' +import { CRON_ROUTE, NEW_CHAT_ROUTE, routeSessionId, sessionRoute, SETTINGS_ROUTE } from './routes' +import { SessionPickerOverlay } from './session-picker-overlay' +import { SessionSwitcher } from './session-switcher' +import { useContextSuggestions } from './session/hooks/use-context-suggestions' +import { useCwdActions } from './session/hooks/use-cwd-actions' +import { useHermesConfig } from './session/hooks/use-hermes-config' +import { useMessageStream } from './session/hooks/use-message-stream' +import { useModelControls } from './session/hooks/use-model-controls' +import { usePreviewRouting } from './session/hooks/use-preview-routing' +import { usePromptActions } from './session/hooks/use-prompt-actions' +import { useRouteResume } from './session/hooks/use-route-resume' +import { useSessionActions } from './session/hooks/use-session-actions' +import { useSessionStateCache } from './session/hooks/use-session-state-cache' +import { AppShell } from './shell/app-shell' +import { useOverlayRouting } from './shell/hooks/use-overlay-routing' +import { useStatusSnapshot } from './shell/hooks/use-status-snapshot' +import { useStatusbarItems } from './shell/hooks/use-statusbar-items' +import { ModelMenuPanel } from './shell/model-menu-panel' +import type { StatusbarItem } from './shell/statusbar-controls' +import type { TitlebarTool } from './shell/titlebar-controls' +import { useGroupRegistry } from './shell/use-group-registry' +import { UpdatesOverlay } from './updates-overlay' + +const AgentsView = lazy(async () => ({ default: (await import('./agents')).AgentsView })) +const ArtifactsView = lazy(async () => ({ default: (await import('./artifacts')).ArtifactsView })) +const CommandCenterView = lazy(async () => ({ default: (await import('./command-center')).CommandCenterView })) +const CronView = lazy(async () => ({ default: (await import('./cron')).CronView })) +const MessagingView = lazy(async () => ({ default: (await import('./messaging')).MessagingView })) +const ProfilesView = lazy(async () => ({ default: (await import('./profiles')).ProfilesView })) +const SettingsView = lazy(async () => ({ default: (await import('./settings')).SettingsView })) +const SkillsView = lazy(async () => ({ default: (await import('./skills')).SkillsView })) + +// Latest cron-job sessions surfaced in the collapsed "Cron jobs" section. The +// Cron sessions are written by a background scheduler tick (the desktop +// backend), so no user action signals the UI. Poll the bounded cron list on +// this cadence while the app is open + visible so new runs surface promptly +// instead of waiting for the next user-triggered refreshSessions(). +const CRON_POLL_INTERVAL_MS = 30_000 +// The recents list is local-only: cron rows have their own section, and each +// messaging platform (telegram, discord, …) is fetched separately into its own +// self-managed sidebar section (refreshMessagingSessions). Excluding both here +// keeps "Load more" paging through interactive local chats instead of +// interleaving gateway threads that bury them. +const SIDEBAR_EXCLUDED_SOURCES = ['cron', 'subagent', 'tool', ...MESSAGING_SESSION_SOURCE_IDS] +// The messaging slice is the inverse: drop cron + every local source so only +// external-platform conversations remain, then split per platform in the UI. +const MESSAGING_EXCLUDED_SOURCES = ['cron', ...LOCAL_SESSION_SOURCE_IDS] + +// Cheap signature compare so the poll only swaps the atom (and re-renders the +// sidebar) when the visible cron rows actually changed. +function sameCronSignature(a: SessionInfo[], b: SessionInfo[]): boolean { + if (a.length !== b.length) { + return false + } + + return a.every((session, i) => session.id === b[i]?.id && session.title === b[i]?.title) +} + +// Rows a session refresh must preserve even if the aggregator omits them: +// in-flight first turns (message_count 0), pinned rows aged off the page, the +// actively-viewed chat (its "working" flag clears a beat before the aggregator +// sees the persisted row), and sessions whose turn just settled (same race, but +// for a chat the user has already navigated away from). Pass `scope` to only +// keep the active row when it belongs to the profile being paged. +function sessionsToKeep(scope?: string): Set { + const keep = new Set([ + ...$workingSessionIds.get(), + ...$pinnedSessionIds.get(), + ...getRecentlySettledSessionIds() + ]) + + const active = $selectedStoredSessionId.get() + + if (active) { + const session = scope ? $sessions.get().find(s => s.id === active) : null + + if (!scope || !session || normalizeProfileKey(session.profile) === scope) { + keep.add(active) + } + } + + return keep +} + +export function DesktopController() { + const queryClient = useQueryClient() + const location = useLocation() + const navigate = useNavigate() + + const busyRef = useRef(false) + const creatingSessionRef = useRef(false) + const refreshSessionsRequestRef = useRef(0) + + const gatewayState = useStore($gatewayState) + const activeSessionId = useStore($activeSessionId) + const currentCwd = useStore($currentCwd) + const freshDraftReady = useStore($freshDraftReady) + const resumeFailedSessionId = useStore($resumeFailedSessionId) + const resumeExhaustedSessionId = useStore($resumeExhaustedSessionId) + const filePreviewTarget = useStore($filePreviewTarget) + const previewTarget = useStore($previewTarget) + const selectedStoredSessionId = useStore($selectedStoredSessionId) + const terminalTakeover = useStore($terminalTakeover) + const reviewOpen = useStore($reviewOpen) + const fileBrowserOpen = useStore($fileBrowserOpen) + const previewPaneOpen = useStore($paneOpen(PREVIEW_PANE_ID)) + const panesFlipped = useStore($panesFlipped) + const profileScope = useStore($profileScope) + // Below SIDEBAR_COLLAPSE_BREAKPOINT_PX there's no room for a docked rail — + // collapse both sidebars (without touching their stored open state) so the + // hover-reveal overlay becomes the way in. Restores once it's wide again. + const narrowViewport = useMediaQuery(SIDEBAR_COLLAPSE_MEDIA_QUERY) + + const routedSessionId = routeSessionId(location.pathname) + const routeToken = `${location.pathname}:${location.search}:${location.hash}` + const routeTokenRef = useRef(routeToken) + routeTokenRef.current = routeToken + const getRouteToken = useCallback(() => routeTokenRef.current, []) + + const { + agentsOpen, + chatOpen, + closeOverlayToPreviousRoute, + commandCenterInitialSection, + commandCenterOpen, + cronOpen, + currentView, + openAgents, + openCommandCenterSection, + profilesOpen, + settingsOpen, + toggleCommandCenter + } = useOverlayRouting() + + const terminalSidebarOpen = chatOpen && terminalTakeover + + const titlebarToolGroups = useGroupRegistry() + const statusbarItemGroups = useGroupRegistry() + const setTitlebarToolGroup = titlebarToolGroups.set + const setStatusbarItemGroup = statusbarItemGroups.set + + const { + activeSessionIdRef, + ensureSessionState, + runtimeIdByStoredSessionIdRef, + selectedStoredSessionIdRef, + sessionStateByRuntimeIdRef, + syncSessionStateToView, + updateSessionState + } = useSessionStateCache({ + activeSessionId, + busyRef, + selectedStoredSessionId, + setAwaitingResponse, + setBusy, + setMessages + }) + + const { connectionRef, gatewayRef, requestGateway } = useGatewayRequest() + + useEffect(() => { + window.hermesDesktop?.setPreviewShortcutActive?.(Boolean(chatOpen && (filePreviewTarget || previewTarget))) + }, [chatOpen, filePreviewTarget, previewTarget]) + + useEffect(() => { + startUpdatePoller() + const unsubscribe = window.hermesDesktop?.onOpenUpdatesRequested?.(() => openUpdatesWindow()) + + return () => { + unsubscribe?.() + stopUpdatePoller() + } + }, []) + + // Remember the open chat so a relaunch reopens it instead of an empty new-chat. + useEffect(() => { + if (routedSessionId) { + setRememberedSessionId(routedSessionId) + } + }, [routedSessionId]) + + // Restore that chat once, on cold start only (we're at the new-chat route and + // haven't navigated yet). A dead/deleted id self-clears via the exhausted latch + // below, so we never boot-loop into an error screen. + const restoredLastSessionRef = useRef(false) + useEffect(() => { + if (restoredLastSessionRef.current) { + return + } + + restoredLastSessionRef.current = true + const last = getRememberedSessionId() + + if (last && location.pathname === NEW_CHAT_ROUTE) { + navigate(sessionRoute(last), { replace: true }) + } + }, [location.pathname, navigate]) + + useEffect(() => { + if (resumeExhaustedSessionId && getRememberedSessionId() === resumeExhaustedSessionId) { + setRememberedSessionId(null) + } + }, [resumeExhaustedSessionId]) + + // Notification click: the main process already focused the window; jump to its + // session. Notifications are tagged with the gateway *runtime* session id, but + // the chat route is keyed by the *stored* id — navigating with the runtime id + // resumes a non-existent stored session ("session not found") and strands the + // user. Translate runtime -> stored before navigating. + useEffect(() => { + const unsubscribe = window.hermesDesktop?.onFocusSession?.(sessionId => { + if (sessionId) { + navigate(sessionRoute(storedSessionIdForNotification(sessionId, runtimeIdByStoredSessionIdRef.current))) + } + }) + + return () => unsubscribe?.() + }, [navigate, runtimeIdByStoredSessionIdRef]) + + // Notification action button (Approve/Reject) — resolve in place, no navigation. + useEffect(() => { + const unsubscribe = window.hermesDesktop?.onNotificationAction?.(({ actionId, sessionId }) => { + void respondToApprovalAction(sessionId ?? null, actionId) + }) + + return () => unsubscribe?.() + }, []) + + // hermes:// deep links (e.g. a docs "Send to App" button for an automation blueprint). + // Build the equivalent /blueprint slash command from the payload and drop + // it into the composer — the user reviews/edits, then sends; the agent (or + // the shared command handler) creates the job. Signal readiness so a link + // that arrived during boot is flushed exactly once. + useEffect(() => { + const unsubscribe = window.hermesDesktop?.onDeepLink?.(payload => { + if (!payload || payload.kind !== 'blueprint' || !payload.name) { + return + } + + const slots = Object.entries(payload.params || {}) + .map(([k, v]) => { + const sval = /\s/.test(v) ? `"${v.replace(/"/g, '\\"')}"` : v + + return `${k}=${sval}` + }) + .join(' ') + + const command = `/blueprint ${payload.name}${slots ? ' ' + slots : ''}` + requestComposerInsert(command, { mode: 'block', target: 'main' }) + requestComposerFocus('main') + }) + + // Tell the main process the renderer is ready to receive deep links. + void window.hermesDesktop?.signalDeepLinkReady?.() + + return () => unsubscribe?.() + }, []) + + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + if (!$filePreviewTarget.get() && !$previewTarget.get()) { + return + } + + if ((event.metaKey || event.ctrlKey) && !event.altKey && !event.shiftKey && event.key.toLowerCase() === 'w') { + event.preventDefault() + event.stopPropagation() + closeActiveRightRailTab() + } + } + + const unsubscribe = window.hermesDesktop?.onClosePreviewRequested?.(closeActiveRightRailTab) + + window.addEventListener('keydown', onKeyDown, { capture: true }) + + return () => { + unsubscribe?.() + window.removeEventListener('keydown', onKeyDown, { capture: true }) + } + }, []) + + // Cron-job sessions as their own list (latest N). Independent of the recents + // page so the two never compete for slots. Cheap + bounded. Kept (even though + // the sidebar now lists cron *jobs*, not run sessions) so a pinned cron run + // still resolves into the Pinned section via sessionByAnyId. + const refreshCronSessions = useCallback(async () => { + try { + const { sessions } = await listAllProfileSessions(CRON_SECTION_LIMIT, 1, 'exclude', 'recent', 'all', { + source: 'cron' + }) + + setCronSessions(prev => (sameCronSignature(prev, sessions) ? prev : sessions)) + } catch { + // Non-fatal: the cron section just stays empty/stale. + } + }, []) + + // Messaging-platform sessions as their own slice, fetched separately from + // local recents so each platform renders a self-managed section and never + // competes with local chats for the recents page budget. One combined fetch + // seeds every platform; the sidebar splits the rows per source. + const refreshMessagingSessions = useCallback(async () => { + try { + const result = await listAllProfileSessions(MESSAGING_SECTION_LIMIT, 1, 'exclude', 'recent', 'all', { + excludeSources: MESSAGING_EXCLUDED_SOURCES + }) + + // Drop any non-messaging source the broad exclude didn't catch (custom + // sources) — those stay in local recents, not a platform section. + const rows = result.sessions.filter(s => isMessagingSource(s.source)) + + setMessagingSessions(prev => (sameCronSignature(prev, rows) ? prev : rows)) + // Hit the cap → at least one platform may have more on disk than loaded, + // so platform sections offer their own per-platform "load more". + setMessagingTruncated(result.sessions.length >= MESSAGING_SECTION_LIMIT) + } catch { + // Non-fatal: the messaging sections just stay empty/stale. + } + }, []) + + // Page a single platform's section independently (mirrors the per-profile + // pager): fetch that source's next window and merge it back in place, leaving + // every other platform's rows untouched. Resolves the platform's exact total. + const loadMoreMessagingForPlatform = useCallback(async (platform: string) => { + const inPlatform = (s: SessionInfo) => normalizeSessionSource(s.source) === platform + const loaded = $messagingSessions.get().filter(inPlatform).length + + const result = await listAllProfileSessions(loaded + SIDEBAR_SESSIONS_PAGE_SIZE, 1, 'exclude', 'recent', 'all', { + source: platform + }) + + const incoming = result.sessions.filter(s => normalizeSessionSource(s.source) === platform) + + setMessagingSessions(prev => [ + ...prev.filter(s => !inPlatform(s)), + ...mergeSessionPage(prev.filter(inPlatform), incoming, sessionsToKeep()) + ]) + + const total = result.total ?? incoming.length + setMessagingPlatformTotals(prev => ({ ...prev, [platform]: Math.max(total, incoming.length) })) + }, []) + + // Cron *jobs* drive the sidebar "Cron jobs" section. Jobs are created + // synchronously (agent tool call or the cron UI), so refreshing here right + // after an agent turn surfaces a new job immediately; the interval poll keeps + // next-run/state fresh as the scheduler advances them. + const refreshCronJobs = useCallback(async () => { + try { + const jobs = await getCronJobs() + + setCronJobs(jobs) + } catch { + // Non-fatal: the cron section just keeps its last-known jobs. + } + }, []) + + const refreshSessions = useCallback(async () => { + const requestId = refreshSessionsRequestRef.current + 1 + refreshSessionsRequestRef.current = requestId + setSessionsLoading(true) + + try { + const limit = $sessionsLimit.get() + + // Require at least one message so abandoned/empty "Untitled" drafts (one + // was created per TUI/desktop launch before the lazy-create fix) don't + // clutter the sidebar. + // Unified cross-profile list (served read-only off each profile's + // state.db; no per-profile backend is spawned). Single-profile users get + // the same rows tagged profile="default". Cron sessions are excluded here + // and fetched separately (refreshCronSessions) so the scheduler's + // always-newest rows can't consume the recents page budget. + // Scope the fetch to the active profile (not always 'all') so a profile + // with few recent sessions isn't windowed out of the cross-profile + // recency page — the empty-history-on-profile-switch bug. + const sessionProfile = profileScope === ALL_PROFILES ? 'all' : profileScope + + const result = await listAllProfileSessions(limit, 1, 'exclude', 'recent', sessionProfile, { + excludeSources: SIDEBAR_EXCLUDED_SOURCES + }) + + if (refreshSessionsRequestRef.current === requestId) { + setSessions(prev => mergeSessionPage(prev, result.sessions, sessionsToKeep())) + setSessionsTotal(typeof result.total === 'number' ? result.total : result.sessions.length) + setSessionProfileTotals(result.profile_totals ?? {}) + } + } finally { + if (refreshSessionsRequestRef.current === requestId) { + setSessionsLoading(false) + } + } + + void refreshCronSessions() + void refreshCronJobs() + void refreshMessagingSessions() + }, [profileScope, refreshCronSessions, refreshCronJobs, refreshMessagingSessions]) + + const loadMoreSessions = useCallback(async () => { + bumpSessionsLimit() + await refreshSessions() + }, [refreshSessions]) + + // Another window mutated the shared session list (e.g. a chat started in the + // pop-out). Re-pull so the sidebar reflects it. Pop-outs have no sidebar, so + // only real windows bother. + useEffect(() => { + if (isSecondaryWindow()) { + return + } + + return onSessionsChanged(() => void refreshSessions().catch(() => undefined)) + }, [refreshSessions]) + + // ALL-profiles view pages one profile at a time: fetch that profile's next + // page and merge it in place, leaving every other profile's rows untouched. + const loadMoreSessionsForProfile = useCallback(async (profile: string) => { + const key = normalizeProfileKey(profile) + const inKey = (s: SessionInfo) => normalizeProfileKey(s.profile) === key + const loaded = $sessions.get().filter(inKey).length + + const result = await listAllProfileSessions(loaded + SIDEBAR_SESSIONS_PAGE_SIZE, 1, 'exclude', 'recent', key, { + excludeSources: SIDEBAR_EXCLUDED_SOURCES + }) + + const keep = sessionsToKeep(key) + + setSessions(prev => [ + ...prev.filter(s => !inKey(s)), + ...mergeSessionPage(prev.filter(inKey), result.sessions, keep) + ]) + + const total = result.profile_totals?.[key] ?? result.total ?? result.sessions.length + setSessionProfileTotals(prev => ({ ...prev, [key]: Math.max(total, result.sessions.length) })) + }, []) + + const toggleSelectedPin = useCallback(() => { + const sessionId = $selectedStoredSessionId.get() + + if (!sessionId) { + return + } + + // Pin on the durable lineage-root id so the pin survives auto-compression. + const session = $sessions.get().find(s => s.id === sessionId || s._lineage_root_id === sessionId) + const pinId = session ? sessionPinId(session) : sessionId + + if ($pinnedSessionIds.get().includes(pinId)) { + unpinSession(pinId) + } else { + pinSession(pinId) + } + }, []) + + const { gatewayLogLines, inferenceStatus, statusSnapshot } = useStatusSnapshot(gatewayState, requestGateway) + + const updateActiveSessionRuntimeInfo = useCallback( + (info: { branch?: string; cwd?: string }) => { + const sessionId = activeSessionIdRef.current + + if (!sessionId) { + return + } + + updateSessionState(sessionId, state => ({ + ...state, + branch: info.branch ?? state.branch, + cwd: info.cwd ?? state.cwd + })) + }, + [activeSessionIdRef, updateSessionState] + ) + + const { refreshProjectBranch } = useCwdActions({ + activeSessionId, + activeSessionIdRef, + onSessionRuntimeInfo: updateActiveSessionRuntimeInfo, + requestGateway + }) + + const { refreshHermesConfig, sttEnabled, voiceMaxRecordingSeconds } = useHermesConfig({ + activeSessionIdRef, + refreshProjectBranch + }) + + const { refreshCurrentModel, selectModel, updateModelOptionsCache } = useModelControls({ + activeSessionId, + queryClient, + requestGateway + }) + + const openProviderSettings = useCallback(() => { + navigate(`${SETTINGS_ROUTE}?tab=providers`) + }, [navigate]) + + const modelMenuContent = useMemo( + () => + gatewayState === 'open' ? ( + + ) : null, + [gatewayRef, gatewayState, requestGateway, selectModel] + ) + + useContextSuggestions({ + activeSessionId, + activeSessionIdRef, + currentCwd, + gatewayState, + requestGateway + }) + + const hydrateFromStoredSession = useCallback( + async ( + attempts = 1, + storedSessionId = selectedStoredSessionIdRef.current, + runtimeSessionId = activeSessionIdRef.current + ) => { + if (!storedSessionId || !runtimeSessionId) { + return + } + + const storedProfile = $sessions + .get() + .find(session => session.id === storedSessionId || session._lineage_root_id === storedSessionId)?.profile + + for (let index = 0; index < Math.max(1, attempts); index += 1) { + try { + const latest = await getSessionMessages(storedSessionId, storedProfile) + const messages = toChatMessages(latest.messages) + updateSessionState( + runtimeSessionId, + state => ({ + ...state, + messages: preserveLocalAssistantErrors(messages, state.messages) + }), + storedSessionId + ) + + // Seed the status stack's todo group from history — but only while + // the plan is still in flight, so reopening an old chat doesn't pin + // its finished todo list above the composer forever. + const todos = latestSessionTodos(messages) + + if (todos && todoListActive(todos)) { + setSessionTodos(runtimeSessionId, todos) + } else { + clearSessionTodos(runtimeSessionId) + } + + return + } catch { + // Best-effort fallback when live stream payloads are empty. + } + + if (index < attempts - 1) { + await new Promise(resolve => window.setTimeout(resolve, 250)) + } + } + }, + [activeSessionIdRef, selectedStoredSessionIdRef, updateSessionState] + ) + + const { handleGatewayEvent } = useMessageStream({ + activeSessionIdRef, + hydrateFromStoredSession, + queryClient, + refreshHermesConfig, + refreshSessions, + sessionStateByRuntimeIdRef, + updateSessionState + }) + + const { handleDesktopGatewayEvent, restartPreviewServer } = usePreviewRouting({ + activeSessionIdRef, + baseHandleGatewayEvent: handleGatewayEvent, + currentCwd, + currentView, + requestGateway, + routedSessionId, + selectedStoredSessionId + }) + + const { + archiveSession, + branchCurrentSession, + branchStoredSession, + createBackendSessionForSend, + openSettings, + removeSession, + resumeSession, + selectSidebarItem, + startFreshSessionDraft + } = useSessionActions({ + activeSessionId, + activeSessionIdRef, + busyRef, + creatingSessionRef, + ensureSessionState, + getRouteToken, + navigate, + requestGateway, + runtimeIdByStoredSessionIdRef, + selectedStoredSessionId, + selectedStoredSessionIdRef, + sessionStateByRuntimeIdRef, + syncSessionStateToView, + updateSessionState + }) + + // Single global listener for every rebindable hotkey (incl. profile switching) + // plus the on-screen keybind editor's capture mode. + useKeybinds({ + startFreshSession: startFreshSessionDraft, + toggleCommandCenter, + toggleSelectedPin + }) + + // A profile switch/create drops to a fresh new-session draft so the previously + // open session doesn't bleed across contexts. Skip the initial value. + const freshSessionRequest = useStore($freshSessionRequest) + const lastFreshRef = useRef(freshSessionRequest) + + useEffect(() => { + if (freshSessionRequest === lastFreshRef.current) { + return + } + + lastFreshRef.current = freshSessionRequest + startFreshSessionDraft() + }, [freshSessionRequest, startFreshSessionDraft]) + + // Swapping the live gateway to another profile must re-pull that profile's + // global model + active-profile pill. Both are nanostores, so the blanket + // invalidateQueries() the profile store fires on swap doesn't touch them — + // without this the statusbar keeps showing the previous profile's model + // (the "forgets the LLM setting" report). gatewayState stays 'open' across a + // swap (background sockets persist), so the open→open effect won't re-run. + const activeGatewayProfile = useStore($activeGatewayProfile) + const lastGatewayProfileRef = useRef(activeGatewayProfile) + + useEffect(() => { + if (activeGatewayProfile === lastGatewayProfileRef.current) { + return + } + + lastGatewayProfileRef.current = activeGatewayProfile + // Force: the new profile has its own default, so reseed even if the composer + // already shows the previous profile's model. + void refreshCurrentModel(true) + void refreshActiveProfile() + }, [activeGatewayProfile, refreshCurrentModel]) + + const composer = useComposerActions({ + activeSessionId, + currentCwd, + requestGateway + }) + + const branchInNewChat = useCallback( + async (messageId?: string) => { + const branched = await branchCurrentSession(messageId) + + if (branched) { + await refreshSessions().catch(() => undefined) + } + + return branched + }, + [branchCurrentSession, refreshSessions] + ) + + // Clear a failed turn's red error banner from the transcript. Errors are + // renderer-local state (never persisted), so dismissing is purely a view + + // session-cache edit. A message that errored before emitting any visible + // text is a bare error placeholder → drop it entirely; one that streamed + // partial output then failed keeps its content and just sheds the error. + // Both the per-runtime cache AND the live $messages view must be updated: + // `preserveLocalAssistantErrors` re-grafts any still-errored message it + // finds in the view onto the next session.info flush, so clearing only the + // cache would let the heartbeat resurrect the banner. + const dismissError = useCallback( + (messageId: string) => { + const runtimeSessionId = activeSessionIdRef.current + + if (!runtimeSessionId) { + return + } + + const clearErrorIn = (messages: ChatMessage[]): ChatMessage[] => + messages.flatMap(message => { + if (message.id !== messageId || !message.error) { + return [message] + } + + if (!chatMessageText(message).trim() && !message.parts.some(part => part.type !== 'text')) { + return [] + } + + return [{ ...message, error: undefined, pending: false }] + }) + + // View first: the flush below reads $messages as the "current" baseline + // for error preservation, so the banner must be gone from it before the + // cache update triggers a re-sync. + setMessages(clearErrorIn($messages.get())) + + updateSessionState(runtimeSessionId, state => ({ + ...state, + messages: clearErrorIn(state.messages) + })) + }, + [activeSessionIdRef, updateSessionState] + ) + + const startSessionInWorkspace = useCallback( + (path: null | string) => { + startFreshSessionDraft() + + // A worktree lane carries its own path; the trunk "+" can be path-less (the + // main checkout is implicit), so fall back to the active project's root + // instead of no-op'ing on null — that was "+ on main does nothing". + const target = path?.trim() || resolveNewSessionCwd() + + if (!target) { + return + } + + // The next message creates the backend session in $currentCwd, so seed + // it (and the branch) from the workspace the user clicked the + on. + setCurrentCwd(target) + void requestGateway<{ branch?: string; cwd?: string }>('config.get', { key: 'project', cwd: target }) + .then(info => { + const resolved = info.cwd || target + + setCurrentCwd(resolved) + setCurrentBranch(info.branch || '') + + // An EXPLICIT target (a worktree/lane path — e.g. just-created via + // "convert a branch" / "new worktree") drills the sidebar into that + // project so the new lane is visible at once. Without this, a brand-new + // worktree session is invisible from the all-projects overview (the + // live overlay skips `.worktrees` rows, and the session.info cwd-follow + // only fires on a same-session move, not a fresh session). The + // path-less trunk "+" keeps the current scope untouched. + if (path?.trim()) { + restoreWorktree(resolved) + void followActiveSessionCwd(resolved) + } + }) + .catch(() => undefined) + }, + [requestGateway, startFreshSessionDraft] + ) + + // Composer "branch off into a new worktree": the composer already created the + // worktree and cleared its draft; open a fresh session anchored to that tree, + // then prefill the task that kicked it off. startSessionInWorkspace owns the + // reset+cwd seed (it runs startFreshSessionDraft, which would otherwise stomp + // the cwd back to the default), so the prefill is dispatched right after — its + // deferred event lands once the fresh composer has remounted and rebound. + const startWorkSessionRequest = useStore($startWorkSessionRequest) + const lastStartWorkTokenRef = useRef(startWorkSessionRequest?.token ?? 0) + + useEffect(() => { + if (!startWorkSessionRequest || startWorkSessionRequest.token === lastStartWorkTokenRef.current) { + return + } + + lastStartWorkTokenRef.current = startWorkSessionRequest.token + startSessionInWorkspace(startWorkSessionRequest.path) + + if (startWorkSessionRequest.draft) { + requestComposerInsert(startWorkSessionRequest.draft, { target: 'main' }) + } + }, [startSessionInWorkspace, startWorkSessionRequest]) + + const handleSkinCommand = useSkinCommand() + + const { + cancelRun, + editMessage, + handleThreadMessagesChange, + reloadFromMessage, + restoreToMessage, + steerPrompt, + submitText, + transcribeVoiceAudio + } = usePromptActions({ + activeSessionId, + activeSessionIdRef, + branchCurrentSession: branchInNewChat, + busyRef, + createBackendSessionForSend, + handleSkinCommand, + refreshSessions, + requestGateway, + resumeStoredSession: resumeSession, + selectedStoredSessionIdRef, + startFreshSessionDraft, + sttEnabled, + updateSessionState + }) + + // The popped-out pet drives two actions back into the app: send a prompt, and + // open the most recent thread. Both are registered ONCE through refs that track + // the latest callbacks — re-registering on every `submitText`/`resumeSession` + // identity change left a brief window where the handler was nulled (cleanup + // before re-register), which could drop a submit fired from the overlay (e.g. + // creating a session from the new-session screen). The ref form keeps a stable, + // always-current handler. Primary window only — it owns the overlay. + const submitTextRef = useRef(submitText) + submitTextRef.current = submitText + const resumeSessionRef = useRef(resumeSession) + resumeSessionRef.current = resumeSession + const requestGatewayRef = useRef(requestGateway) + requestGatewayRef.current = requestGateway + + useEffect(() => { + if (isSecondaryWindow()) { + return + } + + setPetOverlaySubmitHandler(text => void submitTextRef.current(text)) + // Alt+wheel resize from the popped-out pet — persist it through this + // window's gateway (the overlay has none) so it survives restart. + setPetOverlayScaleHandler(scale => setPetScale(requestGatewayRef.current, scale)) + // Mail icon: $sessions is ordered most-recent-first; the pet is global (not + // per session) so "most recent" is the right target. main.cjs already raised + // the window before forwarding this. + setPetOverlayOpenAppHandler(() => { + const recent = $sessions.get()[0] + + if (recent?.id) { + void resumeSessionRef.current(recent.id) + } + }) + + return () => { + setPetOverlaySubmitHandler(null) + setPetOverlayOpenAppHandler(null) + setPetOverlayScaleHandler(null) + } + }, []) + + // Mirror "a session is blocked on the user" (clarify/approval) into the pet's + // awaitingInput flag so it shows the `waiting` pose. Lives on $petActivity so + // it rides the same atom the pop-out overlay mirrors — no session list needed + // there. Every window keeps its own in-window pet in sync. + useEffect(() => { + const sync = () => setPetActivity({ awaitingInput: $attentionSessionIds.get().length > 0 }) + + sync() + + return $attentionSessionIds.listen(sync) + }, []) + + useGatewayBoot({ + handleGatewayEvent: handleDesktopGatewayEvent, + onConnectionReady: c => { + connectionRef.current = c + }, + onGatewayReady: g => { + gatewayRef.current = g + }, + refreshHermesConfig, + refreshSessions + }) + + useEffect(() => { + if (gatewayState === 'open') { + void refreshCurrentModel() + void refreshActiveProfile() + void refreshSessions().catch(() => undefined) + } + }, [gatewayState, refreshCurrentModel, refreshSessions]) + + // "Hey Hermes" wake word: arm the server-side detector for this surface + // (gated on config) and open a fresh session when it fires. Idempotent and + // self-cleaning across reconnects. + useEffect(() => { + if (gatewayState !== 'open') { + return + } + void requestGateway('wake.start', { surface: 'gui' }).catch(() => undefined) + const gw = gatewayRef.current + if (!gw) { + return + } + return gw.on('wake.detected', () => startFreshSessionDraft()) + }, [gatewayState, requestGateway, gatewayRef, startFreshSessionDraft]) + + // Keep the cron jobs section live without a user action: the scheduler ticks + // in the background (advancing next-run/state and creating runs), so poll the + // job list on an interval (and on tab re-focus) while connected. + useEffect(() => { + if (gatewayState !== 'open') { + return + } + + const tick = () => { + if (document.visibilityState === 'visible') { + void refreshCronJobs() + } + } + + const intervalId = window.setInterval(tick, CRON_POLL_INTERVAL_MS) + document.addEventListener('visibilitychange', tick) + + return () => { + window.clearInterval(intervalId) + document.removeEventListener('visibilitychange', tick) + } + }, [gatewayState, refreshCronJobs]) + + useEffect(() => { + if (gatewayState === 'open' && !activeSessionId && freshDraftReady) { + void refreshCurrentModel() + void refreshHermesConfig() + } + }, [activeSessionId, freshDraftReady, gatewayState, refreshCurrentModel, refreshHermesConfig]) + + useRouteResume({ + activeSessionId, + activeSessionIdRef, + creatingSessionRef, + currentView, + freshDraftReady, + gatewayState, + locationPathname: location.pathname, + resumeSession, + resumeFailedSessionId, + resumeExhaustedSessionId, + routedSessionId, + runtimeIdByStoredSessionIdRef, + selectedStoredSessionId, + selectedStoredSessionIdRef, + startFreshSessionDraft + }) + + const { leftStatusbarItems, statusbarItems } = useStatusbarItems({ + agentsOpen, + chatOpen, + commandCenterOpen, + extraLeftItems: statusbarItemGroups.flat.left, + extraRightItems: statusbarItemGroups.flat.right, + gatewayLogLines, + gatewayState, + inferenceStatus, + openAgents, + freshDraftReady, + openCommandCenterSection, + requestGateway, + statusSnapshot, + toggleCommandCenter + }) + + const sidebar = ( + void archiveSession(sessionId)} + onBranchSession={sessionId => void branchStoredSession(sessionId)} + onDeleteSession={sessionId => void removeSession(sessionId)} + onLoadMoreMessaging={loadMoreMessagingForPlatform} + onLoadMoreProfileSessions={loadMoreSessionsForProfile} + onLoadMoreSessions={loadMoreSessions} + onManageCronJob={jobId => { + setCronFocusJobId(jobId) + navigate(CRON_ROUTE) + }} + onNavigate={selectSidebarItem} + onNewSessionInWorkspace={startSessionInWorkspace} + onResumeSession={sessionId => navigate(sessionRoute(sessionId))} + onTriggerCronJob={jobId => { + void triggerCronJob(jobId) + .then(() => refreshCronJobs()) + .catch(() => undefined) + }} + /> + ) + + // One PTY-backed terminal mounted forever; placeholders decide + // where it shows. Lives in main's stacking context (not the root overlay layer) + // so pane resize handles still paint above it. Toggling never rebuilds the shell. + const mainOverlays = ( + + ) + + const overlays = ( + <> + + {!isSecondaryWindow() && } + {!isSecondaryWindow() && ( + { + void refreshHermesConfig() + void refreshCurrentModel() + void queryClient.invalidateQueries({ queryKey: ['model-options'] }) + }} + requestGateway={requestGateway} + /> + )} + + + + + + + + + + + + {settingsOpen && ( + + { + void refreshHermesConfig() + void refreshCurrentModel() + void queryClient.invalidateQueries({ queryKey: ['model-options'] }) + }} + onMainModelChanged={(provider, model) => { + setCurrentProvider(provider) + setCurrentModel(model) + updateModelOptionsCache(provider, model, true) + void refreshCurrentModel() + void queryClient.invalidateQueries({ queryKey: ['model-options'] }) + }} + /> + + )} + + {commandCenterOpen && ( + + navigate(path)} + onOpenSession={sessionId => navigate(sessionRoute(sessionId))} + /> + + )} + + {agentsOpen && ( + + + + )} + + {cronOpen && ( + + navigate(sessionRoute(sessionId))} + /> + + )} + + {profilesOpen && ( + + + + )} + + ) + + const chatView = ( + composer.addContextRefAttachment(`@url:${formatRefValue(url)}`, url)} + onAttachDroppedItems={composer.attachDroppedItems} + onAttachImageBlob={composer.attachImageBlob} + onBranchInNewChat={branchInNewChat} + onCancel={cancelRun} + onDeleteSelectedSession={() => { + if (selectedStoredSessionId) { + void removeSession(selectedStoredSessionId) + } + }} + onDismissError={dismissError} + onEdit={editMessage} + onPasteClipboardImage={opts => composer.pasteClipboardImage(opts)} + onPickFiles={() => void composer.pickContextPaths('file')} + onPickFolders={() => void composer.pickContextPaths('folder')} + onPickImages={() => void composer.pickImages()} + onReload={reloadFromMessage} + onRemoveAttachment={id => void composer.removeAttachment(id)} + onRestoreToMessage={restoreToMessage} + onRetryResume={sessionId => void resumeSession(sessionId, true)} + onSteer={steerPrompt} + onSubmit={submitText} + onThreadMessagesChange={handleThreadMessagesChange} + onToggleSelectedPin={toggleSelectedPin} + onTranscribeAudio={transcribeVoiceAudio} + /> + ) + + // Flipped layout mirrors the default: sessions sidebar → right, file + // browser + preview rail → left. Same panes, swapped sides. + const sidebarSide = panesFlipped ? 'right' : 'left' + const railSide = panesFlipped ? 'left' : 'right' + + // Other sidebars docked as real columns on the terminal's rail. Force-collapsed + // hover-reveal overlays (narrow window) don't take a column, so they don't count. + const railColumnOpen = + (chatOpen && Boolean(previewTarget || filePreviewTarget) && previewPaneOpen) || + (chatOpen && !narrowViewport && fileBrowserOpen) || + (chatOpen && Boolean(currentCwd.trim()) && !narrowViewport && reviewOpen) + + // Once the terminal would share its rail with another sidebar, drop it to a + // full-width row beneath them rather than cramming in one more skinny column. + const terminalAsRow = terminalSidebarOpen && railColumnOpen + + const previewPane = ( + + {chatOpen ? ( + + ) : null} + + ) + + const fileBrowserPane = ( + + {/* Key on the project (cwd) so switching projects unmounts the old tree and + mounts a fresh one straight into its skeleton — no stale-then-blip. */} + composer.insertContextPathInlineRef(path)} + onActivateFolder={path => composer.insertContextPathInlineRef(path, true)} + /> + + ) + + const reviewPane = ( + + + + ) + + const terminalPane = ( + + {/* As a column the terminal clears the titlebar; as a bottom row it sits + below the rail's panes (so it fills its row edge-to-edge) and gets a + left border separating it from the chat — the column-mode separator + lives on the resize sash, which moves to the top edge as a row. */} +
+ +
+ + ) + + return ( + + {!isSecondaryWindow() && ( + + {sidebar} + + )} + + + + + + + + } + path="skills" + /> + + + + } + path="messaging" + /> + + + + } + path="artifacts" + /> + + + + + + } path="new" /> + } path="sessions/:sessionId" /> + } path="*" /> + + + {/* + Order within a side maps to column order. Default (rail on the right): + main | terminal | preview | file-browser. Flipped (rail on the left): + mirror to file-browser | preview | terminal | main so terminal stays + adjacent to the chat. + */} + {panesFlipped ? fileBrowserPane : terminalPane} + {previewPane} + {reviewPane} + {panesFlipped ? terminalPane : fileBrowserPane} + + ) +} + +function LegacySessionRedirect() { + const { sessionId } = useParams() + + return +} diff --git a/cli.py b/cli.py index 8fe26d43a4b2..042b888e2711 100644 --- a/cli.py +++ b/cli.py @@ -12182,10 +12182,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # threading resume logic through the voice machinery. def _maybe_start_wake_word(self): - """Start the wake-word listener at CLI startup if enabled in config.""" + """Start the wake-word listener at CLI startup if this surface owns it.""" try: - from tools.wake_word import load_wake_word_config - if not load_wake_word_config().get("enabled"): + from tools.wake_word import wake_surface_enabled + if not wake_surface_enabled("cli"): return except Exception: return @@ -12340,6 +12340,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): _cprint(f" State: {'LISTENING' if active else 'OFF'}") _cprint(f" Phrase: \"{reqs['phrase']}\"") _cprint(f" Provider: {reqs['provider']}") + _cprint(f" Surface: {cfg.get('surface', 'auto')}") _cprint(f" New session: {'yes' if cfg.get('start_new_session', True) else 'no'}") if not reqs["available"] and reqs.get("hint"): _cprint(f" {_DIM}{reqs['hint']}{_RST}") diff --git a/hermes_cli/config.py b/hermes_cli/config.py index bc96780e4f8c..4881ddd29008 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2379,6 +2379,7 @@ DEFAULT_CONFIG = { # Off by default; toggle with /wake or `wake_word.enabled: true`. "wake_word": { "enabled": False, + "surface": "auto", # which surface owns the listener / opens the new session: "auto" (the running one) | "cli" | "tui" | "gui" "provider": "openwakeword", # "openwakeword" (free, local) | "porcupine" (premium; needs PORCUPINE_ACCESS_KEY) "phrase": "hey jarvis", # cosmetic label only; detection is keyed by the engine model/keyword below "sensitivity": 0.5, # 0.0-1.0 detection threshold (higher = stricter) diff --git a/tests/tools/test_wake_word.py b/tests/tools/test_wake_word.py index 30ece72368bd..658b4e1e7c9a 100644 --- a/tests/tools/test_wake_word.py +++ b/tests/tools/test_wake_word.py @@ -27,6 +27,21 @@ def test_config_defaults_and_clamping(): assert ww.wake_phrase({}) == "hey jarvis" +def test_wake_surface_enabled_gate(): + # Disabled → never, regardless of surface. + assert ww.wake_surface_enabled("cli", {"enabled": False, "surface": "cli"}) is False + # auto → every surface. + for s in ("cli", "tui", "gui"): + assert ww.wake_surface_enabled(s, {"enabled": True, "surface": "auto"}) is True + # Pinned surface → only that one. + cfg = {"enabled": True, "surface": "tui"} + assert ww.wake_surface_enabled("tui", cfg) is True + assert ww.wake_surface_enabled("cli", cfg) is False + assert ww.wake_surface_enabled("gui", cfg) is False + # Missing/blank surface defaults to auto. + assert ww.wake_surface_enabled("gui", {"enabled": True}) is True + + def test_looks_like_path(): assert ww._looks_like_path("models/hey_hermes.onnx") assert ww._looks_like_path("custom.ppn") diff --git a/tools/wake_word.py b/tools/wake_word.py index 6dc473b211c8..574bd4394cfc 100644 --- a/tools/wake_word.py +++ b/tools/wake_word.py @@ -49,12 +49,16 @@ _FIRE_COOLDOWN_SECONDS = 2.0 _DEFAULTS: Dict[str, Any] = { "enabled": False, + "surface": "auto", "provider": "openwakeword", "phrase": "hey jarvis", "sensitivity": 0.5, "start_new_session": True, } +# Surfaces that can host the listener. "auto" means whichever one is running. +SURFACES = ("cli", "tui", "gui") + def load_wake_word_config() -> Dict[str, Any]: """Return the ``wake_word`` config section, shape-guarded to a dict.""" @@ -91,6 +95,20 @@ def wake_phrase(cfg: Optional[Dict[str, Any]] = None) -> str: return str(_get(cfg, "phrase")) or "hey jarvis" +def wake_surface_enabled(surface: str, cfg: Optional[Dict[str, Any]] = None) -> bool: + """Should ``surface`` (``cli`` / ``tui`` / ``gui``) host the listener? + + True when the wake word is enabled and the configured ``surface`` is either + ``auto`` or this exact surface — the single gate every surface consults so + only one place owns the wake word and the new session it opens. + """ + cfg = cfg if cfg is not None else load_wake_word_config() + if not cfg.get("enabled"): + return False + want = str(_get(cfg, "surface")).strip().lower() or "auto" + return want == "auto" or want == surface.strip().lower() + + # --------------------------------------------------------------------------- # Audio capture (lazy — never import sounddevice at module load) # --------------------------------------------------------------------------- diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 1619181086ef..60d17a865a58 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -925,6 +925,11 @@ def _close_sessions_for_transport( def _shutdown_sessions() -> None: + try: + from tools.wake_word import stop_listening as _stop_wake + _stop_wake() + except Exception: + pass with _sessions_lock: sids = list(_sessions) for sid in sids: @@ -17587,6 +17592,136 @@ def _voice_record_key() -> str: return str(record_key) if isinstance(record_key, str) and record_key else "ctrl+b" +# ── Wake word ("Hey Hermes") ────────────────────────────────────────────── +# The detector is process-global (one mic), like voice. It runs server-side so +# both the TUI and desktop GUI share it; clients pass their surface identity to +# wake.start and the shared gate (wake_surface_enabled) decides whether to arm. +# On detection we emit wake.detected; the client opens a new session and starts +# its own voice capture. The detector yields the mic to gateway voice.record +# (pause/resume below) and to the desktop's browser mic (wake.pause/resume RPCs). +_wake_lock = threading.Lock() +_wake_active = False +_wake_event_sid = "" + + +def _wake_is_active() -> bool: + with _wake_lock: + return _wake_active + + +def _wake_resume_if_active() -> None: + if not _wake_is_active(): + return + try: + from tools.wake_word import resume_listening + resume_listening() + except Exception as e: + logger.debug("wake resume failed: %s", e) + + +def _wake_on_detect() -> None: + """Detector-thread callback: tell the client to open a fresh voice session.""" + with _wake_lock: + sid = _wake_event_sid + try: + from tools.wake_word import wake_phrase + phrase = wake_phrase() + except Exception: + phrase = "" + _emit("wake.detected", sid, {"phrase": phrase}) + + +@method("wake.start") +def _(rid, params: dict) -> dict: + """Arm the wake-word listener for the calling surface ("tui" | "gui"). + + Idempotent and gated: returns ``{started: False, reason}`` when the wake + word is disabled, scoped to another surface, or its deps/mic aren't ready. + """ + global _wake_active, _wake_event_sid + surface = str(params.get("surface") or "auto").strip().lower() + try: + from tools.wake_word import ( + check_wake_word_requirements, + load_wake_word_config, + start_listening, + wake_surface_enabled, + ) + except Exception as e: + return _err(rid, 5026, f"wake module unavailable: {e}") + + cfg = load_wake_word_config() + if not wake_surface_enabled(surface, cfg): + return _ok(rid, {"started": False, "reason": "disabled_for_surface"}) + reqs = check_wake_word_requirements(cfg) + if not reqs["available"]: + return _ok(rid, {"started": False, "reason": reqs.get("hint") or "unavailable"}) + + with _wake_lock: + _wake_event_sid = params.get("session_id") or _wake_event_sid + try: + start_listening(_wake_on_detect, config=cfg) + except Exception as e: + return _err(rid, 5026, str(e)) + with _wake_lock: + _wake_active = True + return _ok(rid, {"started": True, "phrase": reqs["phrase"], "provider": reqs["provider"]}) + + +@method("wake.stop") +def _(rid, params: dict) -> dict: + global _wake_active + with _wake_lock: + _wake_active = False + try: + from tools.wake_word import stop_listening + stop_listening() + except Exception: + pass + return _ok(rid, {"stopped": True}) + + +@method("wake.pause") +def _(rid, params: dict) -> dict: + """Release the mic (e.g. while the desktop's browser captures audio).""" + try: + from tools.wake_word import pause_listening + pause_listening() + except Exception: + pass + return _ok(rid, {"paused": True}) + + +@method("wake.resume") +def _(rid, params: dict) -> dict: + """Reclaim the mic after a pause; no-op if the listener isn't armed.""" + active = _wake_is_active() + if active: + _wake_resume_if_active() + return _ok(rid, {"resumed": active}) + + +@method("wake.status") +def _(rid, params: dict) -> dict: + try: + from tools.wake_word import ( + check_wake_word_requirements, + is_listening, + load_wake_word_config, + ) + cfg = load_wake_word_config() + reqs = check_wake_word_requirements(cfg) + return _ok(rid, { + "listening": _wake_is_active() and is_listening(), + "phrase": reqs["phrase"], + "provider": reqs["provider"], + "available": reqs["available"], + "hint": reqs.get("hint", ""), + }) + except Exception as e: + return _err(rid, 5026, str(e)) + + @method("voice.toggle") def _(rid, params: dict) -> dict: """CLI parity for the ``/voice`` slash command. @@ -17735,12 +17870,28 @@ def _(rid, params: dict) -> dict: if isinstance(duration, (int, float)) and not isinstance(duration, bool) else 3.0 ) + # Hand the mic to STT if the wake-word detector holds it; resume + # once a terminal capture event fires (one-shot transcript / silence + # limit), so wake-triggered and manual captures both coexist. + if _wake_is_active(): + try: + from tools.wake_word import pause_listening + pause_listening() + except Exception: + pass + + def _on_transcript(t): + _voice_emit("voice.transcript", {"text": t}) + _wake_resume_if_active() + + def _on_silent(): + _voice_emit("voice.transcript", {"no_speech_limit": True}) + _wake_resume_if_active() + started = start_continuous( - on_transcript=lambda t: _voice_emit("voice.transcript", {"text": t}), + on_transcript=_on_transcript, on_status=lambda s: _voice_emit("voice.status", {"state": s}), - on_silent_limit=lambda: _voice_emit( - "voice.transcript", {"no_speech_limit": True} - ), + on_silent_limit=_on_silent, silence_threshold=safe_threshold, silence_duration=safe_duration, auto_restart=False, @@ -17756,6 +17907,7 @@ def _(rid, params: dict) -> dict: from hermes_cli.voice import stop_continuous stop_continuous(force_transcribe=True) + _wake_resume_if_active() return _ok(rid, {"status": "stopped"}) except ImportError: return _err( diff --git a/ui-tui/src/app/createGatewayEventHandler.ts b/ui-tui/src/app/createGatewayEventHandler.ts index bbd826138607..d52e6b03ee90 100644 --- a/ui-tui/src/app/createGatewayEventHandler.ts +++ b/ui-tui/src/app/createGatewayEventHandler.ts @@ -621,6 +621,10 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: // "too many re-renders" guard in embedded dashboard PTYs. ensureAgentsNudgeConfig() + // Arm "Hey Hermes" if this surface owns it (server gates on config). + // Fire-and-forget + idempotent server-side, so reconnects are harmless. + void rpc('wake.start', { surface: 'tui' }) + rpc('commands.catalog', {}) .then(r => { if (!r?.pairs) { @@ -936,6 +940,23 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: return } + case 'wake.detected': { + // "Hey Hermes": open a fresh session, then arm voice capture so the + // user can speak their request hands-free. Mirrors the CLI flow. + void (async () => { + await newSession() + const sid = getUiState().sid + if (!sid) { + return + } + setVoiceEnabled(true) + await rpc('voice.toggle', { action: 'on' }) + await rpc('voice.record', { action: 'start', session_id: sid }) + })() + + return + } + case 'gateway.start_timeout': { const { cwd, python, stderr_tail: stderrTail } = ev.payload ?? {} const trace = python || cwd ? ` · ${String(python || '')} ${String(cwd || '')}`.trim() : '' diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index 41c5063295e0..30a6efd241bd 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -589,6 +589,7 @@ export type GatewayEvent = } | { payload?: { state?: 'idle' | 'listening' | 'transcribing' }; session_id?: string; type: 'voice.status' } | { payload?: { no_speech_limit?: boolean; text?: string }; session_id?: string; type: 'voice.transcript' } + | { payload?: { phrase?: string }; session_id?: string; type: 'wake.detected' } | { payload?: { reason?: string }; session_id?: string; type: 'dashboard.new_session_requested' } | { payload: { line: string }; session_id?: string; type: 'gateway.stderr' } | { diff --git a/website/docs/user-guide/features/wake-word.md b/website/docs/user-guide/features/wake-word.md index 21d1456cb07e..604631f86aae 100644 --- a/website/docs/user-guide/features/wake-word.md +++ b/website/docs/user-guide/features/wake-word.md @@ -6,11 +6,12 @@ description: "Hands-free 'Hey Hermes' wake word — start a voice session by spe # Wake Word ("Hey Hermes") -The wake word turns Hermes into a hands-free assistant in the CLI: with one -setting on, Hermes listens in the background for a spoken trigger phrase. Say it, -and Hermes starts a fresh session, opens the microphone, captures your command -via the normal [voice pipeline](/user-guide/features/voice-mode), and answers — -exactly like "Hey Siri" or "Alexa". +The wake word turns Hermes into a hands-free assistant across the CLI, TUI, and +desktop app: with one setting on, Hermes listens in the background for a spoken +trigger phrase. Say it, and Hermes starts a fresh session, opens the microphone, +captures your command via the normal [voice pipeline](/user-guide/features/voice-mode), +and answers — exactly like "Hey Siri" or "Alexa". Use `surface` to pick which +one listens. Detection runs **entirely on-device**. The always-on listener only watches for the wake phrase; no audio leaves your machine until you actually speak a command @@ -62,6 +63,7 @@ wake_word: ```yaml wake_word: enabled: false + surface: auto # which surface owns the listener: "auto" | "cli" | "tui" | "gui" provider: openwakeword # "openwakeword" (free, local) | "porcupine" phrase: "hey jarvis" # cosmetic label only — detection is keyed by the model/keyword below sensitivity: 0.5 # 0.0-1.0 — raise to reduce false triggers @@ -76,6 +78,23 @@ wake_word: `sensitivity`, `phrase`, and `start_new_session` apply to both engines. The `openwakeword` and `porcupine` blocks select the actual detection model. +### Surfaces (CLI, TUI, GUI) + +The wake word works in all three Hermes surfaces, and `surface` picks which one +owns the listener and opens the new session when it fires: + +| `surface` | Behavior | +|-----------|----------| +| `auto` (default) | Whichever surface you launch arms the listener. | +| `cli` | Only the classic `hermes` CLI. | +| `tui` | Only `hermes --tui`. | +| `gui` | Only the desktop app. | + +The detector is on-device and single-mic, so only one surface listens at a time +— `surface` is how you pin it. The TUI and desktop GUI share the same Python +backend (`tui_gateway`), which runs the detector server-side and yields the mic +to voice capture while a command records. + ## Using a real "Hey Hermes" The bundled openWakeWord models do **not** include "hey hermes" — `hey_jarvis` @@ -140,8 +159,9 @@ PORCUPINE_ACCESS_KEY=your-key-here ## Notes & limits -- **CLI only.** The wake word lives in the interactive `hermes` CLI, where a - local microphone is available. It does not run in the messaging gateway. +- **Local surfaces only.** The wake word runs in the CLI, TUI, and desktop GUI — + wherever a local microphone is available. It does not run in the messaging + gateway (Telegram, Discord, …), which has no mic. - **One mic at a time.** The detector releases the microphone while a command is recording and reclaims it once the turn ends, so it won't fight voice capture. - **Privacy.** Hotword detection is local. Set `sensitivity` higher if you get