From 75b36a138f43f2201b276a3c5d59f2aae0383fef Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 20 Jun 2026 14:18:36 -0500 Subject: [PATCH] feat(pets): TUI pet pane, picker + gateway RPCs Add the Ink pet sprite pane, the interactive /pet picker overlay, and live pet switching/rescale driven by new tui_gateway RPCs (pet state, pet.scale, per-state frames). Wires pet flash state and the picker into the TUI layout and slash handler. Covered by the slash-handler test. --- tui_gateway/server.py | 424 ++++++++++++++++++ .../src/__tests__/createSlashHandler.test.ts | 35 ++ ui-tui/src/app/createGatewayEventHandler.ts | 7 + ui-tui/src/app/interfaces.ts | 1 + ui-tui/src/app/overlayStore.ts | 5 +- ui-tui/src/app/petFlashStore.ts | 16 + ui-tui/src/app/slash/commands/session.ts | 28 ++ ui-tui/src/app/useInputHandlers.ts | 4 + ui-tui/src/app/usePet.ts | 313 +++++++++++++ ui-tui/src/components/appLayout.tsx | 22 + ui-tui/src/components/appOverlays.tsx | 8 + ui-tui/src/components/petPicker.tsx | 183 ++++++++ ui-tui/src/components/petSprite.tsx | 93 ++++ ui-tui/src/types/hermes-ink.d.ts | 5 +- 14 files changed, 1142 insertions(+), 2 deletions(-) create mode 100644 ui-tui/src/app/petFlashStore.ts create mode 100644 ui-tui/src/app/usePet.ts create mode 100644 ui-tui/src/components/petPicker.tsx create mode 100644 ui-tui/src/components/petSprite.tsx diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 294e543c230..005b26e0cb4 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -177,6 +177,14 @@ _LONG_HANDLERS = frozenset( "billing.step_up", "browser.manage", "cli.exec", + # Pet RPCs hit the network (manifest fetch / spritesheet download) or do + # per-frame PNG decode/encode (pet.cells): inline they serialize on the + # reader thread, so picker previews trickle in one at a time and the + # animation poll stutters. On the pool they run concurrently. + "pet.cells", + "pet.gallery", + "pet.select", + "pet.thumb", "plugins.manage", "session.branch", "session.compress", @@ -692,6 +700,29 @@ def _profile_home(profile: str | None) -> Path | None: return home if (home / "state.db").exists() or home.exists() else None +def _profile_scoped(handler): + """Bind ``params['profile']``'s HERMES_HOME around a pet RPC handler. + + Pets are per-profile: ``display.pet.*`` lives in the profile's config.yaml and + sprites install under its ``pets/`` dir (both resolve via ``get_hermes_home``). + The desktop sends ``profile`` on pet calls so config + pets dir resolve to the + focused profile even in app-global remote mode, where one backend serves every + profile. No-op for the launch profile (own-profile backends already resolve it). + """ + + def wrapper(rid, params): + home = _profile_home(params.get("profile") if isinstance(params, dict) else None) + if home is None: + return handler(rid, params) + token = set_hermes_home_override(home) + try: + return handler(rid, params) + finally: + reset_hermes_home_override(token) + + return wrapper + + # Placeholder ``terminal.cwd`` values that don't name a real directory — the # gateway resolves these to the home dir at runtime, so they must NOT be treated # as an explicit workspace (mirrors gateway/run.py's config bridge). @@ -5181,6 +5212,399 @@ def _(rid, params: dict) -> dict: return _ok(rid, usage) +def _pet_frame_counts(spritesheet) -> dict: + """Real (padding-trimmed) frame count per state, for the desktop canvas. + + Fail-open: a decode hiccup returns ``{}`` and the canvas falls back to its + static ``framesPerState`` rather than breaking the (cosmetic) pet. + """ + try: + from agent.pet import render + + return render.state_frame_counts(str(spritesheet)) + except Exception: # noqa: BLE001 - cosmetic, never break the surface + return {} + + +def _pet_state_rows(spritesheet) -> list[str]: + """Row taxonomy for the concrete active pet sheet. + + Hermes has to support both the legacy 8-row petdex atlas and the current + Codex/petdex 9-row atlas. The desktop canvas gets this list and indexes it + with the same `PetState` names the Python renderer uses. + """ + try: + from PIL import Image + + from agent.pet import constants + + with Image.open(spritesheet) as image: + row_count = max(1, image.height // constants.FRAME_H) + return list(constants.state_rows_for_grid(row_count)) + except Exception: # noqa: BLE001 - cosmetic, never break the surface + from agent.pet import constants + + return list(constants.STATE_ROWS) + + +@method("pet.info") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Return the active petdex pet for surfaces that render sprites. + + Shared by the desktop (canvas) and the TUI (half-block). Carries the + spritesheet bytes (base64) plus the engine's frame geometry + state-row + taxonomy so the renderer is a thin, framework-native consumer. The + activity→state decision is mirrored from ``agent.pet.state`` client-side. + + Agent-independent (reads config + disk), so it works on any session and + before the agent finishes building. Fail-open: returns ``enabled=False`` + on any error rather than erroring the surface. + """ + import base64 + + try: + from agent.pet import constants, store + + try: + from hermes_cli.config import load_config + + cfg = load_config() + display = cfg.get("display", {}) if isinstance(cfg.get("display"), dict) else {} + pet_cfg = display.get("pet", {}) if isinstance(display.get("pet"), dict) else {} + except Exception: + pet_cfg = {} + + enabled = bool(pet_cfg.get("enabled")) + configured_slug = str(pet_cfg.get("slug", "") or "") + pet = store.resolve_active_pet(configured_slug) if enabled else None + + if not enabled or pet is None or not pet.exists: + return _ok(rid, {"enabled": False}) + + raw = pet.spritesheet.read_bytes() + suffix = pet.spritesheet.suffix.lower() + mime = "image/png" if suffix == ".png" else "image/webp" + return _ok( + rid, + { + "enabled": True, + "slug": pet.slug, + "displayName": pet.display_name, + "mime": mime, + "spritesheetBase64": base64.standard_b64encode(raw).decode("ascii"), + "frameW": constants.FRAME_W, + "frameH": constants.FRAME_H, + "framesPerState": constants.FRAMES_PER_STATE, + "framesByState": _pet_frame_counts(pet.spritesheet), + "loopMs": constants.LOOP_MS, + "scale": float(pet_cfg.get("scale", constants.DEFAULT_SCALE) or constants.DEFAULT_SCALE), + "stateRows": _pet_state_rows(pet.spritesheet), + }, + ) + except Exception as exc: # noqa: BLE001 - cosmetic, never break the surface + logger.debug("pet.info failed: %s", exc) + return _ok(rid, {"enabled": False}) + + +@method("pet.cells") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Return half-block cell frames for one pet state (TUI renderer). + + The TUI can't draw a canvas, so the engine downsamples the spritesheet to + a grid of half-block cells and the Ink side paints them with native color + props. Each cell is ``[tr,tg,tb,ta, br,bg,bb,ba]`` (top + bottom pixel). + + Params: ``state`` (idle/run/review/failed/wave/jump), ``cols`` (width). + Fail-open: ``enabled=False`` on any problem. + """ + try: + from agent.pet import constants, render, store + from agent.pet.render import PetRenderer + + try: + from hermes_cli.config import load_config + + cfg = load_config() + display = cfg.get("display", {}) if isinstance(cfg.get("display"), dict) else {} + pet_cfg = display.get("pet", {}) if isinstance(display.get("pet"), dict) else {} + except Exception: + pet_cfg = {} + + if not bool(pet_cfg.get("enabled")): + return _ok(rid, {"enabled": False}) + + pet = store.resolve_active_pet(str(pet_cfg.get("slug", "") or "")) + if pet is None or not pet.exists: + return _ok(rid, {"enabled": False}) + + state = str(params.get("state") or constants.PetState.IDLE.value) + scale = float(pet_cfg.get("scale", constants.DEFAULT_SCALE) or constants.DEFAULT_SCALE) + cols = int(params.get("cols") or 0) or constants.resolve_cols(scale, pet_cfg.get("unicode_cols", 0)) + + # Graphics path: when the TUI is attached to a real TTY (``graphics``) + # and the terminal speaks the kitty protocol, return a Unicode- + # placeholder payload for a crisp image instead of half-blocks. Env + # detection (KITTY_WINDOW_ID / TERM / TERM_PROGRAM) is shared with the + # Ink process since it spawns us; the dashboard PTY (xterm.js) has no + # such env, so it falls through to half-blocks automatically. Only + # kitty is grid-safe in Ink — iTerm/sixel stay on the fallback. + if params.get("graphics"): + configured = str(pet_cfg.get("render_mode", "auto") or "auto").lower() + gmode = render.detect_terminal_graphics() if configured in ("", "auto") else configured + if gmode == "kitty": + image_id = render.kitty_image_id(pet.slug) + # kitty sizes from scaled pixels (_cell_box), so unicode_cols is moot here. + payload = PetRenderer( + str(pet.spritesheet), mode="kitty", scale=scale + ).kitty_payload(state, image_id=image_id) + if payload: + kcount = len(payload["frames"]) or 1 + return _ok( + rid, + { + "enabled": True, + "slug": pet.slug, + "displayName": pet.display_name, + "state": state, + "graphics": "kitty", + "imageId": image_id, + "color": render.kitty_color_hex(image_id), + "cols": payload["cols"], + "rows": payload["rows"], + "placeholder": payload["placeholder"], + "frames": payload["frames"], + "frameMs": constants.LOOP_MS / max(1, kcount), + "scale": scale, + }, + ) + + renderer = PetRenderer( + str(pet.spritesheet), + mode="unicode", + scale=scale, + unicode_cols=cols, + ) + count = renderer.frame_count(state) or 1 + frames = [] + for i in range(count): + grid = renderer.cells(state, i, cols=cols) + frames.append( + [[[*top, *bottom] for (top, bottom) in row] for row in grid] + ) + + return _ok( + rid, + { + "enabled": True, + "slug": pet.slug, + "displayName": pet.display_name, + "state": state, + "cols": cols, + "frameMs": constants.LOOP_MS / max(1, count), + "frames": frames, + "scale": scale, + }, + ) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.cells failed: %s", exc) + return _ok(rid, {"enabled": False}) + + +@method("pet.gallery") +@_profile_scoped +def _(rid, params: dict) -> dict: + """List adoptable pets for the desktop appearance picker. + + Returns the petdex gallery merged with local install state plus the + current config (active slug + enabled). Agent-independent. Fail-open: + returns whatever is installed locally if the gallery can't be reached, so + the picker still works offline. + """ + try: + from agent.pet import store + + try: + from hermes_cli.config import load_config + + cfg = load_config() + display = cfg.get("display", {}) if isinstance(cfg.get("display"), dict) else {} + pet_cfg = display.get("pet", {}) if isinstance(display.get("pet"), dict) else {} + except Exception: + pet_cfg = {} + + installed = {p.slug: p for p in store.installed_pets()} + + gallery: list[dict] = [] + seen: set[str] = set() + try: + from agent.pet.manifest import fetch_manifest + + for entry in fetch_manifest(): + seen.add(entry.slug) + gallery.append( + { + "slug": entry.slug, + "displayName": entry.display_name, + "installed": entry.slug in installed, + "spritesheetUrl": entry.spritesheet_url, + # petdex exposes no popularity metric; "curated" (its + # hand-picked/official set, identified by the asset path) + # is the closest signal, so the picker can surface it first. + "curated": "/curated/" in entry.spritesheet_url, + } + ) + except Exception as exc: # noqa: BLE001 - offline: fall back to installed + logger.debug("pet.gallery manifest fetch failed: %s", exc) + + # Always include locally-installed pets even if the gallery is unreachable. + for slug, pet in installed.items(): + if slug not in seen: + gallery.append( + {"slug": slug, "displayName": pet.display_name, "installed": True, "spritesheetUrl": ""} + ) + + return _ok( + rid, + { + "enabled": bool(pet_cfg.get("enabled")), + "active": str(pet_cfg.get("slug", "") or ""), + "pets": gallery, + }, + ) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.gallery failed: %s", exc) + return _ok(rid, {"enabled": False, "active": "", "pets": []}) + + +@method("pet.select") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Adopt a pet from the desktop picker: install (if needed) + activate. + + Params: ``slug`` (required). Writes ``display.pet.*`` to config and returns + ``{ok, slug, displayName}``. The surface re-pulls ``pet.info`` to render it. + """ + slug = str(params.get("slug") or "").strip() + if not slug: + return _err(rid, 4004, "missing slug") + try: + from agent.pet import store + from agent.pet.manifest import ManifestError + from hermes_cli.pets import _set_active + + try: + pet = store.install_pet(slug) + except (store.PetStoreError, ManifestError) as exc: + return _err(rid, 5031, f"could not adopt '{slug}': {exc}") + _set_active(slug) + return _ok(rid, {"ok": True, "slug": slug, "displayName": pet.display_name}) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.select failed: %s", exc) + return _err(rid, 5031, f"pet.select failed: {exc}") + + +@method("pet.remove") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Uninstall a pet from the desktop picker (delete its on-disk directory). + + Params: ``slug`` (required). If the removed pet was the active one, the + display is turned off so nothing tries to render a now-missing sprite. + Returns ``{ok, slug}`` where ``ok`` reflects whether a directory was deleted. + """ + slug = str(params.get("slug") or "").strip() + if not slug: + return _err(rid, 4004, "missing slug") + try: + from agent.pet import store + from hermes_cli.pets import _clear_active_if + + removed = store.remove_pet(slug) + + # If that was the active pet, stop surfaces pointing at a deleted sprite. + try: + _clear_active_if(slug) + except Exception as exc: # noqa: BLE001 - removal already succeeded + logger.debug("pet.remove config update failed: %s", exc) + + return _ok(rid, {"ok": removed, "slug": slug}) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.remove failed: %s", exc) + return _err(rid, 5031, f"pet.remove failed: {exc}") + + +@method("pet.thumb") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Return a small idle-frame PNG (data URI) for one pet — the picker preview. + + Cropped + cached server-side so the renderer gets a same-origin data URL + instead of a CDN ```` (which the desktop CSP / R2 hotlink rules break). + Params: ``slug`` (required), ``url`` (optional petdex spritesheet URL used + only for not-yet-installed pets). Fail-open: ``{ok: false}`` with no error. + """ + slug = str(params.get("slug") or "").strip() + if not slug: + return _err(rid, 4004, "missing slug") + try: + import base64 + + from agent.pet import store + + data = store.thumbnail_png(slug, source_url=str(params.get("url") or "")) + if not data: + return _ok(rid, {"ok": False, "slug": slug}) + + return _ok( + rid, + { + "ok": True, + "slug": slug, + "dataUri": "data:image/png;base64," + base64.standard_b64encode(data).decode("ascii"), + }, + ) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.thumb failed: %s", exc) + return _ok(rid, {"ok": False, "slug": slug}) + + +@method("pet.disable") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Turn the pet off from the desktop picker (``display.pet.enabled=false``).""" + try: + from hermes_cli.pets import _set_enabled + + _set_enabled(False) + return _ok(rid, {"ok": True}) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.disable failed: %s", exc) + return _err(rid, 5031, f"pet.disable failed: {exc}") + + +@method("pet.scale") +@_profile_scoped +def _(rid, params: dict) -> dict: + """Persist ``display.pet.scale`` from the desktop slider. Params: ``scale``. + + Clamped to the engine bounds. The renderer updates its own ``$petInfo`` for + instant feedback; this just makes the change durable + visible to the other + terminal surfaces on their next read. + """ + try: + from hermes_cli.pets import set_pet_scale + + scale, err = set_pet_scale(params.get("scale")) + if err: + return _err(rid, 4004, err) + return _ok(rid, {"ok": True, "scale": scale}) + except Exception as exc: # noqa: BLE001 + logger.debug("pet.scale failed: %s", exc) + return _err(rid, 5031, f"pet.scale failed: {exc}") + + @method("credits.view") def _(rid, params: dict) -> dict: """Structured Nous credit view for the TUI /credits command. diff --git a/ui-tui/src/__tests__/createSlashHandler.test.ts b/ui-tui/src/__tests__/createSlashHandler.test.ts index a671063e5e9..7124cbfcd75 100644 --- a/ui-tui/src/__tests__/createSlashHandler.test.ts +++ b/ui-tui/src/__tests__/createSlashHandler.test.ts @@ -208,6 +208,41 @@ describe('createSlashHandler', () => { }) }) + it('opens the pet picker for /pet list only', () => { + const ctx = buildCtx() + + expect(createSlashHandler(ctx)('/pet list')).toBe(true) + expect(getOverlayState().petPicker).toBe(true) + expect(ctx.gateway.gw.request).not.toHaveBeenCalled() + + resetOverlayState() + expect(createSlashHandler(ctx)('/pet')).toBe(true) + expect(getOverlayState().petPicker).toBe(false) + expect(ctx.gateway.gw.request).toHaveBeenCalledWith( + 'slash.exec', + expect.objectContaining({ command: 'pet' }) + ) + + resetOverlayState() + expect(createSlashHandler(ctx)('/pet toggle')).toBe(true) + expect(getOverlayState().petPicker).toBe(false) + expect(ctx.gateway.gw.request).toHaveBeenCalledWith( + 'slash.exec', + expect.objectContaining({ command: 'pet toggle' }) + ) + }) + + it('routes /pet to the slash worker without opening the picker', () => { + const ctx = buildCtx() + + expect(createSlashHandler(ctx)('/pet boba')).toBe(true) + expect(getOverlayState().petPicker).toBe(false) + expect(ctx.gateway.gw.request).toHaveBeenCalledWith( + 'slash.exec', + expect.objectContaining({ command: 'pet boba' }) + ) + }) + it('routes /skills inspect to skills.manage', () => { const ctx = buildCtx() diff --git a/ui-tui/src/app/createGatewayEventHandler.ts b/ui-tui/src/app/createGatewayEventHandler.ts index de2f774f149..b0b09a725c3 100644 --- a/ui-tui/src/app/createGatewayEventHandler.ts +++ b/ui-tui/src/app/createGatewayEventHandler.ts @@ -9,6 +9,7 @@ import type { GatewaySkin, SessionMostRecentResponse } from '../gatewayTypes.js' +import { isTodoDone } from '../lib/liveProgress.js' import { rpcErrorMessage } from '../lib/rpc.js' import { openExternalUrl } from '../lib/openExternalUrl.js' import { topLevelSubagents } from '../lib/subagentTree.js' @@ -19,7 +20,9 @@ import type { Msg, SubagentProgress, SubagentStatus } from '../types.js' import { applyDelegationStatus, getDelegationState } from './delegationStore.js' import type { GatewayEventHandlerContext } from './interfaces.js' import { getOverlayState, patchOverlayState } from './overlayStore.js' +import { flashPet } from './petFlashStore.js' import { turnController } from './turnController.js' +import { getTurnState } from './turnStore.js' import { getUiState, patchUiState } from './uiStore.js' const NO_PROVIDER_RE = /\bNo (?:LLM|inference) provider configured\b/i @@ -908,6 +911,9 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: const msgs: Msg[] = finalMessages.length ? finalMessages : [{ role: 'assistant', text: finalText }] msgs.forEach(appendMessage) + // Pet beat: celebrate a finished plan, otherwise a clean-finish wave. + flashPet(isTodoDone(getTurnState().todos) ? 'jump' : 'wave') + if (bellOnComplete && stdout?.isTTY) { stdout.write('\x07') } @@ -924,6 +930,7 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: case 'error': turnController.recordError() + flashPet('failed') { const message = String(ev.payload?.message || 'unknown error') diff --git a/ui-tui/src/app/interfaces.ts b/ui-tui/src/app/interfaces.ts index f570cf2b6ab..1d1f12a3a2a 100644 --- a/ui-tui/src/app/interfaces.ts +++ b/ui-tui/src/app/interfaces.ts @@ -136,6 +136,7 @@ export interface OverlayState { confirm: ConfirmReq | null modelPicker: boolean pager: null | PagerState + petPicker: boolean pluginsHub: boolean secret: null | SecretReq sessions: boolean diff --git a/ui-tui/src/app/overlayStore.ts b/ui-tui/src/app/overlayStore.ts index c0290d71cab..b716b313c90 100644 --- a/ui-tui/src/app/overlayStore.ts +++ b/ui-tui/src/app/overlayStore.ts @@ -11,6 +11,7 @@ const buildOverlayState = (): OverlayState => ({ confirm: null, modelPicker: false, pager: null, + petPicker: false, pluginsHub: false, secret: null, sessions: false, @@ -22,7 +23,7 @@ export const $overlayState = atom(buildOverlayState()) export const $isBlocked = computed( $overlayState, - ({ agents, approval, billing, clarify, confirm, modelPicker, pager, pluginsHub, secret, sessions, skillsHub, sudo }) => + ({ agents, approval, billing, clarify, confirm, modelPicker, pager, petPicker, pluginsHub, secret, sessions, skillsHub, sudo }) => Boolean( agents || approval || @@ -31,6 +32,7 @@ export const $isBlocked = computed( confirm || modelPicker || pager || + petPicker || pluginsHub || secret || sessions || @@ -61,6 +63,7 @@ export const resetFlowOverlays = () => agents: $overlayState.get().agents, agentsInitialHistoryIndex: $overlayState.get().agentsInitialHistoryIndex, modelPicker: $overlayState.get().modelPicker, + petPicker: $overlayState.get().petPicker, pluginsHub: $overlayState.get().pluginsHub, sessions: $overlayState.get().sessions, skillsHub: $overlayState.get().skillsHub diff --git a/ui-tui/src/app/petFlashStore.ts b/ui-tui/src/app/petFlashStore.ts new file mode 100644 index 00000000000..d4865e47a2b --- /dev/null +++ b/ui-tui/src/app/petFlashStore.ts @@ -0,0 +1,16 @@ +import { atom } from 'nanostores' + +import type { PetState } from './usePet.js' + +interface PetFlash { + state: PetState + until: number +} + +// Transient reaction beats (wave/jump/failed) the pet shows for a moment at +// turn end before falling back to its steady state. The gateway event handler +// sets these; usePet reads them with priority over the derived state. +export const $petFlash = atom(null) + +export const flashPet = (state: PetState, ms = 1600) => + $petFlash.set({ state, until: Date.now() + ms }) diff --git a/ui-tui/src/app/slash/commands/session.ts b/ui-tui/src/app/slash/commands/session.ts index b716504b353..a8b4b3ca4e1 100644 --- a/ui-tui/src/app/slash/commands/session.ts +++ b/ui-tui/src/app/slash/commands/session.ts @@ -8,6 +8,7 @@ import type { SessionBranchResponse, SessionCompressResponse, SessionUsageResponse, + SlashExecResponse, VoiceToggleResponse } from '../../../gatewayTypes.js' import { formatVoiceRecordKey, parseVoiceRecordKey } from '../../../lib/platform.js' @@ -340,6 +341,31 @@ export const sessionCommands: SlashCommand[] = [ } }, + { + help: 'toggle / adopt / resize an animated pet', + name: 'pet', + usage: '/pet [toggle | list | scale | ]', + run: (arg, ctx, cmd) => { + const sub = arg.trim().toLowerCase() + + // Gallery picker — the interactive browse surface. + if (sub === 'list') { + return patchOverlayState({ petPicker: true }) + } + + // Bare /pet and /pet toggle flip display.pet.enabled via the slash worker. + ctx.gateway.gw + .request('slash.exec', { command: cmd.slice(1), session_id: ctx.sid }) + .then( + ctx.guarded(r => { + const body = r.output || '/pet: no output' + ctx.transcript.sys(r.warning ? `warning: ${r.warning}\n${body}` : body) + }) + ) + .catch(ctx.guardedErr) + } + }, + { help: 'switch theme skin (fires skin.changed)', name: 'skin', @@ -553,6 +579,7 @@ export const sessionCommands: SlashCommand[] = [ // even with zero API calls or on a resumed session. Render it whenever // present, before the token panel. const creditsLines = r?.credits_lines ?? [] + if (creditsLines.length) { ctx.transcript.panel('Nous credits', [{ text: creditsLines.join('\n') }]) } @@ -561,6 +588,7 @@ export const sessionCommands: SlashCommand[] = [ if (!creditsLines.length) { ctx.transcript.sys('no API calls yet') } + return } diff --git a/ui-tui/src/app/useInputHandlers.ts b/ui-tui/src/app/useInputHandlers.ts index 20d3493f547..f765322163d 100644 --- a/ui-tui/src/app/useInputHandlers.ts +++ b/ui-tui/src/app/useInputHandlers.ts @@ -147,6 +147,10 @@ export function useInputHandlers(ctx: InputHandlerContext): InputHandlerResult { return patchOverlayState({ modelPicker: false }) } + if (overlay.petPicker) { + return patchOverlayState({ petPicker: false }) + } + if (overlay.billing) { return patchOverlayState({ billing: null }) } diff --git a/ui-tui/src/app/usePet.ts b/ui-tui/src/app/usePet.ts new file mode 100644 index 00000000000..8943c1ae7f0 --- /dev/null +++ b/ui-tui/src/app/usePet.ts @@ -0,0 +1,313 @@ +import { useStdout } from '@hermes/ink' +import { useCallback, useEffect, useRef, useState } from 'react' + +import type { PetGrid } from '../components/petSprite.js' + +import { useGateway } from './gatewayContext.js' +import { getOverlayState, $overlayState } from './overlayStore.js' +import { $petFlash } from './petFlashStore.js' +import { $turnState } from './turnStore.js' +import { $uiState } from './uiStore.js' + +export type PetState = 'idle' | 'wave' | 'run' | 'failed' | 'review' | 'jump' | 'waiting' + +interface PetActivity { + busy: boolean + toolRunning: boolean + reasoning: boolean + awaitingInput: boolean +} + +/** + * Resolve the animation state — mirrors `agent.pet.state.derive_pet_state` + * (and the desktop's `derivePetState`) so all surfaces agree. `awaitingInput` + * (a clarify/approval blocking on the user) outranks the in-flight signals + * because the turn is paused on you, not working. + */ +export function derivePetState({ busy, toolRunning, reasoning, awaitingInput }: PetActivity): PetState { + if (awaitingInput) { + return 'waiting' + } + + if (toolRunning) { + return 'run' + } + + if (reasoning) { + return 'review' + } + + if (busy) { + return 'run' + } + + return 'idle' +} + +// The overlays that mean "the agent is blocked on the user" (vs. user-toggled +// pickers like model/sessions, which aren't the agent waiting). +function isAwaitingInput(): boolean { + const o = getOverlayState() + + return Boolean(o.clarify || o.approval || o.sudo || o.secret || o.confirm) +} + +// A kitty Unicode-placeholder frame set: a static placeholder grid (painted by +// Ink in the image-id color) plus per-frame transmit escapes written straight +// to the terminal out-of-band. +interface KittyView { + color: string + placeholder: string[] +} + +interface PetCellsResult { + color?: string + enabled?: boolean + frameMs?: number + // unicode mode: cell grids; kitty mode: transmit-escape strings. + frames?: PetGrid[] | string[] + graphics?: string + imageId?: number + placeholder?: string[] + scale?: number + slug?: string + state?: string +} + +type CacheEntry = + | { kind: 'cells'; frameMs: number; frames: PetGrid[] } + | { kind: 'kitty'; frameMs: number; frames: string[]; placeholder: string[]; color: string } + +const FRAME_MS = 160 +const POLL_MS = 2500 + +// Only the standalone TUI owns a real terminal it can splat image escapes into; +// when piped (or running under the dashboard PTY the gateway resolves to +// half-blocks anyway) we never ask for graphics. +const IS_TTY = Boolean(process.stdout?.isTTY) + +export interface PetRender { + enabled: boolean + grid: PetGrid | null + kitty: KittyView | null +} + +/** + * Drives the TUI pet. Fetches each (slug, state)'s frames via the `pet.cells` + * RPC (cached) and animates the frame index. Two render paths: + * + * - **kitty** (Ghostty/kitty): the engine returns a static placeholder grid + + * per-frame transmit escapes. We paint the placeholder with Ink and write the + * current frame's escape to the terminal out-of-band, so the image animates + * underneath without Ink ever repainting. + * - **cells** (everywhere else): truecolor half-block grids painted by Ink. + * + * A steady poll keeps it reactive to config changes made elsewhere (`/pet`, the + * picker, `hermes pets select`) so adopting/switching/disabling takes effect + * live. The frame cache is keyed by `slug:state` so a switch re-pulls cleanly. + */ +export function usePet(): PetRender { + const { rpc } = useGateway() + const { write } = useStdout() + const [enabled, setEnabled] = useState(false) + const [grid, setGrid] = useState(null) + const [kitty, setKitty] = useState(null) + + const cache = useRef>(new Map()) + const slugRef = useRef('') + const scaleRef = useRef(0) + const imageIdRef = useRef(0) + const stateRef = useRef('idle') + const frameRef = useRef(0) + + const [petState, setPetState] = useState('idle') + + // Recompute the desired state on every turn/ui/flash change. A transient + // flash (wave/jump/failed) wins until it expires; a timer re-runs at expiry. + useEffect(() => { + let expiry: ReturnType | undefined + + const apply = (next: PetState) => { + if (next !== stateRef.current) { + stateRef.current = next + frameRef.current = 0 + setPetState(next) + } + } + + const recompute = () => { + clearTimeout(expiry) + + const flash = $petFlash.get() + const now = Date.now() + + if (flash && now < flash.until) { + apply(flash.state) + expiry = setTimeout(recompute, flash.until - now) + + return + } + + const turn = $turnState.get() + const ui = $uiState.get() + + apply( + derivePetState({ + awaitingInput: isAwaitingInput(), + busy: ui.busy, + reasoning: turn.reasoningActive, + toolRunning: turn.tools.length > 0 + }) + ) + } + + recompute() + const unsubTurn = $turnState.listen(recompute) + const unsubUi = $uiState.listen(recompute) + const unsubFlash = $petFlash.listen(recompute) + const unsubOverlay = $overlayState.listen(recompute) + + return () => { + clearTimeout(expiry) + unsubTurn() + unsubUi() + unsubFlash() + unsubOverlay() + } + }, []) + + // Free the terminal-side image when the pet goes away or the hook unmounts. + const releaseKitty = useCallback(() => { + if (imageIdRef.current) { + try { + write(`\x1b_Ga=d,d=i,i=${imageIdRef.current},q=2\x1b\\`) + } catch { + // best-effort cleanup + } + + imageIdRef.current = 0 + } + }, [write]) + + // Fetch + cache one (slug, state). `pet.cells` resolves the active pet from + // config, so its `slug`/`enabled` are the source of truth. + const sync = useCallback( + async (state: PetState) => { + try { + const res = (await rpc('pet.cells', { graphics: IS_TTY, state })) as PetCellsResult | null + + if (!res) { + return + } + + if (!res.enabled) { + releaseKitty() + slugRef.current = '' + cache.current.clear() + setGrid(null) + setKitty(null) + setEnabled(false) + + return + } + + const slug = res.slug ?? '' + const scale = res.scale ?? 0 + + // A switch OR a live `/pet scale` change invalidates the cached frames + // (they're rendered at the old size), so the steady poll repaints at the + // new scale without a restart. + if (slug !== slugRef.current || (scale > 0 && scale !== scaleRef.current)) { + releaseKitty() + slugRef.current = slug + scaleRef.current = scale + cache.current.clear() + frameRef.current = 0 + } + + if (res.graphics === 'kitty' && res.frames?.length && res.placeholder?.length) { + imageIdRef.current = res.imageId ?? 0 + cache.current.set(`${slug}:${state}`, { + color: res.color ?? '#000001', + frameMs: res.frameMs ?? FRAME_MS, + frames: res.frames as string[], + kind: 'kitty', + placeholder: res.placeholder + }) + } else if (res.frames?.length) { + cache.current.set(`${slug}:${state}`, { + frameMs: res.frameMs ?? FRAME_MS, + frames: res.frames as PetGrid[], + kind: 'cells' + }) + } + + setEnabled(true) + } catch { + // cosmetic — ignore RPC failures + } + }, + [rpc, releaseKitty] + ) + + // Pull frames whenever the state changes (if not already cached for the + // active pet), plus a steady poll that catches adopt/switch/disable. + useEffect(() => { + if (!cache.current.has(`${slugRef.current}:${petState}`)) { + void sync(petState) + } + + const timer = setInterval(() => void sync(stateRef.current), POLL_MS) + + return () => clearInterval(timer) + }, [petState, sync]) + + useEffect(() => releaseKitty, [releaseKitty]) + + // Animation timer. + useEffect(() => { + if (!enabled) { + return + } + + const tick = () => { + const entry = cache.current.get(`${slugRef.current}:${stateRef.current}`) + + if (!entry?.frames.length) { + return // keep the last frame painted while the new state loads + } + + const idx = frameRef.current % entry.frames.length + frameRef.current = idx + 1 + + if (entry.kind === 'kitty') { + // Transmit this frame's image under the shared id; the static + // placeholder cells (set below) render it. No Ink repaint needed. + try { + write(entry.frames[idx] ?? '') + } catch { + // ignore transmit failures + } + + setGrid(null) + setKitty(prev => + prev && prev.color === entry.color && prev.placeholder === entry.placeholder + ? prev + : { color: entry.color, placeholder: entry.placeholder } + ) + + return + } + + setKitty(null) + setGrid(entry.frames[idx] ?? null) + } + + tick() + const interval = setInterval(tick, FRAME_MS) + + return () => clearInterval(interval) + }, [enabled, petState, write]) + + return { enabled, grid, kitty } +} diff --git a/ui-tui/src/components/appLayout.tsx b/ui-tui/src/components/appLayout.tsx index d54f5c6da90..7fa5dec1886 100644 --- a/ui-tui/src/components/appLayout.tsx +++ b/ui-tui/src/components/appLayout.tsx @@ -6,6 +6,7 @@ import { useGateway } from '../app/gatewayContext.js' import type { AppLayoutProps } from '../app/interfaces.js' import { $isBlocked, $overlayState, patchOverlayState } from '../app/overlayStore.js' import { $uiState } from '../app/uiStore.js' +import { usePet } from '../app/usePet.js' import { INLINE_MODE, SHOW_FPS, TERMUX_TUI_MODE } from '../config/env.js' import { PLACEHOLDER } from '../content/placeholders.js' import { prevRenderedMsg } from '../domain/blockLayout.js' @@ -25,10 +26,29 @@ import { Banner, Panel, SessionPanel } from './branding.js' import { FpsOverlay } from './fpsOverlay.js' import { HelpHint } from './helpHint.js' import { MessageLine } from './messageLine.js' +import { PetKitty, PetSprite } from './petSprite.js' import { QueuedMessages } from './queuedMessages.js' import { LiveTodoPanel, StreamingAssistant } from './streamingAssistant.js' import { TextInput, type TextInputMouseApi } from './textInput.js' +// Petdex mascot — sits just above the composer, right-aligned. Renders +// nothing unless a pet is installed + enabled (`hermes pets select `), +// so it's a no-op for everyone else. +const PetPane = memo(function PetPane() { + const { enabled, grid, kitty } = usePet() + + if (!enabled || (!grid && !kitty)) { + return null + } + + return ( + + {kitty ? : null} + {!kitty && grid ? : null} + + ) +}) + const PromptPrefix = memo(function PromptPrefix({ bold = false, color, @@ -420,6 +440,8 @@ export const AppLayout = memo(function AppLayout({ {!overlay.agents && ( <> + + )} + {overlay.petPicker && ( + + patchOverlayState({ petPicker: false })} t={theme} /> + + )} + {overlay.skillsHub && ( patchOverlayState({ skillsHub: false })} t={theme} /> diff --git a/ui-tui/src/components/petPicker.tsx b/ui-tui/src/components/petPicker.tsx new file mode 100644 index 00000000000..dacb553082e --- /dev/null +++ b/ui-tui/src/components/petPicker.tsx @@ -0,0 +1,183 @@ +import { Box, Text, useInput, useStdout } from '@hermes/ink' +import { useEffect, useMemo, useState } from 'react' + +import type { GatewayClient } from '../gatewayClient.js' +import { rpcErrorMessage } from '../lib/rpc.js' +import type { Theme } from '../theme.js' + +import { OverlayHint, windowItems } from './overlayControls.js' + +const VISIBLE = 10 +const MIN_WIDTH = 40 +const MAX_WIDTH = 90 + +interface GalleryPet { + slug: string + displayName: string + installed: boolean + curated?: boolean +} + +interface Gallery { + enabled: boolean + active: string + pets: GalleryPet[] +} + +/** + * Interactive petdex picker overlay. Pulls the gallery via `pet.gallery`, + * filters as you type, and adopts the highlighted pet with `pet.select` + * (install-on-demand). The mascot lights up live once `usePet` next polls — + * no restart. This is the interactive sibling of the text `/pet ` path. + */ +export function PetPicker({ gw, onClose, t }: PetPickerProps) { + const [gallery, setGallery] = useState(null) + const [query, setQuery] = useState('') + const [idx, setIdx] = useState(0) + const [busy, setBusy] = useState(false) + const [err, setErr] = useState('') + const [loading, setLoading] = useState(true) + + const { stdout } = useStdout() + const width = Math.max(MIN_WIDTH, Math.min(MAX_WIDTH, (stdout?.columns ?? 80) - 6)) + + useEffect(() => { + gw.request('pet.gallery') + .then(r => { + setGallery(r) + setErr('') + }) + .catch((e: unknown) => setErr(rpcErrorMessage(e))) + .finally(() => setLoading(false)) + }, [gw]) + + const enabled = gallery?.enabled ?? false + const active = gallery?.active ?? '' + + // Rank by the signals petdex gives us — active, then installed, then curated + // (its official set), then the rest — and hide the clawd placeholders. + const view = useMemo(() => { + const pets = (gallery?.pets ?? []).filter(p => !/^clawd(-|$)/i.test(p.slug)) + const needle = query.trim().toLowerCase() + + const matched = needle + ? pets.filter(p => p.slug.toLowerCase().includes(needle) || p.displayName.toLowerCase().includes(needle)) + : pets + + const rank = (p: GalleryPet) => (enabled && p.slug === active ? 4 : 0) + (p.installed ? 2 : 0) + (p.curated ? 1 : 0) + + return [...matched].sort((a, b) => rank(b) - rank(a)) + }, [gallery, query, enabled, active]) + + const adopt = (slug: string) => { + setBusy(true) + setErr('') + gw.request('pet.select', { slug }) + .then(() => onClose()) + .catch((e: unknown) => { + setErr(rpcErrorMessage(e)) + setBusy(false) + }) + } + + useInput((input, key) => { + if (busy) { + return + } + + if (key.escape) { + return onClose() + } + + if (key.upArrow) { + return setIdx(i => Math.max(0, i - 1)) + } + + if (key.downArrow) { + return setIdx(i => Math.min(view.length - 1, i + 1)) + } + + if (key.return) { + const pet = view[idx] + + return pet ? adopt(pet.slug) : undefined + } + + if (key.backspace || key.delete) { + setQuery(q => q.slice(0, -1)) + + return setIdx(0) + } + + // Printable char → extend the filter (ignore control/chorded keys). + if (input && input.length === 1 && input >= ' ' && !key.ctrl && !key.meta) { + setQuery(q => q + input) + setIdx(0) + } + }) + + if (loading) { + return loading pets… + } + + if (err && !gallery) { + return ( + + error: {err} + Esc cancel + + ) + } + + const { items, offset } = windowItems(view, idx, VISIBLE) + + return ( + + + Pets + + + + {query ? `filter: ${query}` : 'type to filter'} · {view.length} pet{view.length === 1 ? '' : 's'} + + + {offset > 0 && ↑ {offset} more} + + {view.length === 0 ? ( + {query ? `no pets match "${query}"` : 'no pets available'} + ) : ( + items.map((pet, i) => { + const at = offset + i === idx + const isActive = enabled && pet.slug === active + const mark = isActive ? '●' : pet.installed ? '✓' : ' ' + const tag = pet.installed ? '' : pet.curated ? ' · official' : '' + + return ( + + {at ? '▸ ' : ' '} + {mark} {pet.displayName} + + {' '} + ({pet.slug} + {tag}) + + + ) + }) + )} + + {offset + VISIBLE < view.length && ↓ {view.length - offset - VISIBLE} more} + + {err ? error: {err} : null} + {busy ? adopting… : null} + + ↑/↓ select · Enter adopt · type to filter · Esc cancel + + ) +} + +interface PetPickerProps { + gw: GatewayClient + onClose: () => void + t: Theme +} diff --git a/ui-tui/src/components/petSprite.tsx b/ui-tui/src/components/petSprite.tsx new file mode 100644 index 00000000000..5a17f6337d4 --- /dev/null +++ b/ui-tui/src/components/petSprite.tsx @@ -0,0 +1,93 @@ +import { Box, Text } from '@hermes/ink' +import { memo } from 'react' + +// A cell is [tr,tg,tb,ta, br,bg,bb,ba] — the top + bottom pixel of one +// half-block, as produced by the `pet.cells` gateway RPC. +export type PetCell = number[] +export type PetGrid = PetCell[][] + +const UPPER_HALF = '▀' +const LOWER_HALF = '▄' + +const hex = (r: number, g: number, b: number) => + `#${[r, g, b].map(v => Math.max(0, Math.min(255, v | 0)).toString(16).padStart(2, '0')).join('')}` + +/** + * Renders one petdex frame as truecolor half-blocks using native Ink color + * props (no raw ANSI, so width measurement stays correct). The engine + * (`agent/pet/render.py`) does the decode + downscale; this is a thin painter. + */ +export const PetSprite = memo(function PetSprite({ grid }: { grid: PetGrid }) { + if (!grid.length) { + return null + } + + return ( + + {grid.map((row, y) => ( + + {row.map((cell, x) => { + const [tr, tg, tb, ta, br, bg, bb, ba] = cell + const top = (ta ?? 0) >= 32 + const bot = (ba ?? 0) >= 32 + + if (!top && !bot) { + return + } + + // Both halves opaque → fg=top over bg=bottom. One half opaque → + // draw it fg-only so the other stays the terminal bg (no black + // boxes bleeding around transparent sprite edges). + if (top && bot) { + return ( + + {UPPER_HALF} + + ) + } + + return top ? ( + + {UPPER_HALF} + + ) : ( + + {LOWER_HALF} + + ) + })} + + ))} + + ) +}) + +/** + * Renders a kitty Unicode-placeholder grid: each line is a row of U+10EEEE + * cells whose foreground color encodes the image id. The actual pixels are + * drawn by the terminal (the frame image is transmitted out-of-band by + * `usePet`); this only emits the placeholder text Ink can measure as width-1 + * cells. Truecolor-only — the color must reach the terminal verbatim for the + * id to decode, which Ghostty/kitty support. + */ +export const PetKitty = memo(function PetKitty({ + color, + placeholder +}: { + color: string + placeholder: string[] +}) { + if (!placeholder.length) { + return null + } + + return ( + + {placeholder.map((row, y) => ( + + {row} + + ))} + + ) +}) diff --git a/ui-tui/src/types/hermes-ink.d.ts b/ui-tui/src/types/hermes-ink.d.ts index ca2a05dc449..034e04cf6da 100644 --- a/ui-tui/src/types/hermes-ink.d.ts +++ b/ui-tui/src/types/hermes-ink.d.ts @@ -156,7 +156,10 @@ declare module '@hermes/ink' { readonly setSelectionBgColor: (color: string) => void } export function useHasSelection(): boolean - export function useStdout(): { readonly stdout?: NodeJS.WriteStream } + export function useStdout(): { + readonly stdout?: NodeJS.WriteStream + readonly write: (data: string) => boolean + } export function useTerminalFocus(): boolean export function useTerminalTitle(title: string | null): void export function useDeclaredCursor(args: {