feat(pets): TUI pet pane, picker + gateway RPCs

Add the Ink pet sprite pane, the interactive /pet picker overlay, and live
pet switching/rescale driven by new tui_gateway RPCs (pet state, pet.scale,
per-state frames). Wires pet flash state and the picker into the TUI layout
and slash handler. Covered by the slash-handler test.
This commit is contained in:
Brooklyn Nicholson 2026-06-20 14:18:36 -05:00
parent 83aa84ae3b
commit 75b36a138f
14 changed files with 1142 additions and 2 deletions

View file

@ -177,6 +177,14 @@ _LONG_HANDLERS = frozenset(
"billing.step_up",
"browser.manage",
"cli.exec",
# Pet RPCs hit the network (manifest fetch / spritesheet download) or do
# per-frame PNG decode/encode (pet.cells): inline they serialize on the
# reader thread, so picker previews trickle in one at a time and the
# animation poll stutters. On the pool they run concurrently.
"pet.cells",
"pet.gallery",
"pet.select",
"pet.thumb",
"plugins.manage",
"session.branch",
"session.compress",
@ -692,6 +700,29 @@ def _profile_home(profile: str | None) -> Path | None:
return home if (home / "state.db").exists() or home.exists() else None
def _profile_scoped(handler):
"""Bind ``params['profile']``'s HERMES_HOME around a pet RPC handler.
Pets are per-profile: ``display.pet.*`` lives in the profile's config.yaml and
sprites install under its ``pets/`` dir (both resolve via ``get_hermes_home``).
The desktop sends ``profile`` on pet calls so config + pets dir resolve to the
focused profile even in app-global remote mode, where one backend serves every
profile. No-op for the launch profile (own-profile backends already resolve it).
"""
def wrapper(rid, params):
home = _profile_home(params.get("profile") if isinstance(params, dict) else None)
if home is None:
return handler(rid, params)
token = set_hermes_home_override(home)
try:
return handler(rid, params)
finally:
reset_hermes_home_override(token)
return wrapper
# Placeholder ``terminal.cwd`` values that don't name a real directory — the
# gateway resolves these to the home dir at runtime, so they must NOT be treated
# as an explicit workspace (mirrors gateway/run.py's config bridge).
@ -5181,6 +5212,399 @@ def _(rid, params: dict) -> dict:
return _ok(rid, usage)
def _pet_frame_counts(spritesheet) -> dict:
"""Real (padding-trimmed) frame count per state, for the desktop canvas.
Fail-open: a decode hiccup returns ``{}`` and the canvas falls back to its
static ``framesPerState`` rather than breaking the (cosmetic) pet.
"""
try:
from agent.pet import render
return render.state_frame_counts(str(spritesheet))
except Exception: # noqa: BLE001 - cosmetic, never break the surface
return {}
def _pet_state_rows(spritesheet) -> list[str]:
"""Row taxonomy for the concrete active pet sheet.
Hermes has to support both the legacy 8-row petdex atlas and the current
Codex/petdex 9-row atlas. The desktop canvas gets this list and indexes it
with the same `PetState` names the Python renderer uses.
"""
try:
from PIL import Image
from agent.pet import constants
with Image.open(spritesheet) as image:
row_count = max(1, image.height // constants.FRAME_H)
return list(constants.state_rows_for_grid(row_count))
except Exception: # noqa: BLE001 - cosmetic, never break the surface
from agent.pet import constants
return list(constants.STATE_ROWS)
@method("pet.info")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Return the active petdex pet for surfaces that render sprites.
Shared by the desktop (canvas) and the TUI (half-block). Carries the
spritesheet bytes (base64) plus the engine's frame geometry + state-row
taxonomy so the renderer is a thin, framework-native consumer. The
activitystate decision is mirrored from ``agent.pet.state`` client-side.
Agent-independent (reads config + disk), so it works on any session and
before the agent finishes building. Fail-open: returns ``enabled=False``
on any error rather than erroring the surface.
"""
import base64
try:
from agent.pet import constants, store
try:
from hermes_cli.config import load_config
cfg = load_config()
display = cfg.get("display", {}) if isinstance(cfg.get("display"), dict) else {}
pet_cfg = display.get("pet", {}) if isinstance(display.get("pet"), dict) else {}
except Exception:
pet_cfg = {}
enabled = bool(pet_cfg.get("enabled"))
configured_slug = str(pet_cfg.get("slug", "") or "")
pet = store.resolve_active_pet(configured_slug) if enabled else None
if not enabled or pet is None or not pet.exists:
return _ok(rid, {"enabled": False})
raw = pet.spritesheet.read_bytes()
suffix = pet.spritesheet.suffix.lower()
mime = "image/png" if suffix == ".png" else "image/webp"
return _ok(
rid,
{
"enabled": True,
"slug": pet.slug,
"displayName": pet.display_name,
"mime": mime,
"spritesheetBase64": base64.standard_b64encode(raw).decode("ascii"),
"frameW": constants.FRAME_W,
"frameH": constants.FRAME_H,
"framesPerState": constants.FRAMES_PER_STATE,
"framesByState": _pet_frame_counts(pet.spritesheet),
"loopMs": constants.LOOP_MS,
"scale": float(pet_cfg.get("scale", constants.DEFAULT_SCALE) or constants.DEFAULT_SCALE),
"stateRows": _pet_state_rows(pet.spritesheet),
},
)
except Exception as exc: # noqa: BLE001 - cosmetic, never break the surface
logger.debug("pet.info failed: %s", exc)
return _ok(rid, {"enabled": False})
@method("pet.cells")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Return half-block cell frames for one pet state (TUI renderer).
The TUI can't draw a canvas, so the engine downsamples the spritesheet to
a grid of half-block cells and the Ink side paints them with native color
props. Each cell is ``[tr,tg,tb,ta, br,bg,bb,ba]`` (top + bottom pixel).
Params: ``state`` (idle/run/review/failed/wave/jump), ``cols`` (width).
Fail-open: ``enabled=False`` on any problem.
"""
try:
from agent.pet import constants, render, store
from agent.pet.render import PetRenderer
try:
from hermes_cli.config import load_config
cfg = load_config()
display = cfg.get("display", {}) if isinstance(cfg.get("display"), dict) else {}
pet_cfg = display.get("pet", {}) if isinstance(display.get("pet"), dict) else {}
except Exception:
pet_cfg = {}
if not bool(pet_cfg.get("enabled")):
return _ok(rid, {"enabled": False})
pet = store.resolve_active_pet(str(pet_cfg.get("slug", "") or ""))
if pet is None or not pet.exists:
return _ok(rid, {"enabled": False})
state = str(params.get("state") or constants.PetState.IDLE.value)
scale = float(pet_cfg.get("scale", constants.DEFAULT_SCALE) or constants.DEFAULT_SCALE)
cols = int(params.get("cols") or 0) or constants.resolve_cols(scale, pet_cfg.get("unicode_cols", 0))
# Graphics path: when the TUI is attached to a real TTY (``graphics``)
# and the terminal speaks the kitty protocol, return a Unicode-
# placeholder payload for a crisp image instead of half-blocks. Env
# detection (KITTY_WINDOW_ID / TERM / TERM_PROGRAM) is shared with the
# Ink process since it spawns us; the dashboard PTY (xterm.js) has no
# such env, so it falls through to half-blocks automatically. Only
# kitty is grid-safe in Ink — iTerm/sixel stay on the fallback.
if params.get("graphics"):
configured = str(pet_cfg.get("render_mode", "auto") or "auto").lower()
gmode = render.detect_terminal_graphics() if configured in ("", "auto") else configured
if gmode == "kitty":
image_id = render.kitty_image_id(pet.slug)
# kitty sizes from scaled pixels (_cell_box), so unicode_cols is moot here.
payload = PetRenderer(
str(pet.spritesheet), mode="kitty", scale=scale
).kitty_payload(state, image_id=image_id)
if payload:
kcount = len(payload["frames"]) or 1
return _ok(
rid,
{
"enabled": True,
"slug": pet.slug,
"displayName": pet.display_name,
"state": state,
"graphics": "kitty",
"imageId": image_id,
"color": render.kitty_color_hex(image_id),
"cols": payload["cols"],
"rows": payload["rows"],
"placeholder": payload["placeholder"],
"frames": payload["frames"],
"frameMs": constants.LOOP_MS / max(1, kcount),
"scale": scale,
},
)
renderer = PetRenderer(
str(pet.spritesheet),
mode="unicode",
scale=scale,
unicode_cols=cols,
)
count = renderer.frame_count(state) or 1
frames = []
for i in range(count):
grid = renderer.cells(state, i, cols=cols)
frames.append(
[[[*top, *bottom] for (top, bottom) in row] for row in grid]
)
return _ok(
rid,
{
"enabled": True,
"slug": pet.slug,
"displayName": pet.display_name,
"state": state,
"cols": cols,
"frameMs": constants.LOOP_MS / max(1, count),
"frames": frames,
"scale": scale,
},
)
except Exception as exc: # noqa: BLE001
logger.debug("pet.cells failed: %s", exc)
return _ok(rid, {"enabled": False})
@method("pet.gallery")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""List adoptable pets for the desktop appearance picker.
Returns the petdex gallery merged with local install state plus the
current config (active slug + enabled). Agent-independent. Fail-open:
returns whatever is installed locally if the gallery can't be reached, so
the picker still works offline.
"""
try:
from agent.pet import store
try:
from hermes_cli.config import load_config
cfg = load_config()
display = cfg.get("display", {}) if isinstance(cfg.get("display"), dict) else {}
pet_cfg = display.get("pet", {}) if isinstance(display.get("pet"), dict) else {}
except Exception:
pet_cfg = {}
installed = {p.slug: p for p in store.installed_pets()}
gallery: list[dict] = []
seen: set[str] = set()
try:
from agent.pet.manifest import fetch_manifest
for entry in fetch_manifest():
seen.add(entry.slug)
gallery.append(
{
"slug": entry.slug,
"displayName": entry.display_name,
"installed": entry.slug in installed,
"spritesheetUrl": entry.spritesheet_url,
# petdex exposes no popularity metric; "curated" (its
# hand-picked/official set, identified by the asset path)
# is the closest signal, so the picker can surface it first.
"curated": "/curated/" in entry.spritesheet_url,
}
)
except Exception as exc: # noqa: BLE001 - offline: fall back to installed
logger.debug("pet.gallery manifest fetch failed: %s", exc)
# Always include locally-installed pets even if the gallery is unreachable.
for slug, pet in installed.items():
if slug not in seen:
gallery.append(
{"slug": slug, "displayName": pet.display_name, "installed": True, "spritesheetUrl": ""}
)
return _ok(
rid,
{
"enabled": bool(pet_cfg.get("enabled")),
"active": str(pet_cfg.get("slug", "") or ""),
"pets": gallery,
},
)
except Exception as exc: # noqa: BLE001
logger.debug("pet.gallery failed: %s", exc)
return _ok(rid, {"enabled": False, "active": "", "pets": []})
@method("pet.select")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Adopt a pet from the desktop picker: install (if needed) + activate.
Params: ``slug`` (required). Writes ``display.pet.*`` to config and returns
``{ok, slug, displayName}``. The surface re-pulls ``pet.info`` to render it.
"""
slug = str(params.get("slug") or "").strip()
if not slug:
return _err(rid, 4004, "missing slug")
try:
from agent.pet import store
from agent.pet.manifest import ManifestError
from hermes_cli.pets import _set_active
try:
pet = store.install_pet(slug)
except (store.PetStoreError, ManifestError) as exc:
return _err(rid, 5031, f"could not adopt '{slug}': {exc}")
_set_active(slug)
return _ok(rid, {"ok": True, "slug": slug, "displayName": pet.display_name})
except Exception as exc: # noqa: BLE001
logger.debug("pet.select failed: %s", exc)
return _err(rid, 5031, f"pet.select failed: {exc}")
@method("pet.remove")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Uninstall a pet from the desktop picker (delete its on-disk directory).
Params: ``slug`` (required). If the removed pet was the active one, the
display is turned off so nothing tries to render a now-missing sprite.
Returns ``{ok, slug}`` where ``ok`` reflects whether a directory was deleted.
"""
slug = str(params.get("slug") or "").strip()
if not slug:
return _err(rid, 4004, "missing slug")
try:
from agent.pet import store
from hermes_cli.pets import _clear_active_if
removed = store.remove_pet(slug)
# If that was the active pet, stop surfaces pointing at a deleted sprite.
try:
_clear_active_if(slug)
except Exception as exc: # noqa: BLE001 - removal already succeeded
logger.debug("pet.remove config update failed: %s", exc)
return _ok(rid, {"ok": removed, "slug": slug})
except Exception as exc: # noqa: BLE001
logger.debug("pet.remove failed: %s", exc)
return _err(rid, 5031, f"pet.remove failed: {exc}")
@method("pet.thumb")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Return a small idle-frame PNG (data URI) for one pet — the picker preview.
Cropped + cached server-side so the renderer gets a same-origin data URL
instead of a CDN ``<img>`` (which the desktop CSP / R2 hotlink rules break).
Params: ``slug`` (required), ``url`` (optional petdex spritesheet URL used
only for not-yet-installed pets). Fail-open: ``{ok: false}`` with no error.
"""
slug = str(params.get("slug") or "").strip()
if not slug:
return _err(rid, 4004, "missing slug")
try:
import base64
from agent.pet import store
data = store.thumbnail_png(slug, source_url=str(params.get("url") or ""))
if not data:
return _ok(rid, {"ok": False, "slug": slug})
return _ok(
rid,
{
"ok": True,
"slug": slug,
"dataUri": "data:image/png;base64," + base64.standard_b64encode(data).decode("ascii"),
},
)
except Exception as exc: # noqa: BLE001
logger.debug("pet.thumb failed: %s", exc)
return _ok(rid, {"ok": False, "slug": slug})
@method("pet.disable")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Turn the pet off from the desktop picker (``display.pet.enabled=false``)."""
try:
from hermes_cli.pets import _set_enabled
_set_enabled(False)
return _ok(rid, {"ok": True})
except Exception as exc: # noqa: BLE001
logger.debug("pet.disable failed: %s", exc)
return _err(rid, 5031, f"pet.disable failed: {exc}")
@method("pet.scale")
@_profile_scoped
def _(rid, params: dict) -> dict:
"""Persist ``display.pet.scale`` from the desktop slider. Params: ``scale``.
Clamped to the engine bounds. The renderer updates its own ``$petInfo`` for
instant feedback; this just makes the change durable + visible to the other
terminal surfaces on their next read.
"""
try:
from hermes_cli.pets import set_pet_scale
scale, err = set_pet_scale(params.get("scale"))
if err:
return _err(rid, 4004, err)
return _ok(rid, {"ok": True, "scale": scale})
except Exception as exc: # noqa: BLE001
logger.debug("pet.scale failed: %s", exc)
return _err(rid, 5031, f"pet.scale failed: {exc}")
@method("credits.view")
def _(rid, params: dict) -> dict:
"""Structured Nous credit view for the TUI /credits command.

View file

@ -208,6 +208,41 @@ describe('createSlashHandler', () => {
})
})
it('opens the pet picker for /pet list only', () => {
const ctx = buildCtx()
expect(createSlashHandler(ctx)('/pet list')).toBe(true)
expect(getOverlayState().petPicker).toBe(true)
expect(ctx.gateway.gw.request).not.toHaveBeenCalled()
resetOverlayState()
expect(createSlashHandler(ctx)('/pet')).toBe(true)
expect(getOverlayState().petPicker).toBe(false)
expect(ctx.gateway.gw.request).toHaveBeenCalledWith(
'slash.exec',
expect.objectContaining({ command: 'pet' })
)
resetOverlayState()
expect(createSlashHandler(ctx)('/pet toggle')).toBe(true)
expect(getOverlayState().petPicker).toBe(false)
expect(ctx.gateway.gw.request).toHaveBeenCalledWith(
'slash.exec',
expect.objectContaining({ command: 'pet toggle' })
)
})
it('routes /pet <slug> to the slash worker without opening the picker', () => {
const ctx = buildCtx()
expect(createSlashHandler(ctx)('/pet boba')).toBe(true)
expect(getOverlayState().petPicker).toBe(false)
expect(ctx.gateway.gw.request).toHaveBeenCalledWith(
'slash.exec',
expect.objectContaining({ command: 'pet boba' })
)
})
it('routes /skills inspect <name> to skills.manage', () => {
const ctx = buildCtx()

View file

@ -9,6 +9,7 @@ import type {
GatewaySkin,
SessionMostRecentResponse
} from '../gatewayTypes.js'
import { isTodoDone } from '../lib/liveProgress.js'
import { rpcErrorMessage } from '../lib/rpc.js'
import { openExternalUrl } from '../lib/openExternalUrl.js'
import { topLevelSubagents } from '../lib/subagentTree.js'
@ -19,7 +20,9 @@ import type { Msg, SubagentProgress, SubagentStatus } from '../types.js'
import { applyDelegationStatus, getDelegationState } from './delegationStore.js'
import type { GatewayEventHandlerContext } from './interfaces.js'
import { getOverlayState, patchOverlayState } from './overlayStore.js'
import { flashPet } from './petFlashStore.js'
import { turnController } from './turnController.js'
import { getTurnState } from './turnStore.js'
import { getUiState, patchUiState } from './uiStore.js'
const NO_PROVIDER_RE = /\bNo (?:LLM|inference) provider configured\b/i
@ -908,6 +911,9 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev:
const msgs: Msg[] = finalMessages.length ? finalMessages : [{ role: 'assistant', text: finalText }]
msgs.forEach(appendMessage)
// Pet beat: celebrate a finished plan, otherwise a clean-finish wave.
flashPet(isTodoDone(getTurnState().todos) ? 'jump' : 'wave')
if (bellOnComplete && stdout?.isTTY) {
stdout.write('\x07')
}
@ -924,6 +930,7 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev:
case 'error':
turnController.recordError()
flashPet('failed')
{
const message = String(ev.payload?.message || 'unknown error')

View file

@ -136,6 +136,7 @@ export interface OverlayState {
confirm: ConfirmReq | null
modelPicker: boolean
pager: null | PagerState
petPicker: boolean
pluginsHub: boolean
secret: null | SecretReq
sessions: boolean

View file

@ -11,6 +11,7 @@ const buildOverlayState = (): OverlayState => ({
confirm: null,
modelPicker: false,
pager: null,
petPicker: false,
pluginsHub: false,
secret: null,
sessions: false,
@ -22,7 +23,7 @@ export const $overlayState = atom<OverlayState>(buildOverlayState())
export const $isBlocked = computed(
$overlayState,
({ agents, approval, billing, clarify, confirm, modelPicker, pager, pluginsHub, secret, sessions, skillsHub, sudo }) =>
({ agents, approval, billing, clarify, confirm, modelPicker, pager, petPicker, pluginsHub, secret, sessions, skillsHub, sudo }) =>
Boolean(
agents ||
approval ||
@ -31,6 +32,7 @@ export const $isBlocked = computed(
confirm ||
modelPicker ||
pager ||
petPicker ||
pluginsHub ||
secret ||
sessions ||
@ -61,6 +63,7 @@ export const resetFlowOverlays = () =>
agents: $overlayState.get().agents,
agentsInitialHistoryIndex: $overlayState.get().agentsInitialHistoryIndex,
modelPicker: $overlayState.get().modelPicker,
petPicker: $overlayState.get().petPicker,
pluginsHub: $overlayState.get().pluginsHub,
sessions: $overlayState.get().sessions,
skillsHub: $overlayState.get().skillsHub

View file

@ -0,0 +1,16 @@
import { atom } from 'nanostores'
import type { PetState } from './usePet.js'
interface PetFlash {
state: PetState
until: number
}
// Transient reaction beats (wave/jump/failed) the pet shows for a moment at
// turn end before falling back to its steady state. The gateway event handler
// sets these; usePet reads them with priority over the derived state.
export const $petFlash = atom<PetFlash | null>(null)
export const flashPet = (state: PetState, ms = 1600) =>
$petFlash.set({ state, until: Date.now() + ms })

View file

@ -8,6 +8,7 @@ import type {
SessionBranchResponse,
SessionCompressResponse,
SessionUsageResponse,
SlashExecResponse,
VoiceToggleResponse
} from '../../../gatewayTypes.js'
import { formatVoiceRecordKey, parseVoiceRecordKey } from '../../../lib/platform.js'
@ -340,6 +341,31 @@ export const sessionCommands: SlashCommand[] = [
}
},
{
help: 'toggle / adopt / resize an animated pet',
name: 'pet',
usage: '/pet [toggle | list | scale <n> | <slug>]',
run: (arg, ctx, cmd) => {
const sub = arg.trim().toLowerCase()
// Gallery picker — the interactive browse surface.
if (sub === 'list') {
return patchOverlayState({ petPicker: true })
}
// Bare /pet and /pet toggle flip display.pet.enabled via the slash worker.
ctx.gateway.gw
.request<SlashExecResponse>('slash.exec', { command: cmd.slice(1), session_id: ctx.sid })
.then(
ctx.guarded<SlashExecResponse>(r => {
const body = r.output || '/pet: no output'
ctx.transcript.sys(r.warning ? `warning: ${r.warning}\n${body}` : body)
})
)
.catch(ctx.guardedErr)
}
},
{
help: 'switch theme skin (fires skin.changed)',
name: 'skin',
@ -553,6 +579,7 @@ export const sessionCommands: SlashCommand[] = [
// even with zero API calls or on a resumed session. Render it whenever
// present, before the token panel.
const creditsLines = r?.credits_lines ?? []
if (creditsLines.length) {
ctx.transcript.panel('Nous credits', [{ text: creditsLines.join('\n') }])
}
@ -561,6 +588,7 @@ export const sessionCommands: SlashCommand[] = [
if (!creditsLines.length) {
ctx.transcript.sys('no API calls yet')
}
return
}

View file

@ -147,6 +147,10 @@ export function useInputHandlers(ctx: InputHandlerContext): InputHandlerResult {
return patchOverlayState({ modelPicker: false })
}
if (overlay.petPicker) {
return patchOverlayState({ petPicker: false })
}
if (overlay.billing) {
return patchOverlayState({ billing: null })
}

313
ui-tui/src/app/usePet.ts Normal file
View file

@ -0,0 +1,313 @@
import { useStdout } from '@hermes/ink'
import { useCallback, useEffect, useRef, useState } from 'react'
import type { PetGrid } from '../components/petSprite.js'
import { useGateway } from './gatewayContext.js'
import { getOverlayState, $overlayState } from './overlayStore.js'
import { $petFlash } from './petFlashStore.js'
import { $turnState } from './turnStore.js'
import { $uiState } from './uiStore.js'
export type PetState = 'idle' | 'wave' | 'run' | 'failed' | 'review' | 'jump' | 'waiting'
interface PetActivity {
busy: boolean
toolRunning: boolean
reasoning: boolean
awaitingInput: boolean
}
/**
* Resolve the animation state mirrors `agent.pet.state.derive_pet_state`
* (and the desktop's `derivePetState`) so all surfaces agree. `awaitingInput`
* (a clarify/approval blocking on the user) outranks the in-flight signals
* because the turn is paused on you, not working.
*/
export function derivePetState({ busy, toolRunning, reasoning, awaitingInput }: PetActivity): PetState {
if (awaitingInput) {
return 'waiting'
}
if (toolRunning) {
return 'run'
}
if (reasoning) {
return 'review'
}
if (busy) {
return 'run'
}
return 'idle'
}
// The overlays that mean "the agent is blocked on the user" (vs. user-toggled
// pickers like model/sessions, which aren't the agent waiting).
function isAwaitingInput(): boolean {
const o = getOverlayState()
return Boolean(o.clarify || o.approval || o.sudo || o.secret || o.confirm)
}
// A kitty Unicode-placeholder frame set: a static placeholder grid (painted by
// Ink in the image-id color) plus per-frame transmit escapes written straight
// to the terminal out-of-band.
interface KittyView {
color: string
placeholder: string[]
}
interface PetCellsResult {
color?: string
enabled?: boolean
frameMs?: number
// unicode mode: cell grids; kitty mode: transmit-escape strings.
frames?: PetGrid[] | string[]
graphics?: string
imageId?: number
placeholder?: string[]
scale?: number
slug?: string
state?: string
}
type CacheEntry =
| { kind: 'cells'; frameMs: number; frames: PetGrid[] }
| { kind: 'kitty'; frameMs: number; frames: string[]; placeholder: string[]; color: string }
const FRAME_MS = 160
const POLL_MS = 2500
// Only the standalone TUI owns a real terminal it can splat image escapes into;
// when piped (or running under the dashboard PTY the gateway resolves to
// half-blocks anyway) we never ask for graphics.
const IS_TTY = Boolean(process.stdout?.isTTY)
export interface PetRender {
enabled: boolean
grid: PetGrid | null
kitty: KittyView | null
}
/**
* Drives the TUI pet. Fetches each (slug, state)'s frames via the `pet.cells`
* RPC (cached) and animates the frame index. Two render paths:
*
* - **kitty** (Ghostty/kitty): the engine returns a static placeholder grid +
* per-frame transmit escapes. We paint the placeholder with Ink and write the
* current frame's escape to the terminal out-of-band, so the image animates
* underneath without Ink ever repainting.
* - **cells** (everywhere else): truecolor half-block grids painted by Ink.
*
* A steady poll keeps it reactive to config changes made elsewhere (`/pet`, the
* picker, `hermes pets select`) so adopting/switching/disabling takes effect
* live. The frame cache is keyed by `slug:state` so a switch re-pulls cleanly.
*/
export function usePet(): PetRender {
const { rpc } = useGateway()
const { write } = useStdout()
const [enabled, setEnabled] = useState(false)
const [grid, setGrid] = useState<PetGrid | null>(null)
const [kitty, setKitty] = useState<KittyView | null>(null)
const cache = useRef<Map<string, CacheEntry>>(new Map())
const slugRef = useRef('')
const scaleRef = useRef(0)
const imageIdRef = useRef(0)
const stateRef = useRef<PetState>('idle')
const frameRef = useRef(0)
const [petState, setPetState] = useState<PetState>('idle')
// Recompute the desired state on every turn/ui/flash change. A transient
// flash (wave/jump/failed) wins until it expires; a timer re-runs at expiry.
useEffect(() => {
let expiry: ReturnType<typeof setTimeout> | undefined
const apply = (next: PetState) => {
if (next !== stateRef.current) {
stateRef.current = next
frameRef.current = 0
setPetState(next)
}
}
const recompute = () => {
clearTimeout(expiry)
const flash = $petFlash.get()
const now = Date.now()
if (flash && now < flash.until) {
apply(flash.state)
expiry = setTimeout(recompute, flash.until - now)
return
}
const turn = $turnState.get()
const ui = $uiState.get()
apply(
derivePetState({
awaitingInput: isAwaitingInput(),
busy: ui.busy,
reasoning: turn.reasoningActive,
toolRunning: turn.tools.length > 0
})
)
}
recompute()
const unsubTurn = $turnState.listen(recompute)
const unsubUi = $uiState.listen(recompute)
const unsubFlash = $petFlash.listen(recompute)
const unsubOverlay = $overlayState.listen(recompute)
return () => {
clearTimeout(expiry)
unsubTurn()
unsubUi()
unsubFlash()
unsubOverlay()
}
}, [])
// Free the terminal-side image when the pet goes away or the hook unmounts.
const releaseKitty = useCallback(() => {
if (imageIdRef.current) {
try {
write(`\x1b_Ga=d,d=i,i=${imageIdRef.current},q=2\x1b\\`)
} catch {
// best-effort cleanup
}
imageIdRef.current = 0
}
}, [write])
// Fetch + cache one (slug, state). `pet.cells` resolves the active pet from
// config, so its `slug`/`enabled` are the source of truth.
const sync = useCallback(
async (state: PetState) => {
try {
const res = (await rpc('pet.cells', { graphics: IS_TTY, state })) as PetCellsResult | null
if (!res) {
return
}
if (!res.enabled) {
releaseKitty()
slugRef.current = ''
cache.current.clear()
setGrid(null)
setKitty(null)
setEnabled(false)
return
}
const slug = res.slug ?? ''
const scale = res.scale ?? 0
// A switch OR a live `/pet scale` change invalidates the cached frames
// (they're rendered at the old size), so the steady poll repaints at the
// new scale without a restart.
if (slug !== slugRef.current || (scale > 0 && scale !== scaleRef.current)) {
releaseKitty()
slugRef.current = slug
scaleRef.current = scale
cache.current.clear()
frameRef.current = 0
}
if (res.graphics === 'kitty' && res.frames?.length && res.placeholder?.length) {
imageIdRef.current = res.imageId ?? 0
cache.current.set(`${slug}:${state}`, {
color: res.color ?? '#000001',
frameMs: res.frameMs ?? FRAME_MS,
frames: res.frames as string[],
kind: 'kitty',
placeholder: res.placeholder
})
} else if (res.frames?.length) {
cache.current.set(`${slug}:${state}`, {
frameMs: res.frameMs ?? FRAME_MS,
frames: res.frames as PetGrid[],
kind: 'cells'
})
}
setEnabled(true)
} catch {
// cosmetic — ignore RPC failures
}
},
[rpc, releaseKitty]
)
// Pull frames whenever the state changes (if not already cached for the
// active pet), plus a steady poll that catches adopt/switch/disable.
useEffect(() => {
if (!cache.current.has(`${slugRef.current}:${petState}`)) {
void sync(petState)
}
const timer = setInterval(() => void sync(stateRef.current), POLL_MS)
return () => clearInterval(timer)
}, [petState, sync])
useEffect(() => releaseKitty, [releaseKitty])
// Animation timer.
useEffect(() => {
if (!enabled) {
return
}
const tick = () => {
const entry = cache.current.get(`${slugRef.current}:${stateRef.current}`)
if (!entry?.frames.length) {
return // keep the last frame painted while the new state loads
}
const idx = frameRef.current % entry.frames.length
frameRef.current = idx + 1
if (entry.kind === 'kitty') {
// Transmit this frame's image under the shared id; the static
// placeholder cells (set below) render it. No Ink repaint needed.
try {
write(entry.frames[idx] ?? '')
} catch {
// ignore transmit failures
}
setGrid(null)
setKitty(prev =>
prev && prev.color === entry.color && prev.placeholder === entry.placeholder
? prev
: { color: entry.color, placeholder: entry.placeholder }
)
return
}
setKitty(null)
setGrid(entry.frames[idx] ?? null)
}
tick()
const interval = setInterval(tick, FRAME_MS)
return () => clearInterval(interval)
}, [enabled, petState, write])
return { enabled, grid, kitty }
}

View file

@ -6,6 +6,7 @@ import { useGateway } from '../app/gatewayContext.js'
import type { AppLayoutProps } from '../app/interfaces.js'
import { $isBlocked, $overlayState, patchOverlayState } from '../app/overlayStore.js'
import { $uiState } from '../app/uiStore.js'
import { usePet } from '../app/usePet.js'
import { INLINE_MODE, SHOW_FPS, TERMUX_TUI_MODE } from '../config/env.js'
import { PLACEHOLDER } from '../content/placeholders.js'
import { prevRenderedMsg } from '../domain/blockLayout.js'
@ -25,10 +26,29 @@ import { Banner, Panel, SessionPanel } from './branding.js'
import { FpsOverlay } from './fpsOverlay.js'
import { HelpHint } from './helpHint.js'
import { MessageLine } from './messageLine.js'
import { PetKitty, PetSprite } from './petSprite.js'
import { QueuedMessages } from './queuedMessages.js'
import { LiveTodoPanel, StreamingAssistant } from './streamingAssistant.js'
import { TextInput, type TextInputMouseApi } from './textInput.js'
// Petdex mascot — sits just above the composer, right-aligned. Renders
// nothing unless a pet is installed + enabled (`hermes pets select <slug>`),
// so it's a no-op for everyone else.
const PetPane = memo(function PetPane() {
const { enabled, grid, kitty } = usePet()
if (!enabled || (!grid && !kitty)) {
return null
}
return (
<NoSelect flexShrink={0} justifyContent="flex-end" paddingX={1} width="100%">
{kitty ? <PetKitty color={kitty.color} placeholder={kitty.placeholder} /> : null}
{!kitty && grid ? <PetSprite grid={grid} /> : null}
</NoSelect>
)
})
const PromptPrefix = memo(function PromptPrefix({
bold = false,
color,
@ -420,6 +440,8 @@ export const AppLayout = memo(function AppLayout({
{!overlay.agents && (
<>
<PetPane />
<PerfPane id="prompt">
<PromptZone
cols={composer.cols}

View file

@ -12,6 +12,7 @@ import { BillingOverlay } from './billingOverlay.js'
import { MaskedPrompt } from './maskedPrompt.js'
import { ModelPicker } from './modelPicker.js'
import { OverlayHint } from './overlayControls.js'
import { PetPicker } from './petPicker.js'
import { PluginsHub } from './pluginsHub.js'
import { ApprovalPrompt, ClarifyPrompt, ConfirmPrompt } from './prompts.js'
import { SkillsHub } from './skillsHub.js'
@ -140,6 +141,7 @@ export function FloatingOverlays({
const hasAny =
overlay.modelPicker ||
overlay.pager ||
overlay.petPicker ||
overlay.sessions ||
overlay.skillsHub ||
overlay.pluginsHub ||
@ -186,6 +188,12 @@ export function FloatingOverlays({
</FloatBox>
)}
{overlay.petPicker && (
<FloatBox color={theme.color.border}>
<PetPicker gw={gw} onClose={() => patchOverlayState({ petPicker: false })} t={theme} />
</FloatBox>
)}
{overlay.skillsHub && (
<FloatBox color={theme.color.border}>
<SkillsHub gw={gw} onClose={() => patchOverlayState({ skillsHub: false })} t={theme} />

View file

@ -0,0 +1,183 @@
import { Box, Text, useInput, useStdout } from '@hermes/ink'
import { useEffect, useMemo, useState } from 'react'
import type { GatewayClient } from '../gatewayClient.js'
import { rpcErrorMessage } from '../lib/rpc.js'
import type { Theme } from '../theme.js'
import { OverlayHint, windowItems } from './overlayControls.js'
const VISIBLE = 10
const MIN_WIDTH = 40
const MAX_WIDTH = 90
interface GalleryPet {
slug: string
displayName: string
installed: boolean
curated?: boolean
}
interface Gallery {
enabled: boolean
active: string
pets: GalleryPet[]
}
/**
* Interactive petdex picker overlay. Pulls the gallery via `pet.gallery`,
* filters as you type, and adopts the highlighted pet with `pet.select`
* (install-on-demand). The mascot lights up live once `usePet` next polls
* no restart. This is the interactive sibling of the text `/pet <slug>` path.
*/
export function PetPicker({ gw, onClose, t }: PetPickerProps) {
const [gallery, setGallery] = useState<Gallery | null>(null)
const [query, setQuery] = useState('')
const [idx, setIdx] = useState(0)
const [busy, setBusy] = useState(false)
const [err, setErr] = useState('')
const [loading, setLoading] = useState(true)
const { stdout } = useStdout()
const width = Math.max(MIN_WIDTH, Math.min(MAX_WIDTH, (stdout?.columns ?? 80) - 6))
useEffect(() => {
gw.request<Gallery>('pet.gallery')
.then(r => {
setGallery(r)
setErr('')
})
.catch((e: unknown) => setErr(rpcErrorMessage(e)))
.finally(() => setLoading(false))
}, [gw])
const enabled = gallery?.enabled ?? false
const active = gallery?.active ?? ''
// Rank by the signals petdex gives us — active, then installed, then curated
// (its official set), then the rest — and hide the clawd placeholders.
const view = useMemo(() => {
const pets = (gallery?.pets ?? []).filter(p => !/^clawd(-|$)/i.test(p.slug))
const needle = query.trim().toLowerCase()
const matched = needle
? pets.filter(p => p.slug.toLowerCase().includes(needle) || p.displayName.toLowerCase().includes(needle))
: pets
const rank = (p: GalleryPet) => (enabled && p.slug === active ? 4 : 0) + (p.installed ? 2 : 0) + (p.curated ? 1 : 0)
return [...matched].sort((a, b) => rank(b) - rank(a))
}, [gallery, query, enabled, active])
const adopt = (slug: string) => {
setBusy(true)
setErr('')
gw.request('pet.select', { slug })
.then(() => onClose())
.catch((e: unknown) => {
setErr(rpcErrorMessage(e))
setBusy(false)
})
}
useInput((input, key) => {
if (busy) {
return
}
if (key.escape) {
return onClose()
}
if (key.upArrow) {
return setIdx(i => Math.max(0, i - 1))
}
if (key.downArrow) {
return setIdx(i => Math.min(view.length - 1, i + 1))
}
if (key.return) {
const pet = view[idx]
return pet ? adopt(pet.slug) : undefined
}
if (key.backspace || key.delete) {
setQuery(q => q.slice(0, -1))
return setIdx(0)
}
// Printable char → extend the filter (ignore control/chorded keys).
if (input && input.length === 1 && input >= ' ' && !key.ctrl && !key.meta) {
setQuery(q => q + input)
setIdx(0)
}
})
if (loading) {
return <Text color={t.color.muted}>loading pets</Text>
}
if (err && !gallery) {
return (
<Box flexDirection="column" width={width}>
<Text color={t.color.label}>error: {err}</Text>
<OverlayHint t={t}>Esc cancel</OverlayHint>
</Box>
)
}
const { items, offset } = windowItems(view, idx, VISIBLE)
return (
<Box flexDirection="column" width={width}>
<Text bold color={t.color.accent}>
Pets
</Text>
<Text color={t.color.muted} wrap="truncate-end">
{query ? `filter: ${query}` : 'type to filter'} · {view.length} pet{view.length === 1 ? '' : 's'}
</Text>
{offset > 0 && <Text color={t.color.muted}> {offset} more</Text>}
{view.length === 0 ? (
<Text color={t.color.muted}>{query ? `no pets match "${query}"` : 'no pets available'}</Text>
) : (
items.map((pet, i) => {
const at = offset + i === idx
const isActive = enabled && pet.slug === active
const mark = isActive ? '●' : pet.installed ? '✓' : ' '
const tag = pet.installed ? '' : pet.curated ? ' · official' : ''
return (
<Text bold={at} color={at ? t.color.accent : t.color.muted} inverse={at} key={pet.slug} wrap="truncate-end">
{at ? '▸ ' : ' '}
{mark} {pet.displayName}
<Text color={at ? t.color.accent : t.color.muted}>
{' '}
({pet.slug}
{tag})
</Text>
</Text>
)
})
)}
{offset + VISIBLE < view.length && <Text color={t.color.muted}> {view.length - offset - VISIBLE} more</Text>}
{err ? <Text color={t.color.label}>error: {err}</Text> : null}
{busy ? <Text color={t.color.accent}>adopting</Text> : null}
<OverlayHint t={t}>/ select · Enter adopt · type to filter · Esc cancel</OverlayHint>
</Box>
)
}
interface PetPickerProps {
gw: GatewayClient
onClose: () => void
t: Theme
}

View file

@ -0,0 +1,93 @@
import { Box, Text } from '@hermes/ink'
import { memo } from 'react'
// A cell is [tr,tg,tb,ta, br,bg,bb,ba] — the top + bottom pixel of one
// half-block, as produced by the `pet.cells` gateway RPC.
export type PetCell = number[]
export type PetGrid = PetCell[][]
const UPPER_HALF = '▀'
const LOWER_HALF = '▄'
const hex = (r: number, g: number, b: number) =>
`#${[r, g, b].map(v => Math.max(0, Math.min(255, v | 0)).toString(16).padStart(2, '0')).join('')}`
/**
* Renders one petdex frame as truecolor half-blocks using native Ink color
* props (no raw ANSI, so width measurement stays correct). The engine
* (`agent/pet/render.py`) does the decode + downscale; this is a thin painter.
*/
export const PetSprite = memo(function PetSprite({ grid }: { grid: PetGrid }) {
if (!grid.length) {
return null
}
return (
<Box flexDirection="column">
{grid.map((row, y) => (
<Box key={y}>
{row.map((cell, x) => {
const [tr, tg, tb, ta, br, bg, bb, ba] = cell
const top = (ta ?? 0) >= 32
const bot = (ba ?? 0) >= 32
if (!top && !bot) {
return <Text key={x}> </Text>
}
// Both halves opaque → fg=top over bg=bottom. One half opaque →
// draw it fg-only so the other stays the terminal bg (no black
// boxes bleeding around transparent sprite edges).
if (top && bot) {
return (
<Text backgroundColor={hex(br, bg, bb)} color={hex(tr, tg, tb)} key={x}>
{UPPER_HALF}
</Text>
)
}
return top ? (
<Text color={hex(tr, tg, tb)} key={x}>
{UPPER_HALF}
</Text>
) : (
<Text color={hex(br, bg, bb)} key={x}>
{LOWER_HALF}
</Text>
)
})}
</Box>
))}
</Box>
)
})
/**
* Renders a kitty Unicode-placeholder grid: each line is a row of U+10EEEE
* cells whose foreground color encodes the image id. The actual pixels are
* drawn by the terminal (the frame image is transmitted out-of-band by
* `usePet`); this only emits the placeholder text Ink can measure as width-1
* cells. Truecolor-only the color must reach the terminal verbatim for the
* id to decode, which Ghostty/kitty support.
*/
export const PetKitty = memo(function PetKitty({
color,
placeholder
}: {
color: string
placeholder: string[]
}) {
if (!placeholder.length) {
return null
}
return (
<Box flexDirection="column">
{placeholder.map((row, y) => (
<Text color={color} key={y}>
{row}
</Text>
))}
</Box>
)
})

View file

@ -156,7 +156,10 @@ declare module '@hermes/ink' {
readonly setSelectionBgColor: (color: string) => void
}
export function useHasSelection(): boolean
export function useStdout(): { readonly stdout?: NodeJS.WriteStream }
export function useStdout(): {
readonly stdout?: NodeJS.WriteStream
readonly write: (data: string) => boolean
}
export function useTerminalFocus(): boolean
export function useTerminalTitle(title: string | null): void
export function useDeclaredCursor(args: {