mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat(tui): attach images inline at the cursor, delete the token to unattach

Browse source 
Every attach path now drops an `[[ Image N ]]` token where you are typing:
drag-drop, clipboard (bracketed and hotkey), /image, /paste. The composer
owns clipboard attach directly instead of calling back out to useMainApp.

Deleting the token is how you unattach — there is no second control.
updateInput is the one choke point every keystroke passes through, so
syncTokens reconciles there and detaches anything erased. That also fixes
a stale image riding along on the next unrelated turn.

Tokens and the input line get refs alongside state: paste-then-immediately
-Enter submits before React has re-rendered, and the submit path has to see
the token that was just added.
This commit is contained in:
Brooklyn Nicholson 2026-07-30 23:42:53 -05:00
parent fead8c8d6a
commit ca5ee5ed33
6 changed files with 238 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ui-tui/src/app/slash/commands/core.ts
View file

@ -439,7 +439,7 @@ export const coreCommands: SlashCommand[] = [
{
help: 'attach clipboard image',
name: 'paste',
run: (arg, ctx) => (arg ? ctx.transcript.sys('usage: /paste') : ctx.composer.paste())
run: (arg, ctx) => (arg ? ctx.transcript.sys('usage: /paste') : ctx.composer.attachClipboardImage())
},
{

15
ui-tui/src/app/slash/commands/session.ts
View file

@ -1,11 +1,10 @@
import { usageBarsText } from '../../../components/overlayPrimitives.js'
import { attachedImageNotice, introMsg, toTranscriptMessages } from '../../../domain/messages.js'
import { introMsg, toTranscriptMessages } from '../../../domain/messages.js'
import { sessionScopedModelArg, TUI_SESSION_MODEL_FLAG } from '../../../domain/slash.js'
import type {
BackgroundStartResponse,
ConfigGetValueResponse,
ConfigSetResponse,
ImageAttachResponse,
SessionBranchResponse,
SessionCompressResponse,
SessionUsageResponse,
@ -192,17 +191,7 @@ export const sessionCommands: SlashCommand[] = [
{
help: 'attach an image',
name: 'image',
run: (arg, ctx) => {
ctx.gateway.rpc<ImageAttachResponse>('image.attach', { path: arg, session_id: ctx.sid }).then(
ctx.guarded<ImageAttachResponse>(r => {
ctx.transcript.sys(attachedImageNotice(r))
if (r.remainder) {
ctx.composer.setInput(r.remainder)
}
})
)
}
run: (arg, ctx) => ctx.composer.attachImagePath(arg)
},
{

252
ui-tui/src/app/useComposerState.ts
View file

@ -5,10 +5,11 @@ import { join } from 'node:path'
import { useStdin, withInkSuspended } from '@hermes/ink'
import { useStore } from '@nanostores/react'
import { useCallback, useMemo, useState } from 'react'
import { useCallback, useMemo, useRef, useState } from 'react'
import type { PasteEvent } from '../components/textInput.js'
import type { ImageAttachResponse, InputDetectDropResponse } from '../gatewayTypes.js'
import { droppedTokens, imageToken, nextImageIndex } from '../domain/attachments.js'
import type { ClipboardPasteResponse, ImageAttachResponse, InputDetectDropResponse } from '../gatewayTypes.js'
import { useCompletion } from '../hooks/useCompletion.js'
import { useInputHistory } from '../hooks/useInputHistory.js'
import { useQueue } from '../hooks/useQueue.js'
@ -18,30 +19,37 @@ import { readOsc52Clipboard } from '../lib/osc52.js'
import { isRemoteShellSession } from '../lib/terminalSetup.js'
import { pasteTokenLabel, stripTrailingPasteNewlines } from '../lib/text.js'
import type { MaybePromise, PasteSnippet, UseComposerStateOptions, UseComposerStateResult } from './interfaces.js'
import type {
ComposerPasteResult,
ComposerToken,
MaybePromise,
StateSetter,
UseComposerStateOptions,
UseComposerStateResult
} from './interfaces.js'
import { $isBlocked } from './overlayStore.js'
import { getUiState } from './uiStore.js'
const PASTE_SNIP_MAX_COUNT = 32
const PASTE_SNIP_MAX_TOTAL_BYTES = 4 * 1024 * 1024
const TOKEN_MAX_COUNT = 32
const TOKEN_MAX_TOTAL_BYTES = 4 * 1024 * 1024
const trimSnips = (snips: PasteSnippet[]): PasteSnippet[] => {
const trimTokens = (tokens: ComposerToken[]): ComposerToken[] => {
let total = 0
const out: PasteSnippet[] = []
const out: ComposerToken[] = []
for (let i = snips.length - 1; i >= 0; i--) {
const snip = snips[i]!
const size = snip.text.length
for (let i = tokens.length - 1; i >= 0; i--) {
const token = tokens[i]!
const size = token.text?.length ?? 0
if (out.length >= PASTE_SNIP_MAX_COUNT || total + size > PASTE_SNIP_MAX_TOTAL_BYTES) {
if (out.length >= TOKEN_MAX_COUNT || total + size > TOKEN_MAX_TOTAL_BYTES) {
break
}
total += size
out.unshift(snip)
out.unshift(token)
}
return out.length === snips.length ? snips : out
return out.length === tokens.length ? tokens : out
}
/** Insert text at the cursor position, adding spacing to separate from adjacent non-whitespace. */
@ -97,15 +105,26 @@ export function looksLikeDroppedPath(text: string): boolean {
return false
}
export function useComposerState({
gw,
onClipboardPaste,
onImageAttached,
submitRef
}: UseComposerStateOptions): UseComposerStateResult {
const [input, setInput] = useState('')
export function useComposerState({ gw, submitRef, sys }: UseComposerStateOptions): UseComposerStateResult {
const [input, setInputState] = useState('')
const [inputBuf, setInputBuf] = useState<string[]>([])
const [pasteSnips, setPasteSnips] = useState<PasteSnippet[]>([])
const [tokens, setTokens] = useState<ComposerToken[]>([])
// Tokens and the input line are read from keystroke handlers that run several
// times before React re-renders, so the refs — not the state — are the source
// of truth for "what is in the composer right now".
const inputRef = useRef('')
const tokensRef = useRef<ComposerToken[]>([])
const setInput = useCallback<StateSetter<string>>(next => {
inputRef.current = typeof next === 'function' ? next(inputRef.current) : next
setInputState(inputRef.current)
}, [])
const setComposerTokens = useCallback<StateSetter<ComposerToken[]>>(next => {
tokensRef.current = typeof next === 'function' ? next(tokensRef.current) : next
setTokens(tokensRef.current)
}, [])
const isBlocked = useStore($isBlocked)
const { querier } = useStdin() as { querier: Parameters<typeof readOsc52Clipboard>[0] }
@ -128,27 +147,95 @@ export function useComposerState({
const clearIn = useCallback(() => {
setInput('')
setInputBuf([])
setPasteSnips([])
setComposerTokens([])
setQueueEdit(null)
setHistoryIdx(null)
historyDraftRef.current = ''
}, [historyDraftRef, setQueueEdit, setHistoryIdx])
}, [historyDraftRef, setComposerTokens, setHistoryIdx, setInput, setQueueEdit])
/**
* Deleting an `[[ Image N ]]` token IS how you unattach the image — there is
* no separate control. Reconcile on every edit so the gateway's
* `attached_images` never outlives the token the user just erased, which is
* what used to make a stale image ride along on the next unrelated turn.
*/
const syncTokens = useCallback(
(value: string) => {
const gone = droppedTokens(tokensRef.current, value)
if (!gone.length) {
return
}
for (const token of gone) {
if (token.kind === 'image') {
void gw.request('image.detach', { path: token.path, session_id: getUiState().sid }).catch(() => {})
}
}
setComposerTokens(prev => prev.filter(token => !gone.includes(token)))
},
[gw, setComposerTokens]
)
/**
* Attach an image the gateway already resolved: a token at the cursor,
* followed by whatever non-path text came along with it (a drag-drop paste
* of `~/shot.png look at this` keeps the caption).
*/
const attachImageToken = useCallback(
(attached: ImageAttachResponse & { path?: string }, value: string, cursor: number): ComposerPasteResult => {
const index = nextImageIndex(tokensRef.current)
const label = imageToken(index)
setComposerTokens(prev => trimTokens([...prev, { index, kind: 'image', label, path: attached.path ?? '' }]))
const withToken = insertAtCursor(value, cursor, label)
const remainder = attached.remainder?.trim() ?? ''
return remainder ? insertAtCursor(withToken.value, withToken.cursor, remainder) : withToken
},
[setComposerTokens]
)
/**
* Pull an image off the system clipboard into the composer as a token.
*
* `quiet` is the empty-bracketed-paste probe: the terminal delivers an image
* paste as zero text, so we speculatively ask the gateway and stay silent if
* there was nothing there. An explicit `/paste` reports the miss.
*/
const pasteClipboardImage = useCallback(
async (value: string, cursor: number, quiet: boolean): Promise<ComposerPasteResult | null> => {
const sid = getUiState().sid
if (!sid) {
return null
}
const r = await gw
.request<ClipboardPasteResponse & { path?: string }>('clipboard.paste', { session_id: sid })
.catch(() => null)
if (r?.attached) {
return attachImageToken(r, value, cursor)
}
if (!quiet) {
sys(r?.message || 'No image found in clipboard')
}
return null
},
[attachImageToken, gw, sys]
)
const handleResolvedPaste = useCallback(
async ({
bracketed,
cursor,
text,
value
}: Omit<PasteEvent, 'hotkey'>): Promise<null | { cursor: number; value: string }> => {
async ({ bracketed, cursor, text, value }: Omit<PasteEvent, 'hotkey'>): Promise<ComposerPasteResult | null> => {
const cleanedText = stripTrailingPasteNewlines(text)
if (!cleanedText || !/[^\n]/.test(cleanedText)) {
if (bracketed) {
void onClipboardPaste(true)
}
return null
return bracketed ? pasteClipboardImage(value, cursor, true) : null
}
const sid = getUiState().sid
@ -161,14 +248,11 @@ export function useComposerState({
})
if (attached?.name) {
onImageAttached?.(attached)
const remainder = attached.remainder?.trim() ?? ''
if (!remainder) {
return { cursor, value }
}
return insertAtCursor(value, cursor, remainder)
// Drop an `[[ Image N ]]` token where the path was typed. The old
// path printed a notice above the status bar and left the composer
// untouched, so the only trace of the attachment lived outside the
// input the user was editing.
return attachImageToken(attached, value, cursor)
}
} catch {
// Fall back to generic file-drop detection below.
@ -204,7 +288,7 @@ export function useComposerState({
const label = pasteTokenLabel(cleanedText, lineCount)
const inserted = insertAtCursor(value, cursor, label)
setPasteSnips(prev => trimSnips([...prev, { label, text: cleanedText }]))
setComposerTokens(prev => trimTokens([...prev, { kind: 'paste', label, text: cleanedText }]))
void gw
.request<{ path?: string }>('paste.collapse', { text: cleanedText })
@ -215,13 +299,13 @@ export function useComposerState({
return
}
setPasteSnips(prev => prev.map(s => (s.label === label ? { ...s, path } : s)))
setComposerTokens(prev => prev.map(t => (t.label === label ? { ...t, path } : t)))
})
.catch(() => {})
return inserted
},
[gw, onClipboardPaste, onImageAttached]
[attachImageToken, gw, pasteClipboardImage, setComposerTokens]
)
const handleTextPaste = useCallback(
@ -231,7 +315,7 @@ export function useComposerState({
hotkey,
text,
value
}: PasteEvent): MaybePromise<null | { cursor: number; value: string }> => {
}: PasteEvent): MaybePromise<ComposerPasteResult | null> => {
if (hotkey) {
const preferOsc52 = isRemoteShellSession(process.env)
@ -256,15 +340,58 @@ export function useComposerState({
return handleResolvedPaste({ bracketed: false, cursor, text: preferredText, value })
}
void onClipboardPaste(false)
return null
// No text on the clipboard — an image paste looks exactly like this.
return pasteClipboardImage(value, cursor, false)
})
}
return handleResolvedPaste({ bracketed: !!bracketed, cursor, text, value })
},
[handleResolvedPaste, onClipboardPaste, querier]
[handleResolvedPaste, pasteClipboardImage, querier]
)
/**
* `/paste` and `/image` attach without a cursor of their own — the token
* lands at the end of whatever is currently typed.
*/
const appendAttachment = useCallback(
(attach: (value: string, cursor: number) => Promise<ComposerPasteResult | null>) => {
const current = inputRef.current
void attach(current, current.length).then(next => {
if (next) {
setInput(next.value)
}
})
},
[setInput]
)
const attachClipboardImage = useCallback(
() => appendAttachment((value, cursor) => pasteClipboardImage(value, cursor, false)),
[appendAttachment, pasteClipboardImage]
)
const attachImagePath = useCallback(
(path: string) =>
appendAttachment(async (value, cursor) => {
const sid = getUiState().sid
if (!sid || !path.trim()) {
return null
}
const attached = await gw
.request<ImageAttachResponse & { path?: string }>('image.attach', { path, session_id: sid })
.catch((e: Error) => {
sys(`error: ${e.message}`)
return null
})
return attached?.name ? attachImageToken(attached, value, cursor) : null
}),
[appendAttachment, attachImageToken, gw, sys]
)
const openEditor = useCallback(async () => {
@ -297,10 +424,12 @@ export function useComposerState({
} finally {
rmSync(dir, { force: true, recursive: true })
}
}, [input, inputBuf, submitRef])
}, [input, inputBuf, setInput, submitRef])
const actions = useMemo(
() => ({
attachClipboardImage,
attachImagePath,
clearIn,
dequeue,
enqueue,
@ -310,14 +439,17 @@ export function useComposerState({
removeQueue: removeQ,
replaceQueue: replaceQ,
setCompIdx,
setComposerTokens,
setHistoryIdx,
setInput,
setInputBuf,
setPasteSnips,
setQueueEdit,
syncQueue
syncQueue,
syncTokens
}),
[
attachClipboardImage,
attachImagePath,
clearIn,
dequeue,
enqueue,
@ -327,9 +459,12 @@ export function useComposerState({
removeQ,
replaceQ,
setCompIdx,
setComposerTokens,
setHistoryIdx,
setInput,
setQueueEdit,
syncQueue
syncQueue,
syncTokens
]
)
@ -339,7 +474,8 @@ export function useComposerState({
historyRef,
queueEditRef,
queueRef,
submitRef
submitRef,
tokensRef
}),
[historyDraftRef, historyRef, queueEditRef, queueRef, submitRef]
)
@ -352,11 +488,11 @@ export function useComposerState({
historyIdx,
input,
inputBuf,
pasteSnips,
queueEditIdx,
queuedDisplay
queuedDisplay,
tokens
}),
[compIdx, compReplace, completions, historyIdx, input, inputBuf, pasteSnips, queueEditIdx, queuedDisplay]
[compIdx, compReplace, completions, historyIdx, input, inputBuf, queueEditIdx, queuedDisplay, tokens]
)
return {

61
ui-tui/src/app/useMainApp.ts
View file

@ -16,13 +16,11 @@ import { MAX_HISTORY, WHEEL_SCROLL_STEP } from '../config/limits.js'
import { RESIZE_COALESCE_MS } from '../config/timing.js'
import { hasLeadGap, prevRenderedMsg } from '../domain/blockLayout.js'
import { SECTION_NAMES, sectionMode } from '../domain/details.js'
import { attachedImageNotice, imageTokenMeta } from '../domain/messages.js'
import { composeTabTitle, fmtProjectCwdBranch, shortCwd } from '../domain/paths.js'
import { sessionScopedModelArg } from '../domain/slash.js'
import { type GatewayClient } from '../gatewayClient.js'
import type {
ClarifyRespondResponse,
ClipboardPasteResponse,
ConfigSetResponse,
GatewayEvent,
SessionActiveListResponse,
@ -46,7 +44,7 @@ import { createGatewayEventHandler } from './createGatewayEventHandler.js'
import { createSlashHandler } from './createSlashHandler.js'
import { planGatewayRecovery } from './gatewayRecovery.js'
import { getInputSelection } from './inputSelectionStore.js'
import { type GatewayRpc, type TranscriptRow } from './interfaces.js'
import { type GatewayRpc, type StateSetter, type TranscriptRow } from './interfaces.js'
import { $overlayState, patchOverlayState } from './overlayStore.js'
import { $goodVibesTick } from './petFlashStore.js'
import { scrollWithSelectionBy } from './scroll.js'
@ -223,7 +221,7 @@ export function useMainApp(gw: GatewayClient) {
const colsRef = useRef(cols)
const scrollRef = useRef<null | ScrollBoxHandle>(null)
const onEventRef = useRef<(ev: GatewayEvent) => void>(() => {})
const clipboardPasteRef = useRef<(quiet?: boolean) => Promise<void> | void>(() => {})
const sysRef = useRef<(text: string) => void>(() => {})
const submitRef = useRef<(value: string) => void>(() => {})
const terminalHintsShownRef = useRef(new Set<string>())
const historyItemsRef = useRef(historyItems)
@ -296,11 +294,8 @@ export function useMainApp(gw: GatewayClient) {
const composer = useComposerState({
gw,
onClipboardPaste: quiet => clipboardPasteRef.current(quiet),
onImageAttached: info => {
sys(attachedImageNotice(info))
},
submitRef
submitRef,
sys: text => sysRef.current(text)
})
const { actions: composerActions, refs: composerRefs, state: composerState } = composer
@ -711,27 +706,7 @@ export function useMainApp(gw: GatewayClient) {
[appendMessage, overlay.clarify, rpc]
)
const paste = useCallback(
(quiet = false) =>
rpc<ClipboardPasteResponse>('clipboard.paste', { session_id: getUiState().sid }).then(r => {
if (!r) {
return
}
if (r.attached) {
const meta = imageTokenMeta(r)
return sys(`📎 Image #${r.count} attached from clipboard${meta ? ` · ${meta}` : ''}`)
}
if (!quiet) {
sys(r.message || 'No image found in clipboard')
}
}),
[rpc, sys]
)
clipboardPasteRef.current = paste
sysRef.current = sys
const { dispatchSubmission, send, sendQueued, submit } = useSubmission({
appendMessage,
@ -891,10 +866,11 @@ export function useMainApp(gw: GatewayClient) {
() =>
createSlashHandler({
composer: {
attachClipboardImage: composerActions.attachClipboardImage,
attachImagePath: composerActions.attachImagePath,
enqueue: composerActions.enqueue,
hasSelection,
openEditor: composerActions.openEditor,
paste,
queueRef: composerRefs.queueRef,
selection,
setInput: composerActions.setInput
@ -933,7 +909,6 @@ export function useMainApp(gw: GatewayClient) {
maybeWarn,
page,
panel,
paste,
selection,
send,
session,
@ -1137,6 +1112,24 @@ export function useMainApp(gw: GatewayClient) {
]
)
/**
* Every keystroke lands here, so this is where attached payloads are
* reconciled against the tokens still in the text — deleting an
* `[[ Image N ]]` is how the user unattaches it.
*/
const updateInput = useCallback<StateSetter<string>>(
next => {
composerActions.setInput(prev => {
const value = typeof next === 'function' ? next(prev) : next
composerActions.syncTokens(value)
return value
})
},
[composerActions]
)
const appComposer = useMemo(
() => ({
cols,
@ -1150,10 +1143,10 @@ export function useMainApp(gw: GatewayClient) {
queueEditIdx: composerState.queueEditIdx,
queuedDisplay: composerState.queuedDisplay,
submit,
updateInput: composerActions.setInput,
updateInput,
voiceRecordKey
}),
[cols, composerActions, composerState, empty, pagerPageSize, submit, voiceRecordKey]
[cols, composerActions, composerState, empty, pagerPageSize, submit, updateInput, voiceRecordKey]
)
// Pass current progress through unfrozen — streaming update throttling

4
ui-tui/src/app/useSessionLifecycle.ts
View file

@ -178,7 +178,7 @@ export function useSessionLifecycle(opts: UseSessionLifecycleOptions) {
setHistoryItems([])
setLastUserMsg('')
setStickyPrompt('')
composerActions.setPasteSnips([])
composerActions.setComposerTokens([])
// Half-prune: new session has new keys, but keep a warm pool in case
// the user resumes back to the prior session.
evictInkCaches('half')
@ -202,7 +202,7 @@ export function useSessionLifecycle(opts: UseSessionLifecycleOptions) {
setHistoryItems(info ? [introMsg(info)] : [])
setStickyPrompt('')
setLastUserMsg('')
composerActions.setPasteSnips([])
composerActions.setComposerTokens([])
patchTurnState({ activity: [] })
patchUiState({ info, usage: usageFrom(info) })
},

32
ui-tui/src/app/useSubmission.ts
View file

@ -1,32 +1,21 @@
import { type MutableRefObject, useCallback, useEffect, useRef } from 'react'
import { TYPING_IDLE_MS } from '../config/timing.js'
import { expandTokens } from '../domain/attachments.js'
import { completionToApplyOnSubmit, looksLikeSlashCommand } from '../domain/slash.js'
import type { GatewayClient } from '../gatewayClient.js'
import type { SessionSteerResponse, ShellExecResponse } from '../gatewayTypes.js'
import { asRpcResult } from '../lib/rpc.js'
import { hasInterpolation, INTERPOLATION_RE } from '../protocol/interpolation.js'
import { PASTE_SNIPPET_RE } from '../protocol/paste.js'
import type { Msg } from '../types.js'
import type { ComposerActions, ComposerRefs, ComposerState, PasteSnippet } from './interfaces.js'
import type { ComposerActions, ComposerRefs, ComposerState } from './interfaces.js'
import { submitPrompt } from './submissionCore.js'
import { turnController } from './turnController.js'
import { getUiState, patchUiState } from './uiStore.js'
const DOUBLE_ENTER_MS = 450
export const expandSnips = (snips: PasteSnippet[]) => {
const byLabel = new Map<string, string[]>()
for (const { label, text } of snips) {
const hit = byLabel.get(label)
hit ? hit.push(text) : byLabel.set(label, [text])
}
return (value: string) => value.replace(PASTE_SNIPPET_RE, tok => byLabel.get(tok)?.shift() ?? tok)
}
const spliceMatches = (text: string, matches: RegExpMatchArray[], results: string[]) =>
matches.reduceRight((acc, m, i) => acc.slice(0, m.index!) + results[i] + acc.slice(m.index! + m[0].length), text)
@ -68,7 +57,9 @@ export function useSubmission(opts: UseSubmissionOptions) {
const send = useCallback(
(text: string, showUserMessage = true, displayText?: string) => {
const expand = expandSnips(composerState.pasteSnips)
// Read tokens off the ref, not render state: a paste immediately followed
// by Enter submits before React has re-rendered with the new token.
const expand = expandTokens(composerRefs.tokensRef.current)
submitPrompt(
text,
@ -84,7 +75,7 @@ export function useSubmission(opts: UseSubmissionOptions) {
displayText
)
},
[appendMessage, composerActions, composerState.pasteSnips, gw, setLastUserMsg, sys]
[appendMessage, composerActions, composerRefs, gw, setLastUserMsg, sys]
)
const shellExec = useCallback(
@ -217,10 +208,12 @@ export function useSubmission(opts: UseSubmissionOptions) {
return
}
// History stores expanded paste content, not the `[[…]]` label: snips
// are cleared on submit, so recall must be self-contained. Idempotent on
// label-free text, so re-submitting a recalled entry stays stable.
const toHistory = expandSnips(composerState.pasteSnips)(full)
// History stores resolved content, not `[[…]]` labels: tokens are cleared
// on submit, so recall must be self-contained. Image tokens resolve to
// nothing — a detached image can't be re-attached by recalling the text.
// Idempotent on token-free text, so re-submitting a recalled entry is
// stable.
const toHistory = expandTokens(composerRefs.tokensRef.current)(full)
if (looksLikeSlashCommand(full)) {
appendMessage({ kind: 'slash', role: 'system', text: full })
@ -294,7 +287,6 @@ export function useSubmission(opts: UseSubmissionOptions) {
appendMessage,
composerActions,
composerRefs,
composerState.pasteSnips,
handleBusyInput,
interpolate,
send,