mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #75210 from NousResearch/bb/inline-attachments

Browse source 
TUI attachments live in the composer, not above the status bar
This commit is contained in:
brooklyn! 2026-07-30 23:56:57 -05:00 • committed by GitHub
commit dbe14424ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 423 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

96
ui-tui/src/__tests__/attachments.test.ts Normal file
View file

@ -0,0 +1,96 @@
import { describe, expect, it } from 'vitest'
import type { ComposerToken } from '../app/interfaces.js'
import { droppedTokens, expandTokens, imageToken, nextImageIndex } from '../domain/attachments.js'
const paste = (label: string, text: string): ComposerToken => ({ kind: 'paste', label, text })
const image = (index: number, path = `/tmp/img${index}.png`): ComposerToken => ({
index,
kind: 'image',
label: imageToken(index),
path
})
describe('expandTokens (what the agent actually receives)', () => {
it('replaces a collapsed paste label with its full content', () => {
const label = '[[ hello.. [3 lines] .. world ]]'
const expand = expandTokens([paste(label, 'hello\nfoo\nworld')])
expect(expand(`here: ${label} done`)).toBe('here: hello\nfoo\nworld done')
})
it('is a no-op for already-expanded / token-free text (recall round-trip)', () => {
const expanded = 'hello\nfoo\nworld'
expect(expandTokens([])(expanded)).toBe(expanded)
})
it('expands repeated identical labels in submission order', () => {
const label = '[[ x [1 lines] ]]'
const expand = expandTokens([paste(label, 'first'), paste(label, 'second')])
expect(expand(`${label} then ${label}`)).toBe('first then second')
})
it('leaves an unmatched label intact', () => {
const label = '[[ orphan [2 lines] ]]'
expect(expandTokens([])(label)).toBe(label)
})
it('drops an image token from the text — the gateway already holds the file', () => {
const expand = expandTokens([image(1)])
expect(expand(`what is in ${imageToken(1)}`)).toBe('what is in')
})
it('leaves no double space where an image token sat mid-sentence', () => {
const expand = expandTokens([image(1)])
expect(expand(`before ${imageToken(1)} after`)).toBe('before after')
})
it('resolves an image-only message to empty text', () => {
expect(expandTokens([image(1)])(imageToken(1))).toBe('')
})
it('resolves pastes and images in one pass', () => {
const label = '[[ log.. [9 lines] ]]'
const expand = expandTokens([paste(label, 'stack\ntrace'), image(2)])
expect(expand(`${label} and ${imageToken(2)}`)).toBe('stack\ntrace and')
})
})
describe('nextImageIndex (user-facing numbering)', () => {
it('starts at 1', () => {
expect(nextImageIndex([])).toBe(1)
})
it('counts past existing images', () => {
expect(nextImageIndex([image(1), image(2)])).toBe(3)
})
it('ignores paste tokens', () => {
expect(nextImageIndex([paste('[[ x ]]', 'y')])).toBe(1)
})
it('does not reuse an index after an earlier image is deleted', () => {
// [[ Image 1 ]] was erased; the next attach must not become Image 1 again
// or expandTokens would resolve two different files to one label.
expect(nextImageIndex([image(2)])).toBe(3)
})
})
describe('droppedTokens (deleting the token unattaches the thing)', () => {
it('reports an image whose token was erased from the text', () => {
expect(droppedTokens([image(1)], 'just text now')).toEqual([image(1)])
})
it('reports nothing while the token is still present', () => {
expect(droppedTokens([image(1)], `look at ${imageToken(1)}`)).toEqual([])
})
it('keeps a surviving token when a sibling is erased', () => {
expect(droppedTokens([image(1), image(2)], imageToken(2))).toEqual([image(1)])
})
})

34
ui-tui/src/app/interfaces.ts
View file

@ -7,7 +7,6 @@ import type {
BillingCardInfo,
BillingMutationResponse,
BillingStateResponse,
ImageAttachResponse,
SessionCloseResponse,
SubscriptionPreviewResponse,
SubscriptionStateResponse,
@ -364,6 +363,10 @@ export interface ComposerPasteResult {
export type MaybePromise<T> = Promise<T> | T
export interface ComposerActions {
/** Pull an image off the system clipboard in as a token. */
attachClipboardImage: () => void
/** Attach an image by path in as a token. */
attachImagePath: (path: string) => void
clearIn: () => void
dequeue: () => string | undefined
enqueue: (text: string) => void
@ -373,12 +376,14 @@ export interface ComposerActions {
removeQueue: (index: number) => void
replaceQueue: (index: number, text: string) => void
setCompIdx: StateSetter<number>
setComposerTokens: StateSetter<ComposerToken[]>
setHistoryIdx: StateSetter<null | number>
setInput: StateSetter<string>
setInputBuf: StateSetter<string[]>
setPasteSnips: StateSetter<PasteSnippet[]>
setQueueEdit: (index: null | number) => void
syncQueue: () => void
/** Reconcile attached payloads against tokens still present in the text. */
syncTokens: (value: string) => void
}
export interface ComposerRefs {
@ -387,6 +392,7 @@ export interface ComposerRefs {
queueEditRef: MutableRefObject<null | number>
queueRef: MutableRefObject<string[]>
submitRef: MutableRefObject<(value: string) => void>
tokensRef: MutableRefObject<ComposerToken[]>
}
export interface ComposerState {
@ -396,16 +402,15 @@ export interface ComposerState {
historyIdx: null | number
input: string
inputBuf: string[]
pasteSnips: PasteSnippet[]
queueEditIdx: null | number
queuedDisplay: string[]
tokens: ComposerToken[]
}
export interface UseComposerStateOptions {
gw: GatewayClient
onClipboardPaste: (quiet?: boolean) => Promise<void> | void
onImageAttached?: (info: ImageAttachResponse) => void
submitRef: MutableRefObject<(value: string) => void>
sys: (text: string) => void
}
export interface UseComposerStateResult {
@ -495,10 +500,11 @@ export interface GatewayEventHandlerContext {
export interface SlashHandlerContext {
composer: {
attachClipboardImage: () => void
attachImagePath: (path: string) => void
enqueue: (text: string) => void
hasSelection: boolean
openEditor: () => Promise<void>
paste: (quiet?: boolean) => void
queueRef: MutableRefObject<string[]>
selection: SelectionApi
setInput: StateSetter<string>
@ -618,8 +624,14 @@ export interface AppOverlaysProps {
pagerPageSize: number
}
export interface PasteSnippet {
label: string
path?: string
text: string
}
/**
* A `[[ … ]]` token sitting in the composer text, plus the payload it stands
* for. `paste` tokens expand back into their text at submit; `image` tokens
* are a receipt for a file the gateway already holds, and expand to nothing.
*
* `index` is the user-facing number in `[[ Image 2 ]]`; `path` is the gateway
* path, used to detach the image when its token is deleted.
*/
export type ComposerToken =
| { index: number; kind: 'image'; label: string; path: string; text?: undefined }
| { index?: undefined; kind: 'paste'; label: string; path?: string; text: string }

2
ui-tui/src/app/slash/commands/core.ts
View file

@ -439,7 +439,7 @@ export const coreCommands: SlashCommand[] = [
{
help: 'attach clipboard image',
name: 'paste',
run: (arg, ctx) => (arg ? ctx.transcript.sys('usage: /paste') : ctx.composer.paste())
run: (arg, ctx) => (arg ? ctx.transcript.sys('usage: /paste') : ctx.composer.attachClipboardImage())
},
{

15
ui-tui/src/app/slash/commands/session.ts
View file

@ -1,11 +1,10 @@
import { usageBarsText } from '../../../components/overlayPrimitives.js'
import { attachedImageNotice, introMsg, toTranscriptMessages } from '../../../domain/messages.js'
import { introMsg, toTranscriptMessages } from '../../../domain/messages.js'
import { sessionScopedModelArg, TUI_SESSION_MODEL_FLAG } from '../../../domain/slash.js'
import type {
BackgroundStartResponse,
ConfigGetValueResponse,
ConfigSetResponse,
ImageAttachResponse,
SessionBranchResponse,
SessionCompressResponse,
SessionUsageResponse,
@ -192,17 +191,7 @@ export const sessionCommands: SlashCommand[] = [
{
help: 'attach an image',
name: 'image',
run: (arg, ctx) => {
ctx.gateway.rpc<ImageAttachResponse>('image.attach', { path: arg, session_id: ctx.sid }).then(
ctx.guarded<ImageAttachResponse>(r => {
ctx.transcript.sys(attachedImageNotice(r))
if (r.remainder) {
ctx.composer.setInput(r.remainder)
}
})
)
}
run: (arg, ctx) => ctx.composer.attachImagePath(arg)
},
{

12
ui-tui/src/app/submissionCore.ts
View file

@ -1,4 +1,3 @@
import { attachedImageNotice } from '../domain/messages.js'
import type { GatewayClient } from '../gatewayClient.js'
import type { InputDetectDropResponse, PromptSubmitResponse } from '../gatewayTypes.js'
import type { Msg } from '../types.js'
@ -109,6 +108,11 @@ export function submitPrompt(
// Always ask the backend whether this looks like a file drop. The backend's
// _detect_file_drop handles paths with spaces, quotes, Windows drive letters,
// and escaped characters correctly.
//
// No notice is emitted for a match: an image dropped into the composer already
// shows as an `[[ Image N ]]` token, and a matched non-image path is rewritten
// in place. Announcing it a second time above the status bar was the old
// out-of-band attachment UI.
deps.gw
.request<InputDetectDropResponse>('input.detect_drop', { session_id: sid, text })
.then(r => {
@ -116,12 +120,6 @@ export function submitPrompt(
return startSubmit(text, deps.expand(text), showUserMessage)
}
if (r.is_image) {
turnController.pushActivity(attachedImageNotice(r))
} else {
turnController.pushActivity(`detected file: ${r.name}`)
}
startSubmit(r.text || text, deps.expand(r.text || text), showUserMessage)
})
.catch(() => startSubmit(text, deps.expand(text), showUserMessage))

252
ui-tui/src/app/useComposerState.ts
View file

@ -5,10 +5,11 @@ import { join } from 'node:path'
import { useStdin, withInkSuspended } from '@hermes/ink'
import { useStore } from '@nanostores/react'
import { useCallback, useMemo, useState } from 'react'
import { useCallback, useMemo, useRef, useState } from 'react'
import type { PasteEvent } from '../components/textInput.js'
import type { ImageAttachResponse, InputDetectDropResponse } from '../gatewayTypes.js'
import { droppedTokens, imageToken, nextImageIndex } from '../domain/attachments.js'
import type { ClipboardPasteResponse, ImageAttachResponse, InputDetectDropResponse } from '../gatewayTypes.js'
import { useCompletion } from '../hooks/useCompletion.js'
import { useInputHistory } from '../hooks/useInputHistory.js'
import { useQueue } from '../hooks/useQueue.js'
@ -18,30 +19,37 @@ import { readOsc52Clipboard } from '../lib/osc52.js'
import { isRemoteShellSession } from '../lib/terminalSetup.js'
import { pasteTokenLabel, stripTrailingPasteNewlines } from '../lib/text.js'
import type { MaybePromise, PasteSnippet, UseComposerStateOptions, UseComposerStateResult } from './interfaces.js'
import type {
ComposerPasteResult,
ComposerToken,
MaybePromise,
StateSetter,
UseComposerStateOptions,
UseComposerStateResult
} from './interfaces.js'
import { $isBlocked } from './overlayStore.js'
import { getUiState } from './uiStore.js'
const PASTE_SNIP_MAX_COUNT = 32
const PASTE_SNIP_MAX_TOTAL_BYTES = 4 * 1024 * 1024
const TOKEN_MAX_COUNT = 32
const TOKEN_MAX_TOTAL_BYTES = 4 * 1024 * 1024
const trimSnips = (snips: PasteSnippet[]): PasteSnippet[] => {
const trimTokens = (tokens: ComposerToken[]): ComposerToken[] => {
let total = 0
const out: PasteSnippet[] = []
const out: ComposerToken[] = []
for (let i = snips.length - 1; i >= 0; i--) {
const snip = snips[i]!
const size = snip.text.length
for (let i = tokens.length - 1; i >= 0; i--) {
const token = tokens[i]!
const size = token.text?.length ?? 0
if (out.length >= PASTE_SNIP_MAX_COUNT || total + size > PASTE_SNIP_MAX_TOTAL_BYTES) {
if (out.length >= TOKEN_MAX_COUNT || total + size > TOKEN_MAX_TOTAL_BYTES) {
break
}
total += size
out.unshift(snip)
out.unshift(token)
}
return out.length === snips.length ? snips : out
return out.length === tokens.length ? tokens : out
}
/** Insert text at the cursor position, adding spacing to separate from adjacent non-whitespace. */
@ -97,15 +105,26 @@ export function looksLikeDroppedPath(text: string): boolean {
return false
}
export function useComposerState({
gw,
onClipboardPaste,
onImageAttached,
submitRef
}: UseComposerStateOptions): UseComposerStateResult {
const [input, setInput] = useState('')
export function useComposerState({ gw, submitRef, sys }: UseComposerStateOptions): UseComposerStateResult {
const [input, setInputState] = useState('')
const [inputBuf, setInputBuf] = useState<string[]>([])
const [pasteSnips, setPasteSnips] = useState<PasteSnippet[]>([])
const [tokens, setTokens] = useState<ComposerToken[]>([])
// Tokens and the input line are read from keystroke handlers that run several
// times before React re-renders, so the refs — not the state — are the source
// of truth for "what is in the composer right now".
const inputRef = useRef('')
const tokensRef = useRef<ComposerToken[]>([])
const setInput = useCallback<StateSetter<string>>(next => {
inputRef.current = typeof next === 'function' ? next(inputRef.current) : next
setInputState(inputRef.current)
}, [])
const setComposerTokens = useCallback<StateSetter<ComposerToken[]>>(next => {
tokensRef.current = typeof next === 'function' ? next(tokensRef.current) : next
setTokens(tokensRef.current)
}, [])
const isBlocked = useStore($isBlocked)
const { querier } = useStdin() as { querier: Parameters<typeof readOsc52Clipboard>[0] }
@ -128,27 +147,95 @@ export function useComposerState({
const clearIn = useCallback(() => {
setInput('')
setInputBuf([])
setPasteSnips([])
setComposerTokens([])
setQueueEdit(null)
setHistoryIdx(null)
historyDraftRef.current = ''
}, [historyDraftRef, setQueueEdit, setHistoryIdx])
}, [historyDraftRef, setComposerTokens, setHistoryIdx, setInput, setQueueEdit])
/**
* Deleting an `[[ Image N ]]` token IS how you unattach the image — there is
* no separate control. Reconcile on every edit so the gateway's
* `attached_images` never outlives the token the user just erased, which is
* what used to make a stale image ride along on the next unrelated turn.
*/
const syncTokens = useCallback(
(value: string) => {
const gone = droppedTokens(tokensRef.current, value)
if (!gone.length) {
return
}
for (const token of gone) {
if (token.kind === 'image') {
void gw.request('image.detach', { path: token.path, session_id: getUiState().sid }).catch(() => {})
}
}
setComposerTokens(prev => prev.filter(token => !gone.includes(token)))
},
[gw, setComposerTokens]
)
/**
* Attach an image the gateway already resolved: a token at the cursor,
* followed by whatever non-path text came along with it (a drag-drop paste
* of `~/shot.png look at this` keeps the caption).
*/
const attachImageToken = useCallback(
(attached: ImageAttachResponse & { path?: string }, value: string, cursor: number): ComposerPasteResult => {
const index = nextImageIndex(tokensRef.current)
const label = imageToken(index)
setComposerTokens(prev => trimTokens([...prev, { index, kind: 'image', label, path: attached.path ?? '' }]))
const withToken = insertAtCursor(value, cursor, label)
const remainder = attached.remainder?.trim() ?? ''
return remainder ? insertAtCursor(withToken.value, withToken.cursor, remainder) : withToken
},
[setComposerTokens]
)
/**
* Pull an image off the system clipboard into the composer as a token.
*
* `quiet` is the empty-bracketed-paste probe: the terminal delivers an image
* paste as zero text, so we speculatively ask the gateway and stay silent if
* there was nothing there. An explicit `/paste` reports the miss.
*/
const pasteClipboardImage = useCallback(
async (value: string, cursor: number, quiet: boolean): Promise<ComposerPasteResult | null> => {
const sid = getUiState().sid
if (!sid) {
return null
}
const r = await gw
.request<ClipboardPasteResponse & { path?: string }>('clipboard.paste', { session_id: sid })
.catch(() => null)
if (r?.attached) {
return attachImageToken(r, value, cursor)
}
if (!quiet) {
sys(r?.message || 'No image found in clipboard')
}
return null
},
[attachImageToken, gw, sys]
)
const handleResolvedPaste = useCallback(
async ({
bracketed,
cursor,
text,
value
}: Omit<PasteEvent, 'hotkey'>): Promise<null | { cursor: number; value: string }> => {
async ({ bracketed, cursor, text, value }: Omit<PasteEvent, 'hotkey'>): Promise<ComposerPasteResult | null> => {
const cleanedText = stripTrailingPasteNewlines(text)
if (!cleanedText || !/[^\n]/.test(cleanedText)) {
if (bracketed) {
void onClipboardPaste(true)
}
return null
return bracketed ? pasteClipboardImage(value, cursor, true) : null
}
const sid = getUiState().sid
@ -161,14 +248,11 @@ export function useComposerState({
})
if (attached?.name) {
onImageAttached?.(attached)
const remainder = attached.remainder?.trim() ?? ''
if (!remainder) {
return { cursor, value }
}
return insertAtCursor(value, cursor, remainder)
// Drop an `[[ Image N ]]` token where the path was typed. The old
// path printed a notice above the status bar and left the composer
// untouched, so the only trace of the attachment lived outside the
// input the user was editing.
return attachImageToken(attached, value, cursor)
}
} catch {
// Fall back to generic file-drop detection below.
@ -204,7 +288,7 @@ export function useComposerState({
const label = pasteTokenLabel(cleanedText, lineCount)
const inserted = insertAtCursor(value, cursor, label)
setPasteSnips(prev => trimSnips([...prev, { label, text: cleanedText }]))
setComposerTokens(prev => trimTokens([...prev, { kind: 'paste', label, text: cleanedText }]))
void gw
.request<{ path?: string }>('paste.collapse', { text: cleanedText })
@ -215,13 +299,13 @@ export function useComposerState({
return
}
setPasteSnips(prev => prev.map(s => (s.label === label ? { ...s, path } : s)))
setComposerTokens(prev => prev.map(t => (t.label === label ? { ...t, path } : t)))
})
.catch(() => {})
return inserted
},
[gw, onClipboardPaste, onImageAttached]
[attachImageToken, gw, pasteClipboardImage, setComposerTokens]
)
const handleTextPaste = useCallback(
@ -231,7 +315,7 @@ export function useComposerState({
hotkey,
text,
value
}: PasteEvent): MaybePromise<null | { cursor: number; value: string }> => {
}: PasteEvent): MaybePromise<ComposerPasteResult | null> => {
if (hotkey) {
const preferOsc52 = isRemoteShellSession(process.env)
@ -256,15 +340,58 @@ export function useComposerState({
return handleResolvedPaste({ bracketed: false, cursor, text: preferredText, value })
}
void onClipboardPaste(false)
return null
// No text on the clipboard — an image paste looks exactly like this.
return pasteClipboardImage(value, cursor, false)
})
}
return handleResolvedPaste({ bracketed: !!bracketed, cursor, text, value })
},
[handleResolvedPaste, onClipboardPaste, querier]
[handleResolvedPaste, pasteClipboardImage, querier]
)
/**
* `/paste` and `/image` attach without a cursor of their own — the token
* lands at the end of whatever is currently typed.
*/
const appendAttachment = useCallback(
(attach: (value: string, cursor: number) => Promise<ComposerPasteResult | null>) => {
const current = inputRef.current
void attach(current, current.length).then(next => {
if (next) {
setInput(next.value)
}
})
},
[setInput]
)
const attachClipboardImage = useCallback(
() => appendAttachment((value, cursor) => pasteClipboardImage(value, cursor, false)),
[appendAttachment, pasteClipboardImage]
)
const attachImagePath = useCallback(
(path: string) =>
appendAttachment(async (value, cursor) => {
const sid = getUiState().sid
if (!sid || !path.trim()) {
return null
}
const attached = await gw
.request<ImageAttachResponse & { path?: string }>('image.attach', { path, session_id: sid })
.catch((e: Error) => {
sys(`error: ${e.message}`)
return null
})
return attached?.name ? attachImageToken(attached, value, cursor) : null
}),
[appendAttachment, attachImageToken, gw, sys]
)
const openEditor = useCallback(async () => {
@ -297,10 +424,12 @@ export function useComposerState({
} finally {
rmSync(dir, { force: true, recursive: true })
}
}, [input, inputBuf, submitRef])
}, [input, inputBuf, setInput, submitRef])
const actions = useMemo(
() => ({
attachClipboardImage,
attachImagePath,
clearIn,
dequeue,
enqueue,
@ -310,14 +439,17 @@ export function useComposerState({
removeQueue: removeQ,
replaceQueue: replaceQ,
setCompIdx,
setComposerTokens,
setHistoryIdx,
setInput,
setInputBuf,
setPasteSnips,
setQueueEdit,
syncQueue
syncQueue,
syncTokens
}),
[
attachClipboardImage,
attachImagePath,
clearIn,
dequeue,
enqueue,
@ -327,9 +459,12 @@ export function useComposerState({
removeQ,
replaceQ,
setCompIdx,
setComposerTokens,
setHistoryIdx,
setInput,
setQueueEdit,
syncQueue
syncQueue,
syncTokens
]
)
@ -339,7 +474,8 @@ export function useComposerState({
historyRef,
queueEditRef,
queueRef,
submitRef
submitRef,
tokensRef
}),
[historyDraftRef, historyRef, queueEditRef, queueRef, submitRef]
)
@ -352,11 +488,11 @@ export function useComposerState({
historyIdx,
input,
inputBuf,
pasteSnips,
queueEditIdx,
queuedDisplay
queuedDisplay,
tokens
}),
[compIdx, compReplace, completions, historyIdx, input, inputBuf, pasteSnips, queueEditIdx, queuedDisplay]
[compIdx, compReplace, completions, historyIdx, input, inputBuf, queueEditIdx, queuedDisplay, tokens]
)
return {

61
ui-tui/src/app/useMainApp.ts
View file

@ -16,13 +16,11 @@ import { MAX_HISTORY, WHEEL_SCROLL_STEP } from '../config/limits.js'
import { RESIZE_COALESCE_MS } from '../config/timing.js'
import { hasLeadGap, prevRenderedMsg } from '../domain/blockLayout.js'
import { SECTION_NAMES, sectionMode } from '../domain/details.js'
import { attachedImageNotice, imageTokenMeta } from '../domain/messages.js'
import { composeTabTitle, fmtProjectCwdBranch, shortCwd } from '../domain/paths.js'
import { sessionScopedModelArg } from '../domain/slash.js'
import { type GatewayClient } from '../gatewayClient.js'
import type {
ClarifyRespondResponse,
ClipboardPasteResponse,
ConfigSetResponse,
GatewayEvent,
SessionActiveListResponse,
@ -46,7 +44,7 @@ import { createGatewayEventHandler } from './createGatewayEventHandler.js'
import { createSlashHandler } from './createSlashHandler.js'
import { planGatewayRecovery } from './gatewayRecovery.js'
import { getInputSelection } from './inputSelectionStore.js'
import { type GatewayRpc, type TranscriptRow } from './interfaces.js'
import { type GatewayRpc, type StateSetter, type TranscriptRow } from './interfaces.js'
import { $overlayState, patchOverlayState } from './overlayStore.js'
import { $goodVibesTick } from './petFlashStore.js'
import { scrollWithSelectionBy } from './scroll.js'
@ -223,7 +221,7 @@ export function useMainApp(gw: GatewayClient) {
const colsRef = useRef(cols)
const scrollRef = useRef<null | ScrollBoxHandle>(null)
const onEventRef = useRef<(ev: GatewayEvent) => void>(() => {})
const clipboardPasteRef = useRef<(quiet?: boolean) => Promise<void> | void>(() => {})
const sysRef = useRef<(text: string) => void>(() => {})
const submitRef = useRef<(value: string) => void>(() => {})
const terminalHintsShownRef = useRef(new Set<string>())
const historyItemsRef = useRef(historyItems)
@ -296,11 +294,8 @@ export function useMainApp(gw: GatewayClient) {
const composer = useComposerState({
gw,
onClipboardPaste: quiet => clipboardPasteRef.current(quiet),
onImageAttached: info => {
sys(attachedImageNotice(info))
},
submitRef
submitRef,
sys: text => sysRef.current(text)
})
const { actions: composerActions, refs: composerRefs, state: composerState } = composer
@ -711,27 +706,7 @@ export function useMainApp(gw: GatewayClient) {
[appendMessage, overlay.clarify, rpc]
)
const paste = useCallback(
(quiet = false) =>
rpc<ClipboardPasteResponse>('clipboard.paste', { session_id: getUiState().sid }).then(r => {
if (!r) {
return
}
if (r.attached) {
const meta = imageTokenMeta(r)
return sys(`📎 Image #${r.count} attached from clipboard${meta ? ` · ${meta}` : ''}`)
}
if (!quiet) {
sys(r.message || 'No image found in clipboard')
}
}),
[rpc, sys]
)
clipboardPasteRef.current = paste
sysRef.current = sys
const { dispatchSubmission, send, sendQueued, submit } = useSubmission({
appendMessage,
@ -891,10 +866,11 @@ export function useMainApp(gw: GatewayClient) {
() =>
createSlashHandler({
composer: {
attachClipboardImage: composerActions.attachClipboardImage,
attachImagePath: composerActions.attachImagePath,
enqueue: composerActions.enqueue,
hasSelection,
openEditor: composerActions.openEditor,
paste,
queueRef: composerRefs.queueRef,
selection,
setInput: composerActions.setInput
@ -933,7 +909,6 @@ export function useMainApp(gw: GatewayClient) {
maybeWarn,
page,
panel,
paste,
selection,
send,
session,
@ -1137,6 +1112,24 @@ export function useMainApp(gw: GatewayClient) {
]
)
/**
* Every keystroke lands here, so this is where attached payloads are
* reconciled against the tokens still in the text — deleting an
* `[[ Image N ]]` is how the user unattaches it.
*/
const updateInput = useCallback<StateSetter<string>>(
next => {
composerActions.setInput(prev => {
const value = typeof next === 'function' ? next(prev) : next
composerActions.syncTokens(value)
return value
})
},
[composerActions]
)
const appComposer = useMemo(
() => ({
cols,
@ -1150,10 +1143,10 @@ export function useMainApp(gw: GatewayClient) {
queueEditIdx: composerState.queueEditIdx,
queuedDisplay: composerState.queuedDisplay,
submit,
updateInput: composerActions.setInput,
updateInput,
voiceRecordKey
}),
[cols, composerActions, composerState, empty, pagerPageSize, submit, voiceRecordKey]
[cols, composerActions, composerState, empty, pagerPageSize, submit, updateInput, voiceRecordKey]
)
// Pass current progress through unfrozen — streaming update throttling

4
ui-tui/src/app/useSessionLifecycle.ts
View file

@ -178,7 +178,7 @@ export function useSessionLifecycle(opts: UseSessionLifecycleOptions) {
setHistoryItems([])
setLastUserMsg('')
setStickyPrompt('')
composerActions.setPasteSnips([])
composerActions.setComposerTokens([])
// Half-prune: new session has new keys, but keep a warm pool in case
// the user resumes back to the prior session.
evictInkCaches('half')
@ -202,7 +202,7 @@ export function useSessionLifecycle(opts: UseSessionLifecycleOptions) {
setHistoryItems(info ? [introMsg(info)] : [])
setStickyPrompt('')
setLastUserMsg('')
composerActions.setPasteSnips([])
composerActions.setComposerTokens([])
patchTurnState({ activity: [] })
patchUiState({ info, usage: usageFrom(info) })
},

34
ui-tui/src/app/useSubmission.test.ts
View file

@ -1,34 +0,0 @@
import { describe, expect, it } from 'vitest'
import type { PasteSnippet } from './interfaces.js'
import { expandSnips } from './useSubmission.js'
const snip = (label: string, text: string): PasteSnippet => ({ label, text })
describe('expandSnips (paste history recall)', () => {
it('replaces a collapsed paste label with its full content', () => {
const label = '[[ hello.. [3 lines] .. world ]]'
const full = `here: ${label} done`
const expand = expandSnips([snip(label, 'hello\nfoo\nworld')])
expect(expand(full)).toBe('here: hello\nfoo\nworld done')
})
it('is a no-op for already-expanded / label-free text (recall round-trip)', () => {
const expanded = 'hello\nfoo\nworld'
// Re-submitting a recalled history entry has no snips and no labels.
expect(expandSnips([])(expanded)).toBe(expanded)
})
it('expands repeated identical labels in submission order', () => {
const label = '[[ x [1 lines] ]]'
const expand = expandSnips([snip(label, 'first'), snip(label, 'second')])
expect(expand(`${label} then ${label}`)).toBe('first then second')
})
it('leaves an unmatched label intact', () => {
const label = '[[ orphan [2 lines] ]]'
expect(expandSnips([])(label)).toBe(label)
})
})

32
ui-tui/src/app/useSubmission.ts
View file

@ -1,32 +1,21 @@
import { type MutableRefObject, useCallback, useEffect, useRef } from 'react'
import { TYPING_IDLE_MS } from '../config/timing.js'
import { expandTokens } from '../domain/attachments.js'
import { completionToApplyOnSubmit, looksLikeSlashCommand } from '../domain/slash.js'
import type { GatewayClient } from '../gatewayClient.js'
import type { SessionSteerResponse, ShellExecResponse } from '../gatewayTypes.js'
import { asRpcResult } from '../lib/rpc.js'
import { hasInterpolation, INTERPOLATION_RE } from '../protocol/interpolation.js'
import { PASTE_SNIPPET_RE } from '../protocol/paste.js'
import type { Msg } from '../types.js'
import type { ComposerActions, ComposerRefs, ComposerState, PasteSnippet } from './interfaces.js'
import type { ComposerActions, ComposerRefs, ComposerState } from './interfaces.js'
import { submitPrompt } from './submissionCore.js'
import { turnController } from './turnController.js'
import { getUiState, patchUiState } from './uiStore.js'
const DOUBLE_ENTER_MS = 450
export const expandSnips = (snips: PasteSnippet[]) => {
const byLabel = new Map<string, string[]>()
for (const { label, text } of snips) {
const hit = byLabel.get(label)
hit ? hit.push(text) : byLabel.set(label, [text])
}
return (value: string) => value.replace(PASTE_SNIPPET_RE, tok => byLabel.get(tok)?.shift() ?? tok)
}
const spliceMatches = (text: string, matches: RegExpMatchArray[], results: string[]) =>
matches.reduceRight((acc, m, i) => acc.slice(0, m.index!) + results[i] + acc.slice(m.index! + m[0].length), text)
@ -68,7 +57,9 @@ export function useSubmission(opts: UseSubmissionOptions) {
const send = useCallback(
(text: string, showUserMessage = true, displayText?: string) => {
const expand = expandSnips(composerState.pasteSnips)
// Read tokens off the ref, not render state: a paste immediately followed
// by Enter submits before React has re-rendered with the new token.
const expand = expandTokens(composerRefs.tokensRef.current)
submitPrompt(
text,
@ -84,7 +75,7 @@ export function useSubmission(opts: UseSubmissionOptions) {
displayText
)
},
[appendMessage, composerActions, composerState.pasteSnips, gw, setLastUserMsg, sys]
[appendMessage, composerActions, composerRefs, gw, setLastUserMsg, sys]
)
const shellExec = useCallback(
@ -217,10 +208,12 @@ export function useSubmission(opts: UseSubmissionOptions) {
return
}
// History stores expanded paste content, not the `[[…]]` label: snips
// are cleared on submit, so recall must be self-contained. Idempotent on
// label-free text, so re-submitting a recalled entry stays stable.
const toHistory = expandSnips(composerState.pasteSnips)(full)
// History stores resolved content, not `[[…]]` labels: tokens are cleared
// on submit, so recall must be self-contained. Image tokens resolve to
// nothing — a detached image can't be re-attached by recalling the text.
// Idempotent on token-free text, so re-submitting a recalled entry is
// stable.
const toHistory = expandTokens(composerRefs.tokensRef.current)(full)
if (looksLikeSlashCommand(full)) {
appendMessage({ kind: 'slash', role: 'system', text: full })
@ -294,7 +287,6 @@ export function useSubmission(opts: UseSubmissionOptions) {
appendMessage,
composerActions,
composerRefs,
composerState.pasteSnips,
handleBusyInput,
interpolate,
send,

60
ui-tui/src/domain/attachments.ts Normal file
View file

@ -0,0 +1,60 @@
import type { ComposerToken } from '../app/interfaces.js'
import { PASTE_SNIPPET_RE } from '../protocol/paste.js'
/**
* Composer tokens are the ONE way deferred content shows up in the input line:
* a collapsed paste and an attached image both render as `[[ … ]]` sitting in
* the text the user is editing. They are ordinary characters — arrow keys,
* backspace, and selection work on them for free — and they carry their real
* payload out-of-band until submit.
*
* Two consequences the rest of the composer relies on:
* - Deleting the token is how you drop the thing. Nothing else to click.
* - Position in the text is meaningful: the model sees the payload where the
* token sat, not stapled to the front of the turn.
*/
export const imageToken = (index: number) => `[[ Image ${index} ]]`
/** Highest image token index handed out so far, so a new one never collides. */
export const nextImageIndex = (tokens: ComposerToken[]) =>
tokens.reduce((max, t) => (t.kind === 'image' ? Math.max(max, t.index) : max), 0) + 1
/** Tokens whose label is no longer anywhere in the composer text. */
export const droppedTokens = (tokens: ComposerToken[], value: string) => {
const live = new Set(value.match(PASTE_SNIPPET_RE) ?? [])
return tokens.filter(t => !live.has(t.label))
}
/**
* Resolve every token in `value` to what the agent should actually receive.
*
* Repeated identical labels expand in submission order (left to right), which
* is why this walks matches instead of doing a global replace per token.
*
* An image token expands to nothing: the gateway already holds the file in
* `session.attached_images` and splices the real vision content in at submit.
* The token's job was to show the user where it landed, so it also eats one
* adjacent space to avoid leaving a gap in the middle of a sentence.
*/
export const expandTokens = (tokens: ComposerToken[]) => {
const byLabel = new Map<string, ComposerToken[]>()
for (const token of tokens) {
const hit = byLabel.get(token.label)
hit ? hit.push(token) : byLabel.set(token.label, [token])
}
return (value: string) =>
value
.replace(new RegExp(`[ \\t]?(?:${PASTE_SNIPPET_RE.source})`, 'g'), match => {
const token = byLabel.get(match.trimStart())?.shift()
if (!token) {
return match
}
return token.kind === 'paste' ? match.slice(0, match.length - token.label.length) + token.text : ''
})
.trim()
}

23
ui-tui/src/domain/messages.ts
View file

@ -1,24 +1,9 @@
import { LONG_MSG } from '../config/limits.js'
import { buildToolTrailLine, fmtK } from '../lib/text.js'
import { buildToolTrailLine } from '../lib/text.js'
import type { Msg, SessionInfo } from '../types.js'
export const introMsg = (info: SessionInfo): Msg => ({ info, kind: 'intro', role: 'system', text: '' })
export const imageTokenMeta = (info?: ImageMeta | null) => {
const { width, height, token_estimate: t } = info ?? {}
return [width && height ? `${width}x${height}` : '', (t ?? 0) > 0 ? `~${fmtK(t!)} tok` : '']
.filter(Boolean)
.join(' · ')
}
export const attachedImageNotice = (info?: ({ name?: string } & ImageMeta) | null) => {
const meta = imageTokenMeta(info)
const label = info?.name ? `📎 Attached image: ${info.name}` : '📎 Attached image'
return `${label}${meta ? ` · ${meta}` : ''}`
}
export const userDisplay = (text: string) => {
if (text.length <= LONG_MSG) {
return text
@ -112,12 +97,6 @@ export const fmtDuration = (ms: number) => {
return h > 0 ? `${h}h ${m}m` : m > 0 ? `${m}m ${s}s` : `${s}s`
}
interface ImageMeta {
height?: number
token_estimate?: number
width?: number
}
interface TranscriptRow {
context?: string
display_kind?: string