From fead8c8d6ad3fa6f422937e97a35936929159a79 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Thu, 30 Jul 2026 23:42:43 -0500 Subject: [PATCH 1/3] feat(tui): one token type for everything deferred in the composer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A collapsed paste and an attached image are the same idea: a `[[ … ]]` marker sitting in the input line that stands in for a payload resolved at submit. Model both as ComposerToken and give them one expander. Image tokens resolve to nothing — the gateway already holds the file in attached_images — so expandTokens eats an adjacent space to avoid leaving a gap mid-sentence. nextImageIndex never reuses an index after a delete, or two files would collide on one label. --- ui-tui/src/__tests__/attachments.test.ts | 96 ++++++++++++++++++++++++ ui-tui/src/app/interfaces.ts | 34 ++++++--- ui-tui/src/app/useSubmission.test.ts | 34 --------- ui-tui/src/domain/attachments.ts | 60 +++++++++++++++ 4 files changed, 179 insertions(+), 45 deletions(-) create mode 100644 ui-tui/src/__tests__/attachments.test.ts delete mode 100644 ui-tui/src/app/useSubmission.test.ts create mode 100644 ui-tui/src/domain/attachments.ts diff --git a/ui-tui/src/__tests__/attachments.test.ts b/ui-tui/src/__tests__/attachments.test.ts new file mode 100644 index 00000000000..50e5088bcc0 --- /dev/null +++ b/ui-tui/src/__tests__/attachments.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest' + +import type { ComposerToken } from '../app/interfaces.js' +import { droppedTokens, expandTokens, imageToken, nextImageIndex } from '../domain/attachments.js' + +const paste = (label: string, text: string): ComposerToken => ({ kind: 'paste', label, text }) + +const image = (index: number, path = `/tmp/img${index}.png`): ComposerToken => ({ + index, + kind: 'image', + label: imageToken(index), + path +}) + +describe('expandTokens (what the agent actually receives)', () => { + it('replaces a collapsed paste label with its full content', () => { + const label = '[[ hello.. [3 lines] .. world ]]' + const expand = expandTokens([paste(label, 'hello\nfoo\nworld')]) + + expect(expand(`here: ${label} done`)).toBe('here: hello\nfoo\nworld done') + }) + + it('is a no-op for already-expanded / token-free text (recall round-trip)', () => { + const expanded = 'hello\nfoo\nworld' + expect(expandTokens([])(expanded)).toBe(expanded) + }) + + it('expands repeated identical labels in submission order', () => { + const label = '[[ x [1 lines] ]]' + const expand = expandTokens([paste(label, 'first'), paste(label, 'second')]) + + expect(expand(`${label} then ${label}`)).toBe('first then second') + }) + + it('leaves an unmatched label intact', () => { + const label = '[[ orphan [2 lines] ]]' + expect(expandTokens([])(label)).toBe(label) + }) + + it('drops an image token from the text — the gateway already holds the file', () => { + const expand = expandTokens([image(1)]) + + expect(expand(`what is in ${imageToken(1)}`)).toBe('what is in') + }) + + it('leaves no double space where an image token sat mid-sentence', () => { + const expand = expandTokens([image(1)]) + + expect(expand(`before ${imageToken(1)} after`)).toBe('before after') + }) + + it('resolves an image-only message to empty text', () => { + expect(expandTokens([image(1)])(imageToken(1))).toBe('') + }) + + it('resolves pastes and images in one pass', () => { + const label = '[[ log.. [9 lines] ]]' + const expand = expandTokens([paste(label, 'stack\ntrace'), image(2)]) + + expect(expand(`${label} and ${imageToken(2)}`)).toBe('stack\ntrace and') + }) +}) + +describe('nextImageIndex (user-facing numbering)', () => { + it('starts at 1', () => { + expect(nextImageIndex([])).toBe(1) + }) + + it('counts past existing images', () => { + expect(nextImageIndex([image(1), image(2)])).toBe(3) + }) + + it('ignores paste tokens', () => { + expect(nextImageIndex([paste('[[ x ]]', 'y')])).toBe(1) + }) + + it('does not reuse an index after an earlier image is deleted', () => { + // [[ Image 1 ]] was erased; the next attach must not become Image 1 again + // or expandTokens would resolve two different files to one label. + expect(nextImageIndex([image(2)])).toBe(3) + }) +}) + +describe('droppedTokens (deleting the token unattaches the thing)', () => { + it('reports an image whose token was erased from the text', () => { + expect(droppedTokens([image(1)], 'just text now')).toEqual([image(1)]) + }) + + it('reports nothing while the token is still present', () => { + expect(droppedTokens([image(1)], `look at ${imageToken(1)}`)).toEqual([]) + }) + + it('keeps a surviving token when a sibling is erased', () => { + expect(droppedTokens([image(1), image(2)], imageToken(2))).toEqual([image(1)]) + }) +}) diff --git a/ui-tui/src/app/interfaces.ts b/ui-tui/src/app/interfaces.ts index 1a446f69e15..3802769e933 100644 --- a/ui-tui/src/app/interfaces.ts +++ b/ui-tui/src/app/interfaces.ts @@ -7,7 +7,6 @@ import type { BillingCardInfo, BillingMutationResponse, BillingStateResponse, - ImageAttachResponse, SessionCloseResponse, SubscriptionPreviewResponse, SubscriptionStateResponse, @@ -364,6 +363,10 @@ export interface ComposerPasteResult { export type MaybePromise = Promise | T export interface ComposerActions { + /** Pull an image off the system clipboard in as a token. */ + attachClipboardImage: () => void + /** Attach an image by path in as a token. */ + attachImagePath: (path: string) => void clearIn: () => void dequeue: () => string | undefined enqueue: (text: string) => void @@ -373,12 +376,14 @@ export interface ComposerActions { removeQueue: (index: number) => void replaceQueue: (index: number, text: string) => void setCompIdx: StateSetter + setComposerTokens: StateSetter setHistoryIdx: StateSetter setInput: StateSetter setInputBuf: StateSetter - setPasteSnips: StateSetter setQueueEdit: (index: null | number) => void syncQueue: () => void + /** Reconcile attached payloads against tokens still present in the text. */ + syncTokens: (value: string) => void } export interface ComposerRefs { @@ -387,6 +392,7 @@ export interface ComposerRefs { queueEditRef: MutableRefObject queueRef: MutableRefObject submitRef: MutableRefObject<(value: string) => void> + tokensRef: MutableRefObject } export interface ComposerState { @@ -396,16 +402,15 @@ export interface ComposerState { historyIdx: null | number input: string inputBuf: string[] - pasteSnips: PasteSnippet[] queueEditIdx: null | number queuedDisplay: string[] + tokens: ComposerToken[] } export interface UseComposerStateOptions { gw: GatewayClient - onClipboardPaste: (quiet?: boolean) => Promise | void - onImageAttached?: (info: ImageAttachResponse) => void submitRef: MutableRefObject<(value: string) => void> + sys: (text: string) => void } export interface UseComposerStateResult { @@ -495,10 +500,11 @@ export interface GatewayEventHandlerContext { export interface SlashHandlerContext { composer: { + attachClipboardImage: () => void + attachImagePath: (path: string) => void enqueue: (text: string) => void hasSelection: boolean openEditor: () => Promise - paste: (quiet?: boolean) => void queueRef: MutableRefObject selection: SelectionApi setInput: StateSetter @@ -618,8 +624,14 @@ export interface AppOverlaysProps { pagerPageSize: number } -export interface PasteSnippet { - label: string - path?: string - text: string -} +/** + * A `[[ … ]]` token sitting in the composer text, plus the payload it stands + * for. `paste` tokens expand back into their text at submit; `image` tokens + * are a receipt for a file the gateway already holds, and expand to nothing. + * + * `index` is the user-facing number in `[[ Image 2 ]]`; `path` is the gateway + * path, used to detach the image when its token is deleted. + */ +export type ComposerToken = + | { index: number; kind: 'image'; label: string; path: string; text?: undefined } + | { index?: undefined; kind: 'paste'; label: string; path?: string; text: string } diff --git a/ui-tui/src/app/useSubmission.test.ts b/ui-tui/src/app/useSubmission.test.ts deleted file mode 100644 index 34202104dd4..00000000000 --- a/ui-tui/src/app/useSubmission.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import type { PasteSnippet } from './interfaces.js' -import { expandSnips } from './useSubmission.js' - -const snip = (label: string, text: string): PasteSnippet => ({ label, text }) - -describe('expandSnips (paste history recall)', () => { - it('replaces a collapsed paste label with its full content', () => { - const label = '[[ hello.. [3 lines] .. world ]]' - const full = `here: ${label} done` - const expand = expandSnips([snip(label, 'hello\nfoo\nworld')]) - - expect(expand(full)).toBe('here: hello\nfoo\nworld done') - }) - - it('is a no-op for already-expanded / label-free text (recall round-trip)', () => { - const expanded = 'hello\nfoo\nworld' - // Re-submitting a recalled history entry has no snips and no labels. - expect(expandSnips([])(expanded)).toBe(expanded) - }) - - it('expands repeated identical labels in submission order', () => { - const label = '[[ x [1 lines] ]]' - const expand = expandSnips([snip(label, 'first'), snip(label, 'second')]) - - expect(expand(`${label} then ${label}`)).toBe('first then second') - }) - - it('leaves an unmatched label intact', () => { - const label = '[[ orphan [2 lines] ]]' - expect(expandSnips([])(label)).toBe(label) - }) -}) diff --git a/ui-tui/src/domain/attachments.ts b/ui-tui/src/domain/attachments.ts new file mode 100644 index 00000000000..7894cf170a0 --- /dev/null +++ b/ui-tui/src/domain/attachments.ts @@ -0,0 +1,60 @@ +import type { ComposerToken } from '../app/interfaces.js' +import { PASTE_SNIPPET_RE } from '../protocol/paste.js' + +/** + * Composer tokens are the ONE way deferred content shows up in the input line: + * a collapsed paste and an attached image both render as `[[ … ]]` sitting in + * the text the user is editing. They are ordinary characters — arrow keys, + * backspace, and selection work on them for free — and they carry their real + * payload out-of-band until submit. + * + * Two consequences the rest of the composer relies on: + * - Deleting the token is how you drop the thing. Nothing else to click. + * - Position in the text is meaningful: the model sees the payload where the + * token sat, not stapled to the front of the turn. + */ +export const imageToken = (index: number) => `[[ Image ${index} ]]` + +/** Highest image token index handed out so far, so a new one never collides. */ +export const nextImageIndex = (tokens: ComposerToken[]) => + tokens.reduce((max, t) => (t.kind === 'image' ? Math.max(max, t.index) : max), 0) + 1 + +/** Tokens whose label is no longer anywhere in the composer text. */ +export const droppedTokens = (tokens: ComposerToken[], value: string) => { + const live = new Set(value.match(PASTE_SNIPPET_RE) ?? []) + + return tokens.filter(t => !live.has(t.label)) +} + +/** + * Resolve every token in `value` to what the agent should actually receive. + * + * Repeated identical labels expand in submission order (left to right), which + * is why this walks matches instead of doing a global replace per token. + * + * An image token expands to nothing: the gateway already holds the file in + * `session.attached_images` and splices the real vision content in at submit. + * The token's job was to show the user where it landed, so it also eats one + * adjacent space to avoid leaving a gap in the middle of a sentence. + */ +export const expandTokens = (tokens: ComposerToken[]) => { + const byLabel = new Map() + + for (const token of tokens) { + const hit = byLabel.get(token.label) + hit ? hit.push(token) : byLabel.set(token.label, [token]) + } + + return (value: string) => + value + .replace(new RegExp(`[ \\t]?(?:${PASTE_SNIPPET_RE.source})`, 'g'), match => { + const token = byLabel.get(match.trimStart())?.shift() + + if (!token) { + return match + } + + return token.kind === 'paste' ? match.slice(0, match.length - token.label.length) + token.text : '' + }) + .trim() +} From ca5ee5ed331dbbe029157f7bcbb5f8b8f3fc8566 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Thu, 30 Jul 2026 23:42:53 -0500 Subject: [PATCH 2/3] feat(tui): attach images inline at the cursor, delete the token to unattach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every attach path now drops an `[[ Image N ]]` token where you are typing: drag-drop, clipboard (bracketed and hotkey), /image, /paste. The composer owns clipboard attach directly instead of calling back out to useMainApp. Deleting the token is how you unattach — there is no second control. updateInput is the one choke point every keystroke passes through, so syncTokens reconciles there and detaches anything erased. That also fixes a stale image riding along on the next unrelated turn. Tokens and the input line get refs alongside state: paste-then-immediately -Enter submits before React has re-rendered, and the submit path has to see the token that was just added. --- ui-tui/src/app/slash/commands/core.ts | 2 +- ui-tui/src/app/slash/commands/session.ts | 15 +- ui-tui/src/app/useComposerState.ts | 252 +++++++++++++++++------ ui-tui/src/app/useMainApp.ts | 61 +++--- ui-tui/src/app/useSessionLifecycle.ts | 4 +- ui-tui/src/app/useSubmission.ts | 32 ++- 6 files changed, 238 insertions(+), 128 deletions(-) diff --git a/ui-tui/src/app/slash/commands/core.ts b/ui-tui/src/app/slash/commands/core.ts index 00321ecc904..865c9cb5d6e 100644 --- a/ui-tui/src/app/slash/commands/core.ts +++ b/ui-tui/src/app/slash/commands/core.ts @@ -439,7 +439,7 @@ export const coreCommands: SlashCommand[] = [ { help: 'attach clipboard image', name: 'paste', - run: (arg, ctx) => (arg ? ctx.transcript.sys('usage: /paste') : ctx.composer.paste()) + run: (arg, ctx) => (arg ? ctx.transcript.sys('usage: /paste') : ctx.composer.attachClipboardImage()) }, { diff --git a/ui-tui/src/app/slash/commands/session.ts b/ui-tui/src/app/slash/commands/session.ts index a4129490191..bc072ec3802 100644 --- a/ui-tui/src/app/slash/commands/session.ts +++ b/ui-tui/src/app/slash/commands/session.ts @@ -1,11 +1,10 @@ import { usageBarsText } from '../../../components/overlayPrimitives.js' -import { attachedImageNotice, introMsg, toTranscriptMessages } from '../../../domain/messages.js' +import { introMsg, toTranscriptMessages } from '../../../domain/messages.js' import { sessionScopedModelArg, TUI_SESSION_MODEL_FLAG } from '../../../domain/slash.js' import type { BackgroundStartResponse, ConfigGetValueResponse, ConfigSetResponse, - ImageAttachResponse, SessionBranchResponse, SessionCompressResponse, SessionUsageResponse, @@ -192,17 +191,7 @@ export const sessionCommands: SlashCommand[] = [ { help: 'attach an image', name: 'image', - run: (arg, ctx) => { - ctx.gateway.rpc('image.attach', { path: arg, session_id: ctx.sid }).then( - ctx.guarded(r => { - ctx.transcript.sys(attachedImageNotice(r)) - - if (r.remainder) { - ctx.composer.setInput(r.remainder) - } - }) - ) - } + run: (arg, ctx) => ctx.composer.attachImagePath(arg) }, { diff --git a/ui-tui/src/app/useComposerState.ts b/ui-tui/src/app/useComposerState.ts index 40120326a87..188e3c57270 100644 --- a/ui-tui/src/app/useComposerState.ts +++ b/ui-tui/src/app/useComposerState.ts @@ -5,10 +5,11 @@ import { join } from 'node:path' import { useStdin, withInkSuspended } from '@hermes/ink' import { useStore } from '@nanostores/react' -import { useCallback, useMemo, useState } from 'react' +import { useCallback, useMemo, useRef, useState } from 'react' import type { PasteEvent } from '../components/textInput.js' -import type { ImageAttachResponse, InputDetectDropResponse } from '../gatewayTypes.js' +import { droppedTokens, imageToken, nextImageIndex } from '../domain/attachments.js' +import type { ClipboardPasteResponse, ImageAttachResponse, InputDetectDropResponse } from '../gatewayTypes.js' import { useCompletion } from '../hooks/useCompletion.js' import { useInputHistory } from '../hooks/useInputHistory.js' import { useQueue } from '../hooks/useQueue.js' @@ -18,30 +19,37 @@ import { readOsc52Clipboard } from '../lib/osc52.js' import { isRemoteShellSession } from '../lib/terminalSetup.js' import { pasteTokenLabel, stripTrailingPasteNewlines } from '../lib/text.js' -import type { MaybePromise, PasteSnippet, UseComposerStateOptions, UseComposerStateResult } from './interfaces.js' +import type { + ComposerPasteResult, + ComposerToken, + MaybePromise, + StateSetter, + UseComposerStateOptions, + UseComposerStateResult +} from './interfaces.js' import { $isBlocked } from './overlayStore.js' import { getUiState } from './uiStore.js' -const PASTE_SNIP_MAX_COUNT = 32 -const PASTE_SNIP_MAX_TOTAL_BYTES = 4 * 1024 * 1024 +const TOKEN_MAX_COUNT = 32 +const TOKEN_MAX_TOTAL_BYTES = 4 * 1024 * 1024 -const trimSnips = (snips: PasteSnippet[]): PasteSnippet[] => { +const trimTokens = (tokens: ComposerToken[]): ComposerToken[] => { let total = 0 - const out: PasteSnippet[] = [] + const out: ComposerToken[] = [] - for (let i = snips.length - 1; i >= 0; i--) { - const snip = snips[i]! - const size = snip.text.length + for (let i = tokens.length - 1; i >= 0; i--) { + const token = tokens[i]! + const size = token.text?.length ?? 0 - if (out.length >= PASTE_SNIP_MAX_COUNT || total + size > PASTE_SNIP_MAX_TOTAL_BYTES) { + if (out.length >= TOKEN_MAX_COUNT || total + size > TOKEN_MAX_TOTAL_BYTES) { break } total += size - out.unshift(snip) + out.unshift(token) } - return out.length === snips.length ? snips : out + return out.length === tokens.length ? tokens : out } /** Insert text at the cursor position, adding spacing to separate from adjacent non-whitespace. */ @@ -97,15 +105,26 @@ export function looksLikeDroppedPath(text: string): boolean { return false } -export function useComposerState({ - gw, - onClipboardPaste, - onImageAttached, - submitRef -}: UseComposerStateOptions): UseComposerStateResult { - const [input, setInput] = useState('') +export function useComposerState({ gw, submitRef, sys }: UseComposerStateOptions): UseComposerStateResult { + const [input, setInputState] = useState('') const [inputBuf, setInputBuf] = useState([]) - const [pasteSnips, setPasteSnips] = useState([]) + const [tokens, setTokens] = useState([]) + // Tokens and the input line are read from keystroke handlers that run several + // times before React re-renders, so the refs — not the state — are the source + // of truth for "what is in the composer right now". + const inputRef = useRef('') + const tokensRef = useRef([]) + + const setInput = useCallback>(next => { + inputRef.current = typeof next === 'function' ? next(inputRef.current) : next + setInputState(inputRef.current) + }, []) + + const setComposerTokens = useCallback>(next => { + tokensRef.current = typeof next === 'function' ? next(tokensRef.current) : next + setTokens(tokensRef.current) + }, []) + const isBlocked = useStore($isBlocked) const { querier } = useStdin() as { querier: Parameters[0] } @@ -128,27 +147,95 @@ export function useComposerState({ const clearIn = useCallback(() => { setInput('') setInputBuf([]) - setPasteSnips([]) + setComposerTokens([]) setQueueEdit(null) setHistoryIdx(null) historyDraftRef.current = '' - }, [historyDraftRef, setQueueEdit, setHistoryIdx]) + }, [historyDraftRef, setComposerTokens, setHistoryIdx, setInput, setQueueEdit]) + + /** + * Deleting an `[[ Image N ]]` token IS how you unattach the image — there is + * no separate control. Reconcile on every edit so the gateway's + * `attached_images` never outlives the token the user just erased, which is + * what used to make a stale image ride along on the next unrelated turn. + */ + const syncTokens = useCallback( + (value: string) => { + const gone = droppedTokens(tokensRef.current, value) + + if (!gone.length) { + return + } + + for (const token of gone) { + if (token.kind === 'image') { + void gw.request('image.detach', { path: token.path, session_id: getUiState().sid }).catch(() => {}) + } + } + + setComposerTokens(prev => prev.filter(token => !gone.includes(token))) + }, + [gw, setComposerTokens] + ) + + /** + * Attach an image the gateway already resolved: a token at the cursor, + * followed by whatever non-path text came along with it (a drag-drop paste + * of `~/shot.png look at this` keeps the caption). + */ + const attachImageToken = useCallback( + (attached: ImageAttachResponse & { path?: string }, value: string, cursor: number): ComposerPasteResult => { + const index = nextImageIndex(tokensRef.current) + const label = imageToken(index) + + setComposerTokens(prev => trimTokens([...prev, { index, kind: 'image', label, path: attached.path ?? '' }])) + + const withToken = insertAtCursor(value, cursor, label) + const remainder = attached.remainder?.trim() ?? '' + + return remainder ? insertAtCursor(withToken.value, withToken.cursor, remainder) : withToken + }, + [setComposerTokens] + ) + + /** + * Pull an image off the system clipboard into the composer as a token. + * + * `quiet` is the empty-bracketed-paste probe: the terminal delivers an image + * paste as zero text, so we speculatively ask the gateway and stay silent if + * there was nothing there. An explicit `/paste` reports the miss. + */ + const pasteClipboardImage = useCallback( + async (value: string, cursor: number, quiet: boolean): Promise => { + const sid = getUiState().sid + + if (!sid) { + return null + } + + const r = await gw + .request('clipboard.paste', { session_id: sid }) + .catch(() => null) + + if (r?.attached) { + return attachImageToken(r, value, cursor) + } + + if (!quiet) { + sys(r?.message || 'No image found in clipboard') + } + + return null + }, + [attachImageToken, gw, sys] + ) const handleResolvedPaste = useCallback( - async ({ - bracketed, - cursor, - text, - value - }: Omit): Promise => { + async ({ bracketed, cursor, text, value }: Omit): Promise => { const cleanedText = stripTrailingPasteNewlines(text) if (!cleanedText || !/[^\n]/.test(cleanedText)) { - if (bracketed) { - void onClipboardPaste(true) - } - - return null + return bracketed ? pasteClipboardImage(value, cursor, true) : null } const sid = getUiState().sid @@ -161,14 +248,11 @@ export function useComposerState({ }) if (attached?.name) { - onImageAttached?.(attached) - const remainder = attached.remainder?.trim() ?? '' - - if (!remainder) { - return { cursor, value } - } - - return insertAtCursor(value, cursor, remainder) + // Drop an `[[ Image N ]]` token where the path was typed. The old + // path printed a notice above the status bar and left the composer + // untouched, so the only trace of the attachment lived outside the + // input the user was editing. + return attachImageToken(attached, value, cursor) } } catch { // Fall back to generic file-drop detection below. @@ -204,7 +288,7 @@ export function useComposerState({ const label = pasteTokenLabel(cleanedText, lineCount) const inserted = insertAtCursor(value, cursor, label) - setPasteSnips(prev => trimSnips([...prev, { label, text: cleanedText }])) + setComposerTokens(prev => trimTokens([...prev, { kind: 'paste', label, text: cleanedText }])) void gw .request<{ path?: string }>('paste.collapse', { text: cleanedText }) @@ -215,13 +299,13 @@ export function useComposerState({ return } - setPasteSnips(prev => prev.map(s => (s.label === label ? { ...s, path } : s))) + setComposerTokens(prev => prev.map(t => (t.label === label ? { ...t, path } : t))) }) .catch(() => {}) return inserted }, - [gw, onClipboardPaste, onImageAttached] + [attachImageToken, gw, pasteClipboardImage, setComposerTokens] ) const handleTextPaste = useCallback( @@ -231,7 +315,7 @@ export function useComposerState({ hotkey, text, value - }: PasteEvent): MaybePromise => { + }: PasteEvent): MaybePromise => { if (hotkey) { const preferOsc52 = isRemoteShellSession(process.env) @@ -256,15 +340,58 @@ export function useComposerState({ return handleResolvedPaste({ bracketed: false, cursor, text: preferredText, value }) } - void onClipboardPaste(false) - - return null + // No text on the clipboard — an image paste looks exactly like this. + return pasteClipboardImage(value, cursor, false) }) } return handleResolvedPaste({ bracketed: !!bracketed, cursor, text, value }) }, - [handleResolvedPaste, onClipboardPaste, querier] + [handleResolvedPaste, pasteClipboardImage, querier] + ) + + /** + * `/paste` and `/image` attach without a cursor of their own — the token + * lands at the end of whatever is currently typed. + */ + const appendAttachment = useCallback( + (attach: (value: string, cursor: number) => Promise) => { + const current = inputRef.current + + void attach(current, current.length).then(next => { + if (next) { + setInput(next.value) + } + }) + }, + [setInput] + ) + + const attachClipboardImage = useCallback( + () => appendAttachment((value, cursor) => pasteClipboardImage(value, cursor, false)), + [appendAttachment, pasteClipboardImage] + ) + + const attachImagePath = useCallback( + (path: string) => + appendAttachment(async (value, cursor) => { + const sid = getUiState().sid + + if (!sid || !path.trim()) { + return null + } + + const attached = await gw + .request('image.attach', { path, session_id: sid }) + .catch((e: Error) => { + sys(`error: ${e.message}`) + + return null + }) + + return attached?.name ? attachImageToken(attached, value, cursor) : null + }), + [appendAttachment, attachImageToken, gw, sys] ) const openEditor = useCallback(async () => { @@ -297,10 +424,12 @@ export function useComposerState({ } finally { rmSync(dir, { force: true, recursive: true }) } - }, [input, inputBuf, submitRef]) + }, [input, inputBuf, setInput, submitRef]) const actions = useMemo( () => ({ + attachClipboardImage, + attachImagePath, clearIn, dequeue, enqueue, @@ -310,14 +439,17 @@ export function useComposerState({ removeQueue: removeQ, replaceQueue: replaceQ, setCompIdx, + setComposerTokens, setHistoryIdx, setInput, setInputBuf, - setPasteSnips, setQueueEdit, - syncQueue + syncQueue, + syncTokens }), [ + attachClipboardImage, + attachImagePath, clearIn, dequeue, enqueue, @@ -327,9 +459,12 @@ export function useComposerState({ removeQ, replaceQ, setCompIdx, + setComposerTokens, setHistoryIdx, + setInput, setQueueEdit, - syncQueue + syncQueue, + syncTokens ] ) @@ -339,7 +474,8 @@ export function useComposerState({ historyRef, queueEditRef, queueRef, - submitRef + submitRef, + tokensRef }), [historyDraftRef, historyRef, queueEditRef, queueRef, submitRef] ) @@ -352,11 +488,11 @@ export function useComposerState({ historyIdx, input, inputBuf, - pasteSnips, queueEditIdx, - queuedDisplay + queuedDisplay, + tokens }), - [compIdx, compReplace, completions, historyIdx, input, inputBuf, pasteSnips, queueEditIdx, queuedDisplay] + [compIdx, compReplace, completions, historyIdx, input, inputBuf, queueEditIdx, queuedDisplay, tokens] ) return { diff --git a/ui-tui/src/app/useMainApp.ts b/ui-tui/src/app/useMainApp.ts index 5766ebb6a7e..18277fd663c 100644 --- a/ui-tui/src/app/useMainApp.ts +++ b/ui-tui/src/app/useMainApp.ts @@ -16,13 +16,11 @@ import { MAX_HISTORY, WHEEL_SCROLL_STEP } from '../config/limits.js' import { RESIZE_COALESCE_MS } from '../config/timing.js' import { hasLeadGap, prevRenderedMsg } from '../domain/blockLayout.js' import { SECTION_NAMES, sectionMode } from '../domain/details.js' -import { attachedImageNotice, imageTokenMeta } from '../domain/messages.js' import { composeTabTitle, fmtProjectCwdBranch, shortCwd } from '../domain/paths.js' import { sessionScopedModelArg } from '../domain/slash.js' import { type GatewayClient } from '../gatewayClient.js' import type { ClarifyRespondResponse, - ClipboardPasteResponse, ConfigSetResponse, GatewayEvent, SessionActiveListResponse, @@ -46,7 +44,7 @@ import { createGatewayEventHandler } from './createGatewayEventHandler.js' import { createSlashHandler } from './createSlashHandler.js' import { planGatewayRecovery } from './gatewayRecovery.js' import { getInputSelection } from './inputSelectionStore.js' -import { type GatewayRpc, type TranscriptRow } from './interfaces.js' +import { type GatewayRpc, type StateSetter, type TranscriptRow } from './interfaces.js' import { $overlayState, patchOverlayState } from './overlayStore.js' import { $goodVibesTick } from './petFlashStore.js' import { scrollWithSelectionBy } from './scroll.js' @@ -223,7 +221,7 @@ export function useMainApp(gw: GatewayClient) { const colsRef = useRef(cols) const scrollRef = useRef(null) const onEventRef = useRef<(ev: GatewayEvent) => void>(() => {}) - const clipboardPasteRef = useRef<(quiet?: boolean) => Promise | void>(() => {}) + const sysRef = useRef<(text: string) => void>(() => {}) const submitRef = useRef<(value: string) => void>(() => {}) const terminalHintsShownRef = useRef(new Set()) const historyItemsRef = useRef(historyItems) @@ -296,11 +294,8 @@ export function useMainApp(gw: GatewayClient) { const composer = useComposerState({ gw, - onClipboardPaste: quiet => clipboardPasteRef.current(quiet), - onImageAttached: info => { - sys(attachedImageNotice(info)) - }, - submitRef + submitRef, + sys: text => sysRef.current(text) }) const { actions: composerActions, refs: composerRefs, state: composerState } = composer @@ -711,27 +706,7 @@ export function useMainApp(gw: GatewayClient) { [appendMessage, overlay.clarify, rpc] ) - const paste = useCallback( - (quiet = false) => - rpc('clipboard.paste', { session_id: getUiState().sid }).then(r => { - if (!r) { - return - } - - if (r.attached) { - const meta = imageTokenMeta(r) - - return sys(`📎 Image #${r.count} attached from clipboard${meta ? ` · ${meta}` : ''}`) - } - - if (!quiet) { - sys(r.message || 'No image found in clipboard') - } - }), - [rpc, sys] - ) - - clipboardPasteRef.current = paste + sysRef.current = sys const { dispatchSubmission, send, sendQueued, submit } = useSubmission({ appendMessage, @@ -891,10 +866,11 @@ export function useMainApp(gw: GatewayClient) { () => createSlashHandler({ composer: { + attachClipboardImage: composerActions.attachClipboardImage, + attachImagePath: composerActions.attachImagePath, enqueue: composerActions.enqueue, hasSelection, openEditor: composerActions.openEditor, - paste, queueRef: composerRefs.queueRef, selection, setInput: composerActions.setInput @@ -933,7 +909,6 @@ export function useMainApp(gw: GatewayClient) { maybeWarn, page, panel, - paste, selection, send, session, @@ -1137,6 +1112,24 @@ export function useMainApp(gw: GatewayClient) { ] ) + /** + * Every keystroke lands here, so this is where attached payloads are + * reconciled against the tokens still in the text — deleting an + * `[[ Image N ]]` is how the user unattaches it. + */ + const updateInput = useCallback>( + next => { + composerActions.setInput(prev => { + const value = typeof next === 'function' ? next(prev) : next + + composerActions.syncTokens(value) + + return value + }) + }, + [composerActions] + ) + const appComposer = useMemo( () => ({ cols, @@ -1150,10 +1143,10 @@ export function useMainApp(gw: GatewayClient) { queueEditIdx: composerState.queueEditIdx, queuedDisplay: composerState.queuedDisplay, submit, - updateInput: composerActions.setInput, + updateInput, voiceRecordKey }), - [cols, composerActions, composerState, empty, pagerPageSize, submit, voiceRecordKey] + [cols, composerActions, composerState, empty, pagerPageSize, submit, updateInput, voiceRecordKey] ) // Pass current progress through unfrozen — streaming update throttling diff --git a/ui-tui/src/app/useSessionLifecycle.ts b/ui-tui/src/app/useSessionLifecycle.ts index 4cce57bae56..13dab7ce4cb 100644 --- a/ui-tui/src/app/useSessionLifecycle.ts +++ b/ui-tui/src/app/useSessionLifecycle.ts @@ -178,7 +178,7 @@ export function useSessionLifecycle(opts: UseSessionLifecycleOptions) { setHistoryItems([]) setLastUserMsg('') setStickyPrompt('') - composerActions.setPasteSnips([]) + composerActions.setComposerTokens([]) // Half-prune: new session has new keys, but keep a warm pool in case // the user resumes back to the prior session. evictInkCaches('half') @@ -202,7 +202,7 @@ export function useSessionLifecycle(opts: UseSessionLifecycleOptions) { setHistoryItems(info ? [introMsg(info)] : []) setStickyPrompt('') setLastUserMsg('') - composerActions.setPasteSnips([]) + composerActions.setComposerTokens([]) patchTurnState({ activity: [] }) patchUiState({ info, usage: usageFrom(info) }) }, diff --git a/ui-tui/src/app/useSubmission.ts b/ui-tui/src/app/useSubmission.ts index 0ced5f0b8a2..28ffaff9ca3 100644 --- a/ui-tui/src/app/useSubmission.ts +++ b/ui-tui/src/app/useSubmission.ts @@ -1,32 +1,21 @@ import { type MutableRefObject, useCallback, useEffect, useRef } from 'react' import { TYPING_IDLE_MS } from '../config/timing.js' +import { expandTokens } from '../domain/attachments.js' import { completionToApplyOnSubmit, looksLikeSlashCommand } from '../domain/slash.js' import type { GatewayClient } from '../gatewayClient.js' import type { SessionSteerResponse, ShellExecResponse } from '../gatewayTypes.js' import { asRpcResult } from '../lib/rpc.js' import { hasInterpolation, INTERPOLATION_RE } from '../protocol/interpolation.js' -import { PASTE_SNIPPET_RE } from '../protocol/paste.js' import type { Msg } from '../types.js' -import type { ComposerActions, ComposerRefs, ComposerState, PasteSnippet } from './interfaces.js' +import type { ComposerActions, ComposerRefs, ComposerState } from './interfaces.js' import { submitPrompt } from './submissionCore.js' import { turnController } from './turnController.js' import { getUiState, patchUiState } from './uiStore.js' const DOUBLE_ENTER_MS = 450 -export const expandSnips = (snips: PasteSnippet[]) => { - const byLabel = new Map() - - for (const { label, text } of snips) { - const hit = byLabel.get(label) - hit ? hit.push(text) : byLabel.set(label, [text]) - } - - return (value: string) => value.replace(PASTE_SNIPPET_RE, tok => byLabel.get(tok)?.shift() ?? tok) -} - const spliceMatches = (text: string, matches: RegExpMatchArray[], results: string[]) => matches.reduceRight((acc, m, i) => acc.slice(0, m.index!) + results[i] + acc.slice(m.index! + m[0].length), text) @@ -68,7 +57,9 @@ export function useSubmission(opts: UseSubmissionOptions) { const send = useCallback( (text: string, showUserMessage = true, displayText?: string) => { - const expand = expandSnips(composerState.pasteSnips) + // Read tokens off the ref, not render state: a paste immediately followed + // by Enter submits before React has re-rendered with the new token. + const expand = expandTokens(composerRefs.tokensRef.current) submitPrompt( text, @@ -84,7 +75,7 @@ export function useSubmission(opts: UseSubmissionOptions) { displayText ) }, - [appendMessage, composerActions, composerState.pasteSnips, gw, setLastUserMsg, sys] + [appendMessage, composerActions, composerRefs, gw, setLastUserMsg, sys] ) const shellExec = useCallback( @@ -217,10 +208,12 @@ export function useSubmission(opts: UseSubmissionOptions) { return } - // History stores expanded paste content, not the `[[…]]` label: snips - // are cleared on submit, so recall must be self-contained. Idempotent on - // label-free text, so re-submitting a recalled entry stays stable. - const toHistory = expandSnips(composerState.pasteSnips)(full) + // History stores resolved content, not `[[…]]` labels: tokens are cleared + // on submit, so recall must be self-contained. Image tokens resolve to + // nothing — a detached image can't be re-attached by recalling the text. + // Idempotent on token-free text, so re-submitting a recalled entry is + // stable. + const toHistory = expandTokens(composerRefs.tokensRef.current)(full) if (looksLikeSlashCommand(full)) { appendMessage({ kind: 'slash', role: 'system', text: full }) @@ -294,7 +287,6 @@ export function useSubmission(opts: UseSubmissionOptions) { appendMessage, composerActions, composerRefs, - composerState.pasteSnips, handleBusyInput, interpolate, send, From 22af266b4f3b865144279c81bc77c5cb4dbe7fd6 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Thu, 30 Jul 2026 23:42:58 -0500 Subject: [PATCH 3/3] fix(tui): stop announcing attachments outside the composer The token in the input line is the whole receipt. Drop the notices that duplicated it somewhere the user was not looking: the drag-drop and clipboard sys() lines, and the attachedImageNotice / "detected file: X" activity rows above the status bar. attachedImageNotice and imageTokenMeta have no callers left. --- ui-tui/src/app/submissionCore.ts | 12 +++++------- ui-tui/src/domain/messages.ts | 23 +---------------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/ui-tui/src/app/submissionCore.ts b/ui-tui/src/app/submissionCore.ts index c99eca09f2e..aa6cd98db6c 100644 --- a/ui-tui/src/app/submissionCore.ts +++ b/ui-tui/src/app/submissionCore.ts @@ -1,4 +1,3 @@ -import { attachedImageNotice } from '../domain/messages.js' import type { GatewayClient } from '../gatewayClient.js' import type { InputDetectDropResponse, PromptSubmitResponse } from '../gatewayTypes.js' import type { Msg } from '../types.js' @@ -109,6 +108,11 @@ export function submitPrompt( // Always ask the backend whether this looks like a file drop. The backend's // _detect_file_drop handles paths with spaces, quotes, Windows drive letters, // and escaped characters correctly. + // + // No notice is emitted for a match: an image dropped into the composer already + // shows as an `[[ Image N ]]` token, and a matched non-image path is rewritten + // in place. Announcing it a second time above the status bar was the old + // out-of-band attachment UI. deps.gw .request('input.detect_drop', { session_id: sid, text }) .then(r => { @@ -116,12 +120,6 @@ export function submitPrompt( return startSubmit(text, deps.expand(text), showUserMessage) } - if (r.is_image) { - turnController.pushActivity(attachedImageNotice(r)) - } else { - turnController.pushActivity(`detected file: ${r.name}`) - } - startSubmit(r.text || text, deps.expand(r.text || text), showUserMessage) }) .catch(() => startSubmit(text, deps.expand(text), showUserMessage)) diff --git a/ui-tui/src/domain/messages.ts b/ui-tui/src/domain/messages.ts index 45fbf606c0b..b4428520d81 100644 --- a/ui-tui/src/domain/messages.ts +++ b/ui-tui/src/domain/messages.ts @@ -1,24 +1,9 @@ import { LONG_MSG } from '../config/limits.js' -import { buildToolTrailLine, fmtK } from '../lib/text.js' +import { buildToolTrailLine } from '../lib/text.js' import type { Msg, SessionInfo } from '../types.js' export const introMsg = (info: SessionInfo): Msg => ({ info, kind: 'intro', role: 'system', text: '' }) -export const imageTokenMeta = (info?: ImageMeta | null) => { - const { width, height, token_estimate: t } = info ?? {} - - return [width && height ? `${width}x${height}` : '', (t ?? 0) > 0 ? `~${fmtK(t!)} tok` : ''] - .filter(Boolean) - .join(' · ') -} - -export const attachedImageNotice = (info?: ({ name?: string } & ImageMeta) | null) => { - const meta = imageTokenMeta(info) - const label = info?.name ? `📎 Attached image: ${info.name}` : '📎 Attached image' - - return `${label}${meta ? ` · ${meta}` : ''}` -} - export const userDisplay = (text: string) => { if (text.length <= LONG_MSG) { return text @@ -112,12 +97,6 @@ export const fmtDuration = (ms: number) => { return h > 0 ? `${h}h ${m}m` : m > 0 ? `${m}m ${s}s` : `${s}s` } -interface ImageMeta { - height?: number - token_estimate?: number - width?: number -} - interface TranscriptRow { context?: string display_kind?: string