From 153060e206cdbae37d1cb7ba731eef9c8159bf09 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Tue, 9 Jun 2026 17:03:42 -0500 Subject: [PATCH] fix(desktop): render optimistic image thumbnails from in-hand base64 The in-flight user bubble seeded image attachment refs as `@image:`. In remote-gateway mode that path lives on the desktop, not the gateway, so the inline thumbnail fetch hit /api/media and 403'd ("Path outside media roots"), flashing a fallback chip until submit uploaded the bytes. Seed (and keep) image refs as the raw base64 preview data URL instead. It renders inline via extractEmbeddedImages with zero network, and survives the post-sync rewrite (the agent gets the bytes through the attached-image pipeline, not this display ref) so the thumbnail no longer remounts/flashes. Non-image refs are unchanged. Adds optimisticAttachmentRef + unit coverage. --- .../app/session/hooks/use-prompt-actions.ts | 9 +++-- apps/desktop/src/lib/chat-runtime.test.ts | 38 ++++++++++++++++++- apps/desktop/src/lib/chat-runtime.ts | 23 +++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions.ts index 47ee008e788..21eb53f01fd 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions.ts @@ -6,7 +6,7 @@ import { getProfiles, transcribeAudio } from '@/hermes' import { translateNow, type Translations, useI18n } from '@/i18n' import { branchGroupForUser, type ChatMessage, chatMessageText, textPart } from '@/lib/chat-messages' import { - attachmentDisplayText, + optimisticAttachmentRef, parseCommandDispatch, parseSlashCommand, pathLabel, @@ -418,7 +418,9 @@ export function usePromptActions({ // Refs are recomputed after sync (file.attach rewrites @file: refs to // workspace-relative paths the remote gateway can resolve). Seed the // optimistic message with the pre-sync refs, then rewrite once synced. - let attachmentRefs = attachments.map(attachmentDisplayText).filter((r): r is string => Boolean(r)) + // Images use their base64 preview so the thumbnail renders inline without + // a (remote-mode 403-prone) /api/media fetch — see optimisticAttachmentRef. + let attachmentRefs = attachments.map(optimisticAttachmentRef).filter((r): r is string => Boolean(r)) const buildContextText = (atts: ComposerAttachment[]): string => { const contextRefs = atts .map(a => a.refText) @@ -550,7 +552,8 @@ export function usePromptActions({ }) // Rewrite the optimistic message + prompt text with the synced refs so // the gateway receives @file: paths that resolve in its workspace. - attachmentRefs = syncedAttachments.map(attachmentDisplayText).filter((r): r is string => Boolean(r)) + // (Images keep their inline base64 preview — see optimisticAttachmentRef.) + attachmentRefs = syncedAttachments.map(optimisticAttachmentRef).filter((r): r is string => Boolean(r)) rewriteOptimistic(sessionId) const text = buildContextText(syncedAttachments) diff --git a/apps/desktop/src/lib/chat-runtime.test.ts b/apps/desktop/src/lib/chat-runtime.test.ts index c06ea6f324c..c2a9099a1a8 100644 --- a/apps/desktop/src/lib/chat-runtime.test.ts +++ b/apps/desktop/src/lib/chat-runtime.test.ts @@ -1,6 +1,42 @@ import { describe, expect, it } from 'vitest' -import { coerceThinkingText } from './chat-runtime' +import type { ComposerAttachment } from '@/store/composer' + +import { coerceThinkingText, optimisticAttachmentRef } from './chat-runtime' + +const DATA_URL = 'data:image/png;base64,iVBORw0KGgoAAAANS' + +function attachment(overrides: Partial & Pick): ComposerAttachment { + return { id: 'a', label: 'file.png', ...overrides } +} + +describe('optimisticAttachmentRef', () => { + it('renders an image from its in-hand base64 preview (no @image: path ref)', () => { + const ref = optimisticAttachmentRef(attachment({ kind: 'image', detail: '/tmp/shot.png', previewUrl: DATA_URL })) + + // The raw data URL flows through extractEmbeddedImages → inline thumbnail, + // dodging the remote /api/media 403 an @image: ref would hit. + expect(ref).toBe(DATA_URL) + }) + + it('falls back to an @image: path ref when no preview is available', () => { + expect(optimisticAttachmentRef(attachment({ kind: 'image', detail: '/tmp/shot.png' }))).toBe('@image:/tmp/shot.png') + }) + + it('ignores a non-data preview url and uses the path ref', () => { + const ref = optimisticAttachmentRef( + attachment({ kind: 'image', detail: '/tmp/shot.png', previewUrl: 'https://example.com/x.png' }) + ) + + expect(ref).toBe('@image:/tmp/shot.png') + }) + + it('passes non-image attachments straight through to attachmentDisplayText', () => { + expect(optimisticAttachmentRef(attachment({ kind: 'file', refText: '@file:src/a.ts', previewUrl: DATA_URL }))).toBe( + '@file:src/a.ts' + ) + }) +}) describe('coerceThinkingText', () => { it('strips streaming status prefixes from thinking deltas', () => { diff --git a/apps/desktop/src/lib/chat-runtime.ts b/apps/desktop/src/lib/chat-runtime.ts index 2599fb0dad3..3246f490d08 100644 --- a/apps/desktop/src/lib/chat-runtime.ts +++ b/apps/desktop/src/lib/chat-runtime.ts @@ -165,6 +165,29 @@ export function attachmentDisplayText(attachment: ComposerAttachment): string | return null } +/** + * Display ref for the optimistic (in-flight) user bubble. + * + * Images prefer their in-hand base64 preview (a `data:` URL) over a file path. + * `DirectiveContent` runs `extractEmbeddedImages` first, so a raw `data:` URL + * renders as an inline thumbnail with zero network. An `@image:` ref + * would instead route through `/api/media`, which in remote mode 403s ("Path + * outside media roots") on a local path the gateway can't read yet — flashing a + * fallback chip until submit uploads the bytes. The preview also survives the + * post-sync rewrite (bytes go to the agent via the attached-image pipeline, not + * this display ref), so the thumbnail stays stable instead of remounting. + * + * Everything else (files, folders, terminals, post-sync `@file:` refs) falls + * through to `attachmentDisplayText`. + */ +export function optimisticAttachmentRef(attachment: ComposerAttachment): string | null { + if (attachment.kind === 'image' && attachment.previewUrl?.startsWith('data:')) { + return attachment.previewUrl + } + + return attachmentDisplayText(attachment) +} + export function personalityNamesFromConfig(config: unknown): string[] { const root = config && typeof config === 'object' ? (config as Record) : {} const agent = root.agent && typeof root.agent === 'object' ? (root.agent as Record) : {}