diff --git a/apps/desktop/src/app/session/hooks/use-message-stream.ts b/apps/desktop/src/app/session/hooks/use-message-stream.ts index 99da03f08bf..2ccf73aa41b 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream.ts @@ -16,6 +16,11 @@ import { } from '@/lib/chat-messages' import { coerceGatewayText, coerceThinkingText, normalizePersonalityValue } from '@/lib/chat-runtime' import { gatewayEventRequiresSessionId } from '@/lib/gateway-events' +import { + dedupeGeneratedImageEchoesInParts, + generatedImageEchoSources, + stripGeneratedImageEchoes +} from '@/lib/generated-images' import { triggerHaptic } from '@/lib/haptics' import { isProviderSetupErrorMessage } from '@/lib/provider-setup-errors' import { parseTodos } from '@/lib/todos' @@ -343,7 +348,7 @@ export function useMessageStream({ if (queued.assistant) { mutateStream( id, - parts => appendAssistantTextPart(parts, queued.assistant), + parts => dedupeGeneratedImageEchoesInParts(appendAssistantTextPart(parts, queued.assistant)), () => [assistantTextPart(queued.assistant)] ) } @@ -507,7 +512,7 @@ export function useMessageStream({ mutateStream( sessionId, - parts => upsertToolPart(parts, payload, phase), + parts => dedupeGeneratedImageEchoesInParts(upsertToolPart(parts, payload, phase)), () => upsertToolPart([], payload, phase), { pending: m => phase !== 'complete' || (m.pending ?? false) } ) @@ -540,9 +545,11 @@ export function useMessageStream({ const finalText = renderMediaTags(text).trim() const completionError = completionErrorText(finalText) const normalize = (value: string) => value.replace(/\s+/g, ' ').trim() - const dedupeReference = normalize(finalText) const replaceTextPart = (parts: ChatMessagePart[]) => { + const visibleFinalText = stripGeneratedImageEchoes(finalText, generatedImageEchoSources(parts)).trim() + const dedupeReference = normalize(visibleFinalText) + const kept = parts.filter(part => { if (part.type === 'text') { return false @@ -557,7 +564,7 @@ export function useMessageStream({ return !(r && (dedupeReference.startsWith(r) || r.startsWith(dedupeReference))) }) - return finalText ? [...kept, assistantTextPart(finalText)] : kept + return visibleFinalText ? [...kept, assistantTextPart(visibleFinalText)] : kept } const completeMessage = (message: ChatMessage): ChatMessage => diff --git a/apps/desktop/src/components/assistant-ui/streaming.test.tsx b/apps/desktop/src/components/assistant-ui/streaming.test.tsx index 423a6f862e0..a5090416df8 100644 --- a/apps/desktop/src/components/assistant-ui/streaming.test.tsx +++ b/apps/desktop/src/components/assistant-ui/streaming.test.tsx @@ -216,6 +216,32 @@ function assistantTodoMessage( } as ThreadMessage } +function assistantImageMessage(running = false): ThreadMessage { + return { + id: `assistant-image-${running ? 'running' : 'done'}`, + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'image-1', + toolName: 'image_generate', + args: { prompt: 'draw a cat' }, + argsText: JSON.stringify({ prompt: 'draw a cat' }), + ...(running ? {} : { result: { image: 'https://cdn.example/cat.png', success: true } }) + } + ], + status: running ? { type: 'running' } : { type: 'complete', reason: 'stop' }, + createdAt, + metadata: { + unstable_state: null, + unstable_annotations: [], + unstable_data: [], + steps: [], + custom: {} + } + } as ThreadMessage +} + function StreamingHarness() { const [messages, setMessages] = useState([userMessage()]) const [isRunning, setIsRunning] = useState(true) @@ -640,14 +666,19 @@ describe('assistant-ui streaming renderer', () => { it('renders an incomplete streaming reasoning fenced code block as a code card', async () => { const { container } = render() const ui = within(container) + const thinkingToggle = ui.getByRole('button', { name: /thinking/i }) - fireEvent.click(ui.getByRole('button', { name: /thinking/i })) + if (thinkingToggle.getAttribute('aria-expanded') !== 'true') { + fireEvent.click(thinkingToggle) + } await waitFor(() => { expect(container.querySelector('[data-slot="code-card"]')).toBeTruthy() }) - expect(container.querySelector('[data-slot="aui_reasoning-text"]')?.textContent).toContain('const answer = 42') + await waitFor(() => { + expect(container.querySelector('[data-slot="aui_reasoning-text"]')?.textContent).toContain('const answer = 42') + }) expect(container.textContent).not.toContain('```ts') }) @@ -700,4 +731,16 @@ describe('assistant-ui streaming renderer', () => { expect(container.querySelector('[data-slot="aui_todo-hoisted"]')).toBeNull() }) + + it('renders completed image generation results in the tool slot', async () => { + const { container } = render() + + await waitFor(() => { + expect(screen.getByRole('img', { name: 'Generated image' }).getAttribute('src')).toBe( + 'https://cdn.example/cat.png' + ) + }) + expect(container.querySelector('[data-slot="aui_generated-image"]')).toBeTruthy() + expect(screen.queryByRole('status', { name: /rendering image/i })).toBeNull() + }) }) diff --git a/apps/desktop/src/components/assistant-ui/thread.tsx b/apps/desktop/src/components/assistant-ui/thread.tsx index f2a574d475b..697d5fd1e50 100644 --- a/apps/desktop/src/components/assistant-ui/thread.tsx +++ b/apps/desktop/src/components/assistant-ui/thread.tsx @@ -70,8 +70,7 @@ import { UserMessageText } from '@/components/assistant-ui/user-message-text' import { useElapsedSeconds } from '@/components/chat/activity-timer' import { ActivityTimerText } from '@/components/chat/activity-timer-text' import { DisclosureRow } from '@/components/chat/disclosure-row' -import { GeneratedImageProvider, useGeneratedImageContext } from '@/components/chat/generated-image-context' -import { ImageGenerationPlaceholder } from '@/components/chat/image-generation-placeholder' +import { GeneratedImage } from '@/components/chat/generated-image-result' import { Intro, type IntroProps } from '@/components/chat/intro' import { PreviewAttachment } from '@/components/chat/preview-attachment' import { Codicon } from '@/components/ui/codicon' @@ -200,18 +199,16 @@ export const Thread: FC<{ ) : undefined return ( - -
- : null} - sessionKey={sessionKey} - /> - {loading === 'session' && } -
-
+
+ : null} + sessionKey={sessionKey} + /> + {loading === 'session' && } +
) } @@ -404,21 +401,12 @@ const StreamStallIndicator: FC = () => { ) } -const ImageGenerateTool: FC = ({ result }) => { - const generatedImage = useGeneratedImageContext() - const running = result === undefined - - useEffect(() => { - generatedImage?.setPending(running) - }, [generatedImage, running]) - - if (!running) { - return null - } +const ImageGenerateTool: FC = ({ args, result }) => { + const aspectRatio = typeof args?.aspect_ratio === 'string' ? args.aspect_ratio : undefined return (
- +
) } diff --git a/apps/desktop/src/components/chat/generated-image-context.tsx b/apps/desktop/src/components/chat/generated-image-context.tsx deleted file mode 100644 index 8b020bb7db6..00000000000 --- a/apps/desktop/src/components/chat/generated-image-context.tsx +++ /dev/null @@ -1,19 +0,0 @@ -'use client' - -import { createContext, type ReactNode, useContext, useMemo, useState } from 'react' - -type Value = { - isPending: boolean - setPending: (pending: boolean) => void -} - -const Ctx = createContext(null) - -export function GeneratedImageProvider({ children }: { children: ReactNode }) { - const [isPending, setPending] = useState(false) - const value = useMemo(() => ({ isPending, setPending }), [isPending]) - - return {children} -} - -export const useGeneratedImageContext = () => useContext(Ctx) diff --git a/apps/desktop/src/components/chat/generated-image-result.tsx b/apps/desktop/src/components/chat/generated-image-result.tsx new file mode 100644 index 00000000000..e4313d20c51 --- /dev/null +++ b/apps/desktop/src/components/chat/generated-image-result.tsx @@ -0,0 +1,174 @@ +'use client' + +import { type FC, useEffect, useState } from 'react' + +import { DiffusionCanvas } from '@/components/chat/image-generation-placeholder' +import { ImageActionButton, ImageLightbox } from '@/components/chat/zoomable-image' +import { useImageDownload } from '@/hooks/use-image-download' +import { useI18n } from '@/i18n' +import { generatedImageFromResult } from '@/lib/generated-images' +import { filePathFromMediaPath, gatewayMediaDataUrl, isRemoteGateway, mediaExternalUrl, mediaName } from '@/lib/media' +import { cn } from '@/lib/utils' + +// Aspect hint from the tool args sizes the frame *before* the image loads, so +// the placeholder and the resolved image occupy the same box — no layout shift. +const ASPECT_HINTS: Record = { + landscape: 16 / 9, + square: 1, + portrait: 9 / 16 +} + +function hintedRatio(aspectRatio?: string): number { + return ASPECT_HINTS[String(aspectRatio ?? '').toLowerCase().trim()] ?? ASPECT_HINTS.landscape +} + +function isInlineSrc(path: string): boolean { + return /^(?:https?|data):/i.test(path) +} + +async function resolveImageSrc(path: string): Promise { + if (isInlineSrc(path)) { + return path + } + + if (window.hermesDesktop && isRemoteGateway()) { + return gatewayMediaDataUrl(path) + } + + if (!window.hermesDesktop?.readFileDataUrl) { + return mediaExternalUrl(path) + } + + return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path)) +} + +export const GeneratedImage: FC<{ aspectRatio?: string; result?: unknown }> = ({ aspectRatio, result }) => { + const { t } = useI18n() + const copy = t.desktop + const image = result === undefined ? null : generatedImageFromResult(result) + const pending = result === undefined + + const [ratio, setRatio] = useState(() => hintedRatio(aspectRatio)) + const [src, setSrc] = useState(() => (image && isInlineSrc(image) ? image : '')) + const [loaded, setLoaded] = useState(false) + const [canvasGone, setCanvasGone] = useState(false) + const [failed, setFailed] = useState(false) + const [lightboxOpen, setLightboxOpen] = useState(false) + const { download, saving } = useImageDownload(src) + + useEffect(() => setRatio(hintedRatio(aspectRatio)), [aspectRatio]) + + // Resolve the deliverable path (local read / gateway proxy / remote URL). The + // stays mounted under the placeholder and only fades in once it decodes, + // so the frame keeps its hinted size and never jumps. + useEffect(() => { + let cancelled = false + setFailed(false) + setLoaded(false) + setCanvasGone(false) + setSrc(image && isInlineSrc(image) ? image : '') + + if (!image || isInlineSrc(image)) { + return + } + + void resolveImageSrc(image) + .then(resolved => !cancelled && setSrc(resolved)) + .catch(() => !cancelled && setFailed(true)) + + return () => { + cancelled = true + } + }, [image]) + + // Completed but no usable image (generation failed): the agent's prose carries + // the explanation, so render nothing here. + if (!pending && !image) { + return null + } + + if (failed && image) { + return ( + { + event.preventDefault() + void window.hermesDesktop?.openExternal(mediaExternalUrl(image)) + }} + > + {copy.openImage}: {mediaName(image)} + + ) + } + + return ( + <> + + {!canvasGone && ( +
loaded && setCanvasGone(true)} + > + +
+ )} + {src && ( + + )} + {loaded && src && ( + + )} +
+ {src && ( + + )} + + ) +} diff --git a/apps/desktop/src/components/chat/image-generation-placeholder.tsx b/apps/desktop/src/components/chat/image-generation-placeholder.tsx index 202efcc131b..d69a48bb5de 100644 --- a/apps/desktop/src/components/chat/image-generation-placeholder.tsx +++ b/apps/desktop/src/components/chat/image-generation-placeholder.tsx @@ -1,7 +1,6 @@ import { type FC, useCallback, useEffect, useRef } from 'react' import { useResizeObserver } from '@/hooks/use-resize-observer' -import { useI18n } from '@/i18n' type Rgb = { r: number; g: number; b: number } @@ -241,7 +240,7 @@ const drawAsciiDiffusion = ( ctx.fillRect(0, 0, width, height) } -const DiffusionCanvas: FC = () => { +export const DiffusionCanvas: FC = () => { const canvasRef = useRef(null) const sizeRef = useRef({ width: 0, height: 0 }) const themeRef = useRef(FALLBACKS) @@ -305,15 +304,3 @@ const DiffusionCanvas: FC = () => { return } - -export const ImageGenerationPlaceholder: FC = () => { - const { t } = useI18n() - - return ( -
-
- -
-
- ) -} diff --git a/apps/desktop/src/components/chat/zoomable-image.tsx b/apps/desktop/src/components/chat/zoomable-image.tsx index c73068050ea..243f9f3415a 100644 --- a/apps/desktop/src/components/chat/zoomable-image.tsx +++ b/apps/desktop/src/components/chat/zoomable-image.tsx @@ -3,55 +3,17 @@ import { type ComponentProps, useState } from 'react' import { Dialog, DialogContent } from '@/components/ui/dialog' +import { useImageDownload } from '@/hooks/use-image-download' import { useI18n } from '@/i18n' import { Download } from '@/lib/icons' import { cn } from '@/lib/utils' -import { notify, notifyError } from '@/store/notifications' - -function imageFilename(src?: string): string { - if (!src) { - return 'image' - } - - try { - const { pathname } = new URL(src, window.location.href) - - return pathname.split('/').filter(Boolean).pop() || 'image' - } catch { - return src.split(/[\\/]/).filter(Boolean).pop() || 'image' - } -} - -function isMissingIpcHandler(error: unknown): boolean { - const message = error instanceof Error ? error.message : typeof error === 'string' ? error : '' - - return message.includes("No handler registered for 'hermes:saveImageFromUrl'") -} - -async function startBrowserDownload(src: string) { - const response = await fetch(src) - - if (!response.ok) { - throw new Error(`Could not fetch image: ${response.status}`) - } - - const blobUrl = URL.createObjectURL(await response.blob()) - const link = document.createElement('a') - link.href = blobUrl - link.download = imageFilename(src) - link.rel = 'noopener noreferrer' - document.body.appendChild(link) - link.click() - link.remove() - window.setTimeout(() => URL.revokeObjectURL(blobUrl), 30_000) -} export interface ZoomableImageProps extends ComponentProps<'img'> { containerClassName?: string slot?: string } -interface ImageActionCopy { +export interface ImageActionCopy { downloadImage: string savingImage: string } @@ -59,70 +21,10 @@ interface ImageActionCopy { export function ZoomableImage({ className, containerClassName, src, alt, slot, ...props }: ZoomableImageProps) { const { t } = useI18n() const copy = t.desktop - const [saving, setSaving] = useState(false) + const { download, saving } = useImageDownload(src) const [lightboxOpen, setLightboxOpen] = useState(false) const canOpen = Boolean(src) - async function handleDownload() { - if (!src || saving) { - return - } - - setSaving(true) - - try { - if (window.hermesDesktop?.saveImageFromUrl) { - const saved = await window.hermesDesktop.saveImageFromUrl(src) - - if (saved) { - notify({ kind: 'success', title: copy.imageSaved, message: imageFilename(src) }) - } - - return - } - - await startBrowserDownload(src) - } catch (error) { - if (isMissingIpcHandler(error)) { - try { - await startBrowserDownload(src) - notify({ - kind: 'info', - title: copy.downloadStarted, - message: copy.restartToUseSaveImage - }) - } catch (fallbackError) { - notifyError(fallbackError, copy.restartToSaveImages) - } - - return - } - - notifyError(error, copy.imageDownloadFailed) - } finally { - setSaving(false) - } - } - - const lightbox = src ? ( - - -
- {alt setLightboxOpen(false)} - src={src} - /> - -
-
-
- ) : null - return ( <> {alt - {src && } + {src && ( + + )} - {lightbox} + {src && ( + + )} ) } -function ImageActionButton({ +export function ImageLightbox({ + alt, copy, onClick, + onOpenChange, + open, saving, - variant + src }: { + alt?: string + copy: ImageActionCopy + onClick: () => void + onOpenChange: (open: boolean) => void + open: boolean + saving: boolean + src: string +}) { + return ( + + +
+ {alt onOpenChange(false)} + src={src} + /> + +
+
+
+ ) +} + +export function ImageActionButton({ + className, + copy, + onClick, + saving +}: { + className?: string copy: ImageActionCopy onClick: () => void saving: boolean - variant: 'inline' | 'lightbox' }) { return (