mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-18 14:52:04 +00:00
fix(desktop): keep generated images in the tool slot, not inline
The image-generate tool showed a placeholder, then the model echoed a (often different) image inline in its prose — a second, jarring copy in the wrong place, dimmed as tool scaffolding, with a misplaced download button. Now the generated image lives only in the tool slot: - Strip every embedded image/media link from the assistant prose of a message that produced an image (the model frequently restates the remote URL while the result holds the local path), preserving the agent's words. Applied on hydration, live deltas, and completion. - One stable frame sized from the aspect_ratio arg up front, so the diffusion placeholder and the decoded image share the same box and crossfade with no layout shift; the box derives its height from the true ratio on load (no letterboxing). - Exempt generated images from the tool-block dim-until-hover rule. - Extract a shared useImageDownload hook + ImageLightbox so the tool image and markdown images share one implementation.
This commit is contained in:
Brooklyn Nicholson
parent
0a7a81835b
commit
b15dc58064
13 changed files with 645 additions and 173 deletions
|
|
@ -1,19 +0,0 @@
|
|||
'use client'
|
||||
|
||||
import { createContext, type ReactNode, useContext, useMemo, useState } from 'react'
|
||||
|
||||
type Value = {
|
||||
isPending: boolean
|
||||
setPending: (pending: boolean) => void
|
||||
}
|
||||
|
||||
const Ctx = createContext<Value | null>(null)
|
||||
|
||||
export function GeneratedImageProvider({ children }: { children: ReactNode }) {
|
||||
const [isPending, setPending] = useState(false)
|
||||
const value = useMemo(() => ({ isPending, setPending }), [isPending])
|
||||
|
||||
return <Ctx.Provider value={value}>{children}</Ctx.Provider>
|
||||
}
|
||||
|
||||
export const useGeneratedImageContext = () => useContext(Ctx)
|
||||
174
apps/desktop/src/components/chat/generated-image-result.tsx
Normal file
174
apps/desktop/src/components/chat/generated-image-result.tsx
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
'use client'
|
||||
|
||||
import { type FC, useEffect, useState } from 'react'
|
||||
|
||||
import { DiffusionCanvas } from '@/components/chat/image-generation-placeholder'
|
||||
import { ImageActionButton, ImageLightbox } from '@/components/chat/zoomable-image'
|
||||
import { useImageDownload } from '@/hooks/use-image-download'
|
||||
import { useI18n } from '@/i18n'
|
||||
import { generatedImageFromResult } from '@/lib/generated-images'
|
||||
import { filePathFromMediaPath, gatewayMediaDataUrl, isRemoteGateway, mediaExternalUrl, mediaName } from '@/lib/media'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
// Aspect hint from the tool args sizes the frame *before* the image loads, so
|
||||
// the placeholder and the resolved image occupy the same box — no layout shift.
|
||||
const ASPECT_HINTS: Record<string, number> = {
|
||||
landscape: 16 / 9,
|
||||
square: 1,
|
||||
portrait: 9 / 16
|
||||
}
|
||||
|
||||
function hintedRatio(aspectRatio?: string): number {
|
||||
return ASPECT_HINTS[String(aspectRatio ?? '').toLowerCase().trim()] ?? ASPECT_HINTS.landscape
|
||||
}
|
||||
|
||||
function isInlineSrc(path: string): boolean {
|
||||
return /^(?:https?|data):/i.test(path)
|
||||
}
|
||||
|
||||
async function resolveImageSrc(path: string): Promise<string> {
|
||||
if (isInlineSrc(path)) {
|
||||
return path
|
||||
}
|
||||
|
||||
if (window.hermesDesktop && isRemoteGateway()) {
|
||||
return gatewayMediaDataUrl(path)
|
||||
}
|
||||
|
||||
if (!window.hermesDesktop?.readFileDataUrl) {
|
||||
return mediaExternalUrl(path)
|
||||
}
|
||||
|
||||
return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path))
|
||||
}
|
||||
|
||||
export const GeneratedImage: FC<{ aspectRatio?: string; result?: unknown }> = ({ aspectRatio, result }) => {
|
||||
const { t } = useI18n()
|
||||
const copy = t.desktop
|
||||
const image = result === undefined ? null : generatedImageFromResult(result)
|
||||
const pending = result === undefined
|
||||
|
||||
const [ratio, setRatio] = useState(() => hintedRatio(aspectRatio))
|
||||
const [src, setSrc] = useState(() => (image && isInlineSrc(image) ? image : ''))
|
||||
const [loaded, setLoaded] = useState(false)
|
||||
const [canvasGone, setCanvasGone] = useState(false)
|
||||
const [failed, setFailed] = useState(false)
|
||||
const [lightboxOpen, setLightboxOpen] = useState(false)
|
||||
const { download, saving } = useImageDownload(src)
|
||||
|
||||
useEffect(() => setRatio(hintedRatio(aspectRatio)), [aspectRatio])
|
||||
|
||||
// Resolve the deliverable path (local read / gateway proxy / remote URL). The
|
||||
// <img> stays mounted under the placeholder and only fades in once it decodes,
|
||||
// so the frame keeps its hinted size and never jumps.
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
setFailed(false)
|
||||
setLoaded(false)
|
||||
setCanvasGone(false)
|
||||
setSrc(image && isInlineSrc(image) ? image : '')
|
||||
|
||||
if (!image || isInlineSrc(image)) {
|
||||
return
|
||||
}
|
||||
|
||||
void resolveImageSrc(image)
|
||||
.then(resolved => !cancelled && setSrc(resolved))
|
||||
.catch(() => !cancelled && setFailed(true))
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [image])
|
||||
|
||||
// Completed but no usable image (generation failed): the agent's prose carries
|
||||
// the explanation, so render nothing here.
|
||||
if (!pending && !image) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (failed && image) {
|
||||
return (
|
||||
<a
|
||||
className="mt-2 inline-block font-semibold text-foreground underline underline-offset-4 decoration-current/20 wrap-anywhere"
|
||||
href="#"
|
||||
onClick={event => {
|
||||
event.preventDefault()
|
||||
void window.hermesDesktop?.openExternal(mediaExternalUrl(image))
|
||||
}}
|
||||
>
|
||||
{copy.openImage}: {mediaName(image)}
|
||||
</a>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<span
|
||||
aria-label={pending ? t.assistant.tool.renderingImage : undefined}
|
||||
aria-live={pending ? 'polite' : undefined}
|
||||
className="group/image relative block max-w-full overflow-hidden rounded-2xl transition-[width,height] duration-300 ease-out"
|
||||
data-slot="aui_generated-image"
|
||||
role={pending ? 'status' : undefined}
|
||||
style={{
|
||||
aspectRatio: ratio,
|
||||
// Width is capped so the derived height (width / ratio) never exceeds
|
||||
// --image-preview-height; the box then matches the image exactly with
|
||||
// no letterboxing.
|
||||
width: `min(calc(var(--image-preview-height) * ${ratio}), var(--image-preview-max-width), 100%)`
|
||||
}}
|
||||
>
|
||||
{!canvasGone && (
|
||||
<div
|
||||
className={cn('absolute inset-0 transition-opacity duration-500 ease-out', loaded && 'opacity-0')}
|
||||
onTransitionEnd={() => loaded && setCanvasGone(true)}
|
||||
>
|
||||
<DiffusionCanvas />
|
||||
</div>
|
||||
)}
|
||||
{src && (
|
||||
<button
|
||||
className="absolute inset-0 block size-full cursor-zoom-in"
|
||||
onClick={() => setLightboxOpen(true)}
|
||||
title={copy.openImage}
|
||||
type="button"
|
||||
>
|
||||
<img
|
||||
alt="Generated image"
|
||||
className={cn(
|
||||
'absolute inset-0 size-full object-contain opacity-0 transition-opacity duration-500 ease-out',
|
||||
loaded && 'opacity-100'
|
||||
)}
|
||||
draggable={false}
|
||||
onError={() => setFailed(true)}
|
||||
onLoad={event => {
|
||||
const { naturalHeight, naturalWidth } = event.currentTarget
|
||||
|
||||
if (naturalWidth && naturalHeight) {
|
||||
setRatio(naturalWidth / naturalHeight)
|
||||
}
|
||||
|
||||
setLoaded(true)
|
||||
}}
|
||||
src={src}
|
||||
/>
|
||||
</button>
|
||||
)}
|
||||
{loaded && src && (
|
||||
<ImageActionButton className="group-hover/image:opacity-100" copy={copy} onClick={download} saving={saving} />
|
||||
)}
|
||||
</span>
|
||||
{src && (
|
||||
<ImageLightbox
|
||||
alt="Generated image"
|
||||
copy={copy}
|
||||
onClick={download}
|
||||
onOpenChange={setLightboxOpen}
|
||||
open={lightboxOpen}
|
||||
saving={saving}
|
||||
src={src}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
import { type FC, useCallback, useEffect, useRef } from 'react'
|
||||
|
||||
import { useResizeObserver } from '@/hooks/use-resize-observer'
|
||||
import { useI18n } from '@/i18n'
|
||||
|
||||
type Rgb = { r: number; g: number; b: number }
|
||||
|
||||
|
|
@ -241,7 +240,7 @@ const drawAsciiDiffusion = (
|
|||
ctx.fillRect(0, 0, width, height)
|
||||
}
|
||||
|
||||
const DiffusionCanvas: FC = () => {
|
||||
export const DiffusionCanvas: FC = () => {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
||||
const sizeRef = useRef({ width: 0, height: 0 })
|
||||
const themeRef = useRef<Theme>(FALLBACKS)
|
||||
|
|
@ -305,15 +304,3 @@ const DiffusionCanvas: FC = () => {
|
|||
|
||||
return <canvas className="absolute inset-0 h-full w-full" ref={canvasRef} />
|
||||
}
|
||||
|
||||
export const ImageGenerationPlaceholder: FC = () => {
|
||||
const { t } = useI18n()
|
||||
|
||||
return (
|
||||
<div aria-label={t.assistant.tool.renderingImage} aria-live="polite" className="w-full max-w-136 self-start" role="status">
|
||||
<div className="relative h-(--image-preview-height) overflow-hidden rounded-4xl border border-border/55 shadow-[inset_0_0.0625rem_0_color-mix(in_srgb,white_45%,transparent),inset_0_0_0_0.0625rem_color-mix(in_srgb,var(--dt-border)_34%,transparent),inset_0_-0.75rem_1.75rem_color-mix(in_srgb,var(--dt-primary)_5%,transparent)]">
|
||||
<DiffusionCanvas />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,55 +3,17 @@
|
|||
import { type ComponentProps, useState } from 'react'
|
||||
|
||||
import { Dialog, DialogContent } from '@/components/ui/dialog'
|
||||
import { useImageDownload } from '@/hooks/use-image-download'
|
||||
import { useI18n } from '@/i18n'
|
||||
import { Download } from '@/lib/icons'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { notify, notifyError } from '@/store/notifications'
|
||||
|
||||
function imageFilename(src?: string): string {
|
||||
if (!src) {
|
||||
return 'image'
|
||||
}
|
||||
|
||||
try {
|
||||
const { pathname } = new URL(src, window.location.href)
|
||||
|
||||
return pathname.split('/').filter(Boolean).pop() || 'image'
|
||||
} catch {
|
||||
return src.split(/[\\/]/).filter(Boolean).pop() || 'image'
|
||||
}
|
||||
}
|
||||
|
||||
function isMissingIpcHandler(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : typeof error === 'string' ? error : ''
|
||||
|
||||
return message.includes("No handler registered for 'hermes:saveImageFromUrl'")
|
||||
}
|
||||
|
||||
async function startBrowserDownload(src: string) {
|
||||
const response = await fetch(src)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Could not fetch image: ${response.status}`)
|
||||
}
|
||||
|
||||
const blobUrl = URL.createObjectURL(await response.blob())
|
||||
const link = document.createElement('a')
|
||||
link.href = blobUrl
|
||||
link.download = imageFilename(src)
|
||||
link.rel = 'noopener noreferrer'
|
||||
document.body.appendChild(link)
|
||||
link.click()
|
||||
link.remove()
|
||||
window.setTimeout(() => URL.revokeObjectURL(blobUrl), 30_000)
|
||||
}
|
||||
|
||||
export interface ZoomableImageProps extends ComponentProps<'img'> {
|
||||
containerClassName?: string
|
||||
slot?: string
|
||||
}
|
||||
|
||||
interface ImageActionCopy {
|
||||
export interface ImageActionCopy {
|
||||
downloadImage: string
|
||||
savingImage: string
|
||||
}
|
||||
|
|
@ -59,70 +21,10 @@ interface ImageActionCopy {
|
|||
export function ZoomableImage({ className, containerClassName, src, alt, slot, ...props }: ZoomableImageProps) {
|
||||
const { t } = useI18n()
|
||||
const copy = t.desktop
|
||||
const [saving, setSaving] = useState(false)
|
||||
const { download, saving } = useImageDownload(src)
|
||||
const [lightboxOpen, setLightboxOpen] = useState(false)
|
||||
const canOpen = Boolean(src)
|
||||
|
||||
async function handleDownload() {
|
||||
if (!src || saving) {
|
||||
return
|
||||
}
|
||||
|
||||
setSaving(true)
|
||||
|
||||
try {
|
||||
if (window.hermesDesktop?.saveImageFromUrl) {
|
||||
const saved = await window.hermesDesktop.saveImageFromUrl(src)
|
||||
|
||||
if (saved) {
|
||||
notify({ kind: 'success', title: copy.imageSaved, message: imageFilename(src) })
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
await startBrowserDownload(src)
|
||||
} catch (error) {
|
||||
if (isMissingIpcHandler(error)) {
|
||||
try {
|
||||
await startBrowserDownload(src)
|
||||
notify({
|
||||
kind: 'info',
|
||||
title: copy.downloadStarted,
|
||||
message: copy.restartToUseSaveImage
|
||||
})
|
||||
} catch (fallbackError) {
|
||||
notifyError(fallbackError, copy.restartToSaveImages)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
notifyError(error, copy.imageDownloadFailed)
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}
|
||||
|
||||
const lightbox = src ? (
|
||||
<Dialog onOpenChange={setLightboxOpen} open={lightboxOpen}>
|
||||
<DialogContent
|
||||
className="block w-auto max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] overflow-visible border-0 bg-transparent p-0 shadow-none"
|
||||
showCloseButton={false}
|
||||
>
|
||||
<div className="group/lightbox relative inline-block">
|
||||
<img
|
||||
alt={alt ?? ''}
|
||||
className="block max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] cursor-zoom-out select-auto rounded-lg object-contain shadow-2xl"
|
||||
onClick={() => setLightboxOpen(false)}
|
||||
src={src}
|
||||
/>
|
||||
<ImageActionButton copy={copy} onClick={handleDownload} saving={saving} variant="lightbox" />
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
) : null
|
||||
|
||||
return (
|
||||
<>
|
||||
<span
|
||||
|
|
@ -138,30 +40,79 @@ export function ZoomableImage({ className, containerClassName, src, alt, slot, .
|
|||
>
|
||||
<img alt={alt ?? ''} className={className} src={src} {...props} />
|
||||
</button>
|
||||
{src && <ImageActionButton copy={copy} onClick={handleDownload} saving={saving} variant="inline" />}
|
||||
{src && (
|
||||
<ImageActionButton className="group-hover/image:opacity-100" copy={copy} onClick={download} saving={saving} />
|
||||
)}
|
||||
</span>
|
||||
{lightbox}
|
||||
{src && (
|
||||
<ImageLightbox
|
||||
alt={alt}
|
||||
copy={copy}
|
||||
onClick={download}
|
||||
onOpenChange={setLightboxOpen}
|
||||
open={lightboxOpen}
|
||||
saving={saving}
|
||||
src={src}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
function ImageActionButton({
|
||||
export function ImageLightbox({
|
||||
alt,
|
||||
copy,
|
||||
onClick,
|
||||
onOpenChange,
|
||||
open,
|
||||
saving,
|
||||
variant
|
||||
src
|
||||
}: {
|
||||
alt?: string
|
||||
copy: ImageActionCopy
|
||||
onClick: () => void
|
||||
onOpenChange: (open: boolean) => void
|
||||
open: boolean
|
||||
saving: boolean
|
||||
src: string
|
||||
}) {
|
||||
return (
|
||||
<Dialog onOpenChange={onOpenChange} open={open}>
|
||||
<DialogContent
|
||||
className="block w-auto max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] overflow-visible border-0 bg-transparent p-0 shadow-none"
|
||||
showCloseButton={false}
|
||||
>
|
||||
<div className="group/lightbox relative inline-block">
|
||||
<img
|
||||
alt={alt ?? ''}
|
||||
className="block max-h-[calc(100vh-12rem)] max-w-[calc(100vw-12rem)] cursor-zoom-out select-auto rounded-lg object-contain shadow-2xl"
|
||||
onClick={() => onOpenChange(false)}
|
||||
src={src}
|
||||
/>
|
||||
<ImageActionButton className="group-hover/lightbox:opacity-100" copy={copy} onClick={onClick} saving={saving} />
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
|
||||
export function ImageActionButton({
|
||||
className,
|
||||
copy,
|
||||
onClick,
|
||||
saving
|
||||
}: {
|
||||
className?: string
|
||||
copy: ImageActionCopy
|
||||
onClick: () => void
|
||||
saving: boolean
|
||||
variant: 'inline' | 'lightbox'
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
aria-label={saving ? copy.savingImage : copy.downloadImage}
|
||||
className={cn(
|
||||
'absolute right-2 top-2 grid size-8 place-items-center rounded-full border border-border/70 bg-background/80 text-muted-foreground opacity-0 shadow-sm backdrop-blur transition-opacity hover:bg-accent hover:text-foreground focus-visible:opacity-100 disabled:opacity-50',
|
||||
variant === 'inline' ? 'group-hover/image:opacity-100' : 'group-hover/lightbox:opacity-100'
|
||||
className
|
||||
)}
|
||||
disabled={saving}
|
||||
onClick={event => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue