From 7cbdcf1ba6f79b2e509963e142b4a77915eea523 Mon Sep 17 00:00:00 2001 From: Gille <4317663+helix4u@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:16:20 -0600 Subject: [PATCH] fix(desktop): render remote markdown images in chat (#57944) * fix(desktop): render remote markdown images in chat * test(desktop): cover remote markdown image resolution --- .../assistant-ui/markdown-text.media.test.tsx | 51 ++++++++++++ .../components/assistant-ui/markdown-text.tsx | 83 ++++++++++++++----- apps/desktop/src/lib/media.remote.test.ts | 66 ++++++++++++++- apps/desktop/src/lib/media.ts | 24 ++++++ 4 files changed, 204 insertions(+), 20 deletions(-) create mode 100644 apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx new file mode 100644 index 000000000000..ca64acd4c746 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx @@ -0,0 +1,51 @@ +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { $connection } from '@/store/session' + +import { MarkdownTextContent } from './markdown-text' + +const REMOTE_IMAGE_PATH = '/home/user/project/images/remote-preview.png' +const REMOTE_IMAGE_DATA_URL = 'data:image/png;base64,cmVtb3RlLWltYWdl' + +describe('MarkdownTextContent remote images', () => { + const api = vi.fn(async ({ path }: { path: string }) => { + if (path.startsWith('/api/fs/read-data-url?')) { + return { dataUrl: REMOTE_IMAGE_DATA_URL } + } + + throw new Error(`unexpected path ${path}`) + }) + let originalDesktop: typeof window.hermesDesktop + + beforeEach(() => { + api.mockClear() + originalDesktop = window.hermesDesktop + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: { api } + }) + $connection.set({ mode: 'remote', profile: 'remote-work' } as never) + }) + + afterEach(() => { + cleanup() + $connection.set(null) + Object.defineProperty(window, 'hermesDesktop', { + configurable: true, + value: originalDesktop + }) + }) + + it('passes the gateway bridge data URL through Streamdown to the zoomable image', async () => { + render() + + const image = await screen.findByRole('img', { name: 'Remote preview' }) + + expect(image.getAttribute('src')).toBe(REMOTE_IMAGE_DATA_URL) + expect(api).toHaveBeenCalledWith({ + path: '/api/fs/read-data-url?path=%2Fhome%2Fuser%2Fproject%2Fimages%2Fremote-preview.png', + profile: 'remote-work' + }) + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.tsx index a199b5aa1e7d..f5d78f531334 100644 --- a/apps/desktop/src/components/assistant-ui/markdown-text.tsx +++ b/apps/desktop/src/components/assistant-ui/markdown-text.tsx @@ -20,14 +20,14 @@ import { parseMarkdownIntoBlocksCached } from '@/lib/markdown-blocks' import { preprocessMarkdown } from '@/lib/markdown-preprocess' import { downloadGatewayMediaFile, - filePathFromMediaPath, - gatewayMediaDataUrl, + isInlineMediaSrc, isRemoteGateway, mediaExternalUrl, mediaKind, mediaName, mediaPathFromMarkdownHref, - mediaStreamUrl + mediaStreamUrl, + resolveMediaDisplaySrc } from '@/lib/media' import { previewTargetFromMarkdownHref } from '@/lib/preview-targets' import { cn } from '@/lib/utils' @@ -60,27 +60,13 @@ function preprocessWithTailRepair(text: string): string { } async function mediaSrc(path: string): Promise { - if (/^(?:https?|data):/i.test(path)) { - return path - } - // Stream audio/video through the custom protocol: data URLs are capped and // load the whole file into memory, which broke playback for larger videos. if (window.hermesDesktop && ['audio', 'video'].includes(mediaKind(path))) { return mediaStreamUrl(path) } - // Remote gateway: the image lives on the gateway machine, so read it over the - // authenticated API rather than this machine's disk. - if (window.hermesDesktop && isRemoteGateway()) { - return gatewayMediaDataUrl(path) - } - - if (!window.hermesDesktop?.readFileDataUrl) { - return mediaExternalUrl(path) - } - - return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path)) + return resolveMediaDisplaySrc(path) } function useOpenMediaFile(path: string) { @@ -286,6 +272,65 @@ function MarkdownLink({ children, className, href, ...props }: ComponentProps<'a } function MarkdownImage({ className, src, alt, ...props }: ComponentProps<'img'>) { + const rawSrc = typeof src === 'string' ? src : '' + const [resolvedSrc, setResolvedSrc] = useState(() => (rawSrc && isInlineMediaSrc(rawSrc) ? rawSrc : '')) + const [failed, setFailed] = useState(false) + const { open, openFailed } = useOpenMediaFile(rawSrc) + const name = mediaName(rawSrc || String(alt || 'image')) + + useEffect(() => { + let cancelled = false + + setFailed(false) + setResolvedSrc(rawSrc && isInlineMediaSrc(rawSrc) ? rawSrc : '') + + if (!rawSrc || isInlineMediaSrc(rawSrc)) { + return () => { + cancelled = true + } + } + + void resolveMediaDisplaySrc(rawSrc) + .then(value => { + if (!cancelled) { + setResolvedSrc(value) + } + }) + .catch(() => { + if (!cancelled) { + setFailed(true) + } + }) + + return () => { + cancelled = true + } + }, [rawSrc]) + + if (!rawSrc) { + return null + } + + if (failed) { + return ( + + Couldn't load {name}.{' '} + + {openFailed && } + + ) + } + + if (!resolvedSrc) { + return Loading {name}... + } + return ( ) )} containerClassName="my-2 block w-fit max-w-full" slot="aui_markdown-image" - src={src} + src={resolvedSrc} {...props} /> ) diff --git a/apps/desktop/src/lib/media.remote.test.ts b/apps/desktop/src/lib/media.remote.test.ts index e10ca5f59ec1..79676c691431 100644 --- a/apps/desktop/src/lib/media.remote.test.ts +++ b/apps/desktop/src/lib/media.remote.test.ts @@ -8,8 +8,10 @@ import { downloadGatewayMediaFile, filePathFromMediaPath, gatewayMediaDataUrl, + isInlineMediaSrc, isRemoteGateway, - mediaExternalUrl + mediaExternalUrl, + resolveMediaDisplaySrc } from './media' describe('isRemoteGateway', () => { @@ -75,6 +77,68 @@ describe('mediaExternalUrl', () => { }) }) +describe('resolveMediaDisplaySrc', () => { + const api = vi.fn(async ({ path }: { path: string }) => { + if (path.startsWith('/api/fs/read-data-url?')) { + return { dataUrl: 'data:image/png;base64,ZHVtbXk=' } + } + + throw new Error(`unexpected path ${path}`) + }) + + beforeEach(() => { + api.mockClear() + }) + + afterEach(() => { + vi.unstubAllGlobals() + $connection.set(null) + }) + + it('recognizes inline image URLs', () => { + expect(isInlineMediaSrc('https://example.com/a.png')).toBe(true) + expect(isInlineMediaSrc('data:image/png;base64,ZHVtbXk=')).toBe(true) + expect(isInlineMediaSrc('/Users/me/a.png')).toBe(false) + }) + + it('leaves web, data, and relative markdown image sources unchanged', async () => { + vi.stubGlobal('window', { hermesDesktop: { api } }) + $connection.set({ mode: 'remote', profile: 'remote-work' } as never) + + await expect(resolveMediaDisplaySrc('https://example.com/a.png')).resolves.toBe('https://example.com/a.png') + await expect(resolveMediaDisplaySrc('data:image/png;base64,ZHVtbXk=')).resolves.toBe( + 'data:image/png;base64,ZHVtbXk=' + ) + await expect(resolveMediaDisplaySrc('images/a.png')).resolves.toBe('images/a.png') + await expect(resolveMediaDisplaySrc('./images/a.png')).resolves.toBe('./images/a.png') + await expect(resolveMediaDisplaySrc('../images/a.png')).resolves.toBe('../images/a.png') + expect(api).not.toHaveBeenCalled() + }) + + it('reads remote gateway-local file paths through the desktop fs bridge', async () => { + vi.stubGlobal('window', { hermesDesktop: { api } }) + $connection.set({ mode: 'remote', profile: 'remote-work' } as never) + + await expect(resolveMediaDisplaySrc('/Users/me/project/a b.png')).resolves.toBe('data:image/png;base64,ZHVtbXk=') + expect(api).toHaveBeenCalledWith({ + path: '/api/fs/read-data-url?path=%2FUsers%2Fme%2Fproject%2Fa%20b.png', + profile: 'remote-work' + }) + }) + + it('reads local desktop file paths from the local desktop shell', async () => { + const readFileDataUrl = vi.fn(async () => 'data:image/png;base64,bG9jYWw=') + + vi.stubGlobal('window', { hermesDesktop: { readFileDataUrl } }) + $connection.set({ mode: 'local' } as never) + + await expect(resolveMediaDisplaySrc('file:///Users/me/project/a%20b.png')).resolves.toBe( + 'data:image/png;base64,bG9jYWw=' + ) + expect(readFileDataUrl).toHaveBeenCalledWith('/Users/me/project/a b.png') + }) +}) + describe('gatewayMediaDataUrl', () => { const api = vi.fn(async ({ path }: { path: string }) => { if (path.startsWith('/api/fs/read-data-url?')) { diff --git a/apps/desktop/src/lib/media.ts b/apps/desktop/src/lib/media.ts index ffaea2c7c2bc..c392caa4d7bd 100644 --- a/apps/desktop/src/lib/media.ts +++ b/apps/desktop/src/lib/media.ts @@ -58,6 +58,30 @@ export function mediaMarkdownHref(path: string): string { return `#media:${encodeURIComponent(path)}` } +export function isInlineMediaSrc(path: string): boolean { + return /^(?:https?|data):/i.test(path) +} + +function isFileMediaPath(path: string): boolean { + return /^(?:file:|\/|~\/|[a-z]:[\\/]|\\\\)/i.test(path) +} + +export async function resolveMediaDisplaySrc(path: string): Promise { + if (isInlineMediaSrc(path) || !isFileMediaPath(path)) { + return path + } + + if (window.hermesDesktop && isRemoteGateway()) { + return gatewayMediaDataUrl(path) + } + + if (!window.hermesDesktop?.readFileDataUrl) { + return mediaExternalUrl(path) + } + + return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path)) +} + // Resolve a media path to a URL the shell can open. Remote mode rewrites // gateway-local paths to an authenticated /api/files/download URL (the file // lives on the gateway, not this disk); local mode keeps the file:// form.