fix(desktop): render remote markdown images in chat (#57944)

* fix(desktop): render remote markdown images in chat

* test(desktop): cover remote markdown image resolution

apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx

@@ -0,0 +1,51 @@
+import { cleanup, render, screen } from '@testing-library/react'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { $connection } from '@/store/session'
+
+import { MarkdownTextContent } from './markdown-text'
+
+const REMOTE_IMAGE_PATH = '/home/user/project/images/remote-preview.png'
+const REMOTE_IMAGE_DATA_URL = 'data:image/png;base64,cmVtb3RlLWltYWdl'
+
+describe('MarkdownTextContent remote images', () => {
+  const api = vi.fn(async ({ path }: { path: string }) => {
+    if (path.startsWith('/api/fs/read-data-url?')) {
+      return { dataUrl: REMOTE_IMAGE_DATA_URL }
+    }
+
+    throw new Error(`unexpected path ${path}`)
+  })
+  let originalDesktop: typeof window.hermesDesktop
+
+  beforeEach(() => {
+    api.mockClear()
+    originalDesktop = window.hermesDesktop
+    Object.defineProperty(window, 'hermesDesktop', {
+      configurable: true,
+      value: { api }
+    })
+    $connection.set({ mode: 'remote', profile: 'remote-work' } as never)
+  })
+
+  afterEach(() => {
+    cleanup()
+    $connection.set(null)
+    Object.defineProperty(window, 'hermesDesktop', {
+      configurable: true,
+      value: originalDesktop
+    })
+  })
+
+  it('passes the gateway bridge data URL through Streamdown to the zoomable image', async () => {
+    render(<MarkdownTextContent isRunning={false} text={`![Remote preview](${REMOTE_IMAGE_PATH})`} />)
+
+    const image = await screen.findByRole('img', { name: 'Remote preview' })
+
+    expect(image.getAttribute('src')).toBe(REMOTE_IMAGE_DATA_URL)
+    expect(api).toHaveBeenCalledWith({
+      path: '/api/fs/read-data-url?path=%2Fhome%2Fuser%2Fproject%2Fimages%2Fremote-preview.png',
+      profile: 'remote-work'
+    })
+  })
+})

apps/desktop/src/components/assistant-ui/markdown-text.tsx

@@ -20,14 +20,14 @@ import { parseMarkdownIntoBlocksCached } from '@/lib/markdown-blocks'
 import { preprocessMarkdown } from '@/lib/markdown-preprocess'
 import {
   downloadGatewayMediaFile,
-  filePathFromMediaPath,
-  gatewayMediaDataUrl,
+  isInlineMediaSrc,
   isRemoteGateway,
   mediaExternalUrl,
   mediaKind,
   mediaName,
   mediaPathFromMarkdownHref,
-  mediaStreamUrl
+  mediaStreamUrl,
+  resolveMediaDisplaySrc
 } from '@/lib/media'
 import { previewTargetFromMarkdownHref } from '@/lib/preview-targets'
 import { cn } from '@/lib/utils'
@@ -60,27 +60,13 @@ function preprocessWithTailRepair(text: string): string {
 }
 
 async function mediaSrc(path: string): Promise<string> {
-  if (/^(?:https?|data):/i.test(path)) {
-    return path
-  }
-
   // Stream audio/video through the custom protocol: data URLs are capped and
   // load the whole file into memory, which broke playback for larger videos.
   if (window.hermesDesktop && ['audio', 'video'].includes(mediaKind(path))) {
     return mediaStreamUrl(path)
   }
 
-  // Remote gateway: the image lives on the gateway machine, so read it over the
-  // authenticated API rather than this machine's disk.
-  if (window.hermesDesktop && isRemoteGateway()) {
-    return gatewayMediaDataUrl(path)
-  }
-
-  if (!window.hermesDesktop?.readFileDataUrl) {
-    return mediaExternalUrl(path)
-  }
-
-  return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path))
+  return resolveMediaDisplaySrc(path)
 }
 
 function useOpenMediaFile(path: string) {
@@ -286,6 +272,65 @@ function MarkdownLink({ children, className, href, ...props }: ComponentProps<'a
 }
 
 function MarkdownImage({ className, src, alt, ...props }: ComponentProps<'img'>) {
+  const rawSrc = typeof src === 'string' ? src : ''
+  const [resolvedSrc, setResolvedSrc] = useState(() => (rawSrc && isInlineMediaSrc(rawSrc) ? rawSrc : ''))
+  const [failed, setFailed] = useState(false)
+  const { open, openFailed } = useOpenMediaFile(rawSrc)
+  const name = mediaName(rawSrc || String(alt || 'image'))
+
+  useEffect(() => {
+    let cancelled = false
+
+    setFailed(false)
+    setResolvedSrc(rawSrc && isInlineMediaSrc(rawSrc) ? rawSrc : '')
+
+    if (!rawSrc || isInlineMediaSrc(rawSrc)) {
+      return () => {
+        cancelled = true
+      }
+    }
+
+    void resolveMediaDisplaySrc(rawSrc)
+      .then(value => {
+        if (!cancelled) {
+          setResolvedSrc(value)
+        }
+      })
+      .catch(() => {
+        if (!cancelled) {
+          setFailed(true)
+        }
+      })
+
+    return () => {
+      cancelled = true
+    }
+  }, [rawSrc])
+
+  if (!rawSrc) {
+    return null
+  }
+
+  if (failed) {
+    return (
+      <span className="my-2 block text-sm text-muted-foreground">
+        Couldn&apos;t load {name}.{' '}
+        <button
+          className="bg-transparent font-medium text-foreground underline underline-offset-4 decoration-current/20 hover:text-foreground"
+          onClick={open}
+          type="button"
+        >
+          Open image
+        </button>
+        {openFailed && <OpenMediaFailedNote name={name} />}
+      </span>
+    )
+  }
+
+  if (!resolvedSrc) {
+    return <span className="my-2 block text-sm text-muted-foreground">Loading {name}...</span>
+  }
+
   return (
     <ZoomableImage
       alt={alt}
@@ -295,7 +340,7 @@ function MarkdownImage({ className, src, alt, ...props }: ComponentProps<'img'>)
       )}
       containerClassName="my-2 block w-fit max-w-full"
       slot="aui_markdown-image"
-      src={src}
+      src={resolvedSrc}
       {...props}
     />
   )

apps/desktop/src/lib/media.remote.test.ts

@@ -8,8 +8,10 @@ import {
   downloadGatewayMediaFile,
   filePathFromMediaPath,
   gatewayMediaDataUrl,
+  isInlineMediaSrc,
   isRemoteGateway,
-  mediaExternalUrl
+  mediaExternalUrl,
+  resolveMediaDisplaySrc
 } from './media'
 
 describe('isRemoteGateway', () => {
@@ -75,6 +77,68 @@ describe('mediaExternalUrl', () => {
   })
 })
 
+describe('resolveMediaDisplaySrc', () => {
+  const api = vi.fn(async ({ path }: { path: string }) => {
+    if (path.startsWith('/api/fs/read-data-url?')) {
+      return { dataUrl: 'data:image/png;base64,ZHVtbXk=' }
+    }
+
+    throw new Error(`unexpected path ${path}`)
+  })
+
+  beforeEach(() => {
+    api.mockClear()
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+    $connection.set(null)
+  })
+
+  it('recognizes inline image URLs', () => {
+    expect(isInlineMediaSrc('https://example.com/a.png')).toBe(true)
+    expect(isInlineMediaSrc('data:image/png;base64,ZHVtbXk=')).toBe(true)
+    expect(isInlineMediaSrc('/Users/me/a.png')).toBe(false)
+  })
+
+  it('leaves web, data, and relative markdown image sources unchanged', async () => {
+    vi.stubGlobal('window', { hermesDesktop: { api } })
+    $connection.set({ mode: 'remote', profile: 'remote-work' } as never)
+
+    await expect(resolveMediaDisplaySrc('https://example.com/a.png')).resolves.toBe('https://example.com/a.png')
+    await expect(resolveMediaDisplaySrc('data:image/png;base64,ZHVtbXk=')).resolves.toBe(
+      'data:image/png;base64,ZHVtbXk='
+    )
+    await expect(resolveMediaDisplaySrc('images/a.png')).resolves.toBe('images/a.png')
+    await expect(resolveMediaDisplaySrc('./images/a.png')).resolves.toBe('./images/a.png')
+    await expect(resolveMediaDisplaySrc('../images/a.png')).resolves.toBe('../images/a.png')
+    expect(api).not.toHaveBeenCalled()
+  })
+
+  it('reads remote gateway-local file paths through the desktop fs bridge', async () => {
+    vi.stubGlobal('window', { hermesDesktop: { api } })
+    $connection.set({ mode: 'remote', profile: 'remote-work' } as never)
+
+    await expect(resolveMediaDisplaySrc('/Users/me/project/a b.png')).resolves.toBe('data:image/png;base64,ZHVtbXk=')
+    expect(api).toHaveBeenCalledWith({
+      path: '/api/fs/read-data-url?path=%2FUsers%2Fme%2Fproject%2Fa%20b.png',
+      profile: 'remote-work'
+    })
+  })
+
+  it('reads local desktop file paths from the local desktop shell', async () => {
+    const readFileDataUrl = vi.fn(async () => 'data:image/png;base64,bG9jYWw=')
+
+    vi.stubGlobal('window', { hermesDesktop: { readFileDataUrl } })
+    $connection.set({ mode: 'local' } as never)
+
+    await expect(resolveMediaDisplaySrc('file:///Users/me/project/a%20b.png')).resolves.toBe(
+      'data:image/png;base64,bG9jYWw='
+    )
+    expect(readFileDataUrl).toHaveBeenCalledWith('/Users/me/project/a b.png')
+  })
+})
+
 describe('gatewayMediaDataUrl', () => {
   const api = vi.fn(async ({ path }: { path: string }) => {
     if (path.startsWith('/api/fs/read-data-url?')) {

apps/desktop/src/lib/media.ts

@@ -58,6 +58,30 @@ export function mediaMarkdownHref(path: string): string {
   return `#media:${encodeURIComponent(path)}`
 }
 
+export function isInlineMediaSrc(path: string): boolean {
+  return /^(?:https?|data):/i.test(path)
+}
+
+function isFileMediaPath(path: string): boolean {
+  return /^(?:file:|\/|~\/|[a-z]:[\\/]|\\\\)/i.test(path)
+}
+
+export async function resolveMediaDisplaySrc(path: string): Promise<string> {
+  if (isInlineMediaSrc(path) || !isFileMediaPath(path)) {
+    return path
+  }
+
+  if (window.hermesDesktop && isRemoteGateway()) {
+    return gatewayMediaDataUrl(path)
+  }
+
+  if (!window.hermesDesktop?.readFileDataUrl) {
+    return mediaExternalUrl(path)
+  }
+
+  return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path))
+}
+
 // Resolve a media path to a URL the shell can open. Remote mode rewrites
 // gateway-local paths to an authenticated /api/files/download URL (the file
 // lives on the gateway, not this disk); local mode keeps the file:// form.