diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx
new file mode 100644
index 000000000000..ca64acd4c746
--- /dev/null
+++ b/apps/desktop/src/components/assistant-ui/markdown-text.media.test.tsx
@@ -0,0 +1,51 @@
+import { cleanup, render, screen } from '@testing-library/react'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { $connection } from '@/store/session'
+
+import { MarkdownTextContent } from './markdown-text'
+
+const REMOTE_IMAGE_PATH = '/home/user/project/images/remote-preview.png'
+const REMOTE_IMAGE_DATA_URL = 'data:image/png;base64,cmVtb3RlLWltYWdl'
+
+describe('MarkdownTextContent remote images', () => {
+ const api = vi.fn(async ({ path }: { path: string }) => {
+ if (path.startsWith('/api/fs/read-data-url?')) {
+ return { dataUrl: REMOTE_IMAGE_DATA_URL }
+ }
+
+ throw new Error(`unexpected path ${path}`)
+ })
+ let originalDesktop: typeof window.hermesDesktop
+
+ beforeEach(() => {
+ api.mockClear()
+ originalDesktop = window.hermesDesktop
+ Object.defineProperty(window, 'hermesDesktop', {
+ configurable: true,
+ value: { api }
+ })
+ $connection.set({ mode: 'remote', profile: 'remote-work' } as never)
+ })
+
+ afterEach(() => {
+ cleanup()
+ $connection.set(null)
+ Object.defineProperty(window, 'hermesDesktop', {
+ configurable: true,
+ value: originalDesktop
+ })
+ })
+
+ it('passes the gateway bridge data URL through Streamdown to the zoomable image', async () => {
+ render()
+
+ const image = await screen.findByRole('img', { name: 'Remote preview' })
+
+ expect(image.getAttribute('src')).toBe(REMOTE_IMAGE_DATA_URL)
+ expect(api).toHaveBeenCalledWith({
+ path: '/api/fs/read-data-url?path=%2Fhome%2Fuser%2Fproject%2Fimages%2Fremote-preview.png',
+ profile: 'remote-work'
+ })
+ })
+})
diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.tsx
index a199b5aa1e7d..f5d78f531334 100644
--- a/apps/desktop/src/components/assistant-ui/markdown-text.tsx
+++ b/apps/desktop/src/components/assistant-ui/markdown-text.tsx
@@ -20,14 +20,14 @@ import { parseMarkdownIntoBlocksCached } from '@/lib/markdown-blocks'
import { preprocessMarkdown } from '@/lib/markdown-preprocess'
import {
downloadGatewayMediaFile,
- filePathFromMediaPath,
- gatewayMediaDataUrl,
+ isInlineMediaSrc,
isRemoteGateway,
mediaExternalUrl,
mediaKind,
mediaName,
mediaPathFromMarkdownHref,
- mediaStreamUrl
+ mediaStreamUrl,
+ resolveMediaDisplaySrc
} from '@/lib/media'
import { previewTargetFromMarkdownHref } from '@/lib/preview-targets'
import { cn } from '@/lib/utils'
@@ -60,27 +60,13 @@ function preprocessWithTailRepair(text: string): string {
}
async function mediaSrc(path: string): Promise {
- if (/^(?:https?|data):/i.test(path)) {
- return path
- }
-
// Stream audio/video through the custom protocol: data URLs are capped and
// load the whole file into memory, which broke playback for larger videos.
if (window.hermesDesktop && ['audio', 'video'].includes(mediaKind(path))) {
return mediaStreamUrl(path)
}
- // Remote gateway: the image lives on the gateway machine, so read it over the
- // authenticated API rather than this machine's disk.
- if (window.hermesDesktop && isRemoteGateway()) {
- return gatewayMediaDataUrl(path)
- }
-
- if (!window.hermesDesktop?.readFileDataUrl) {
- return mediaExternalUrl(path)
- }
-
- return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path))
+ return resolveMediaDisplaySrc(path)
}
function useOpenMediaFile(path: string) {
@@ -286,6 +272,65 @@ function MarkdownLink({ children, className, href, ...props }: ComponentProps<'a
}
function MarkdownImage({ className, src, alt, ...props }: ComponentProps<'img'>) {
+ const rawSrc = typeof src === 'string' ? src : ''
+ const [resolvedSrc, setResolvedSrc] = useState(() => (rawSrc && isInlineMediaSrc(rawSrc) ? rawSrc : ''))
+ const [failed, setFailed] = useState(false)
+ const { open, openFailed } = useOpenMediaFile(rawSrc)
+ const name = mediaName(rawSrc || String(alt || 'image'))
+
+ useEffect(() => {
+ let cancelled = false
+
+ setFailed(false)
+ setResolvedSrc(rawSrc && isInlineMediaSrc(rawSrc) ? rawSrc : '')
+
+ if (!rawSrc || isInlineMediaSrc(rawSrc)) {
+ return () => {
+ cancelled = true
+ }
+ }
+
+ void resolveMediaDisplaySrc(rawSrc)
+ .then(value => {
+ if (!cancelled) {
+ setResolvedSrc(value)
+ }
+ })
+ .catch(() => {
+ if (!cancelled) {
+ setFailed(true)
+ }
+ })
+
+ return () => {
+ cancelled = true
+ }
+ }, [rawSrc])
+
+ if (!rawSrc) {
+ return null
+ }
+
+ if (failed) {
+ return (
+
+ Couldn't load {name}.{' '}
+
+ {openFailed && }
+
+ )
+ }
+
+ if (!resolvedSrc) {
+ return Loading {name}...
+ }
+
return (
)
)}
containerClassName="my-2 block w-fit max-w-full"
slot="aui_markdown-image"
- src={src}
+ src={resolvedSrc}
{...props}
/>
)
diff --git a/apps/desktop/src/lib/media.remote.test.ts b/apps/desktop/src/lib/media.remote.test.ts
index e10ca5f59ec1..79676c691431 100644
--- a/apps/desktop/src/lib/media.remote.test.ts
+++ b/apps/desktop/src/lib/media.remote.test.ts
@@ -8,8 +8,10 @@ import {
downloadGatewayMediaFile,
filePathFromMediaPath,
gatewayMediaDataUrl,
+ isInlineMediaSrc,
isRemoteGateway,
- mediaExternalUrl
+ mediaExternalUrl,
+ resolveMediaDisplaySrc
} from './media'
describe('isRemoteGateway', () => {
@@ -75,6 +77,68 @@ describe('mediaExternalUrl', () => {
})
})
+describe('resolveMediaDisplaySrc', () => {
+ const api = vi.fn(async ({ path }: { path: string }) => {
+ if (path.startsWith('/api/fs/read-data-url?')) {
+ return { dataUrl: 'data:image/png;base64,ZHVtbXk=' }
+ }
+
+ throw new Error(`unexpected path ${path}`)
+ })
+
+ beforeEach(() => {
+ api.mockClear()
+ })
+
+ afterEach(() => {
+ vi.unstubAllGlobals()
+ $connection.set(null)
+ })
+
+ it('recognizes inline image URLs', () => {
+ expect(isInlineMediaSrc('https://example.com/a.png')).toBe(true)
+ expect(isInlineMediaSrc('data:image/png;base64,ZHVtbXk=')).toBe(true)
+ expect(isInlineMediaSrc('/Users/me/a.png')).toBe(false)
+ })
+
+ it('leaves web, data, and relative markdown image sources unchanged', async () => {
+ vi.stubGlobal('window', { hermesDesktop: { api } })
+ $connection.set({ mode: 'remote', profile: 'remote-work' } as never)
+
+ await expect(resolveMediaDisplaySrc('https://example.com/a.png')).resolves.toBe('https://example.com/a.png')
+ await expect(resolveMediaDisplaySrc('data:image/png;base64,ZHVtbXk=')).resolves.toBe(
+ 'data:image/png;base64,ZHVtbXk='
+ )
+ await expect(resolveMediaDisplaySrc('images/a.png')).resolves.toBe('images/a.png')
+ await expect(resolveMediaDisplaySrc('./images/a.png')).resolves.toBe('./images/a.png')
+ await expect(resolveMediaDisplaySrc('../images/a.png')).resolves.toBe('../images/a.png')
+ expect(api).not.toHaveBeenCalled()
+ })
+
+ it('reads remote gateway-local file paths through the desktop fs bridge', async () => {
+ vi.stubGlobal('window', { hermesDesktop: { api } })
+ $connection.set({ mode: 'remote', profile: 'remote-work' } as never)
+
+ await expect(resolveMediaDisplaySrc('/Users/me/project/a b.png')).resolves.toBe('data:image/png;base64,ZHVtbXk=')
+ expect(api).toHaveBeenCalledWith({
+ path: '/api/fs/read-data-url?path=%2FUsers%2Fme%2Fproject%2Fa%20b.png',
+ profile: 'remote-work'
+ })
+ })
+
+ it('reads local desktop file paths from the local desktop shell', async () => {
+ const readFileDataUrl = vi.fn(async () => 'data:image/png;base64,bG9jYWw=')
+
+ vi.stubGlobal('window', { hermesDesktop: { readFileDataUrl } })
+ $connection.set({ mode: 'local' } as never)
+
+ await expect(resolveMediaDisplaySrc('file:///Users/me/project/a%20b.png')).resolves.toBe(
+ 'data:image/png;base64,bG9jYWw='
+ )
+ expect(readFileDataUrl).toHaveBeenCalledWith('/Users/me/project/a b.png')
+ })
+})
+
describe('gatewayMediaDataUrl', () => {
const api = vi.fn(async ({ path }: { path: string }) => {
if (path.startsWith('/api/fs/read-data-url?')) {
diff --git a/apps/desktop/src/lib/media.ts b/apps/desktop/src/lib/media.ts
index ffaea2c7c2bc..c392caa4d7bd 100644
--- a/apps/desktop/src/lib/media.ts
+++ b/apps/desktop/src/lib/media.ts
@@ -58,6 +58,30 @@ export function mediaMarkdownHref(path: string): string {
return `#media:${encodeURIComponent(path)}`
}
+export function isInlineMediaSrc(path: string): boolean {
+ return /^(?:https?|data):/i.test(path)
+}
+
+function isFileMediaPath(path: string): boolean {
+ return /^(?:file:|\/|~\/|[a-z]:[\\/]|\\\\)/i.test(path)
+}
+
+export async function resolveMediaDisplaySrc(path: string): Promise {
+ if (isInlineMediaSrc(path) || !isFileMediaPath(path)) {
+ return path
+ }
+
+ if (window.hermesDesktop && isRemoteGateway()) {
+ return gatewayMediaDataUrl(path)
+ }
+
+ if (!window.hermesDesktop?.readFileDataUrl) {
+ return mediaExternalUrl(path)
+ }
+
+ return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path))
+}
+
// Resolve a media path to a URL the shell can open. Remote mode rewrites
// gateway-local paths to an authenticated /api/files/download URL (the file
// lives on the gateway, not this disk); local mode keeps the file:// form.