From 03406ae2553e802f11399129c3f376a096bbec4f Mon Sep 17 00:00:00 2001 From: helix4u <4317663+helix4u@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:21:06 -0600 Subject: [PATCH] fix(desktop): restore remote artifact rendering --- .../src/app/artifacts/artifact-utils.ts | 282 +++++++++++++++++ apps/desktop/src/app/artifacts/index.test.ts | 33 +- apps/desktop/src/app/artifacts/index.tsx | 299 ++---------------- .../components/assistant-ui/markdown-text.tsx | 80 +++-- apps/desktop/src/lib/media.remote.test.ts | 71 ++++- apps/desktop/src/lib/media.ts | 35 +- 6 files changed, 497 insertions(+), 303 deletions(-) create mode 100644 apps/desktop/src/app/artifacts/artifact-utils.ts diff --git a/apps/desktop/src/app/artifacts/artifact-utils.ts b/apps/desktop/src/app/artifacts/artifact-utils.ts new file mode 100644 index 00000000000..730ec0e0016 --- /dev/null +++ b/apps/desktop/src/app/artifacts/artifact-utils.ts @@ -0,0 +1,282 @@ +import { readDesktopFileDataUrl } from '@/lib/desktop-fs' +import { filePathFromMediaPath, isRemoteGateway, mediaExternalUrl } from '@/lib/media' +import type { SessionInfo, SessionMessage } from '@/types/hermes' + +export type ArtifactKind = 'image' | 'file' | 'link' +export type ArtifactFilter = 'all' | ArtifactKind +export const ARTIFACT_FILTERS: readonly ArtifactFilter[] = ['all', 'image', 'file', 'link'] + +export interface ArtifactRecord { + id: string + kind: ArtifactKind + value: string + href: string + label: string + sessionId: string + sessionTitle: string + timestamp: number +} + +const MARKDOWN_IMAGE_RE = /!\[([^\]]*)\]\(([^)\s]+)\)/g +const MARKDOWN_LINK_RE = /\[([^\]]+)\]\(([^)\s]+)\)/g +const URL_RE = /https?:\/\/[^\s<>"')]+/g +const PATH_RE = /(^|[\s("'`])((?:\/|~\/|\.\.?\/)[^\s"'`<>]+(?:\.[a-z0-9]{1,8})?)/gi +const IMAGE_EXT_RE = /\.(?:png|jpe?g|gif|webp|svg|bmp)(?:\?.*)?$/i +const FILE_EXT_RE = /\.(?:png|jpe?g|gif|webp|svg|bmp|pdf|txt|json|md|csv|zip|tar|gz|mp3|wav|mp4|mov)(?:\?.*)?$/i +const KEY_HINT_RE = /(path|file|url|image|artifact|output|download|result|target)/i + +function artifactSessionTitle(session: SessionInfo): string { + return session.title?.trim() || session.preview?.trim() || 'Untitled session' +} + +function normalizeValue(value: string): string { + return value.trim().replace(/[),.;]+$/, '') +} + +function parseMaybeJson(value: string): unknown { + if (!value.trim()) { + return null + } + + try { + return JSON.parse(value) + } catch { + return null + } +} + +function looksLikePathOrUrl(value: string): boolean { + return ( + value.startsWith('http://') || + value.startsWith('https://') || + value.startsWith('file://') || + value.startsWith('data:image/') || + value.startsWith('/') || + value.startsWith('./') || + value.startsWith('../') || + value.startsWith('~/') + ) +} + +function looksLikeArtifact(value: string): boolean { + if (/^(?:https?:\/\/|data:image\/)/.test(value)) { + return true + } + + if (looksLikePathOrUrl(value) && (IMAGE_EXT_RE.test(value) || FILE_EXT_RE.test(value))) { + return true + } + + return value.startsWith('/') && value.includes('.') +} + +function artifactKind(value: string): ArtifactKind { + if (value.startsWith('data:image/') || IMAGE_EXT_RE.test(value)) { + return 'image' + } + + if ( + value.startsWith('/') || + value.startsWith('./') || + value.startsWith('../') || + value.startsWith('~/') || + value.startsWith('file://') + ) { + return 'file' + } + + return 'link' +} + +function artifactHref(value: string): string { + if (value.startsWith('http://') || value.startsWith('https://') || value.startsWith('data:')) { + return value + } + + if (value.startsWith('file://') || value.startsWith('/')) { + return mediaExternalUrl(value) + } + + return value +} + +export async function artifactImageSrc(value: string, href = artifactHref(value)): Promise { + if (/^(?:https?|data):/i.test(value)) { + return href + } + + if (typeof window !== 'undefined' && window.hermesDesktop && isRemoteGateway()) { + return readDesktopFileDataUrl(filePathFromMediaPath(value)) + } + + return href +} + +function artifactLabel(value: string): string { + try { + const url = new URL(value) + const item = url.pathname.split('/').filter(Boolean).pop() + + return item || value + } catch { + const parts = value.split(/[\\/]/).filter(Boolean) + + return parts.pop() || value + } +} + +function messageText(message: SessionMessage): string { + if (typeof message.content === 'string' && message.content.trim()) { + return message.content + } + + if (typeof message.text === 'string' && message.text.trim()) { + return message.text + } + + if (typeof message.context === 'string' && message.context.trim()) { + return message.context + } + + return '' +} + +function collectStringValues( + value: unknown, + keyPath: string, + collector: (value: string, keyPath: string) => void +): void { + if (typeof value === 'string') { + collector(value, keyPath) + + return + } + + if (Array.isArray(value)) { + value.forEach((entry, index) => collectStringValues(entry, `${keyPath}.${index}`, collector)) + + return + } + + if (!value || typeof value !== 'object') { + return + } + + for (const [key, child] of Object.entries(value as Record)) { + collectStringValues(child, keyPath ? `${keyPath}.${key}` : key, collector) + } +} + +function collectArtifactsFromText(text: string, pushValue: (value: string) => void): void { + for (const match of text.matchAll(MARKDOWN_IMAGE_RE)) { + pushValue(match[2] || '') + } + + for (const match of text.matchAll(MARKDOWN_LINK_RE)) { + const start = match.index ?? 0 + + if (start > 0 && text[start - 1] === '!') { + continue + } + + const value = match[2] || '' + + if (looksLikeArtifact(value)) { + pushValue(value) + } + } + + for (const match of text.matchAll(URL_RE)) { + const value = match[0] || '' + + if (looksLikeArtifact(value)) { + pushValue(value) + } + } + + for (const match of text.matchAll(PATH_RE)) { + pushValue(match[2] || '') + } +} + +function collectArtifactsFromMessage(message: SessionMessage, pushValue: (value: string) => void): void { + const text = messageText(message) + + if (text) { + collectArtifactsFromText(text, pushValue) + } + + if (message.role !== 'tool' && !Array.isArray(message.tool_calls)) { + return + } + + if (Array.isArray(message.tool_calls)) { + for (const call of message.tool_calls) { + collectStringValues(call, 'tool_call', (value, keyPath) => { + const normalized = normalizeValue(value) + + if (!normalized) { + return + } + + if (KEY_HINT_RE.test(keyPath) && (looksLikePathOrUrl(normalized) || FILE_EXT_RE.test(normalized))) { + pushValue(normalized) + } + }) + } + } + + const parsed = parseMaybeJson(text) + + if (parsed !== null) { + collectStringValues(parsed, 'tool_result', (value, keyPath) => { + const normalized = normalizeValue(value) + + if (!normalized) { + return + } + + if ((KEY_HINT_RE.test(keyPath) || looksLikePathOrUrl(normalized)) && looksLikeArtifact(normalized)) { + pushValue(normalized) + } + }) + } +} + +export function collectArtifactsForSession(session: SessionInfo, messages: SessionMessage[]): ArtifactRecord[] { + const found = new Map() + const title = artifactSessionTitle(session) + + for (const message of messages) { + if (message.role !== 'assistant' && message.role !== 'tool') { + continue + } + + collectArtifactsFromMessage(message, candidate => { + const value = normalizeValue(candidate) + + if (!value || !looksLikeArtifact(value)) { + return + } + + const key = `${session.id}:${value}` + + if (found.has(key)) { + return + } + + found.set(key, { + id: key, + kind: artifactKind(value), + value, + href: artifactHref(value), + label: artifactLabel(value), + sessionId: session.id, + sessionTitle: title, + timestamp: message.timestamp || session.last_active || session.started_at || Date.now() + }) + }) + } + + return Array.from(found.values()) +} diff --git a/apps/desktop/src/app/artifacts/index.test.ts b/apps/desktop/src/app/artifacts/index.test.ts index ebca956a2c9..cd98db3243a 100644 --- a/apps/desktop/src/app/artifacts/index.test.ts +++ b/apps/desktop/src/app/artifacts/index.test.ts @@ -1,8 +1,9 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { $connection } from '@/store/session' import type { SessionInfo, SessionMessage } from '@/types/hermes' -import { collectArtifactsForSession } from './index' +import { artifactImageSrc, collectArtifactsForSession } from './artifact-utils' function makeSession(overrides: Partial = {}): SessionInfo { return { @@ -24,6 +25,12 @@ function makeSession(overrides: Partial = {}): SessionInfo { } describe('collectArtifactsForSession', () => { + afterEach(() => { + vi.unstubAllGlobals() + vi.clearAllMocks() + $connection.set(null) + }) + it('indexes plain https links from assistant text', () => { const artifacts = collectArtifactsForSession(makeSession(), [ { @@ -59,4 +66,26 @@ describe('collectArtifactsForSession', () => { value: 'https://example.com/changelog/latest' }) }) + + it('resolves remote image artifact thumbnails through the desktop fs bridge', async () => { + const api = vi.fn(async ({ path }: { path: string }) => { + if (path.startsWith('/api/fs/read-data-url?')) { + return { dataUrl: 'data:image/jpeg;base64,cmVtb3Rl' } + } + + throw new Error(`unexpected path ${path}`) + }) + + vi.stubGlobal('window', { hermesDesktop: { api } }) + $connection.set({ baseUrl: 'https://gw', mode: 'remote', token: 'secret' } as never) + + const path = '/Users/me/.hermes/skills/work-esab/references/images/manual-step03.jpeg' + const downloadHref = `https://gw/api/files/download?path=${encodeURIComponent(path)}&token=secret` + + await expect(artifactImageSrc(path, downloadHref)).resolves.toBe('data:image/jpeg;base64,cmVtb3Rl') + + expect(api).toHaveBeenCalledWith({ + path: '/api/fs/read-data-url?path=%2FUsers%2Fme%2F.hermes%2Fskills%2Fwork-esab%2Freferences%2Fimages%2Fmanual-step03.jpeg' + }) + }) }) diff --git a/apps/desktop/src/app/artifacts/index.tsx b/apps/desktop/src/app/artifacts/index.tsx index f7d9e3238e3..fea5b7f58c9 100644 --- a/apps/desktop/src/app/artifacts/index.tsx +++ b/apps/desktop/src/app/artifacts/index.tsx @@ -21,14 +21,18 @@ import { TextTab, TextTabMeta } from '@/components/ui/text-tab' import { Tip } from '@/components/ui/tooltip' import { getSessionMessages, listAllProfileSessions } from '@/hermes' import { type Translations, useI18n } from '@/i18n' -import { sessionTitle } from '@/lib/chat-runtime' import { ExternalLink, ExternalLinkIcon, hostPathLabel, urlSlugTitleLabel, useLinkTitle } from '@/lib/external-link' import { FileImage, FileText, FolderOpen, Link2 } from '@/lib/icons' -import { mediaExternalUrl } from '@/lib/media' import { cn } from '@/lib/utils' import { notifyError } from '@/store/notifications' -import type { SessionInfo, SessionMessage } from '@/types/hermes' +import { + ARTIFACT_FILTERS, + artifactImageSrc, + collectArtifactsForSession, + type ArtifactFilter, + type ArtifactRecord +} from './artifact-utils' import { useRefreshHotkey } from '../hooks/use-refresh-hotkey' import { useRouteEnumParam } from '../hooks/use-route-enum-param' import { PAGE_INSET_NEG_X, PAGE_INSET_X } from '../layout-constants' @@ -36,29 +40,6 @@ import { PageSearchShell } from '../page-search-shell' import { sessionRoute } from '../routes' import type { SetStatusbarItemGroup } from '../shell/statusbar-controls' -type ArtifactKind = 'image' | 'file' | 'link' -type ArtifactFilter = 'all' | ArtifactKind -const ARTIFACT_FILTERS: readonly ArtifactFilter[] = ['all', 'image', 'file', 'link'] - -interface ArtifactRecord { - id: string - kind: ArtifactKind - value: string - href: string - label: string - sessionId: string - sessionTitle: string - timestamp: number -} - -const MARKDOWN_IMAGE_RE = /!\[([^\]]*)\]\(([^)\s]+)\)/g -const MARKDOWN_LINK_RE = /\[([^\]]+)\]\(([^)\s]+)\)/g -const URL_RE = /https?:\/\/[^\s<>"')]+/g -const PATH_RE = /(^|[\s("'`])((?:\/|~\/|\.\.?\/)[^\s"'`<>]+(?:\.[a-z0-9]{1,8})?)/gi -const IMAGE_EXT_RE = /\.(?:png|jpe?g|gif|webp|svg|bmp)(?:\?.*)?$/i -const FILE_EXT_RE = /\.(?:png|jpe?g|gif|webp|svg|bmp|pdf|txt|json|md|csv|zip|tar|gz|mp3|wav|mp4|mov)(?:\?.*)?$/i -const KEY_HINT_RE = /(path|file|url|image|artifact|output|download|result|target)/i - const ARTIFACT_TIME_FMT = new Intl.DateTimeFormat(undefined, { day: 'numeric', hour: 'numeric', @@ -66,246 +47,6 @@ const ARTIFACT_TIME_FMT = new Intl.DateTimeFormat(undefined, { month: 'short' }) -function normalizeValue(value: string): string { - return value.trim().replace(/[),.;]+$/, '') -} - -function parseMaybeJson(value: string): unknown { - if (!value.trim()) { - return null - } - - try { - return JSON.parse(value) - } catch { - return null - } -} - -function looksLikePathOrUrl(value: string): boolean { - return ( - value.startsWith('http://') || - value.startsWith('https://') || - value.startsWith('file://') || - value.startsWith('data:image/') || - value.startsWith('/') || - value.startsWith('./') || - value.startsWith('../') || - value.startsWith('~/') - ) -} - -function looksLikeArtifact(value: string): boolean { - if (/^(?:https?:\/\/|data:image\/)/.test(value)) { - return true - } - - if (looksLikePathOrUrl(value) && (IMAGE_EXT_RE.test(value) || FILE_EXT_RE.test(value))) { - return true - } - - return value.startsWith('/') && value.includes('.') -} - -function artifactKind(value: string): ArtifactKind { - if (value.startsWith('data:image/') || IMAGE_EXT_RE.test(value)) { - return 'image' - } - - if ( - value.startsWith('/') || - value.startsWith('./') || - value.startsWith('../') || - value.startsWith('~/') || - value.startsWith('file://') - ) { - return 'file' - } - - return 'link' -} - -function artifactHref(value: string): string { - if (value.startsWith('http://') || value.startsWith('https://') || value.startsWith('data:')) { - return value - } - - if (value.startsWith('file://') || value.startsWith('/')) { - return mediaExternalUrl(value) - } - - return value -} - -function artifactLabel(value: string): string { - try { - const url = new URL(value) - const item = url.pathname.split('/').filter(Boolean).pop() - - return item || value - } catch { - const parts = value.split(/[\\/]/).filter(Boolean) - - return parts.pop() || value - } -} - -function messageText(message: SessionMessage): string { - if (typeof message.content === 'string' && message.content.trim()) { - return message.content - } - - if (typeof message.text === 'string' && message.text.trim()) { - return message.text - } - - if (typeof message.context === 'string' && message.context.trim()) { - return message.context - } - - return '' -} - -function collectStringValues( - value: unknown, - keyPath: string, - collector: (value: string, keyPath: string) => void -): void { - if (typeof value === 'string') { - collector(value, keyPath) - - return - } - - if (Array.isArray(value)) { - value.forEach((entry, index) => collectStringValues(entry, `${keyPath}.${index}`, collector)) - - return - } - - if (!value || typeof value !== 'object') { - return - } - - for (const [key, child] of Object.entries(value as Record)) { - collectStringValues(child, keyPath ? `${keyPath}.${key}` : key, collector) - } -} - -function collectArtifactsFromText(text: string, pushValue: (value: string) => void): void { - for (const match of text.matchAll(MARKDOWN_IMAGE_RE)) { - pushValue(match[2] || '') - } - - for (const match of text.matchAll(MARKDOWN_LINK_RE)) { - const start = match.index ?? 0 - - if (start > 0 && text[start - 1] === '!') { - continue - } - - const value = match[2] || '' - - if (looksLikeArtifact(value)) { - pushValue(value) - } - } - - for (const match of text.matchAll(URL_RE)) { - const value = match[0] || '' - - if (looksLikeArtifact(value)) { - pushValue(value) - } - } - - for (const match of text.matchAll(PATH_RE)) { - pushValue(match[2] || '') - } -} - -function collectArtifactsFromMessage(message: SessionMessage, pushValue: (value: string) => void): void { - const text = messageText(message) - - if (text) { - collectArtifactsFromText(text, pushValue) - } - - if (message.role !== 'tool' && !Array.isArray(message.tool_calls)) { - return - } - - if (Array.isArray(message.tool_calls)) { - for (const call of message.tool_calls) { - collectStringValues(call, 'tool_call', (value, keyPath) => { - const normalized = normalizeValue(value) - - if (!normalized) { - return - } - - if (KEY_HINT_RE.test(keyPath) && (looksLikePathOrUrl(normalized) || FILE_EXT_RE.test(normalized))) { - pushValue(normalized) - } - }) - } - } - - const parsed = parseMaybeJson(text) - - if (parsed !== null) { - collectStringValues(parsed, 'tool_result', (value, keyPath) => { - const normalized = normalizeValue(value) - - if (!normalized) { - return - } - - if ((KEY_HINT_RE.test(keyPath) || looksLikePathOrUrl(normalized)) && looksLikeArtifact(normalized)) { - pushValue(normalized) - } - }) - } -} - -export function collectArtifactsForSession(session: SessionInfo, messages: SessionMessage[]): ArtifactRecord[] { - const found = new Map() - const title = sessionTitle(session) - - for (const message of messages) { - if (message.role !== 'assistant' && message.role !== 'tool') { - continue - } - - collectArtifactsFromMessage(message, candidate => { - const value = normalizeValue(candidate) - - if (!value || !looksLikeArtifact(value)) { - return - } - - const key = `${session.id}:${value}` - - if (found.has(key)) { - return - } - - found.set(key, { - id: key, - kind: artifactKind(value), - value, - href: artifactHref(value), - label: artifactLabel(value), - sessionId: session.id, - sessionTitle: title, - timestamp: message.timestamp || session.last_active || session.started_at || Date.now() - }) - }) - } - - return Array.from(found.values()) -} - function formatArtifactTime(timestamp: number): string { return ARTIFACT_TIME_FMT.format(new Date(timestamp)) } @@ -684,6 +425,28 @@ function ArtifactImageCard({ artifact, failedImage, onImageError, onOpenChat }: const { t } = useI18n() const a = t.artifacts const kindLabel = artifact.kind === 'image' ? a.kindImage : artifact.kind === 'file' ? a.kindFile : a.kindLink + const [src, setSrc] = useState('') + + useEffect(() => { + let active = true + + setSrc('') + void artifactImageSrc(artifact.value, artifact.href) + .then(nextSrc => { + if (active) { + setSrc(nextSrc) + } + }) + .catch(() => { + if (active) { + onImageError(artifact.id) + } + }) + + return () => { + active = false + } + }, [artifact.href, artifact.id, artifact.value, onImageError]) return (
@@ -693,7 +456,7 @@ function ArtifactImageCard({ artifact, failedImage, onImageError, onOpenChat }: failedImage && 'cursor-default' )} > - {!failedImage && ( + {!failedImage && src && ( onImageError(artifact.id)} slot="artifact-media" - src={artifact.href} + src={src} /> )} diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.tsx index beabcbf8cc2..bceadb8e6ca 100644 --- a/apps/desktop/src/components/assistant-ui/markdown-text.tsx +++ b/apps/desktop/src/components/assistant-ui/markdown-text.tsx @@ -27,6 +27,7 @@ import { normalizeExternalUrl, openExternalLink, PrettyLink } from '@/lib/extern import { createMemoizedMathPlugin } from '@/lib/katex-memo' import { preprocessMarkdown } from '@/lib/markdown-preprocess' import { + downloadGatewayMediaFile, filePathFromMediaPath, gatewayMediaDataUrl, isRemoteGateway, @@ -129,21 +130,50 @@ async function mediaSrc(path: string): Promise { return window.hermesDesktop.readFileDataUrl(filePathFromMediaPath(path)) } -function OpenMediaButton({ kind, path }: { kind: 'audio' | 'video'; path: string }) { +function useOpenMediaFile(path: string) { + const [openFailed, setOpenFailed] = useState(false) + + const open = () => { + if (window.hermesDesktop && isRemoteGateway()) { + setOpenFailed(false) + void downloadGatewayMediaFile(path).catch(() => setOpenFailed(true)) + } else { + openExternalLink(mediaExternalUrl(path)) + } + } + + return { open, openFailed } +} + +function OpenMediaFailedNote({ name }: { name: string }) { return ( - + + Couldn't fetch {name} from the gateway (missing, unreadable, or too large). + + ) +} + +function OpenMediaButton({ kind, path }: { kind: 'audio' | 'video'; path: string }) { + const { open, openFailed } = useOpenMediaFile(path) + + return ( + + + {openFailed && } + ) } function MediaAttachment({ path }: { path: string }) { const [src, setSrc] = useState('') const [failed, setFailed] = useState(false) + const { open, openFailed } = useOpenMediaFile(path) const kind = mediaKind(path) const name = mediaName(path) @@ -153,6 +183,15 @@ function MediaAttachment({ path }: { path: string }) { setFailed(false) setSrc('') + + if (kind === 'file') { + setFailed(true) + + return () => { + cancelled = true + } + } + void mediaSrc(path) .then(value => { if (value.startsWith('blob:')) { @@ -178,7 +217,7 @@ function MediaAttachment({ path }: { path: string }) { URL.revokeObjectURL(objectUrl) } } - }, [path]) + }, [kind, path]) if (kind === 'image' && src) { return ( @@ -214,16 +253,19 @@ function MediaAttachment({ path }: { path: string }) { } return ( - { - event.preventDefault() - openExternalLink(mediaExternalUrl(path)) - }} - > - {failed ? `Open ${name}` : `Loading ${name}...`} - + + { + event.preventDefault() + open() + }} + > + {failed ? `Open ${name}` : `Loading ${name}...`} + + {openFailed && } + ) } diff --git a/apps/desktop/src/lib/media.remote.test.ts b/apps/desktop/src/lib/media.remote.test.ts index 53e5c2212c3..074d11ec42a 100644 --- a/apps/desktop/src/lib/media.remote.test.ts +++ b/apps/desktop/src/lib/media.remote.test.ts @@ -2,7 +2,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { $connection } from '@/store/session' -import { filePathFromMediaPath, gatewayMediaDataUrl, isRemoteGateway, mediaExternalUrl } from './media' +import { + downloadGatewayMediaFile, + filePathFromMediaPath, + gatewayMediaDataUrl, + isRemoteGateway, + mediaExternalUrl +} from './media' describe('isRemoteGateway', () => { afterEach(() => { @@ -68,23 +74,78 @@ describe('mediaExternalUrl', () => { }) describe('gatewayMediaDataUrl', () => { - const api = vi.fn(async () => ({ data_url: 'data:image/png;base64,ZHVtbXk=' })) + const api = vi.fn(async ({ path }: { path: string }) => { + if (path.startsWith('/api/fs/read-data-url?')) { + return { dataUrl: 'data:image/png;base64,ZHVtbXk=' } + } + + throw new Error(`unexpected path ${path}`) + }) beforeEach(() => { api.mockClear() vi.stubGlobal('window', { hermesDesktop: { api } }) + $connection.set({ mode: 'remote' } as never) }) afterEach(() => { vi.unstubAllGlobals() + $connection.set(null) }) - it('requests the encoded gateway path and returns the data URL', async () => { - const url = await gatewayMediaDataUrl('/home/u/.hermes/images/a b.png') + it('reads gateway media through the desktop fs bridge instead of /api/media roots', async () => { + const url = await gatewayMediaDataUrl('/home/u/.hermes/skills/demo/images/a b.png') expect(url).toBe('data:image/png;base64,ZHVtbXk=') expect(api).toHaveBeenCalledWith({ - path: '/api/media?path=%2Fhome%2Fu%2F.hermes%2Fimages%2Fa%20b.png' + path: '/api/fs/read-data-url?path=%2Fhome%2Fu%2F.hermes%2Fskills%2Fdemo%2Fimages%2Fa%20b.png' }) }) }) + +describe('downloadGatewayMediaFile', () => { + const api = vi.fn(async ({ path }: { path: string }) => { + if (path.startsWith('/api/fs/read-data-url?')) { + return { dataUrl: 'data:text/markdown;base64,IyByZXBvcnQ=' } + } + + throw new Error(`unexpected path ${path}`) + }) + let clickSpy: ReturnType + + beforeEach(() => { + api.mockClear() + vi.stubGlobal('window', { hermesDesktop: { api }, setTimeout: vi.fn() }) + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ blob: async () => new Blob(['# report'], { type: 'text/markdown' }) })) + ) + URL.createObjectURL = vi.fn(() => 'blob:remote-artifact') + URL.revokeObjectURL = vi.fn() + clickSpy = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {}) + $connection.set({ mode: 'remote' } as never) + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.clearAllMocks() + clickSpy.mockRestore() + $connection.set(null) + }) + + it('downloads gateway files through the desktop fs bridge', async () => { + await downloadGatewayMediaFile('file:///Users/me/project/report.md') + + expect(api).toHaveBeenCalledWith({ + path: '/api/fs/read-data-url?path=%2FUsers%2Fme%2Fproject%2Freport.md' + }) + expect(clickSpy).toHaveBeenCalledOnce() + }) + + it('rejects when the gateway refuses the file read', async () => { + api.mockRejectedValueOnce(new Error('403 File is not readable')) + + await expect(downloadGatewayMediaFile('/Users/me/project/report.md')).rejects.toThrow('403') + expect(clickSpy).not.toHaveBeenCalled() + }) +}) diff --git a/apps/desktop/src/lib/media.ts b/apps/desktop/src/lib/media.ts index 9c50ce6c757..24988d97f15 100644 --- a/apps/desktop/src/lib/media.ts +++ b/apps/desktop/src/lib/media.ts @@ -1,3 +1,4 @@ +import { readDesktopFileDataUrl } from '@/lib/desktop-fs' import { $connection } from '@/store/session' export type MediaKind = 'audio' | 'image' | 'video' | 'file' @@ -114,18 +115,34 @@ export function isRemoteGateway(): boolean { return $connection.get()?.mode === 'remote' } -// Fetch a gateway-local image as a data URL via the authenticated REST bridge. -// Used in remote mode where readFileDataUrl (which reads THIS machine's disk) -// can't see files the agent wrote on the gateway. Requires the gateway to -// expose GET /api/media (hermes_cli/web_server.py). +// Fetch gateway-local media as a data URL via the authenticated desktop FS +// bridge. Remote Desktop artifacts can live anywhere the gateway can read +// (workspace, skills, ~/.hermes/cache, etc.); /api/media is intentionally +// narrower and rejects non-images plus images outside its media roots. export async function gatewayMediaDataUrl(path: string): Promise { - const file = filePathFromMediaPath(path) + return readDesktopFileDataUrl(filePathFromMediaPath(path)) +} - const result = await window.hermesDesktop!.api<{ data_url: string }>({ - path: `/api/media?path=${encodeURIComponent(file)}` - }) +// Remote-mode replacement for opening gateway-local file paths with file://. +// The file lives on the gateway, so fetch it over the authenticated fs bridge +// and hand the bytes to the local browser shell as a download. +export async function downloadGatewayMediaFile(path: string): Promise { + const dataUrl = await readDesktopFileDataUrl(filePathFromMediaPath(path)) - return result.data_url + if (!dataUrl) { + throw new Error('Gateway returned no file data') + } + + const response = await fetch(dataUrl) + const blobUrl = URL.createObjectURL(await response.blob()) + const anchor = document.createElement('a') + anchor.href = blobUrl + anchor.download = mediaName(path) + anchor.rel = 'noopener noreferrer' + document.body.appendChild(anchor) + anchor.click() + anchor.remove() + window.setTimeout(() => URL.revokeObjectURL(blobUrl), 30_000) } export function mediaDisplayLabel(path: string): string {