From 81ac562bf0e589d068fc999f6be2355456f3bab8 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Fri, 26 Jun 2026 03:21:59 -0500 Subject: [PATCH 1/5] feat(desktop): inline embed detection + module primitives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure, synchronous URL→descriptor matchers for YouTube, Vimeo, Instagram, Pinterest, TikTok, X, Spotify, Google Maps and OpenStreetMap, plus the shared embed primitives (error boundary, fail card, escape-html, dark-mode hook, sizing token). Declares the mermaid + dompurify deps used by the fenced renderers. --- apps/desktop/package.json | 2 + .../assistant-ui/embeds/embed-size.ts | 4 + .../assistant-ui/embeds/escape-html.ts | 3 + .../components/assistant-ui/embeds/fail.tsx | 7 + .../embeds/providers/detect.test.ts | 133 ++++++++++++++++++ .../assistant-ui/embeds/providers/index.ts | 54 +++++++ .../embeds/providers/instagram.ts | 26 ++++ .../assistant-ui/embeds/providers/maps.ts | 95 +++++++++++++ .../embeds/providers/pinterest.ts | 28 ++++ .../assistant-ui/embeds/providers/spotify.ts | 34 +++++ .../assistant-ui/embeds/providers/tiktok.ts | 28 ++++ .../assistant-ui/embeds/providers/twitter.ts | 27 ++++ .../assistant-ui/embeds/providers/types.ts | 60 ++++++++ .../assistant-ui/embeds/providers/vimeo.ts | 32 +++++ .../assistant-ui/embeds/providers/youtube.ts | 65 +++++++++ .../assistant-ui/embeds/rich-boundary.tsx | 34 +++++ .../components/assistant-ui/embeds/types.ts | 8 ++ .../assistant-ui/embeds/use-is-dark.ts | 24 ++++ package-lock.json | 2 + 19 files changed, 666 insertions(+) create mode 100644 apps/desktop/src/components/assistant-ui/embeds/embed-size.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/escape-html.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/fail.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/detect.test.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/index.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/instagram.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/maps.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/pinterest.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/spotify.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/tiktok.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/twitter.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/types.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/vimeo.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/providers/youtube.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/rich-boundary.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/types.ts create mode 100644 apps/desktop/src/components/assistant-ui/embeds/use-is-dark.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 56a7374eeef..6d0ca01f33f 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -81,11 +81,13 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "dnd-core": "^14.0.1", + "dompurify": "^3.4.11", "hast-util-from-html-isomorphic": "^2.0.0", "hast-util-to-text": "^4.0.2", "ignore": "^7.0.5", "katex": "^0.16.45", "leva": "^0.10.1", + "mermaid": "^11.15.0", "motion": "^12.38.0", "nanostores": "^1.3.0", "node-pty": "1.1.0", diff --git a/apps/desktop/src/components/assistant-ui/embeds/embed-size.ts b/apps/desktop/src/components/assistant-ui/embeds/embed-size.ts new file mode 100644 index 00000000000..3f3da9fb30f --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/embed-size.ts @@ -0,0 +1,4 @@ +// Shared height cap for inline embeds. Ratio embeds cap their width off this in +// UrlEmbed so height follows the aspect ratio; fenced renderers (mermaid, svg) +// reuse it directly. Pure CSS — no measuring. +export const EMBED_MAX_H = '33dvh' diff --git a/apps/desktop/src/components/assistant-ui/embeds/escape-html.ts b/apps/desktop/src/components/assistant-ui/embeds/escape-html.ts new file mode 100644 index 00000000000..c8545bca52e --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/escape-html.ts @@ -0,0 +1,3 @@ +export function escapeHtml(value: string): string { + return value.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/fail.tsx b/apps/desktop/src/components/assistant-ui/embeds/fail.tsx new file mode 100644 index 00000000000..833f67f8390 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/fail.tsx @@ -0,0 +1,7 @@ +export function EmbedFail({ label }: { label: string }) { + return ( + + Failed to load {label} embed + + ) +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/detect.test.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/detect.test.ts new file mode 100644 index 00000000000..74be1caa14d --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/detect.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest' + +import type { FrameEmbed, TweetEmbed } from './types' + +import { detectEmbed, isEmbeddableUrl } from './index' + +function frame(url: string): FrameEmbed { + const descriptor = detectEmbed(url) + + if (!descriptor || descriptor.renderer !== 'frame') { + throw new Error(`expected a frame embed for ${url}`) + } + + return descriptor +} + +describe('detectEmbed — YouTube', () => { + it.each([ + 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + 'https://youtu.be/dQw4w9WgXcQ', + 'https://www.youtube.com/shorts/dQw4w9WgXcQ', + 'https://m.youtube.com/watch?v=dQw4w9WgXcQ', + 'https://www.youtube.com/embed/dQw4w9WgXcQ', + 'https://www.youtube.com/live/dQw4w9WgXcQ' + ])('resolves %s to the privacy-enhanced embed of the same id', url => { + const embed = frame(url) + + expect(embed.provider).toBe('youtube') + expect(embed.id).toBe('youtube:dQw4w9WgXcQ') + expect(embed.embedUrl).toContain('youtube-nocookie.com/embed/dQw4w9WgXcQ') + }) + + it('carries a start time from t/start through to the embed', () => { + expect(frame('https://youtu.be/dQw4w9WgXcQ?t=90').embedUrl).toContain('start=90') + expect(frame('https://youtu.be/dQw4w9WgXcQ?t=1m30s').embedUrl).toContain('start=90') + }) + + it('rejects ids that are not 11 chars', () => { + expect(detectEmbed('https://www.youtube.com/watch?v=short')).toBeNull() + }) +}) + +describe('detectEmbed — other frame providers', () => { + it('resolves Vimeo numeric ids across path shapes', () => { + expect(frame('https://vimeo.com/76979871').embedUrl).toBe('https://player.vimeo.com/video/76979871') + expect(frame('https://vimeo.com/channels/staffpicks/76979871').id).toBe('vimeo:76979871') + }) + + it('resolves Instagram posts and reels', () => { + expect(frame('https://www.instagram.com/p/CabcDEF123/').embedUrl).toBe( + 'https://www.instagram.com/p/CabcDEF123/embed' + ) + expect(frame('https://www.instagram.com/reel/CabcDEF123/').embedUrl).toContain('/reel/CabcDEF123/embed') + expect(frame('https://www.instagram.com/reels/CabcDEF123/').embedUrl).toContain('/reel/CabcDEF123/embed') + }) + + it('resolves Pinterest pins across locale hosts', () => { + expect(frame('https://www.pinterest.com/pin/1234567890/').embedUrl).toBe( + 'https://assets.pinterest.com/ext/embed.html?id=1234567890' + ) + expect(frame('https://fr.pinterest.com/pin/1234567890/').provider).toBe('pinterest') + }) + + it('resolves TikTok videos to the official player', () => { + expect(frame('https://www.tiktok.com/@user/video/7212345678901234567').embedUrl).toBe( + 'https://www.tiktok.com/player/v1/7212345678901234567' + ) + }) + + it('resolves Spotify tracks, collections, and locale-prefixed urls', () => { + expect(frame('https://open.spotify.com/track/4cOdK2wGLETKBW3PvgPWqT').embedUrl).toBe( + 'https://open.spotify.com/embed/track/4cOdK2wGLETKBW3PvgPWqT' + ) + expect(frame('https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M').provider).toBe('spotify') + expect(frame('https://open.spotify.com/intl-de/album/1DFixLWuPkv3KT3TnV35m3').id).toBe( + 'spotify:album:1DFixLWuPkv3KT3TnV35m3' + ) + expect(detectEmbed('https://open.spotify.com/track/')).toBeNull() + }) +}) + +describe('detectEmbed — maps', () => { + it('resolves Google Maps coordinates with zoom', () => { + const embed = frame('https://www.google.com/maps/@40.7128,-74.0060,12z') + + expect(embed.provider).toBe('googlemaps') + expect(embed.embedUrl).toContain('output=embed') + expect(embed.embedUrl).toContain('q=40.7128%2C-74.006') + expect(embed.embedUrl).toContain('z=12') + }) + + it('resolves a Google Maps place name', () => { + expect(frame('https://www.google.com/maps/place/Eiffel+Tower/').embedUrl).toContain('q=Eiffel+Tower') + }) + + it('resolves OpenStreetMap fragment state to a bbox embed', () => { + const embed = frame('https://www.openstreetmap.org/#map=12/40.7128/-74.0060') + + expect(embed.provider).toBe('openstreetmap') + expect(embed.embedUrl).toContain('export/embed.html') + expect(embed.embedUrl).toContain('marker=40.7128%2C-74.006') + expect(embed.embedUrl).toContain('bbox=') + }) +}) + +describe('detectEmbed — Twitter/X', () => { + it('resolves twitter.com and x.com status urls to a tweet descriptor', () => { + for (const url of ['https://twitter.com/jack/status/20', 'https://x.com/jack/status/20']) { + const descriptor = detectEmbed(url) + + expect(descriptor?.renderer).toBe('tweet') + expect((descriptor as TweetEmbed).tweetId).toBe('20') + } + }) +}) + +describe('detectEmbed — non-matches', () => { + it.each([ + 'https://example.com/watch?v=dQw4w9WgXcQ', + 'https://github.com/NousResearch/hermes', + 'not-a-url', + 'ftp://youtube.com/watch?v=dQw4w9WgXcQ', + 'mailto:someone@youtube.com' + ])('returns null for %s', url => { + expect(detectEmbed(url)).toBeNull() + expect(isEmbeddableUrl(url)).toBe(false) + }) + + it('handles empty input without throwing', () => { + expect(detectEmbed(undefined)).toBeNull() + expect(detectEmbed('')).toBeNull() + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/index.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/index.ts new file mode 100644 index 00000000000..b45983ada0d --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/index.ts @@ -0,0 +1,54 @@ +import { instagram } from './instagram' +import { maps } from './maps' +import { pinterest } from './pinterest' +import { spotify } from './spotify' +import { tiktok } from './tiktok' +import { twitter } from './twitter' +import type { EmbedDescriptor, EmbedMatcher } from './types' +import { vimeo } from './vimeo' +import { youtube } from './youtube' + +export type { EmbedDescriptor, EmbedProvider, EmbedRenderer, FrameEmbed, TweetEmbed } from './types' + +// All provider hosts are disjoint, so order is irrelevant — first match wins. +const MATCHERS: EmbedMatcher[] = [youtube, vimeo, instagram, pinterest, tiktok, twitter, spotify, maps] + +function parseUrl(raw: string): URL | null { + try { + const url = new URL(raw) + + return url.protocol === 'http:' || url.protocol === 'https:' ? url : null + } catch { + return null + } +} + +/** + * Resolve a URL to a rich-embed descriptor, or null when no provider matches. + * Pure and synchronous — safe to call during render. + */ +export function detectEmbed(rawUrl: string | null | undefined): EmbedDescriptor | null { + if (!rawUrl) { + return null + } + + const url = parseUrl(rawUrl) + + if (!url) { + return null + } + + for (const match of MATCHERS) { + const descriptor = match(url) + + if (descriptor) { + return descriptor + } + } + + return null +} + +export function isEmbeddableUrl(rawUrl: string | null | undefined): boolean { + return detectEmbed(rawUrl) !== null +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/instagram.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/instagram.ts new file mode 100644 index 00000000000..3d45577a396 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/instagram.ts @@ -0,0 +1,26 @@ +import { bareHost, type EmbedMatcher } from './types' + +export const instagram: EmbedMatcher = url => { + if (bareHost(url.hostname) !== 'instagram.com') { + return null + } + + const [typeRaw, code] = url.pathname.split('/').filter(Boolean) + const type = typeRaw === 'reels' ? 'reel' : typeRaw + + if (!code || !['p', 'reel', 'tv'].includes(type || '') || !/^[A-Za-z0-9_-]+$/.test(code)) { + return null + } + + return { + embedUrl: `https://www.instagram.com/${type}/${code}/embed`, + // Placeholder height for content-visibility; embed.js self-sizes in-document. + height: 450, + id: `instagram:${code}`, + label: 'Instagram', + maxWidth: 400, + provider: 'instagram', + renderer: 'frame', + sourceUrl: url.toString() + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/maps.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/maps.ts new file mode 100644 index 00000000000..2aa8e615c95 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/maps.ts @@ -0,0 +1,95 @@ +import { bareHost, type EmbedMatcher, type FrameEmbed } from './types' + +// `@lat,lng` (optionally `,z`) as it appears in Google Maps URLs. +const LATLNG_RE = /@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)(?:,(\d+(?:\.\d+)?)z)?/ + +function googleMapsEmbed(url: URL): FrameEmbed | null { + const host = bareHost(url.hostname) + + if (host !== 'google.com' && host !== 'maps.google.com' && !host.startsWith('google.')) { + return null + } + + const isMapsPath = host.startsWith('maps.') || url.pathname.startsWith('/maps') + + if (!isMapsPath) { + return null + } + + // Prefer explicit coordinates; then a `q=` query; then a `/place/`. + const coords = url.pathname.match(LATLNG_RE) + const placeName = url.pathname.match(/\/place\/([^/@]+)/) + const query = url.searchParams.get('q') || url.searchParams.get('query') + let q = '' + let zoom = '' + + if (coords) { + q = `${coords[1]},${coords[2]}` + zoom = coords[3] ? String(Math.round(Number(coords[3]))) : '' + } else if (query) { + q = query + } else if (placeName) { + q = decodeURIComponent(placeName[1].replace(/\+/g, ' ')) + } + + if (!q) { + return null + } + + // `output=embed` is the long-standing keyless Maps embed surface. + const params = new URLSearchParams({ output: 'embed', q }) + + if (zoom) { + params.set('z', zoom) + } + + return { + aspectRatio: 16 / 10, + embedUrl: `https://maps.google.com/maps?${params.toString()}`, + id: `googlemaps:${q}${zoom ? `@${zoom}` : ''}`, + label: 'Google Maps', + maxWidth: 640, + provider: 'googlemaps', + renderer: 'frame', + sourceUrl: url.toString() + } +} + +function openStreetMapEmbed(url: URL): FrameEmbed | null { + if (bareHost(url.hostname) !== 'openstreetmap.org') { + return null + } + + // State lives in the fragment: `#map=//`. + const match = url.hash.match(/map=(\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)/) + + if (!match) { + return null + } + + const zoom = Number(match[1]) + const lat = Number(match[2]) + const lng = Number(match[3]) + // Degrees spanned at this zoom; halved for the bbox half-extent. + const lonDelta = 360 / 2 ** zoom + const latDelta = lonDelta / 2 + + const bbox = [lng - lonDelta / 2, lat - latDelta / 2, lng + lonDelta / 2, lat + latDelta / 2] + .map(value => value.toFixed(5)) + .join(',') + + const params = new URLSearchParams({ bbox, layer: 'mapnik', marker: `${lat},${lng}` }) + + return { + aspectRatio: 16 / 10, + embedUrl: `https://www.openstreetmap.org/export/embed.html?${params.toString()}`, + id: `openstreetmap:${lat},${lng}@${zoom}`, + label: 'OpenStreetMap', + maxWidth: 640, + provider: 'openstreetmap', + renderer: 'frame', + sourceUrl: url.toString() + } +} + +export const maps: EmbedMatcher = url => googleMapsEmbed(url) || openStreetMapEmbed(url) diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/pinterest.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/pinterest.ts new file mode 100644 index 00000000000..f1928247eb4 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/pinterest.ts @@ -0,0 +1,28 @@ +import { bareHost, type EmbedMatcher } from './types' + +export const pinterest: EmbedMatcher = url => { + // Pinterest runs many locale TLDs (pinterest.co.uk, fr.pinterest.com, ...). + if (!bareHost(url.hostname).includes('pinterest.')) { + return null + } + + const segments = url.pathname.split('/').filter(Boolean) + + if (segments[0] !== 'pin' || !/^\d+$/.test(segments[1] || '')) { + return null + } + + const id = segments[1] + + return { + embedUrl: `https://assets.pinterest.com/ext/embed.html?id=${id}`, + // Pinterest's "small" pin size — the default card is too dominant inline. + height: 380, + id: `pinterest:${id}`, + label: 'Pinterest', + maxWidth: 236, + provider: 'pinterest', + renderer: 'frame', + sourceUrl: url.toString() + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/spotify.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/spotify.ts new file mode 100644 index 00000000000..526bcdc4949 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/spotify.ts @@ -0,0 +1,34 @@ +import { bareHost, type EmbedMatcher } from './types' + +// Spotify's embed has only two layouts: compact (≤152) and full (352). Any +// in-between height renders the compact player and pads the rest with grey, so +// we snap to the compact size for every type — tight, no dead space. +const COMPACT_HEIGHT = 152 +const EMBED_TYPES = new Set(['album', 'artist', 'episode', 'playlist', 'show', 'track']) + +export const spotify: EmbedMatcher = url => { + if (bareHost(url.hostname) !== 'open.spotify.com') { + return null + } + + // Drop an optional locale prefix (`/intl-de/track/...`). + const segments = url.pathname.split('/').filter(Boolean) + const start = segments[0]?.startsWith('intl-') ? 1 : 0 + const type = segments[start] || '' + const id = segments[start + 1] || '' + + if (!EMBED_TYPES.has(type) || !/^[A-Za-z0-9]+$/.test(id)) { + return null + } + + return { + embedUrl: `https://open.spotify.com/embed/${type}/${id}`, + height: COMPACT_HEIGHT, + id: `spotify:${type}:${id}`, + label: 'Spotify', + maxWidth: 480, + provider: 'spotify', + renderer: 'frame', + sourceUrl: url.toString() + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/tiktok.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/tiktok.ts new file mode 100644 index 00000000000..a6ea3179ac5 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/tiktok.ts @@ -0,0 +1,28 @@ +import { bareHost, type EmbedMatcher } from './types' + +export const tiktok: EmbedMatcher = url => { + if (bareHost(url.hostname) !== 'tiktok.com') { + return null + } + + const segments = url.pathname.split('/').filter(Boolean) + const videoIndex = segments.indexOf('video') + const id = videoIndex >= 0 ? segments[videoIndex + 1] : '' + + if (!/^\d+$/.test(id || '')) { + return null + } + + return { + // The official player is a clean dark video iframe (no white blockquote + // chrome), so it goes through the plain-iframe frame path, sized 9:16. + aspectRatio: 9 / 16, + embedUrl: `https://www.tiktok.com/player/v1/${id}`, + id: `tiktok:${id}`, + label: 'TikTok', + maxWidth: 365, + provider: 'tiktok', + renderer: 'frame', + sourceUrl: url.toString() + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/twitter.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/twitter.ts new file mode 100644 index 00000000000..938c1a021b7 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/twitter.ts @@ -0,0 +1,27 @@ +import { bareHost, type EmbedMatcher } from './types' + +export const twitter: EmbedMatcher = url => { + const host = bareHost(url.hostname) + + if (host !== 'twitter.com' && host !== 'x.com') { + return null + } + + const segments = url.pathname.split('/').filter(Boolean) + const statusIndex = segments.indexOf('status') + const id = statusIndex >= 0 ? segments[statusIndex + 1] : '' + + if (!/^\d+$/.test(id || '')) { + return null + } + + return { + id: `twitter:${id}`, + label: 'X', + maxWidth: 480, + provider: 'twitter', + renderer: 'tweet', + sourceUrl: url.toString(), + tweetId: id + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/types.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/types.ts new file mode 100644 index 00000000000..e9f4ce381d3 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/types.ts @@ -0,0 +1,60 @@ +// Embed provider model. Detection is pure, synchronous, and dependency-free so +// it is safe to run during render and trivial to unit-test. Rendering lives in +// the lazy renderers (see ../registry.tsx) keyed off `renderer`. + +export type EmbedProvider = + | 'googlemaps' + | 'instagram' + | 'openstreetmap' + | 'pinterest' + | 'spotify' + | 'tiktok' + | 'twitter' + | 'vimeo' + | 'youtube' + +/** Which lazy renderer materialises the descriptor. */ +export type EmbedRenderer = 'frame' | 'tweet' + +interface EmbedLayout { + /** Frame aspect ratio (width / height). For video/maps. */ + aspectRatio?: number + /** Fixed pixel height for non-ratio embeds (Instagram, Pinterest, Spotify). */ + height?: number + /** Max rendered width in px; falls back to the conversation column. */ + maxWidth?: number +} + +interface BaseEmbed extends EmbedLayout { + /** Stable id for React keys / dedupe. */ + id: string + /** Human-facing provider name (e.g. "YouTube"). */ + label: string + provider: EmbedProvider + renderer: EmbedRenderer + /** Canonical URL opened in the system browser from the card. */ + sourceUrl: string +} + +/** A provider whose embed is a single iframe URL (video, post, map, ...). */ +export interface FrameEmbed extends BaseEmbed { + /** URL loaded inside the iframe. */ + embedUrl: string + renderer: 'frame' +} + +/** Twitter/X ships no iframe URL — only a widget script (see social-embed.tsx). */ +export interface TweetEmbed extends BaseEmbed { + renderer: 'tweet' + tweetId: string +} + +export type EmbedDescriptor = FrameEmbed | TweetEmbed + +/** A provider matcher. Receives a parsed http(s) URL; returns null if unmatched. */ +export type EmbedMatcher = (url: URL) => EmbedDescriptor | null + +/** Strip a leading `www.`/`m.`/`mobile.` so host checks read cleanly. */ +export function bareHost(host: string): string { + return host.replace(/^(?:www|m|mobile)\./i, '').toLowerCase() +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/vimeo.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/vimeo.ts new file mode 100644 index 00000000000..1ef73ff78e6 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/vimeo.ts @@ -0,0 +1,32 @@ +import { bareHost, type EmbedMatcher } from './types' + +export const vimeo: EmbedMatcher = url => { + const host = bareHost(url.hostname) + + if (host !== 'vimeo.com' && host !== 'player.vimeo.com') { + return null + } + + // The clip id is the last all-digits segment, covering vimeo.com/123, + // /channels/x/123, /groups/x/videos/123, and player/video/123. + const id = url.pathname + .split('/') + .filter(Boolean) + .reverse() + .find(segment => /^\d+$/.test(segment)) + + if (!id) { + return null + } + + return { + aspectRatio: 16 / 9, + embedUrl: `https://player.vimeo.com/video/${id}`, + id: `vimeo:${id}`, + label: 'Vimeo', + maxWidth: 640, + provider: 'vimeo', + renderer: 'frame', + sourceUrl: url.toString() + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/providers/youtube.ts b/apps/desktop/src/components/assistant-ui/embeds/providers/youtube.ts new file mode 100644 index 00000000000..40444540e89 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/providers/youtube.ts @@ -0,0 +1,65 @@ +import { bareHost, type EmbedMatcher } from './types' + +const YOUTUBE_ID_RE = /^[A-Za-z0-9_-]{11}$/ + +// `t`/`start` accept either raw seconds ("90") or the "1m30s" form. +function startSeconds(value: string | null): number | undefined { + if (!value) { + return undefined + } + + if (/^\d+$/.test(value)) { + return Number(value) + } + + const match = value.match(/^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/) + + if (!match || !match[0]) { + return undefined + } + + const seconds = Number(match[1] || 0) * 3600 + Number(match[2] || 0) * 60 + Number(match[3] || 0) + + return seconds > 0 ? seconds : undefined +} + +export const youtube: EmbedMatcher = url => { + const host = bareHost(url.hostname) + const segments = url.pathname.split('/').filter(Boolean) + let id = '' + + if (host === 'youtu.be') { + id = segments[0] || '' + } else if (host === 'youtube.com' || host === 'youtube-nocookie.com') { + if (segments[0] === 'watch') { + id = url.searchParams.get('v') || '' + } else if (['embed', 'shorts', 'live', 'v'].includes(segments[0] || '')) { + id = segments[1] || '' + } + } else { + return null + } + + if (!YOUTUBE_ID_RE.test(id)) { + return null + } + + const params = new URLSearchParams({ modestbranding: '1', rel: '0' }) + + const start = startSeconds(url.searchParams.get('t') || url.searchParams.get('start')) + + if (start) { + params.set('start', String(start)) + } + + return { + aspectRatio: 16 / 9, + embedUrl: `https://www.youtube-nocookie.com/embed/${id}?${params.toString()}`, + id: `youtube:${id}`, + label: 'YouTube', + maxWidth: 640, + provider: 'youtube', + renderer: 'frame', + sourceUrl: url.toString() + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/rich-boundary.tsx b/apps/desktop/src/components/assistant-ui/embeds/rich-boundary.tsx new file mode 100644 index 00000000000..ee5c5b313a9 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/rich-boundary.tsx @@ -0,0 +1,34 @@ +import { Component, type ReactNode } from 'react' + +interface Props { + children: ReactNode + /** Rendered in place of the subtree when a render throws. */ + fallback: ReactNode + /** Changing this clears a caught error (e.g. new source for a re-parse). */ + resetKey?: string +} + +/** + * Local boundary for rich renderers (Mermaid parse throws, malformed SVG, a + * provider widget blowing up). A failed embed must never blank the transcript — + * we show the `fallback` (typically the raw source) and recover when `resetKey` + * changes. Unlike MessageRenderBoundary this swallows ALL render errors, because + * the blast radius is one self-contained block, not the message tree. + */ +export class RichBoundary extends Component { + state = { failed: false } + + static getDerivedStateFromError() { + return { failed: true } + } + + componentDidUpdate(prev: Props) { + if (this.state.failed && prev.resetKey !== this.props.resetKey) { + this.setState({ failed: false }) + } + } + + render() { + return this.state.failed ? this.props.fallback : this.props.children + } +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/types.ts b/apps/desktop/src/components/assistant-ui/embeds/types.ts new file mode 100644 index 00000000000..7457bcd9e21 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/types.ts @@ -0,0 +1,8 @@ +// Shared prop contract for fenced-block renderers (mermaid, svg). Kept in its +// own module so renderers and the registry can both import it without a cycle. +export interface RichFenceProps { + code: string + /** True while the surrounding message is still streaming. Renderers that can + * throw on partial input (e.g. mermaid) defer until this is false. */ + streaming?: boolean +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/use-is-dark.ts b/apps/desktop/src/components/assistant-ui/embeds/use-is-dark.ts new file mode 100644 index 00000000000..178a0c0fd5c --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/use-is-dark.ts @@ -0,0 +1,24 @@ +import { useEffect, useState } from 'react' + +// Tracks the app's dark/light mode off the `dark` class on (set by +// themes/context.tsx). Embeds that theme their own content (tweets) read this. +export function useIsDark(): boolean { + const [dark, setDark] = useState( + () => typeof document !== 'undefined' && document.documentElement.classList.contains('dark') + ) + + useEffect(() => { + if (typeof document === 'undefined') { + return + } + + const root = document.documentElement + const observer = new MutationObserver(() => setDark(root.classList.contains('dark'))) + + observer.observe(root, { attributeFilter: ['class'], attributes: true }) + + return () => observer.disconnect() + }, []) + + return dark +} diff --git a/package-lock.json b/package-lock.json index 070ed298f99..85e5679784d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -95,11 +95,13 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "dnd-core": "^14.0.1", + "dompurify": "^3.4.11", "hast-util-from-html-isomorphic": "^2.0.0", "hast-util-to-text": "^4.0.2", "ignore": "^7.0.5", "katex": "^0.16.45", "leva": "^0.10.1", + "mermaid": "^11.15.0", "motion": "^12.38.0", "nanostores": "^1.3.0", "node-pty": "1.1.0", From 0c190083cd9a7a88d67229b746d7e7efc1bfb527 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Fri, 26 Jun 2026 03:22:08 -0500 Subject: [PATCH 2/5] feat(desktop): lazy embed renderers + fenced diagrams/alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-kind renderers, each a lazy split chunk: plain-iframe video/maps (wheel chains to the transcript; maps gate scroll behind ⌘), the in-document blockquote-script path for X/Instagram, the dark Spotify player, and the YouTube iframe. Adds Mermaid and DOMPurify-sanitised SVG fences and GFM alert callouts, all sized to 33dvh and theme-matched to avoid white color-scheme artifacts. Main-process stamps a Referer on YouTube embed requests. --- apps/desktop/electron/embed-referer.cjs | 48 +++++++ apps/desktop/electron/main.cjs | 2 + .../assistant-ui/embeds/alert.test.tsx | 39 ++++++ .../components/assistant-ui/embeds/alert.tsx | 125 +++++++++++++++++ .../assistant-ui/embeds/frame-embed.tsx | 55 ++++++++ .../assistant-ui/embeds/mermaid-embed.tsx | 97 +++++++++++++ .../assistant-ui/embeds/registry.tsx | 39 ++++++ .../assistant-ui/embeds/scroll-gate.tsx | 33 +++++ .../assistant-ui/embeds/social-embed.tsx | 131 ++++++++++++++++++ .../assistant-ui/embeds/spotify-embed.tsx | 45 ++++++ .../assistant-ui/embeds/svg-embed.tsx | 32 +++++ .../assistant-ui/embeds/url-embed.tsx | 64 +++++++++ .../assistant-ui/embeds/youtube-embed.tsx | 47 +++++++ 13 files changed, 757 insertions(+) create mode 100644 apps/desktop/electron/embed-referer.cjs create mode 100644 apps/desktop/src/components/assistant-ui/embeds/alert.test.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/alert.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/frame-embed.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/mermaid-embed.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/registry.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/scroll-gate.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/social-embed.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/spotify-embed.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/svg-embed.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/url-embed.tsx create mode 100644 apps/desktop/src/components/assistant-ui/embeds/youtube-embed.tsx diff --git a/apps/desktop/electron/embed-referer.cjs b/apps/desktop/electron/embed-referer.cjs new file mode 100644 index 00000000000..2825eda40e7 --- /dev/null +++ b/apps/desktop/electron/embed-referer.cjs @@ -0,0 +1,48 @@ +'use strict' + +const { session } = require('electron') + +const EMBED_SESSION_PARTITION = 'persist:hermes-embed' +const EMBED_REFERER = 'https://www.youtube.com/' +const YOUTUBE_REFERER_HOST_RE = + /(^|\.)(youtube\.com|youtube-nocookie\.com|googlevideo\.com|ytimg\.com|youtubei\.googleapis\.com)$/i + +function installEmbedRefererForSession(embedSession) { + if (!embedSession) { + return + } + + embedSession.webRequest.onBeforeSendHeaders((details, callback) => { + let host = '' + + try { + host = new URL(details.url).hostname + } catch { + host = '' + } + + if (!YOUTUBE_REFERER_HOST_RE.test(host)) { + callback({ requestHeaders: details.requestHeaders }) + return + } + + const headers = { ...details.requestHeaders } + + if (!headers.Referer && !headers.referer) { + headers.Referer = EMBED_REFERER + } + + callback({ requestHeaders: headers }) + }) +} + +/** Stamp Referer on YouTube requests in the embed webview partition only. */ +function installEmbedReferer() { + try { + installEmbedRefererForSession(session.fromPartition(EMBED_SESSION_PARTITION)) + } catch { + // Non-fatal: embeds still render; YouTube may show referer errors. + } +} + +module.exports = { installEmbedReferer } diff --git a/apps/desktop/electron/main.cjs b/apps/desktop/electron/main.cjs index d7184bc9743..5d859bca649 100644 --- a/apps/desktop/electron/main.cjs +++ b/apps/desktop/electron/main.cjs @@ -24,6 +24,7 @@ const https = require('node:https') const path = require('node:path') const { pathToFileURL } = require('node:url') const { execFileSync, spawn } = require('node:child_process') +const { installEmbedReferer } = require('./embed-referer.cjs') const { detectRemoteDisplay, isWindowsBinaryPathInWsl, isWslEnvironment } = require('./bootstrap-platform.cjs') const { runBootstrap } = require('./bootstrap-runner.cjs') const { @@ -7447,6 +7448,7 @@ app.whenReady().then(() => { } installMediaPermissions() registerMediaProtocol() + installEmbedReferer() registerDeepLinkProtocol() ensureWslWindowsFonts() configureSpellChecker() diff --git a/apps/desktop/src/components/assistant-ui/embeds/alert.test.tsx b/apps/desktop/src/components/assistant-ui/embeds/alert.test.tsx new file mode 100644 index 00000000000..b003b914a89 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/alert.test.tsx @@ -0,0 +1,39 @@ +import { createElement } from 'react' +import { describe, expect, it } from 'vitest' + +import { extractAlert } from './alert' + +describe('extractAlert', () => { + it('detects each GFM alert kind from the leading marker', () => { + for (const [marker, type] of [ + ['[!NOTE]', 'note'], + ['[!TIP]', 'tip'], + ['[!IMPORTANT]', 'important'], + ['[!WARNING]', 'warning'], + ['[!CAUTION]', 'caution'] + ] as const) { + const node = createElement('p', null, `${marker}\nBody text`) + const result = extractAlert(node) + + expect(result?.type).toBe(type) + } + }) + + it('is case-insensitive on the marker', () => { + expect(extractAlert(createElement('p', null, '[!note] hi'))?.type).toBe('note') + }) + + it('returns null for a plain blockquote', () => { + expect(extractAlert(createElement('p', null, 'just a quote'))).toBeNull() + expect(extractAlert('no marker here')).toBeNull() + }) + + it('strips the marker token from the body', () => { + const result = extractAlert(createElement('p', null, '[!WARNING]\nDanger ahead')) + + expect(result).not.toBeNull() + // The marker must not survive into the rendered body. + expect(JSON.stringify(result?.body)).not.toContain('[!WARNING]') + expect(JSON.stringify(result?.body)).toContain('Danger ahead') + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/embeds/alert.tsx b/apps/desktop/src/components/assistant-ui/embeds/alert.tsx new file mode 100644 index 00000000000..d2c55e43d71 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/alert.tsx @@ -0,0 +1,125 @@ +import { cloneElement, isValidElement, type ReactNode } from 'react' + +import { AlertCircle, AlertTriangle, type IconComponent, Info, Zap } from '@/lib/icons' +import { cn } from '@/lib/utils' + +export type AlertType = 'caution' | 'important' | 'note' | 'tip' | 'warning' + +interface AlertStyle { + accent: string + icon: IconComponent + label: string +} + +// GitHub's five alert kinds, mapped to our icon set + a tinted accent. +const ALERT_STYLES: Record = { + caution: { accent: 'text-rose-600 dark:text-rose-400', icon: AlertTriangle, label: 'Caution' }, + important: { accent: 'text-violet-600 dark:text-violet-400', icon: AlertCircle, label: 'Important' }, + note: { accent: 'text-blue-600 dark:text-blue-400', icon: Info, label: 'Note' }, + tip: { accent: 'text-emerald-600 dark:text-emerald-400', icon: Zap, label: 'Tip' }, + warning: { accent: 'text-amber-600 dark:text-amber-400', icon: AlertTriangle, label: 'Warning' } +} + +const MARKER_RE = /^\s*\[!(note|tip|important|warning|caution)\]\s*\n?/i + +function firstText(node: ReactNode): string { + if (typeof node === 'string') { + return node + } + + if (typeof node === 'number') { + return String(node) + } + + if (Array.isArray(node)) { + for (const child of node) { + const text = firstText(child) + + if (text.trim()) { + return text + } + } + + return '' + } + + if (isValidElement(node)) { + return firstText((node.props as { children?: ReactNode }).children) + } + + return '' +} + +// Remove the leading `[!TYPE]` token from the first text node that carries it, +// leaving the rest of the blockquote body intact. One-shot via the `state` flag. +function stripMarker(node: ReactNode, state: { done: boolean }): ReactNode { + if (state.done) { + return node + } + + if (typeof node === 'string') { + const replaced = node.replace(MARKER_RE, '') + + if (replaced !== node) { + state.done = true + + return replaced + } + + return node + } + + if (Array.isArray(node)) { + return node.map((child, index) => ) + } + + if (isValidElement(node)) { + const children = (node.props as { children?: ReactNode }).children + + if (children == null) { + return node + } + + return cloneElement(node, undefined, stripMarker(children, state)) + } + + return node +} + +// Tiny helper so the array branch can return keyed nodes without wrapping +// strings in extra elements (React renders the raw node). +function Fragmentless({ node }: { node: ReactNode }) { + return <>{node} +} + +/** + * Detect a GitHub-style alert blockquote (`> [!NOTE]`). Returns the alert kind + * and the body with the marker stripped, or null for a plain blockquote. + */ +export function extractAlert(children: ReactNode): { body: ReactNode; type: AlertType } | null { + const match = firstText(children).match(MARKER_RE) + + if (!match) { + return null + } + + return { body: stripMarker(children, { done: false }), type: match[1].toLowerCase() as AlertType } +} + +export function MarkdownAlert({ children, type }: { children: ReactNode; type: AlertType }) { + const style = ALERT_STYLES[type] + const Icon = style.icon + + return ( +
+
+ + {style.label} +
+ {children} +
+ ) +} diff --git a/apps/desktop/src/components/assistant-ui/embeds/frame-embed.tsx b/apps/desktop/src/components/assistant-ui/embeds/frame-embed.tsx new file mode 100644 index 00000000000..7d99c0dd7a2 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/embeds/frame-embed.tsx @@ -0,0 +1,55 @@ +'use client' + +import { type CSSProperties } from 'react' + +import type { FrameEmbed } from './providers/types' +import { ScrollGate } from './scroll-gate' +import { useIsDark } from './use-is-dark' + +const ALLOW = 'autoplay; encrypted-media; picture-in-picture; clipboard-write; fullscreen' + +// Plain iframes (not webviews): a non-scrollable cross-origin iframe lets the +// wheel chain to the transcript instead of capturing it. Maps are the one +// exception — they're interactive, so a ScrollGate blocks them until ⌘ is held. +export default function FrameEmbedRenderer({ descriptor }: { descriptor: FrameEmbed }) { + const isDark = useIsDark() + const isMap = descriptor.provider === 'googlemaps' || descriptor.provider === 'openstreetmap' + // color-scheme makes the iframe's default (unpainted) backdrop follow the + // theme instead of flashing white at the corners / during load. + const colorScheme = isDark ? 'dark' : 'light' + + const style: CSSProperties = descriptor.aspectRatio + ? { aspectRatio: descriptor.aspectRatio, colorScheme } + : { colorScheme, height: descriptor.height } + + if (isMap) { + return ( +
+