diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 6cc000a1fa0..ecd47ab2943 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -900,6 +900,11 @@ class ManagedFileUpload(BaseModel): overwrite: bool = True +class ChatImageUpload(BaseModel): + data_url: str + filename: Optional[str] = None + + class ManagedDirectoryCreate(BaseModel): path: str @@ -1766,6 +1771,91 @@ def _decode_data_url(data_url: str) -> tuple[bytes, str]: return data, mime_type +_CHAT_IMAGE_UPLOAD_MAX_BYTES = 25 * 1024 * 1024 +_CHAT_IMAGE_ALLOWED_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}) +_CHAT_IMAGE_MAGIC: tuple[tuple[bytes, str], ...] = ( + (b"\x89PNG\r\n\x1a\n", ".png"), + (b"\xff\xd8\xff", ".jpg"), + (b"GIF87a", ".gif"), + (b"GIF89a", ".gif"), + (b"BM", ".bmp"), +) + + +def _sanitize_chat_image_filename(filename: str | None) -> str: + candidate = Path(str(filename or "").strip()).name + candidate = re.sub(r"[\x00-\x1f]+", "_", candidate) + candidate = candidate.strip().strip(".") + return candidate or "pasted-image" + + +def _chat_image_extension(data: bytes) -> str | None: + head = data[:16] + if head.startswith(b"RIFF") and head[8:12] == b"WEBP": + return ".webp" + for sig, ext in _CHAT_IMAGE_MAGIC: + if head.startswith(sig): + return ext + return None + + +def _decode_chat_image_upload(payload: ChatImageUpload) -> tuple[bytes, str, str]: + data, mime_type = _decode_data_url(payload.data_url) + if not mime_type.lower().startswith("image/"): + raise HTTPException(status_code=400, detail="Upload payload must be an image") + if len(data) > _CHAT_IMAGE_UPLOAD_MAX_BYTES: + mb = _CHAT_IMAGE_UPLOAD_MAX_BYTES // (1024 * 1024) + raise HTTPException(status_code=413, detail=f"Image is too large; cap is {mb} MB") + + ext = _chat_image_extension(data) + if ext not in _CHAT_IMAGE_ALLOWED_EXTENSIONS: + raise HTTPException(status_code=400, detail="Unsupported image type") + return data, mime_type, ext + + +@app.post("/api/chat/image-upload") +async def upload_chat_image(payload: ChatImageUpload, profile: Optional[str] = None): + """Persist a browser-provided chat image where the embedded TUI can read it. + + The dashboard /chat page runs Hermes inside an xterm.js PTY. Browser + clipboard image bytes are not visible to the server-side clipboard, so the + page uploads them here, then drives the TUI's ``/image `` command + with the returned gateway-visible path. Files land under + ``HERMES_HOME/images/`` — the same directory ``clipboard.paste`` / + ``image.attach`` already use. + """ + data, mime_type, ext = _decode_chat_image_upload(payload) + with _profile_scope(profile) as scoped_home: + home = scoped_home or get_hermes_home() + img_dir = Path(home) / "images" + try: + img_dir.mkdir(parents=True, exist_ok=True) + except PermissionError: + raise HTTPException(status_code=403, detail="Image directory is not writable") + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Could not create image directory: {exc}") + + stem = Path(_sanitize_chat_image_filename(payload.filename)).stem or "pasted-image" + stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", stem).strip("._-") or "pasted-image" + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + target = img_dir / f"dashboard_{ts}_{secrets.token_hex(4)}_{stem}{ext}" + + try: + target.write_bytes(data) + except PermissionError: + raise HTTPException(status_code=403, detail="Image directory is not writable") + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Could not write image: {exc}") + + return { + "ok": True, + "path": str(target), + "name": target.name, + "bytes": len(data), + "mime_type": mime_type, + } + + @app.get("/api/files") async def list_managed_files(request: Request, path: Optional[str] = None): policy, target, display_path = _resolve_managed_path(path, request) diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 519ccf1518e..8caffe2dbdb 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -863,6 +863,107 @@ class TestWebServerEndpoints: ) assert resp.status_code == 401 + # ── POST /api/chat/image-upload (browser clipboard/drop images) ───── + + def test_chat_image_upload_writes_to_default_profile_images(self): + from hermes_constants import get_hermes_home + + data_url = ( + "data:image/png;base64," + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk" + "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + + resp = self.client.post( + "/api/chat/image-upload", + json={"data_url": data_url, "filename": "../../clip.png"}, + ) + + assert resp.status_code == 200 + data = resp.json() + target = Path(data["path"]) + assert data["ok"] is True + assert data["mime_type"] == "image/png" + assert target.parent == get_hermes_home() / "images" + assert target.name.startswith("dashboard_") + assert target.name.endswith("_clip.png") + assert target.is_file() + assert target.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") + + def test_chat_image_upload_writes_to_requested_profile_images(self): + from hermes_cli import profiles as profiles_mod + + worker_home = profiles_mod.get_profile_dir("worker") + worker_home.mkdir(parents=True) + + resp = self.client.post( + "/api/chat/image-upload?profile=worker", + json={ + "data_url": "data:image/gif;base64,R0lGODlhAQABAAAAACwAAAAAAQABAAA=", + "filename": "drop.gif", + }, + ) + + assert resp.status_code == 200 + target = Path(resp.json()["path"]) + assert target.parent == worker_home / "images" + assert target.is_file() + assert target.read_bytes().startswith(b"GIF89a") + + def test_chat_image_upload_rejects_non_image_payload(self): + resp = self.client.post( + "/api/chat/image-upload", + json={"data_url": "data:text/plain;base64,aGVsbG8="}, + ) + + assert resp.status_code == 400 + assert "image" in resp.json()["detail"].lower() + + def test_chat_image_upload_rejects_spoofed_image_payload(self): + resp = self.client.post( + "/api/chat/image-upload", + json={"data_url": "data:image/png;base64,aGVsbG8=", "filename": "fake.png"}, + ) + + assert resp.status_code == 400 + assert "unsupported image type" in resp.json()["detail"].lower() + + def test_chat_image_upload_rejects_unknown_profile(self): + resp = self.client.post( + "/api/chat/image-upload?profile=missing-profile", + json={"data_url": "data:image/gif;base64,R0lGODlhAQABAAAAACwAAAAAAQABAAA="}, + ) + + assert resp.status_code == 404 + assert "does not exist" in resp.json()["detail"] + + def test_chat_image_upload_enforces_image_size_cap(self, monkeypatch): + import hermes_cli.web_server as web_server + + monkeypatch.setattr(web_server, "_CHAT_IMAGE_UPLOAD_MAX_BYTES", 4) + + resp = self.client.post( + "/api/chat/image-upload", + json={ + "data_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=", + "filename": "large.png", + }, + ) + + assert resp.status_code == 413 + assert "too large" in resp.json()["detail"].lower() + + def test_chat_image_upload_requires_auth(self): + from hermes_cli.web_server import _SESSION_HEADER_NAME + + resp = self.client.post( + "/api/chat/image-upload", + json={"data_url": "data:image/gif;base64,R0lGODlhAQABAAAAACwAAAAAAQABAAA="}, + headers={_SESSION_HEADER_NAME: "wrong-token"}, + ) + + assert resp.status_code == 401 + # ── Dashboard font override ───────────────────────────────────────── def test_get_dashboard_font_defaults_to_theme(self): diff --git a/web/src/lib/chatImagePaste.test.ts b/web/src/lib/chatImagePaste.test.ts new file mode 100644 index 00000000000..047dec17528 --- /dev/null +++ b/web/src/lib/chatImagePaste.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; + +import { + firstImageFromClipboard, + imageFilesFromTransfer, + transferMayContainImage, +} from "./chatImagePaste"; + +// Minimal DataTransfer stand-ins. jsdom's DataTransfer doesn't let us seed +// items/files, so we hand-roll the shape the helpers read. +function makeItem(kind: string, type: string, file: File | null) { + return { kind, type, getAsFile: () => file } as unknown as DataTransferItem; +} + +function makeData(opts: { + items?: DataTransferItem[]; + files?: File[]; +}): DataTransfer { + const items = opts.items ?? []; + const files = opts.files ?? []; + const itemList: Record = { length: items.length }; + items.forEach((it, i) => { + itemList[i] = it; + }); + const fileList: Record = { length: files.length }; + files.forEach((f, i) => { + fileList[i] = f; + }); + return { + items: itemList, + files: fileList, + } as unknown as DataTransfer; +} + +const png = new File([new Uint8Array([1, 2, 3])], "x.png", { + type: "image/png", +}); +const gif = new File([new Uint8Array([4, 5])], "y.gif", { + type: "image/gif", +}); + +describe("firstImageFromClipboard", () => { + it("returns null for null clipboard data", () => { + expect(firstImageFromClipboard(null)).toBeNull(); + }); + + it("finds an image via items[].getAsFile()", () => { + const data = makeData({ items: [makeItem("file", "image/png", png)] }); + expect(firstImageFromClipboard(data)).toBe(png); + }); + + it("ignores non-file and non-image items", () => { + const data = makeData({ + items: [ + makeItem("string", "text/plain", null), + makeItem("file", "application/pdf", new File([], "a.pdf")), + ], + }); + expect(firstImageFromClipboard(data)).toBeNull(); + }); + + it("falls back to files[] when items are absent (Safari/Firefox)", () => { + const data = makeData({ files: [png] }); + expect(firstImageFromClipboard(data)).toBe(png); + }); + + it("returns null when nothing image-like is present", () => { + const data = makeData({ + files: [new File([], "notes.txt", { type: "text/plain" })], + }); + expect(firstImageFromClipboard(data)).toBeNull(); + }); +}); + +describe("imageFilesFromTransfer", () => { + it("dedupes the same file when present in both items and files", () => { + const data = makeData({ + items: [makeItem("file", "image/png", png)], + files: [png, gif], + }); + expect(imageFilesFromTransfer(data)).toEqual([png, gif]); + }); +}); + +describe("transferMayContainImage", () => { + it("is true for image items even when type is empty (some browsers)", () => { + const data = makeData({ + items: [makeItem("file", "", png)], + }); + expect(transferMayContainImage(data)).toBe(true); + }); + + it("is false for text-only transfers", () => { + const data = makeData({ + items: [makeItem("string", "text/plain", null)], + }); + expect(transferMayContainImage(data)).toBe(false); + }); +}); diff --git a/web/src/lib/chatImagePaste.ts b/web/src/lib/chatImagePaste.ts new file mode 100644 index 00000000000..3767cf80374 --- /dev/null +++ b/web/src/lib/chatImagePaste.ts @@ -0,0 +1,164 @@ +import { authedFetch } from "@/lib/api"; + +// Clipboard image MIME → file extension. Mirrors the set the TUI's /image +// attach path and the gateway's image sniffer accept. +const IMAGE_MIME_EXT: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/webp": "webp", + "image/bmp": "bmp", +}; + +// Anthropic caps a single vision image at ~25 MB; reject earlier client-side +// with a clear message rather than round-tripping a doomed upload. +const MAX_IMAGE_BYTES = 25 * 1024 * 1024; + +export interface ChatImageUploadResult { + /** Absolute path under HERMES_HOME/images the gateway wrote. */ + path: string; + /** Byte size of the uploaded image. */ + bytes: number; + /** Basename written on the server. */ + name: string; + mime_type: string; +} + +function imageFileKey(file: File): string { + return `${file.name}\0${file.type}\0${file.size}\0${file.lastModified}`; +} + +function addImageFile(files: File[], seen: Set, file: File | null) { + if (!file || !file.type.startsWith("image/")) return; + const key = imageFileKey(file); + if (seen.has(key)) return; + seen.add(key); + files.push(file); +} + +/** Pull every image file out of a DataTransfer (clipboard or drop). */ +export function imageFilesFromTransfer( + data: DataTransfer | null, +): File[] { + if (!data) return []; + const files: File[] = []; + const seen = new Set(); + + if (data.items?.length) { + for (let i = 0; i < data.items.length; i++) { + const item = data.items[i]; + if (item.kind === "file" && item.type.startsWith("image/")) { + addImageFile(files, seen, item.getAsFile()); + } + } + } + + if (data.files?.length) { + for (let i = 0; i < data.files.length; i++) { + addImageFile(files, seen, data.files[i]); + } + } + + return files; +} + +/** Pull the first image blob out of a DataTransfer, or null if none present. */ +export function firstImageFromClipboard( + data: DataTransfer | null, +): File | null { + return imageFilesFromTransfer(data)[0] ?? null; +} + +/** True when a drag payload may contain an image (for dragover preventDefault). */ +export function transferMayContainImage(data: DataTransfer | null): boolean { + if (!data) return false; + if (data.items?.length) { + for (let i = 0; i < data.items.length; i++) { + const item = data.items[i]; + if ( + item.kind === "file" && + (!item.type || item.type.startsWith("image/")) + ) { + return true; + } + } + return false; + } + if (data.files?.length) { + for (let i = 0; i < data.files.length; i++) { + if (data.files[i].type.startsWith("image/")) return true; + } + } + return false; +} + +function fileToDataUrl(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => + reject(reader.error ?? new Error("image read failed")); + reader.onload = () => { + const result = reader.result; + if (typeof result === "string") { + resolve(result); + } else { + reject(new Error("image read failed")); + } + }; + reader.readAsDataURL(file); + }); +} + +/** + * Upload a browser clipboard/drop image to ``HERMES_HOME/images`` via the + * dedicated chat upload endpoint and return the absolute gateway path. + * + * The dashboard Chat tab is an xterm mirror of a TUI running INSIDE the + * gateway. The container has no access to the browser's clipboard, so the + * server-side ``clipboard.paste`` path can never see a pasted image. + * Upload the bytes the browser already holds, then hand the path to the + * TUI's ``/image`` command. + */ +export async function uploadChatImage( + blob: Blob, + profile = "", +): Promise { + if (blob.size === 0) throw new Error("clipboard image is empty"); + if (blob.size > MAX_IMAGE_BYTES) { + const mb = Math.round(MAX_IMAGE_BYTES / (1024 * 1024)); + throw new Error(`image too large (max ${mb} MB)`); + } + + const mime = blob.type || "image/png"; + const ext = IMAGE_MIME_EXT[mime] || "png"; + const filename = + blob instanceof File && blob.name + ? blob.name + : `clipboard.${ext}`; + const file = + blob instanceof File + ? blob + : new File([blob], filename, { type: mime }); + + const dataUrl = await fileToDataUrl(file); + const qs = profile ? `?profile=${encodeURIComponent(profile)}` : ""; + const res = await authedFetch(`/api/chat/image-upload${qs}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + data_url: dataUrl, + filename, + }), + }); + + if (!res.ok) { + const text = await res.text().catch(() => res.statusText); + throw new Error(text || `HTTP ${res.status}`); + } + + const uploaded = (await res.json()) as ChatImageUploadResult; + if (!uploaded?.path) { + throw new Error("image upload did not return a path"); + } + return uploaded; +} diff --git a/web/src/pages/ChatPage.tsx b/web/src/pages/ChatPage.tsx index 68f9221aaa4..dc65af6ca7c 100644 --- a/web/src/pages/ChatPage.tsx +++ b/web/src/pages/ChatPage.tsx @@ -49,6 +49,11 @@ import { normalizePtyMobileInput, shouldTreatInputAsMobileReplacement, } from "@/lib/pty-mobile-input"; +import { + imageFilesFromTransfer, + transferMayContainImage, + uploadChatImage, +} from "@/lib/chatImagePaste"; import { PluginSlot } from "@/plugins"; import { useTheme } from "@/themes"; import { useProfileScope } from "@/contexts/useProfileScope"; @@ -486,7 +491,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { // --- Clipboard integration --------------------------------------- // - // Three independent paths all route to the system clipboard: + // Four independent paths all route to the system clipboard: // // 1. **Selection → Ctrl+C (or Cmd+C on macOS).** Ink's own handler // in useInputHandlers.ts turns Ctrl+C into a copy when the @@ -500,9 +505,15 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { // ever stops listening (e.g. overlays / pickers) or if the user // has selected with the mouse outside of Ink's selection model. // - // 3. **Ctrl/Cmd+Shift+V.** Reads the system clipboard and feeds - // it to the terminal as keyboard input. xterm's paste() wraps - // it with bracketed-paste if the host has that mode enabled. + // 3. **Ctrl/Cmd+Shift+V.** Prefers clipboard.read() for images + // (upload → `/image`), else readText() into term.paste(). + // preventDefault here suppresses the DOM paste event, so image + // handling must live in this key path — not only the host + // listener below. + // + // 4. **DOM paste / drop on the host.** Bare Ctrl+V and context-menu + // paste fire a ClipboardEvent; drag-drop lands files. Image + // payloads upload to HERMES_HOME/images then drive `/image`. // // OSC 52 reads (terminal asking to read the clipboard) are not // supported — that would let any content the TUI renders exfiltrate @@ -532,6 +543,73 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { const isMac = typeof navigator !== "undefined" && /Mac/i.test(navigator.platform); + // ── Image paste / drop ─────────────────────────────────────────────── + // The Chat tab is an xterm mirror of a TUI inside the gateway. Server-side + // clipboard.paste / xclip never see the browser clipboard, so image paste + // must upload browser bytes to HERMES_HOME/images, then drive `/image` + // over the PTY (same burst-then-Return timing as handleCopyLast). + let imageUploadDisposed = false; + const pasteDelay = () => + new Promise((resolve) => window.setTimeout(resolve, 40)); + const reportImageUploadError = (err: unknown) => { + const message = err instanceof Error ? err.message : String(err); + console.warn("[dashboard chat] image upload failed:", message); + setBanner(`Image upload failed: ${message}`); + }; + const driveImageAttach = async (paths: string[]) => { + for (const path of paths) { + if (imageUploadDisposed) return; + const ws = wsRef.current; + if (!ws || ws.readyState !== WebSocket.OPEN) { + setBanner( + "Image uploaded, but chat is not connected — try again.", + ); + return; + } + ws.send(`/image ${path}`); + await new Promise((resolve) => window.setTimeout(resolve, 100)); + const s = wsRef.current; + if (!s || s.readyState !== WebSocket.OPEN) return; + s.send("\r"); + await pasteDelay(); + } + term.focus(); + }; + const uploadAndAttachImages = (files: File[]) => { + if (!files.length) return; + void (async () => { + const paths: string[] = []; + for (const file of files) { + const uploaded = await uploadChatImage(file, scopedProfile); + if (imageUploadDisposed) return; + paths.push(uploaded.path); + } + await driveImageAttach(paths); + })().catch(reportImageUploadError); + }; + const handleBrowserPaste = (ev: ClipboardEvent) => { + const files = imageFilesFromTransfer(ev.clipboardData); + if (!files.length) return; + ev.preventDefault(); + ev.stopPropagation(); + uploadAndAttachImages(files); + }; + const handleBrowserDragOver = (ev: DragEvent) => { + if (!transferMayContainImage(ev.dataTransfer)) return; + ev.preventDefault(); + if (ev.dataTransfer) ev.dataTransfer.dropEffect = "copy"; + }; + const handleBrowserDrop = (ev: DragEvent) => { + const files = imageFilesFromTransfer(ev.dataTransfer); + if (!files.length) return; + ev.preventDefault(); + ev.stopPropagation(); + uploadAndAttachImages(files); + }; + host.addEventListener("paste", handleBrowserPaste, { capture: true }); + host.addEventListener("dragover", handleBrowserDragOver, { capture: true }); + host.addEventListener("drop", handleBrowserDrop, { capture: true }); + term.attachCustomKeyEventHandler((ev) => { if (ev.type !== "keydown") return true; @@ -563,15 +641,42 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { } if (pasteModifier && ev.key.toLowerCase() === "v") { - navigator.clipboard - .readText() - .then((text) => { - if (text) term.paste(text); - }) - .catch((err) => { - console.warn("[dashboard clipboard] paste failed:", err.message); - }); + // preventDefault suppresses the DOM paste event, so image paste must + // be handled here via clipboard.read() — readText() alone misses + // image-only clipboards (the Discord / #24860 failure mode). ev.preventDefault(); + void (async () => { + try { + const read = navigator.clipboard?.read; + if (typeof read === "function") { + const items = await read.call(navigator.clipboard); + const files: File[] = []; + for (const item of items) { + const type = item.types.find((t) => t.startsWith("image/")); + if (!type) continue; + const blob = await item.getType(type); + const ext = type.split("/")[1]?.split("+")[0] || "png"; + files.push( + new File([blob], `clipboard.${ext}`, { type }), + ); + } + if (files.length) { + uploadAndAttachImages(files); + return; + } + } + } catch { + /* fall through to text paste */ + } + try { + const text = await navigator.clipboard.readText(); + if (text) term.paste(text); + } catch (err) { + const message = + err instanceof Error ? err.message : String(err); + console.warn("[dashboard clipboard] paste failed:", message); + } + })(); return false; } @@ -1020,10 +1125,14 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { return () => { unmounting = true; + imageUploadDisposed = true; syncMetricsRef.current = null; onDataDisposable?.dispose(); onResizeDisposable?.dispose(); mobileInputCleanup?.(); + host.removeEventListener("paste", handleBrowserPaste, true); + host.removeEventListener("dragover", handleBrowserDragOver, true); + host.removeEventListener("drop", handleBrowserDrop, true); if (metricsDebounce) clearTimeout(metricsDebounce); window.removeEventListener("resize", scheduleSyncTerminalMetrics); window.visualViewport?.removeEventListener(