fix(web): paste/drop images into dashboard Chat via HERMES_HOME/images

Dashboard Chat is an xterm mirror of a TUI inside the gateway, so
server-side clipboard.paste never sees the browser clipboard. Upload
pasted/dropped images to the profile's images/ dir (same place
clipboard.paste / image.attach use), then drive /image over the PTY.

Uses a dedicated /api/chat/image-upload endpoint (magic-byte check,
25MB cap, profile scope) instead of relative managed-files uploads that
400 on local dashboards without a locked root. Ctrl/Cmd+Shift+V also
tries clipboard.read() for images before falling back to text, since
preventDefault on that chord suppresses the DOM paste event.

Salvages #57912 (client composition + /image PTY drive) and folds in
#48563's upload endpoint + drop path.

Co-authored-by: bird <6666242+bird@users.noreply.github.com>
Co-authored-by: tt-a1i <53142663+tt-a1i@users.noreply.github.com>
This commit is contained in:
Brooklyn Nicholson 2026-07-10 02:20:04 -05:00
parent 1318cd9b0d
commit 301acc9eaa
5 changed files with 575 additions and 12 deletions

View file

@ -0,0 +1,99 @@
import { describe, expect, it } from "vitest";
import {
firstImageFromClipboard,
imageFilesFromTransfer,
transferMayContainImage,
} from "./chatImagePaste";
// Minimal DataTransfer stand-ins. jsdom's DataTransfer doesn't let us seed
// items/files, so we hand-roll the shape the helpers read.
function makeItem(kind: string, type: string, file: File | null) {
return { kind, type, getAsFile: () => file } as unknown as DataTransferItem;
}
function makeData(opts: {
items?: DataTransferItem[];
files?: File[];
}): DataTransfer {
const items = opts.items ?? [];
const files = opts.files ?? [];
const itemList: Record<string | number, unknown> = { length: items.length };
items.forEach((it, i) => {
itemList[i] = it;
});
const fileList: Record<string | number, unknown> = { length: files.length };
files.forEach((f, i) => {
fileList[i] = f;
});
return {
items: itemList,
files: fileList,
} as unknown as DataTransfer;
}
const png = new File([new Uint8Array([1, 2, 3])], "x.png", {
type: "image/png",
});
const gif = new File([new Uint8Array([4, 5])], "y.gif", {
type: "image/gif",
});
describe("firstImageFromClipboard", () => {
it("returns null for null clipboard data", () => {
expect(firstImageFromClipboard(null)).toBeNull();
});
it("finds an image via items[].getAsFile()", () => {
const data = makeData({ items: [makeItem("file", "image/png", png)] });
expect(firstImageFromClipboard(data)).toBe(png);
});
it("ignores non-file and non-image items", () => {
const data = makeData({
items: [
makeItem("string", "text/plain", null),
makeItem("file", "application/pdf", new File([], "a.pdf")),
],
});
expect(firstImageFromClipboard(data)).toBeNull();
});
it("falls back to files[] when items are absent (Safari/Firefox)", () => {
const data = makeData({ files: [png] });
expect(firstImageFromClipboard(data)).toBe(png);
});
it("returns null when nothing image-like is present", () => {
const data = makeData({
files: [new File([], "notes.txt", { type: "text/plain" })],
});
expect(firstImageFromClipboard(data)).toBeNull();
});
});
describe("imageFilesFromTransfer", () => {
it("dedupes the same file when present in both items and files", () => {
const data = makeData({
items: [makeItem("file", "image/png", png)],
files: [png, gif],
});
expect(imageFilesFromTransfer(data)).toEqual([png, gif]);
});
});
describe("transferMayContainImage", () => {
it("is true for image items even when type is empty (some browsers)", () => {
const data = makeData({
items: [makeItem("file", "", png)],
});
expect(transferMayContainImage(data)).toBe(true);
});
it("is false for text-only transfers", () => {
const data = makeData({
items: [makeItem("string", "text/plain", null)],
});
expect(transferMayContainImage(data)).toBe(false);
});
});

View file

@ -0,0 +1,164 @@
import { authedFetch } from "@/lib/api";
// Clipboard image MIME → file extension. Mirrors the set the TUI's /image
// attach path and the gateway's image sniffer accept.
const IMAGE_MIME_EXT: Record<string, string> = {
"image/png": "png",
"image/jpeg": "jpg",
"image/gif": "gif",
"image/webp": "webp",
"image/bmp": "bmp",
};
// Anthropic caps a single vision image at ~25 MB; reject earlier client-side
// with a clear message rather than round-tripping a doomed upload.
const MAX_IMAGE_BYTES = 25 * 1024 * 1024;
export interface ChatImageUploadResult {
/** Absolute path under HERMES_HOME/images the gateway wrote. */
path: string;
/** Byte size of the uploaded image. */
bytes: number;
/** Basename written on the server. */
name: string;
mime_type: string;
}
function imageFileKey(file: File): string {
return `${file.name}\0${file.type}\0${file.size}\0${file.lastModified}`;
}
function addImageFile(files: File[], seen: Set<string>, file: File | null) {
if (!file || !file.type.startsWith("image/")) return;
const key = imageFileKey(file);
if (seen.has(key)) return;
seen.add(key);
files.push(file);
}
/** Pull every image file out of a DataTransfer (clipboard or drop). */
export function imageFilesFromTransfer(
data: DataTransfer | null,
): File[] {
if (!data) return [];
const files: File[] = [];
const seen = new Set<string>();
if (data.items?.length) {
for (let i = 0; i < data.items.length; i++) {
const item = data.items[i];
if (item.kind === "file" && item.type.startsWith("image/")) {
addImageFile(files, seen, item.getAsFile());
}
}
}
if (data.files?.length) {
for (let i = 0; i < data.files.length; i++) {
addImageFile(files, seen, data.files[i]);
}
}
return files;
}
/** Pull the first image blob out of a DataTransfer, or null if none present. */
export function firstImageFromClipboard(
data: DataTransfer | null,
): File | null {
return imageFilesFromTransfer(data)[0] ?? null;
}
/** True when a drag payload may contain an image (for dragover preventDefault). */
export function transferMayContainImage(data: DataTransfer | null): boolean {
if (!data) return false;
if (data.items?.length) {
for (let i = 0; i < data.items.length; i++) {
const item = data.items[i];
if (
item.kind === "file" &&
(!item.type || item.type.startsWith("image/"))
) {
return true;
}
}
return false;
}
if (data.files?.length) {
for (let i = 0; i < data.files.length; i++) {
if (data.files[i].type.startsWith("image/")) return true;
}
}
return false;
}
function fileToDataUrl(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onerror = () =>
reject(reader.error ?? new Error("image read failed"));
reader.onload = () => {
const result = reader.result;
if (typeof result === "string") {
resolve(result);
} else {
reject(new Error("image read failed"));
}
};
reader.readAsDataURL(file);
});
}
/**
* Upload a browser clipboard/drop image to ``HERMES_HOME/images`` via the
* dedicated chat upload endpoint and return the absolute gateway path.
*
* The dashboard Chat tab is an xterm mirror of a TUI running INSIDE the
* gateway. The container has no access to the browser's clipboard, so the
* server-side ``clipboard.paste`` path can never see a pasted image.
* Upload the bytes the browser already holds, then hand the path to the
* TUI's ``/image`` command.
*/
export async function uploadChatImage(
blob: Blob,
profile = "",
): Promise<ChatImageUploadResult> {
if (blob.size === 0) throw new Error("clipboard image is empty");
if (blob.size > MAX_IMAGE_BYTES) {
const mb = Math.round(MAX_IMAGE_BYTES / (1024 * 1024));
throw new Error(`image too large (max ${mb} MB)`);
}
const mime = blob.type || "image/png";
const ext = IMAGE_MIME_EXT[mime] || "png";
const filename =
blob instanceof File && blob.name
? blob.name
: `clipboard.${ext}`;
const file =
blob instanceof File
? blob
: new File([blob], filename, { type: mime });
const dataUrl = await fileToDataUrl(file);
const qs = profile ? `?profile=${encodeURIComponent(profile)}` : "";
const res = await authedFetch(`/api/chat/image-upload${qs}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
data_url: dataUrl,
filename,
}),
});
if (!res.ok) {
const text = await res.text().catch(() => res.statusText);
throw new Error(text || `HTTP ${res.status}`);
}
const uploaded = (await res.json()) as ChatImageUploadResult;
if (!uploaded?.path) {
throw new Error("image upload did not return a path");
}
return uploaded;
}

View file

@ -49,6 +49,11 @@ import {
normalizePtyMobileInput,
shouldTreatInputAsMobileReplacement,
} from "@/lib/pty-mobile-input";
import {
imageFilesFromTransfer,
transferMayContainImage,
uploadChatImage,
} from "@/lib/chatImagePaste";
import { PluginSlot } from "@/plugins";
import { useTheme } from "@/themes";
import { useProfileScope } from "@/contexts/useProfileScope";
@ -486,7 +491,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
// --- Clipboard integration ---------------------------------------
//
// Three independent paths all route to the system clipboard:
// Four independent paths all route to the system clipboard:
//
// 1. **Selection → Ctrl+C (or Cmd+C on macOS).** Ink's own handler
// in useInputHandlers.ts turns Ctrl+C into a copy when the
@ -500,9 +505,15 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
// ever stops listening (e.g. overlays / pickers) or if the user
// has selected with the mouse outside of Ink's selection model.
//
// 3. **Ctrl/Cmd+Shift+V.** Reads the system clipboard and feeds
// it to the terminal as keyboard input. xterm's paste() wraps
// it with bracketed-paste if the host has that mode enabled.
// 3. **Ctrl/Cmd+Shift+V.** Prefers clipboard.read() for images
// (upload → `/image`), else readText() into term.paste().
// preventDefault here suppresses the DOM paste event, so image
// handling must live in this key path — not only the host
// listener below.
//
// 4. **DOM paste / drop on the host.** Bare Ctrl+V and context-menu
// paste fire a ClipboardEvent; drag-drop lands files. Image
// payloads upload to HERMES_HOME/images then drive `/image`.
//
// OSC 52 reads (terminal asking to read the clipboard) are not
// supported — that would let any content the TUI renders exfiltrate
@ -532,6 +543,73 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
const isMac =
typeof navigator !== "undefined" && /Mac/i.test(navigator.platform);
// ── Image paste / drop ───────────────────────────────────────────────
// The Chat tab is an xterm mirror of a TUI inside the gateway. Server-side
// clipboard.paste / xclip never see the browser clipboard, so image paste
// must upload browser bytes to HERMES_HOME/images, then drive `/image`
// over the PTY (same burst-then-Return timing as handleCopyLast).
let imageUploadDisposed = false;
const pasteDelay = () =>
new Promise<void>((resolve) => window.setTimeout(resolve, 40));
const reportImageUploadError = (err: unknown) => {
const message = err instanceof Error ? err.message : String(err);
console.warn("[dashboard chat] image upload failed:", message);
setBanner(`Image upload failed: ${message}`);
};
const driveImageAttach = async (paths: string[]) => {
for (const path of paths) {
if (imageUploadDisposed) return;
const ws = wsRef.current;
if (!ws || ws.readyState !== WebSocket.OPEN) {
setBanner(
"Image uploaded, but chat is not connected — try again.",
);
return;
}
ws.send(`/image ${path}`);
await new Promise<void>((resolve) => window.setTimeout(resolve, 100));
const s = wsRef.current;
if (!s || s.readyState !== WebSocket.OPEN) return;
s.send("\r");
await pasteDelay();
}
term.focus();
};
const uploadAndAttachImages = (files: File[]) => {
if (!files.length) return;
void (async () => {
const paths: string[] = [];
for (const file of files) {
const uploaded = await uploadChatImage(file, scopedProfile);
if (imageUploadDisposed) return;
paths.push(uploaded.path);
}
await driveImageAttach(paths);
})().catch(reportImageUploadError);
};
const handleBrowserPaste = (ev: ClipboardEvent) => {
const files = imageFilesFromTransfer(ev.clipboardData);
if (!files.length) return;
ev.preventDefault();
ev.stopPropagation();
uploadAndAttachImages(files);
};
const handleBrowserDragOver = (ev: DragEvent) => {
if (!transferMayContainImage(ev.dataTransfer)) return;
ev.preventDefault();
if (ev.dataTransfer) ev.dataTransfer.dropEffect = "copy";
};
const handleBrowserDrop = (ev: DragEvent) => {
const files = imageFilesFromTransfer(ev.dataTransfer);
if (!files.length) return;
ev.preventDefault();
ev.stopPropagation();
uploadAndAttachImages(files);
};
host.addEventListener("paste", handleBrowserPaste, { capture: true });
host.addEventListener("dragover", handleBrowserDragOver, { capture: true });
host.addEventListener("drop", handleBrowserDrop, { capture: true });
term.attachCustomKeyEventHandler((ev) => {
if (ev.type !== "keydown") return true;
@ -563,15 +641,42 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
}
if (pasteModifier && ev.key.toLowerCase() === "v") {
navigator.clipboard
.readText()
.then((text) => {
if (text) term.paste(text);
})
.catch((err) => {
console.warn("[dashboard clipboard] paste failed:", err.message);
});
// preventDefault suppresses the DOM paste event, so image paste must
// be handled here via clipboard.read() — readText() alone misses
// image-only clipboards (the Discord / #24860 failure mode).
ev.preventDefault();
void (async () => {
try {
const read = navigator.clipboard?.read;
if (typeof read === "function") {
const items = await read.call(navigator.clipboard);
const files: File[] = [];
for (const item of items) {
const type = item.types.find((t) => t.startsWith("image/"));
if (!type) continue;
const blob = await item.getType(type);
const ext = type.split("/")[1]?.split("+")[0] || "png";
files.push(
new File([blob], `clipboard.${ext}`, { type }),
);
}
if (files.length) {
uploadAndAttachImages(files);
return;
}
}
} catch {
/* fall through to text paste */
}
try {
const text = await navigator.clipboard.readText();
if (text) term.paste(text);
} catch (err) {
const message =
err instanceof Error ? err.message : String(err);
console.warn("[dashboard clipboard] paste failed:", message);
}
})();
return false;
}
@ -1020,10 +1125,14 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
return () => {
unmounting = true;
imageUploadDisposed = true;
syncMetricsRef.current = null;
onDataDisposable?.dispose();
onResizeDisposable?.dispose();
mobileInputCleanup?.();
host.removeEventListener("paste", handleBrowserPaste, true);
host.removeEventListener("dragover", handleBrowserDragOver, true);
host.removeEventListener("drop", handleBrowserDrop, true);
if (metricsDebounce) clearTimeout(metricsDebounce);
window.removeEventListener("resize", scheduleSyncTerminalMetrics);
window.visualViewport?.removeEventListener(