mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-17 14:42:06 +00:00
fix(web): paste/drop images into dashboard Chat via HERMES_HOME/images
Dashboard Chat is an xterm mirror of a TUI inside the gateway, so server-side clipboard.paste never sees the browser clipboard. Upload pasted/dropped images to the profile's images/ dir (same place clipboard.paste / image.attach use), then drive /image over the PTY. Uses a dedicated /api/chat/image-upload endpoint (magic-byte check, 25MB cap, profile scope) instead of relative managed-files uploads that 400 on local dashboards without a locked root. Ctrl/Cmd+Shift+V also tries clipboard.read() for images before falling back to text, since preventDefault on that chord suppresses the DOM paste event. Salvages #57912 (client composition + /image PTY drive) and folds in #48563's upload endpoint + drop path. Co-authored-by: bird <6666242+bird@users.noreply.github.com> Co-authored-by: tt-a1i <53142663+tt-a1i@users.noreply.github.com>
This commit is contained in:
parent
1318cd9b0d
commit
301acc9eaa
5 changed files with 575 additions and 12 deletions
99
web/src/lib/chatImagePaste.test.ts
Normal file
99
web/src/lib/chatImagePaste.test.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
firstImageFromClipboard,
|
||||
imageFilesFromTransfer,
|
||||
transferMayContainImage,
|
||||
} from "./chatImagePaste";
|
||||
|
||||
// Minimal DataTransfer stand-ins. jsdom's DataTransfer doesn't let us seed
|
||||
// items/files, so we hand-roll the shape the helpers read.
|
||||
function makeItem(kind: string, type: string, file: File | null) {
|
||||
return { kind, type, getAsFile: () => file } as unknown as DataTransferItem;
|
||||
}
|
||||
|
||||
function makeData(opts: {
|
||||
items?: DataTransferItem[];
|
||||
files?: File[];
|
||||
}): DataTransfer {
|
||||
const items = opts.items ?? [];
|
||||
const files = opts.files ?? [];
|
||||
const itemList: Record<string | number, unknown> = { length: items.length };
|
||||
items.forEach((it, i) => {
|
||||
itemList[i] = it;
|
||||
});
|
||||
const fileList: Record<string | number, unknown> = { length: files.length };
|
||||
files.forEach((f, i) => {
|
||||
fileList[i] = f;
|
||||
});
|
||||
return {
|
||||
items: itemList,
|
||||
files: fileList,
|
||||
} as unknown as DataTransfer;
|
||||
}
|
||||
|
||||
const png = new File([new Uint8Array([1, 2, 3])], "x.png", {
|
||||
type: "image/png",
|
||||
});
|
||||
const gif = new File([new Uint8Array([4, 5])], "y.gif", {
|
||||
type: "image/gif",
|
||||
});
|
||||
|
||||
describe("firstImageFromClipboard", () => {
|
||||
it("returns null for null clipboard data", () => {
|
||||
expect(firstImageFromClipboard(null)).toBeNull();
|
||||
});
|
||||
|
||||
it("finds an image via items[].getAsFile()", () => {
|
||||
const data = makeData({ items: [makeItem("file", "image/png", png)] });
|
||||
expect(firstImageFromClipboard(data)).toBe(png);
|
||||
});
|
||||
|
||||
it("ignores non-file and non-image items", () => {
|
||||
const data = makeData({
|
||||
items: [
|
||||
makeItem("string", "text/plain", null),
|
||||
makeItem("file", "application/pdf", new File([], "a.pdf")),
|
||||
],
|
||||
});
|
||||
expect(firstImageFromClipboard(data)).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to files[] when items are absent (Safari/Firefox)", () => {
|
||||
const data = makeData({ files: [png] });
|
||||
expect(firstImageFromClipboard(data)).toBe(png);
|
||||
});
|
||||
|
||||
it("returns null when nothing image-like is present", () => {
|
||||
const data = makeData({
|
||||
files: [new File([], "notes.txt", { type: "text/plain" })],
|
||||
});
|
||||
expect(firstImageFromClipboard(data)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("imageFilesFromTransfer", () => {
|
||||
it("dedupes the same file when present in both items and files", () => {
|
||||
const data = makeData({
|
||||
items: [makeItem("file", "image/png", png)],
|
||||
files: [png, gif],
|
||||
});
|
||||
expect(imageFilesFromTransfer(data)).toEqual([png, gif]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("transferMayContainImage", () => {
|
||||
it("is true for image items even when type is empty (some browsers)", () => {
|
||||
const data = makeData({
|
||||
items: [makeItem("file", "", png)],
|
||||
});
|
||||
expect(transferMayContainImage(data)).toBe(true);
|
||||
});
|
||||
|
||||
it("is false for text-only transfers", () => {
|
||||
const data = makeData({
|
||||
items: [makeItem("string", "text/plain", null)],
|
||||
});
|
||||
expect(transferMayContainImage(data)).toBe(false);
|
||||
});
|
||||
});
|
||||
164
web/src/lib/chatImagePaste.ts
Normal file
164
web/src/lib/chatImagePaste.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import { authedFetch } from "@/lib/api";
|
||||
|
||||
// Clipboard image MIME → file extension. Mirrors the set the TUI's /image
|
||||
// attach path and the gateway's image sniffer accept.
|
||||
const IMAGE_MIME_EXT: Record<string, string> = {
|
||||
"image/png": "png",
|
||||
"image/jpeg": "jpg",
|
||||
"image/gif": "gif",
|
||||
"image/webp": "webp",
|
||||
"image/bmp": "bmp",
|
||||
};
|
||||
|
||||
// Anthropic caps a single vision image at ~25 MB; reject earlier client-side
|
||||
// with a clear message rather than round-tripping a doomed upload.
|
||||
const MAX_IMAGE_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
export interface ChatImageUploadResult {
|
||||
/** Absolute path under HERMES_HOME/images the gateway wrote. */
|
||||
path: string;
|
||||
/** Byte size of the uploaded image. */
|
||||
bytes: number;
|
||||
/** Basename written on the server. */
|
||||
name: string;
|
||||
mime_type: string;
|
||||
}
|
||||
|
||||
function imageFileKey(file: File): string {
|
||||
return `${file.name}\0${file.type}\0${file.size}\0${file.lastModified}`;
|
||||
}
|
||||
|
||||
function addImageFile(files: File[], seen: Set<string>, file: File | null) {
|
||||
if (!file || !file.type.startsWith("image/")) return;
|
||||
const key = imageFileKey(file);
|
||||
if (seen.has(key)) return;
|
||||
seen.add(key);
|
||||
files.push(file);
|
||||
}
|
||||
|
||||
/** Pull every image file out of a DataTransfer (clipboard or drop). */
|
||||
export function imageFilesFromTransfer(
|
||||
data: DataTransfer | null,
|
||||
): File[] {
|
||||
if (!data) return [];
|
||||
const files: File[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
if (data.items?.length) {
|
||||
for (let i = 0; i < data.items.length; i++) {
|
||||
const item = data.items[i];
|
||||
if (item.kind === "file" && item.type.startsWith("image/")) {
|
||||
addImageFile(files, seen, item.getAsFile());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (data.files?.length) {
|
||||
for (let i = 0; i < data.files.length; i++) {
|
||||
addImageFile(files, seen, data.files[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Pull the first image blob out of a DataTransfer, or null if none present. */
|
||||
export function firstImageFromClipboard(
|
||||
data: DataTransfer | null,
|
||||
): File | null {
|
||||
return imageFilesFromTransfer(data)[0] ?? null;
|
||||
}
|
||||
|
||||
/** True when a drag payload may contain an image (for dragover preventDefault). */
|
||||
export function transferMayContainImage(data: DataTransfer | null): boolean {
|
||||
if (!data) return false;
|
||||
if (data.items?.length) {
|
||||
for (let i = 0; i < data.items.length; i++) {
|
||||
const item = data.items[i];
|
||||
if (
|
||||
item.kind === "file" &&
|
||||
(!item.type || item.type.startsWith("image/"))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (data.files?.length) {
|
||||
for (let i = 0; i < data.files.length; i++) {
|
||||
if (data.files[i].type.startsWith("image/")) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function fileToDataUrl(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onerror = () =>
|
||||
reject(reader.error ?? new Error("image read failed"));
|
||||
reader.onload = () => {
|
||||
const result = reader.result;
|
||||
if (typeof result === "string") {
|
||||
resolve(result);
|
||||
} else {
|
||||
reject(new Error("image read failed"));
|
||||
}
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a browser clipboard/drop image to ``HERMES_HOME/images`` via the
|
||||
* dedicated chat upload endpoint and return the absolute gateway path.
|
||||
*
|
||||
* The dashboard Chat tab is an xterm mirror of a TUI running INSIDE the
|
||||
* gateway. The container has no access to the browser's clipboard, so the
|
||||
* server-side ``clipboard.paste`` path can never see a pasted image.
|
||||
* Upload the bytes the browser already holds, then hand the path to the
|
||||
* TUI's ``/image`` command.
|
||||
*/
|
||||
export async function uploadChatImage(
|
||||
blob: Blob,
|
||||
profile = "",
|
||||
): Promise<ChatImageUploadResult> {
|
||||
if (blob.size === 0) throw new Error("clipboard image is empty");
|
||||
if (blob.size > MAX_IMAGE_BYTES) {
|
||||
const mb = Math.round(MAX_IMAGE_BYTES / (1024 * 1024));
|
||||
throw new Error(`image too large (max ${mb} MB)`);
|
||||
}
|
||||
|
||||
const mime = blob.type || "image/png";
|
||||
const ext = IMAGE_MIME_EXT[mime] || "png";
|
||||
const filename =
|
||||
blob instanceof File && blob.name
|
||||
? blob.name
|
||||
: `clipboard.${ext}`;
|
||||
const file =
|
||||
blob instanceof File
|
||||
? blob
|
||||
: new File([blob], filename, { type: mime });
|
||||
|
||||
const dataUrl = await fileToDataUrl(file);
|
||||
const qs = profile ? `?profile=${encodeURIComponent(profile)}` : "";
|
||||
const res = await authedFetch(`/api/chat/image-upload${qs}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
data_url: dataUrl,
|
||||
filename,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => res.statusText);
|
||||
throw new Error(text || `HTTP ${res.status}`);
|
||||
}
|
||||
|
||||
const uploaded = (await res.json()) as ChatImageUploadResult;
|
||||
if (!uploaded?.path) {
|
||||
throw new Error("image upload did not return a path");
|
||||
}
|
||||
return uploaded;
|
||||
}
|
||||
|
|
@ -49,6 +49,11 @@ import {
|
|||
normalizePtyMobileInput,
|
||||
shouldTreatInputAsMobileReplacement,
|
||||
} from "@/lib/pty-mobile-input";
|
||||
import {
|
||||
imageFilesFromTransfer,
|
||||
transferMayContainImage,
|
||||
uploadChatImage,
|
||||
} from "@/lib/chatImagePaste";
|
||||
import { PluginSlot } from "@/plugins";
|
||||
import { useTheme } from "@/themes";
|
||||
import { useProfileScope } from "@/contexts/useProfileScope";
|
||||
|
|
@ -486,7 +491,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
|
|||
|
||||
// --- Clipboard integration ---------------------------------------
|
||||
//
|
||||
// Three independent paths all route to the system clipboard:
|
||||
// Four independent paths all route to the system clipboard:
|
||||
//
|
||||
// 1. **Selection → Ctrl+C (or Cmd+C on macOS).** Ink's own handler
|
||||
// in useInputHandlers.ts turns Ctrl+C into a copy when the
|
||||
|
|
@ -500,9 +505,15 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
|
|||
// ever stops listening (e.g. overlays / pickers) or if the user
|
||||
// has selected with the mouse outside of Ink's selection model.
|
||||
//
|
||||
// 3. **Ctrl/Cmd+Shift+V.** Reads the system clipboard and feeds
|
||||
// it to the terminal as keyboard input. xterm's paste() wraps
|
||||
// it with bracketed-paste if the host has that mode enabled.
|
||||
// 3. **Ctrl/Cmd+Shift+V.** Prefers clipboard.read() for images
|
||||
// (upload → `/image`), else readText() into term.paste().
|
||||
// preventDefault here suppresses the DOM paste event, so image
|
||||
// handling must live in this key path — not only the host
|
||||
// listener below.
|
||||
//
|
||||
// 4. **DOM paste / drop on the host.** Bare Ctrl+V and context-menu
|
||||
// paste fire a ClipboardEvent; drag-drop lands files. Image
|
||||
// payloads upload to HERMES_HOME/images then drive `/image`.
|
||||
//
|
||||
// OSC 52 reads (terminal asking to read the clipboard) are not
|
||||
// supported — that would let any content the TUI renders exfiltrate
|
||||
|
|
@ -532,6 +543,73 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
|
|||
const isMac =
|
||||
typeof navigator !== "undefined" && /Mac/i.test(navigator.platform);
|
||||
|
||||
// ── Image paste / drop ───────────────────────────────────────────────
|
||||
// The Chat tab is an xterm mirror of a TUI inside the gateway. Server-side
|
||||
// clipboard.paste / xclip never see the browser clipboard, so image paste
|
||||
// must upload browser bytes to HERMES_HOME/images, then drive `/image`
|
||||
// over the PTY (same burst-then-Return timing as handleCopyLast).
|
||||
let imageUploadDisposed = false;
|
||||
const pasteDelay = () =>
|
||||
new Promise<void>((resolve) => window.setTimeout(resolve, 40));
|
||||
const reportImageUploadError = (err: unknown) => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn("[dashboard chat] image upload failed:", message);
|
||||
setBanner(`Image upload failed: ${message}`);
|
||||
};
|
||||
const driveImageAttach = async (paths: string[]) => {
|
||||
for (const path of paths) {
|
||||
if (imageUploadDisposed) return;
|
||||
const ws = wsRef.current;
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) {
|
||||
setBanner(
|
||||
"Image uploaded, but chat is not connected — try again.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
ws.send(`/image ${path}`);
|
||||
await new Promise<void>((resolve) => window.setTimeout(resolve, 100));
|
||||
const s = wsRef.current;
|
||||
if (!s || s.readyState !== WebSocket.OPEN) return;
|
||||
s.send("\r");
|
||||
await pasteDelay();
|
||||
}
|
||||
term.focus();
|
||||
};
|
||||
const uploadAndAttachImages = (files: File[]) => {
|
||||
if (!files.length) return;
|
||||
void (async () => {
|
||||
const paths: string[] = [];
|
||||
for (const file of files) {
|
||||
const uploaded = await uploadChatImage(file, scopedProfile);
|
||||
if (imageUploadDisposed) return;
|
||||
paths.push(uploaded.path);
|
||||
}
|
||||
await driveImageAttach(paths);
|
||||
})().catch(reportImageUploadError);
|
||||
};
|
||||
const handleBrowserPaste = (ev: ClipboardEvent) => {
|
||||
const files = imageFilesFromTransfer(ev.clipboardData);
|
||||
if (!files.length) return;
|
||||
ev.preventDefault();
|
||||
ev.stopPropagation();
|
||||
uploadAndAttachImages(files);
|
||||
};
|
||||
const handleBrowserDragOver = (ev: DragEvent) => {
|
||||
if (!transferMayContainImage(ev.dataTransfer)) return;
|
||||
ev.preventDefault();
|
||||
if (ev.dataTransfer) ev.dataTransfer.dropEffect = "copy";
|
||||
};
|
||||
const handleBrowserDrop = (ev: DragEvent) => {
|
||||
const files = imageFilesFromTransfer(ev.dataTransfer);
|
||||
if (!files.length) return;
|
||||
ev.preventDefault();
|
||||
ev.stopPropagation();
|
||||
uploadAndAttachImages(files);
|
||||
};
|
||||
host.addEventListener("paste", handleBrowserPaste, { capture: true });
|
||||
host.addEventListener("dragover", handleBrowserDragOver, { capture: true });
|
||||
host.addEventListener("drop", handleBrowserDrop, { capture: true });
|
||||
|
||||
term.attachCustomKeyEventHandler((ev) => {
|
||||
if (ev.type !== "keydown") return true;
|
||||
|
||||
|
|
@ -563,15 +641,42 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
|
|||
}
|
||||
|
||||
if (pasteModifier && ev.key.toLowerCase() === "v") {
|
||||
navigator.clipboard
|
||||
.readText()
|
||||
.then((text) => {
|
||||
if (text) term.paste(text);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.warn("[dashboard clipboard] paste failed:", err.message);
|
||||
});
|
||||
// preventDefault suppresses the DOM paste event, so image paste must
|
||||
// be handled here via clipboard.read() — readText() alone misses
|
||||
// image-only clipboards (the Discord / #24860 failure mode).
|
||||
ev.preventDefault();
|
||||
void (async () => {
|
||||
try {
|
||||
const read = navigator.clipboard?.read;
|
||||
if (typeof read === "function") {
|
||||
const items = await read.call(navigator.clipboard);
|
||||
const files: File[] = [];
|
||||
for (const item of items) {
|
||||
const type = item.types.find((t) => t.startsWith("image/"));
|
||||
if (!type) continue;
|
||||
const blob = await item.getType(type);
|
||||
const ext = type.split("/")[1]?.split("+")[0] || "png";
|
||||
files.push(
|
||||
new File([blob], `clipboard.${ext}`, { type }),
|
||||
);
|
||||
}
|
||||
if (files.length) {
|
||||
uploadAndAttachImages(files);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* fall through to text paste */
|
||||
}
|
||||
try {
|
||||
const text = await navigator.clipboard.readText();
|
||||
if (text) term.paste(text);
|
||||
} catch (err) {
|
||||
const message =
|
||||
err instanceof Error ? err.message : String(err);
|
||||
console.warn("[dashboard clipboard] paste failed:", message);
|
||||
}
|
||||
})();
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -1020,10 +1125,14 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
|
|||
|
||||
return () => {
|
||||
unmounting = true;
|
||||
imageUploadDisposed = true;
|
||||
syncMetricsRef.current = null;
|
||||
onDataDisposable?.dispose();
|
||||
onResizeDisposable?.dispose();
|
||||
mobileInputCleanup?.();
|
||||
host.removeEventListener("paste", handleBrowserPaste, true);
|
||||
host.removeEventListener("dragover", handleBrowserDragOver, true);
|
||||
host.removeEventListener("drop", handleBrowserDrop, true);
|
||||
if (metricsDebounce) clearTimeout(metricsDebounce);
|
||||
window.removeEventListener("resize", scheduleSyncTerminalMetrics);
|
||||
window.visualViewport?.removeEventListener(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue