fix(web): paste/drop images into dashboard Chat via HERMES_HOME/images

Dashboard Chat is an xterm mirror of a TUI inside the gateway, so
server-side clipboard.paste never sees the browser clipboard. Upload
pasted/dropped images to the profile's images/ dir (same place
clipboard.paste / image.attach use), then drive /image over the PTY.

Uses a dedicated /api/chat/image-upload endpoint (magic-byte check,
25MB cap, profile scope) instead of relative managed-files uploads that
400 on local dashboards without a locked root. Ctrl/Cmd+Shift+V also
tries clipboard.read() for images before falling back to text, since
preventDefault on that chord suppresses the DOM paste event.

Salvages #57912 (client composition + /image PTY drive) and folds in
#48563's upload endpoint + drop path.

Co-authored-by: bird <6666242+bird@users.noreply.github.com>
Co-authored-by: tt-a1i <53142663+tt-a1i@users.noreply.github.com>
This commit is contained in:
Brooklyn Nicholson 2026-07-10 02:20:04 -05:00
parent 1318cd9b0d
commit 301acc9eaa
5 changed files with 575 additions and 12 deletions

View file

@ -900,6 +900,11 @@ class ManagedFileUpload(BaseModel):
overwrite: bool = True
class ChatImageUpload(BaseModel):
data_url: str
filename: Optional[str] = None
class ManagedDirectoryCreate(BaseModel):
path: str
@ -1766,6 +1771,91 @@ def _decode_data_url(data_url: str) -> tuple[bytes, str]:
return data, mime_type
_CHAT_IMAGE_UPLOAD_MAX_BYTES = 25 * 1024 * 1024
_CHAT_IMAGE_ALLOWED_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"})
_CHAT_IMAGE_MAGIC: tuple[tuple[bytes, str], ...] = (
(b"\x89PNG\r\n\x1a\n", ".png"),
(b"\xff\xd8\xff", ".jpg"),
(b"GIF87a", ".gif"),
(b"GIF89a", ".gif"),
(b"BM", ".bmp"),
)
def _sanitize_chat_image_filename(filename: str | None) -> str:
candidate = Path(str(filename or "").strip()).name
candidate = re.sub(r"[\x00-\x1f]+", "_", candidate)
candidate = candidate.strip().strip(".")
return candidate or "pasted-image"
def _chat_image_extension(data: bytes) -> str | None:
head = data[:16]
if head.startswith(b"RIFF") and head[8:12] == b"WEBP":
return ".webp"
for sig, ext in _CHAT_IMAGE_MAGIC:
if head.startswith(sig):
return ext
return None
def _decode_chat_image_upload(payload: ChatImageUpload) -> tuple[bytes, str, str]:
data, mime_type = _decode_data_url(payload.data_url)
if not mime_type.lower().startswith("image/"):
raise HTTPException(status_code=400, detail="Upload payload must be an image")
if len(data) > _CHAT_IMAGE_UPLOAD_MAX_BYTES:
mb = _CHAT_IMAGE_UPLOAD_MAX_BYTES // (1024 * 1024)
raise HTTPException(status_code=413, detail=f"Image is too large; cap is {mb} MB")
ext = _chat_image_extension(data)
if ext not in _CHAT_IMAGE_ALLOWED_EXTENSIONS:
raise HTTPException(status_code=400, detail="Unsupported image type")
return data, mime_type, ext
@app.post("/api/chat/image-upload")
async def upload_chat_image(payload: ChatImageUpload, profile: Optional[str] = None):
"""Persist a browser-provided chat image where the embedded TUI can read it.
The dashboard /chat page runs Hermes inside an xterm.js PTY. Browser
clipboard image bytes are not visible to the server-side clipboard, so the
page uploads them here, then drives the TUI's ``/image <path>`` command
with the returned gateway-visible path. Files land under
``HERMES_HOME/images/`` the same directory ``clipboard.paste`` /
``image.attach`` already use.
"""
data, mime_type, ext = _decode_chat_image_upload(payload)
with _profile_scope(profile) as scoped_home:
home = scoped_home or get_hermes_home()
img_dir = Path(home) / "images"
try:
img_dir.mkdir(parents=True, exist_ok=True)
except PermissionError:
raise HTTPException(status_code=403, detail="Image directory is not writable")
except OSError as exc:
raise HTTPException(status_code=500, detail=f"Could not create image directory: {exc}")
stem = Path(_sanitize_chat_image_filename(payload.filename)).stem or "pasted-image"
stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", stem).strip("._-") or "pasted-image"
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
target = img_dir / f"dashboard_{ts}_{secrets.token_hex(4)}_{stem}{ext}"
try:
target.write_bytes(data)
except PermissionError:
raise HTTPException(status_code=403, detail="Image directory is not writable")
except OSError as exc:
raise HTTPException(status_code=500, detail=f"Could not write image: {exc}")
return {
"ok": True,
"path": str(target),
"name": target.name,
"bytes": len(data),
"mime_type": mime_type,
}
@app.get("/api/files")
async def list_managed_files(request: Request, path: Optional[str] = None):
policy, target, display_path = _resolve_managed_path(path, request)

View file

@ -863,6 +863,107 @@ class TestWebServerEndpoints:
)
assert resp.status_code == 401
# ── POST /api/chat/image-upload (browser clipboard/drop images) ─────
def test_chat_image_upload_writes_to_default_profile_images(self):
from hermes_constants import get_hermes_home
data_url = (
"data:image/png;base64,"
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk"
"+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
)
resp = self.client.post(
"/api/chat/image-upload",
json={"data_url": data_url, "filename": "../../clip.png"},
)
assert resp.status_code == 200
data = resp.json()
target = Path(data["path"])
assert data["ok"] is True
assert data["mime_type"] == "image/png"
assert target.parent == get_hermes_home() / "images"
assert target.name.startswith("dashboard_")
assert target.name.endswith("_clip.png")
assert target.is_file()
assert target.read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
def test_chat_image_upload_writes_to_requested_profile_images(self):
from hermes_cli import profiles as profiles_mod
worker_home = profiles_mod.get_profile_dir("worker")
worker_home.mkdir(parents=True)
resp = self.client.post(
"/api/chat/image-upload?profile=worker",
json={
"data_url": "data:image/gif;base64,R0lGODlhAQABAAAAACwAAAAAAQABAAA=",
"filename": "drop.gif",
},
)
assert resp.status_code == 200
target = Path(resp.json()["path"])
assert target.parent == worker_home / "images"
assert target.is_file()
assert target.read_bytes().startswith(b"GIF89a")
def test_chat_image_upload_rejects_non_image_payload(self):
resp = self.client.post(
"/api/chat/image-upload",
json={"data_url": "data:text/plain;base64,aGVsbG8="},
)
assert resp.status_code == 400
assert "image" in resp.json()["detail"].lower()
def test_chat_image_upload_rejects_spoofed_image_payload(self):
resp = self.client.post(
"/api/chat/image-upload",
json={"data_url": "data:image/png;base64,aGVsbG8=", "filename": "fake.png"},
)
assert resp.status_code == 400
assert "unsupported image type" in resp.json()["detail"].lower()
def test_chat_image_upload_rejects_unknown_profile(self):
resp = self.client.post(
"/api/chat/image-upload?profile=missing-profile",
json={"data_url": "data:image/gif;base64,R0lGODlhAQABAAAAACwAAAAAAQABAAA="},
)
assert resp.status_code == 404
assert "does not exist" in resp.json()["detail"]
def test_chat_image_upload_enforces_image_size_cap(self, monkeypatch):
import hermes_cli.web_server as web_server
monkeypatch.setattr(web_server, "_CHAT_IMAGE_UPLOAD_MAX_BYTES", 4)
resp = self.client.post(
"/api/chat/image-upload",
json={
"data_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=",
"filename": "large.png",
},
)
assert resp.status_code == 413
assert "too large" in resp.json()["detail"].lower()
def test_chat_image_upload_requires_auth(self):
from hermes_cli.web_server import _SESSION_HEADER_NAME
resp = self.client.post(
"/api/chat/image-upload",
json={"data_url": "data:image/gif;base64,R0lGODlhAQABAAAAACwAAAAAAQABAAA="},
headers={_SESSION_HEADER_NAME: "wrong-token"},
)
assert resp.status_code == 401
# ── Dashboard font override ─────────────────────────────────────────
def test_get_dashboard_font_defaults_to_theme(self):

View file

@ -0,0 +1,99 @@
import { describe, expect, it } from "vitest";
import {
firstImageFromClipboard,
imageFilesFromTransfer,
transferMayContainImage,
} from "./chatImagePaste";
// Minimal DataTransfer stand-ins. jsdom's DataTransfer doesn't let us seed
// items/files, so we hand-roll the shape the helpers read.
function makeItem(kind: string, type: string, file: File | null) {
return { kind, type, getAsFile: () => file } as unknown as DataTransferItem;
}
function makeData(opts: {
items?: DataTransferItem[];
files?: File[];
}): DataTransfer {
const items = opts.items ?? [];
const files = opts.files ?? [];
const itemList: Record<string | number, unknown> = { length: items.length };
items.forEach((it, i) => {
itemList[i] = it;
});
const fileList: Record<string | number, unknown> = { length: files.length };
files.forEach((f, i) => {
fileList[i] = f;
});
return {
items: itemList,
files: fileList,
} as unknown as DataTransfer;
}
const png = new File([new Uint8Array([1, 2, 3])], "x.png", {
type: "image/png",
});
const gif = new File([new Uint8Array([4, 5])], "y.gif", {
type: "image/gif",
});
describe("firstImageFromClipboard", () => {
it("returns null for null clipboard data", () => {
expect(firstImageFromClipboard(null)).toBeNull();
});
it("finds an image via items[].getAsFile()", () => {
const data = makeData({ items: [makeItem("file", "image/png", png)] });
expect(firstImageFromClipboard(data)).toBe(png);
});
it("ignores non-file and non-image items", () => {
const data = makeData({
items: [
makeItem("string", "text/plain", null),
makeItem("file", "application/pdf", new File([], "a.pdf")),
],
});
expect(firstImageFromClipboard(data)).toBeNull();
});
it("falls back to files[] when items are absent (Safari/Firefox)", () => {
const data = makeData({ files: [png] });
expect(firstImageFromClipboard(data)).toBe(png);
});
it("returns null when nothing image-like is present", () => {
const data = makeData({
files: [new File([], "notes.txt", { type: "text/plain" })],
});
expect(firstImageFromClipboard(data)).toBeNull();
});
});
describe("imageFilesFromTransfer", () => {
it("dedupes the same file when present in both items and files", () => {
const data = makeData({
items: [makeItem("file", "image/png", png)],
files: [png, gif],
});
expect(imageFilesFromTransfer(data)).toEqual([png, gif]);
});
});
describe("transferMayContainImage", () => {
it("is true for image items even when type is empty (some browsers)", () => {
const data = makeData({
items: [makeItem("file", "", png)],
});
expect(transferMayContainImage(data)).toBe(true);
});
it("is false for text-only transfers", () => {
const data = makeData({
items: [makeItem("string", "text/plain", null)],
});
expect(transferMayContainImage(data)).toBe(false);
});
});

View file

@ -0,0 +1,164 @@
import { authedFetch } from "@/lib/api";
// Clipboard image MIME → file extension. Mirrors the set the TUI's /image
// attach path and the gateway's image sniffer accept.
const IMAGE_MIME_EXT: Record<string, string> = {
"image/png": "png",
"image/jpeg": "jpg",
"image/gif": "gif",
"image/webp": "webp",
"image/bmp": "bmp",
};
// Anthropic caps a single vision image at ~25 MB; reject earlier client-side
// with a clear message rather than round-tripping a doomed upload.
const MAX_IMAGE_BYTES = 25 * 1024 * 1024;
export interface ChatImageUploadResult {
/** Absolute path under HERMES_HOME/images the gateway wrote. */
path: string;
/** Byte size of the uploaded image. */
bytes: number;
/** Basename written on the server. */
name: string;
mime_type: string;
}
function imageFileKey(file: File): string {
return `${file.name}\0${file.type}\0${file.size}\0${file.lastModified}`;
}
function addImageFile(files: File[], seen: Set<string>, file: File | null) {
if (!file || !file.type.startsWith("image/")) return;
const key = imageFileKey(file);
if (seen.has(key)) return;
seen.add(key);
files.push(file);
}
/** Pull every image file out of a DataTransfer (clipboard or drop). */
export function imageFilesFromTransfer(
data: DataTransfer | null,
): File[] {
if (!data) return [];
const files: File[] = [];
const seen = new Set<string>();
if (data.items?.length) {
for (let i = 0; i < data.items.length; i++) {
const item = data.items[i];
if (item.kind === "file" && item.type.startsWith("image/")) {
addImageFile(files, seen, item.getAsFile());
}
}
}
if (data.files?.length) {
for (let i = 0; i < data.files.length; i++) {
addImageFile(files, seen, data.files[i]);
}
}
return files;
}
/** Pull the first image blob out of a DataTransfer, or null if none present. */
export function firstImageFromClipboard(
data: DataTransfer | null,
): File | null {
return imageFilesFromTransfer(data)[0] ?? null;
}
/** True when a drag payload may contain an image (for dragover preventDefault). */
export function transferMayContainImage(data: DataTransfer | null): boolean {
if (!data) return false;
if (data.items?.length) {
for (let i = 0; i < data.items.length; i++) {
const item = data.items[i];
if (
item.kind === "file" &&
(!item.type || item.type.startsWith("image/"))
) {
return true;
}
}
return false;
}
if (data.files?.length) {
for (let i = 0; i < data.files.length; i++) {
if (data.files[i].type.startsWith("image/")) return true;
}
}
return false;
}
function fileToDataUrl(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onerror = () =>
reject(reader.error ?? new Error("image read failed"));
reader.onload = () => {
const result = reader.result;
if (typeof result === "string") {
resolve(result);
} else {
reject(new Error("image read failed"));
}
};
reader.readAsDataURL(file);
});
}
/**
* Upload a browser clipboard/drop image to ``HERMES_HOME/images`` via the
* dedicated chat upload endpoint and return the absolute gateway path.
*
* The dashboard Chat tab is an xterm mirror of a TUI running INSIDE the
* gateway. The container has no access to the browser's clipboard, so the
* server-side ``clipboard.paste`` path can never see a pasted image.
* Upload the bytes the browser already holds, then hand the path to the
* TUI's ``/image`` command.
*/
export async function uploadChatImage(
blob: Blob,
profile = "",
): Promise<ChatImageUploadResult> {
if (blob.size === 0) throw new Error("clipboard image is empty");
if (blob.size > MAX_IMAGE_BYTES) {
const mb = Math.round(MAX_IMAGE_BYTES / (1024 * 1024));
throw new Error(`image too large (max ${mb} MB)`);
}
const mime = blob.type || "image/png";
const ext = IMAGE_MIME_EXT[mime] || "png";
const filename =
blob instanceof File && blob.name
? blob.name
: `clipboard.${ext}`;
const file =
blob instanceof File
? blob
: new File([blob], filename, { type: mime });
const dataUrl = await fileToDataUrl(file);
const qs = profile ? `?profile=${encodeURIComponent(profile)}` : "";
const res = await authedFetch(`/api/chat/image-upload${qs}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
data_url: dataUrl,
filename,
}),
});
if (!res.ok) {
const text = await res.text().catch(() => res.statusText);
throw new Error(text || `HTTP ${res.status}`);
}
const uploaded = (await res.json()) as ChatImageUploadResult;
if (!uploaded?.path) {
throw new Error("image upload did not return a path");
}
return uploaded;
}

View file

@ -49,6 +49,11 @@ import {
normalizePtyMobileInput,
shouldTreatInputAsMobileReplacement,
} from "@/lib/pty-mobile-input";
import {
imageFilesFromTransfer,
transferMayContainImage,
uploadChatImage,
} from "@/lib/chatImagePaste";
import { PluginSlot } from "@/plugins";
import { useTheme } from "@/themes";
import { useProfileScope } from "@/contexts/useProfileScope";
@ -486,7 +491,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
// --- Clipboard integration ---------------------------------------
//
// Three independent paths all route to the system clipboard:
// Four independent paths all route to the system clipboard:
//
// 1. **Selection → Ctrl+C (or Cmd+C on macOS).** Ink's own handler
// in useInputHandlers.ts turns Ctrl+C into a copy when the
@ -500,9 +505,15 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
// ever stops listening (e.g. overlays / pickers) or if the user
// has selected with the mouse outside of Ink's selection model.
//
// 3. **Ctrl/Cmd+Shift+V.** Reads the system clipboard and feeds
// it to the terminal as keyboard input. xterm's paste() wraps
// it with bracketed-paste if the host has that mode enabled.
// 3. **Ctrl/Cmd+Shift+V.** Prefers clipboard.read() for images
// (upload → `/image`), else readText() into term.paste().
// preventDefault here suppresses the DOM paste event, so image
// handling must live in this key path — not only the host
// listener below.
//
// 4. **DOM paste / drop on the host.** Bare Ctrl+V and context-menu
// paste fire a ClipboardEvent; drag-drop lands files. Image
// payloads upload to HERMES_HOME/images then drive `/image`.
//
// OSC 52 reads (terminal asking to read the clipboard) are not
// supported — that would let any content the TUI renders exfiltrate
@ -532,6 +543,73 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
const isMac =
typeof navigator !== "undefined" && /Mac/i.test(navigator.platform);
// ── Image paste / drop ───────────────────────────────────────────────
// The Chat tab is an xterm mirror of a TUI inside the gateway. Server-side
// clipboard.paste / xclip never see the browser clipboard, so image paste
// must upload browser bytes to HERMES_HOME/images, then drive `/image`
// over the PTY (same burst-then-Return timing as handleCopyLast).
let imageUploadDisposed = false;
const pasteDelay = () =>
new Promise<void>((resolve) => window.setTimeout(resolve, 40));
const reportImageUploadError = (err: unknown) => {
const message = err instanceof Error ? err.message : String(err);
console.warn("[dashboard chat] image upload failed:", message);
setBanner(`Image upload failed: ${message}`);
};
const driveImageAttach = async (paths: string[]) => {
for (const path of paths) {
if (imageUploadDisposed) return;
const ws = wsRef.current;
if (!ws || ws.readyState !== WebSocket.OPEN) {
setBanner(
"Image uploaded, but chat is not connected — try again.",
);
return;
}
ws.send(`/image ${path}`);
await new Promise<void>((resolve) => window.setTimeout(resolve, 100));
const s = wsRef.current;
if (!s || s.readyState !== WebSocket.OPEN) return;
s.send("\r");
await pasteDelay();
}
term.focus();
};
const uploadAndAttachImages = (files: File[]) => {
if (!files.length) return;
void (async () => {
const paths: string[] = [];
for (const file of files) {
const uploaded = await uploadChatImage(file, scopedProfile);
if (imageUploadDisposed) return;
paths.push(uploaded.path);
}
await driveImageAttach(paths);
})().catch(reportImageUploadError);
};
const handleBrowserPaste = (ev: ClipboardEvent) => {
const files = imageFilesFromTransfer(ev.clipboardData);
if (!files.length) return;
ev.preventDefault();
ev.stopPropagation();
uploadAndAttachImages(files);
};
const handleBrowserDragOver = (ev: DragEvent) => {
if (!transferMayContainImage(ev.dataTransfer)) return;
ev.preventDefault();
if (ev.dataTransfer) ev.dataTransfer.dropEffect = "copy";
};
const handleBrowserDrop = (ev: DragEvent) => {
const files = imageFilesFromTransfer(ev.dataTransfer);
if (!files.length) return;
ev.preventDefault();
ev.stopPropagation();
uploadAndAttachImages(files);
};
host.addEventListener("paste", handleBrowserPaste, { capture: true });
host.addEventListener("dragover", handleBrowserDragOver, { capture: true });
host.addEventListener("drop", handleBrowserDrop, { capture: true });
term.attachCustomKeyEventHandler((ev) => {
if (ev.type !== "keydown") return true;
@ -563,15 +641,42 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
}
if (pasteModifier && ev.key.toLowerCase() === "v") {
navigator.clipboard
.readText()
.then((text) => {
if (text) term.paste(text);
})
.catch((err) => {
console.warn("[dashboard clipboard] paste failed:", err.message);
});
// preventDefault suppresses the DOM paste event, so image paste must
// be handled here via clipboard.read() — readText() alone misses
// image-only clipboards (the Discord / #24860 failure mode).
ev.preventDefault();
void (async () => {
try {
const read = navigator.clipboard?.read;
if (typeof read === "function") {
const items = await read.call(navigator.clipboard);
const files: File[] = [];
for (const item of items) {
const type = item.types.find((t) => t.startsWith("image/"));
if (!type) continue;
const blob = await item.getType(type);
const ext = type.split("/")[1]?.split("+")[0] || "png";
files.push(
new File([blob], `clipboard.${ext}`, { type }),
);
}
if (files.length) {
uploadAndAttachImages(files);
return;
}
}
} catch {
/* fall through to text paste */
}
try {
const text = await navigator.clipboard.readText();
if (text) term.paste(text);
} catch (err) {
const message =
err instanceof Error ? err.message : String(err);
console.warn("[dashboard clipboard] paste failed:", message);
}
})();
return false;
}
@ -1020,10 +1125,14 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) {
return () => {
unmounting = true;
imageUploadDisposed = true;
syncMetricsRef.current = null;
onDataDisposable?.dispose();
onResizeDisposable?.dispose();
mobileInputCleanup?.();
host.removeEventListener("paste", handleBrowserPaste, true);
host.removeEventListener("dragover", handleBrowserDragOver, true);
host.removeEventListener("drop", handleBrowserDrop, true);
if (metricsDebounce) clearTimeout(metricsDebounce);
window.removeEventListener("resize", scheduleSyncTerminalMetrics);
window.visualViewport?.removeEventListener(