From 52cf24e441de10780aecf92c8c142b4cc11836c0 Mon Sep 17 00:00:00 2001 From: Jeff Date: Wed, 10 Jun 2026 21:40:24 -0400 Subject: [PATCH] fix(tools): downscale computer_use captures before aux-vision routing Full-resolution desktop captures (1920x1032+) tokenize to thousands of vision tokens and overflow small local vision models' context windows - the aux call came back "the vision API rejected the image" and the model got no description at all. Cap the long side at 1456px before writing the temp image for vision_analyze: SOM badges stay legible, the request fits comfortably, and per-capture vision latency drops roughly in half. Also drop the hardcoded "macOS" from the describe prompt now that captures come from Windows hosts too. Co-Authored-By: Claude Fable 5 --- tools/computer_use/tool.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 1eebd4a47062..46addb34d7c7 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -710,10 +710,29 @@ def _route_capture_through_aux_vision( cache_dir = get_hermes_dir("cache/vision", "temp_vision_images") cache_dir.mkdir(parents=True, exist_ok=True) temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}" + + # Downscale before handing to the aux vision model. Full-resolution + # desktop captures (e.g. 1920x1032) tokenize to thousands of vision + # tokens and overflow small local-model context windows ("the vision + # API rejected the image"); ~1456px keeps SOM badges legible while + # fitting comfortably and cutting latency. + _MAX_VISION_DIM = 1456 + try: + from io import BytesIO as _BytesIO + from PIL import Image as _Image + img = _Image.open(_BytesIO(raw)) + if max(img.size) > _MAX_VISION_DIM: + img.thumbnail((_MAX_VISION_DIM, _MAX_VISION_DIM)) + out = _BytesIO() + img.save(out, format="JPEG" if ext == ".jpg" else "PNG") + raw = out.getvalue() + except Exception as exc: + logger.debug("computer_use: vision downscale skipped: %s", exc) + temp_image_path.write_bytes(raw) prompt = ( - "Describe what is visible in this macOS application screenshot in " + "Describe what is visible in this application screenshot in " "concise but specific terms. Mention the app name and window " "title if visible, the overall layout, any labelled buttons, " "menus or text fields, and any prominent text content the user "