diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 33f9841ce052..34203520487d 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -440,9 +440,9 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str: elif is_windows: offscreen_line = ( "- If an element is behind another window, cua-driver still " - "drives it — no need to raise it. Some targets (Chromium-based " - "apps, GTK) ignore background input; pass `dispatch='foreground'` " - "for those.\n\n" + "drives it — no need to raise it. Some apps may still force " + "foreground behavior internally; if an action does not land, " + "re-capture and adapt instead of retrying blindly.\n\n" ) else: offscreen_line = ( diff --git a/tests/tools/test_computer_use.py b/tests/tools/test_computer_use.py index 87b2b01c7bc3..0ab5d2fcfa48 100644 --- a/tests/tools/test_computer_use.py +++ b/tests/tools/test_computer_use.py @@ -1195,7 +1195,10 @@ class TestLazyMcpInstall: def test_feature_registered_in_allowlist(self): from tools import lazy_deps - assert lazy_deps.feature_specs("tool.computer_use") == ("mcp==1.26.0",) + assert lazy_deps.feature_specs("tool.computer_use") == ( + "mcp==1.26.0", + "starlette==1.0.1", + ) def test_start_lazy_installs_mcp(self): from tools.computer_use import cua_backend diff --git a/tests/tools/test_computer_use_capture_routing.py b/tests/tools/test_computer_use_capture_routing.py index e6e517bcf13d..ab2b80b9e05a 100644 --- a/tests/tools/test_computer_use_capture_routing.py +++ b/tests/tools/test_computer_use_capture_routing.py @@ -298,15 +298,17 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - # Aux failure → fall back to multimodal envelope (so the user still - # gets *something* useful even if vision is broken). - assert isinstance(resp, dict) - assert resp.get("_multimodal") is True + # Aux failure with routing requested degrades to the AX/SOM text + # payload. Falling through to a multimodal envelope can hand pixels to + # a text-only model and fail the provider request. + assert isinstance(resp, str) + body = json.loads(resp) + assert body.get("vision_unavailable") is True # Temp file must still be cleaned up. assert observed_path["path"] assert not os.path.exists(observed_path["path"]) - def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir): + def test_empty_aux_analysis_degrades_to_text_payload(self, tmp_cache_dir): from tools.computer_use import tool as cu_tool cap = _make_capture(mode="som") @@ -323,12 +325,15 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - # Empty analysis is treated as failure — we'd rather show pixels - # than embed an empty 'vision_analysis' string into the result. - assert isinstance(resp, dict) - assert resp.get("_multimodal") is True + # Empty analysis is treated as failure; with routing requested the + # capture degrades to the AX/SOM text payload (elements stay usable) + # rather than embedding an empty 'vision_analysis' string. + assert isinstance(resp, str) + body = json.loads(resp) + assert body.get("vision_unavailable") is True + assert body.get("elements") is not None - def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir): + def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir): from tools.computer_use import tool as cu_tool cap = _make_capture(mode="som") @@ -345,8 +350,9 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - assert isinstance(resp, dict) - assert resp.get("_multimodal") is True + assert isinstance(resp, str) + body = json.loads(resp) + assert body.get("vision_unavailable") is True # --------------------------------------------------------------------------- diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index b39ccf06aa90..e53ecc17b000 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -16,14 +16,15 @@ from typing import Any, Dict COMPUTER_USE_SCHEMA: Dict[str, Any] = { "name": "computer_use", "description": ( - "Drive the macOS desktop in the background — screenshots, mouse, " - "keyboard, scroll, drag — without stealing the user's cursor, " - "keyboard focus, or Space. Preferred workflow: call with " + "Drive the desktop in the background via cua-driver — screenshots, " + "mouse, keyboard, scroll, drag — without stealing the user's cursor " + "or keyboard focus. Supported on macOS and Windows. Preferred " + "workflow: call with " "action='capture' (mode='som' gives numbered element overlays), " "then click by `element` index for reliability. Pixel coordinates " "are supported for models trained on them. Works on any window — " - "hidden, minimized, on another Space, or behind another app. " - "macOS only; requires cua-driver to be installed." + "hidden, minimized, or behind another app. Requires cua-driver to " + "be installed." ), "parameters": { "type": "object", @@ -70,9 +71,9 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "type": "string", "description": ( "Optional. Limit capture/action to a specific app " - "(by name, e.g. 'Safari', or bundle ID, " - "'com.apple.Safari'). If omitted, operates on the " - "frontmost app's window or the whole screen." + "(by name, e.g. 'Safari' or 'Notepad', or bundle ID " + "where the platform supports it). If omitted, operates " + "on the frontmost app's window or the whole screen." ), }, "max_elements": { @@ -126,7 +127,10 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "type": "array", "items": { "type": "string", - "enum": ["cmd", "shift", "option", "alt", "ctrl", "fn"], + "enum": [ + "cmd", "shift", "option", "alt", "ctrl", "fn", + "win", "windows", "super", "meta", + ], }, "description": "Modifier keys held during the action.", }, diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index db6458445e77..d2ed3cd7734e 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -90,9 +90,19 @@ _BLOCKED_KEY_COMBOS = { frozenset({"cmd", "ctrl", "q"}), # lock screen frozenset({"cmd", "shift", "q"}), # log out frozenset({"cmd", "option", "shift", "q"}), # force log out + # Windows secure/session shortcuts. The Windows driver accepts Win-key + # combos, and Alt is canonicalized to option below, so block the + # destructive variants before any backend sees them. + frozenset({"win", "l"}), + frozenset({"ctrl", "option", "delete"}), + frozenset({"ctrl", "option", "del"}), + frozenset({"option", "f4"}), } -_KEY_ALIASES = {"command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option"} +_KEY_ALIASES = { + "command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option", + "windows": "win", "super": "win", "meta": "win", +} def _canon_key_combo(keys: str) -> frozenset: @@ -574,10 +584,37 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME routed = _route_capture_through_aux_vision(cap, summary) if routed is not None: return routed - # Aux routing was requested but failed (no vision client, aux - # call raised, etc.). Fall through to the multimodal envelope — - # better to surface a tool-result error from the main model - # than to silently drop the screenshot entirely. + # Aux routing was requested but failed (vision node down, aux call + # raised, empty analysis, etc.). Routing being requested means the + # main model may not be able to consume images; falling through to + # the multimodal envelope can break the capture with a provider + # error. Degrade to the AX/SOM text payload instead so element + # indices remain usable while vision is unavailable. + summary_lines.append( + " (vision unavailable: the auxiliary vision model could not " + "be reached; screenshot omitted. Element-index actions still " + "work — drive via the element list above.)" + ) + if truncated_elements: + summary_lines.append( + f" (response truncated to {len(visible_elements)} of " + f"{total_elements} elements; raise max_elements or pass " + "app= to narrow)" + ) + payload = { + "mode": cap.mode, + "width": response_width, + "height": response_height, + "app": cap.app, + "window_title": cap.window_title, + "elements": [_element_to_dict(e) for e in visible_elements], + "total_elements": total_elements, + "summary": "\n".join(summary_lines), + "vision_unavailable": True, + } + if truncated_elements: + payload["truncated_elements"] = truncated_elements + return json.dumps(payload) # Detect actual image format from base64 magic bytes so the MIME type # matches what the data contains (cua-driver may return JPEG or PNG). @@ -625,6 +662,33 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME # auxiliary.vision routing for captured screenshots (#24015) # --------------------------------------------------------------------------- +# Longest image side handed to the aux vision model. Full-resolution desktop +# captures tokenize heavily and can overflow small local-model context windows; +# ~1456px keeps SOM badges legible while cutting per-capture vision latency. +_MAX_VISION_DIM = 1456 + + +def _shrink_capture_for_vision(raw: bytes, ext: str, + max_dim: int = _MAX_VISION_DIM) -> bytes: + """Downscale encoded image bytes so the longest side is <= max_dim. + + Returns the original bytes unchanged when the image already fits or when + Pillow is unavailable/fails — no worse than the pre-shrink behavior. + """ + try: + from io import BytesIO + from PIL import Image + img = Image.open(BytesIO(raw)) + if max(img.size) <= max_dim: + return raw + img.thumbnail((max_dim, max_dim)) + out = BytesIO() + img.save(out, format="JPEG" if ext == ".jpg" else "PNG") + return out.getvalue() + except Exception as exc: + logger.debug("computer_use: vision downscale skipped: %s", exc) + return raw + def _should_route_through_aux_vision() -> bool: """Return True when ``_capture_response`` should hand the PNG to aux vision. @@ -702,6 +766,7 @@ def _route_capture_through_aux_vision( cache_dir = get_hermes_dir("cache/vision", "temp_vision_images") cache_dir.mkdir(parents=True, exist_ok=True) temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}" + raw = _shrink_capture_for_vision(raw, ext) temp_image_path.write_bytes(raw) prompt = ( @@ -720,7 +785,7 @@ def _route_capture_through_aux_vision( except Exception as exc: logger.warning( "computer_use: auxiliary.vision pre-analysis failed (%s); " - "falling back to native multimodal envelope", + "returning to caller without aux analysis", exc, ) return None diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 444996953e5f..699a342b1acc 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -185,7 +185,10 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # extras in pyproject.toml. The one-liner installer pulls this in via # `[all]`; lazy-installing here covers lean / partial / broken-extra # installs so computer_use never dead-ends on `No module named 'mcp'`. - "tool.computer_use": ("mcp==1.26.0",), + "tool.computer_use": ( + "mcp==1.26.0", + "starlette==1.0.1", # CVE-2026-48710 — keep in sync with pyproject [computer-use] + ), }