fix(computer-use): fold duplicate PR polish into cua-driver salvage

agent/prompt_builder.py

@@ -440,9 +440,9 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str:
     elif is_windows:
         offscreen_line = (
             "- If an element is behind another window, cua-driver still "
-            "drives it — no need to raise it. Some targets (Chromium-based "
-            "apps, GTK) ignore background input; pass `dispatch='foreground'` "
-            "for those.\n\n"
+            "drives it — no need to raise it. Some apps may still force "
+            "foreground behavior internally; if an action does not land, "
+            "re-capture and adapt instead of retrying blindly.\n\n"
         )
     else:
         offscreen_line = (

tests/tools/test_computer_use.py

@@ -1195,7 +1195,10 @@ class TestLazyMcpInstall:
 
     def test_feature_registered_in_allowlist(self):
         from tools import lazy_deps
-        assert lazy_deps.feature_specs("tool.computer_use") == ("mcp==1.26.0",)
+        assert lazy_deps.feature_specs("tool.computer_use") == (
+            "mcp==1.26.0",
+            "starlette==1.0.1",
+        )
 
     def test_start_lazy_installs_mcp(self):
         from tools.computer_use import cua_backend

tests/tools/test_computer_use_capture_routing.py

@@ -298,15 +298,17 @@ class TestCaptureResponseRoutedToAuxVision:
                    new_callable=lambda: fake_vat):
             resp = cu_tool._capture_response(cap)
 
-        # Aux failure → fall back to multimodal envelope (so the user still
-        # gets *something* useful even if vision is broken).
-        assert isinstance(resp, dict)
-        assert resp.get("_multimodal") is True
+        # Aux failure with routing requested degrades to the AX/SOM text
+        # payload. Falling through to a multimodal envelope can hand pixels to
+        # a text-only model and fail the provider request.
+        assert isinstance(resp, str)
+        body = json.loads(resp)
+        assert body.get("vision_unavailable") is True
         # Temp file must still be cleaned up.
         assert observed_path["path"]
         assert not os.path.exists(observed_path["path"])
 
-    def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir):
+    def test_empty_aux_analysis_degrades_to_text_payload(self, tmp_cache_dir):
         from tools.computer_use import tool as cu_tool
 
         cap = _make_capture(mode="som")
@@ -323,12 +325,15 @@ class TestCaptureResponseRoutedToAuxVision:
                    new_callable=lambda: fake_vat):
             resp = cu_tool._capture_response(cap)
 
-        # Empty analysis is treated as failure — we'd rather show pixels
-        # than embed an empty 'vision_analysis' string into the result.
-        assert isinstance(resp, dict)
-        assert resp.get("_multimodal") is True
+        # Empty analysis is treated as failure; with routing requested the
+        # capture degrades to the AX/SOM text payload (elements stay usable)
+        # rather than embedding an empty 'vision_analysis' string.
+        assert isinstance(resp, str)
+        body = json.loads(resp)
+        assert body.get("vision_unavailable") is True
+        assert body.get("elements") is not None
 
-    def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir):
+    def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir):
         from tools.computer_use import tool as cu_tool
 
         cap = _make_capture(mode="som")
@@ -345,8 +350,9 @@ class TestCaptureResponseRoutedToAuxVision:
                    new_callable=lambda: fake_vat):
             resp = cu_tool._capture_response(cap)
 
-        assert isinstance(resp, dict)
-        assert resp.get("_multimodal") is True
+        assert isinstance(resp, str)
+        body = json.loads(resp)
+        assert body.get("vision_unavailable") is True
 
 
 # ---------------------------------------------------------------------------

tools/computer_use/schema.py

@@ -16,14 +16,15 @@ from typing import Any, Dict
 COMPUTER_USE_SCHEMA: Dict[str, Any] = {
     "name": "computer_use",
     "description": (
-        "Drive the macOS desktop in the background — screenshots, mouse, "
-        "keyboard, scroll, drag — without stealing the user's cursor, "
-        "keyboard focus, or Space. Preferred workflow: call with "
+        "Drive the desktop in the background via cua-driver — screenshots, "
+        "mouse, keyboard, scroll, drag — without stealing the user's cursor "
+        "or keyboard focus. Supported on macOS and Windows. Preferred "
+        "workflow: call with "
         "action='capture' (mode='som' gives numbered element overlays), "
         "then click by `element` index for reliability. Pixel coordinates "
         "are supported for models trained on them. Works on any window — "
-        "hidden, minimized, on another Space, or behind another app. "
-        "macOS only; requires cua-driver to be installed."
+        "hidden, minimized, or behind another app. Requires cua-driver to "
+        "be installed."
     ),
     "parameters": {
         "type": "object",
@@ -70,9 +71,9 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = {
                 "type": "string",
                 "description": (
                     "Optional. Limit capture/action to a specific app "
-                    "(by name, e.g. 'Safari', or bundle ID, "
-                    "'com.apple.Safari'). If omitted, operates on the "
-                    "frontmost app's window or the whole screen."
+                    "(by name, e.g. 'Safari' or 'Notepad', or bundle ID "
+                    "where the platform supports it). If omitted, operates "
+                    "on the frontmost app's window or the whole screen."
                 ),
             },
             "max_elements": {
@@ -126,7 +127,10 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = {
                 "type": "array",
                 "items": {
                     "type": "string",
-                    "enum": ["cmd", "shift", "option", "alt", "ctrl", "fn"],
+                    "enum": [
+                        "cmd", "shift", "option", "alt", "ctrl", "fn",
+                        "win", "windows", "super", "meta",
+                    ],
                 },
                 "description": "Modifier keys held during the action.",
             },

tools/computer_use/tool.py

@@ -90,9 +90,19 @@ _BLOCKED_KEY_COMBOS = {
     frozenset({"cmd", "ctrl", "q"}),             # lock screen
     frozenset({"cmd", "shift", "q"}),            # log out
     frozenset({"cmd", "option", "shift", "q"}),  # force log out
+    # Windows secure/session shortcuts. The Windows driver accepts Win-key
+    # combos, and Alt is canonicalized to option below, so block the
+    # destructive variants before any backend sees them.
+    frozenset({"win", "l"}),
+    frozenset({"ctrl", "option", "delete"}),
+    frozenset({"ctrl", "option", "del"}),
+    frozenset({"option", "f4"}),
 }
 
-_KEY_ALIASES = {"command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option"}
+_KEY_ALIASES = {
+    "command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option",
+    "windows": "win", "super": "win", "meta": "win",
+}
 
 
 def _canon_key_combo(keys: str) -> frozenset:
@@ -574,10 +584,37 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
             routed = _route_capture_through_aux_vision(cap, summary)
             if routed is not None:
                 return routed
-            # Aux routing was requested but failed (no vision client, aux
-            # call raised, etc.). Fall through to the multimodal envelope —
-            # better to surface a tool-result error from the main model
-            # than to silently drop the screenshot entirely.
+            # Aux routing was requested but failed (vision node down, aux call
+            # raised, empty analysis, etc.). Routing being requested means the
+            # main model may not be able to consume images; falling through to
+            # the multimodal envelope can break the capture with a provider
+            # error. Degrade to the AX/SOM text payload instead so element
+            # indices remain usable while vision is unavailable.
+            summary_lines.append(
+                "  (vision unavailable: the auxiliary vision model could not "
+                "be reached; screenshot omitted. Element-index actions still "
+                "work — drive via the element list above.)"
+            )
+            if truncated_elements:
+                summary_lines.append(
+                    f"  (response truncated to {len(visible_elements)} of "
+                    f"{total_elements} elements; raise max_elements or pass "
+                    "app= to narrow)"
+                )
+            payload = {
+                "mode": cap.mode,
+                "width": response_width,
+                "height": response_height,
+                "app": cap.app,
+                "window_title": cap.window_title,
+                "elements": [_element_to_dict(e) for e in visible_elements],
+                "total_elements": total_elements,
+                "summary": "\n".join(summary_lines),
+                "vision_unavailable": True,
+            }
+            if truncated_elements:
+                payload["truncated_elements"] = truncated_elements
+            return json.dumps(payload)
 
         # Detect actual image format from base64 magic bytes so the MIME type
         # matches what the data contains (cua-driver may return JPEG or PNG).
@@ -625,6 +662,33 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
 # auxiliary.vision routing for captured screenshots (#24015)
 # ---------------------------------------------------------------------------
 
+# Longest image side handed to the aux vision model. Full-resolution desktop
+# captures tokenize heavily and can overflow small local-model context windows;
+# ~1456px keeps SOM badges legible while cutting per-capture vision latency.
+_MAX_VISION_DIM = 1456
+
+
+def _shrink_capture_for_vision(raw: bytes, ext: str,
+                               max_dim: int = _MAX_VISION_DIM) -> bytes:
+    """Downscale encoded image bytes so the longest side is <= max_dim.
+
+    Returns the original bytes unchanged when the image already fits or when
+    Pillow is unavailable/fails — no worse than the pre-shrink behavior.
+    """
+    try:
+        from io import BytesIO
+        from PIL import Image
+        img = Image.open(BytesIO(raw))
+        if max(img.size) <= max_dim:
+            return raw
+        img.thumbnail((max_dim, max_dim))
+        out = BytesIO()
+        img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
+        return out.getvalue()
+    except Exception as exc:
+        logger.debug("computer_use: vision downscale skipped: %s", exc)
+        return raw
+
 def _should_route_through_aux_vision() -> bool:
     """Return True when ``_capture_response`` should hand the PNG to aux vision.
 
@@ -702,6 +766,7 @@ def _route_capture_through_aux_vision(
         cache_dir = get_hermes_dir("cache/vision", "temp_vision_images")
         cache_dir.mkdir(parents=True, exist_ok=True)
         temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}"
+        raw = _shrink_capture_for_vision(raw, ext)
         temp_image_path.write_bytes(raw)
 
         prompt = (
@@ -720,7 +785,7 @@ def _route_capture_through_aux_vision(
     except Exception as exc:
         logger.warning(
             "computer_use: auxiliary.vision pre-analysis failed (%s); "
-            "falling back to native multimodal envelope",
+            "returning to caller without aux analysis",
             exc,
         )
         return None

tools/lazy_deps.py

@@ -185,7 +185,10 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = {
     # extras in pyproject.toml. The one-liner installer pulls this in via
     # `[all]`; lazy-installing here covers lean / partial / broken-extra
     # installs so computer_use never dead-ends on `No module named 'mcp'`.
-    "tool.computer_use": ("mcp==1.26.0",),
+    "tool.computer_use": (
+        "mcp==1.26.0",
+        "starlette==1.0.1",  # CVE-2026-48710 — keep in sync with pyproject [computer-use]
+    ),
 }