diff --git a/tests/tools/test_computer_use_capture_routing.py b/tests/tools/test_computer_use_capture_routing.py index b2c5c81bf412..5f5e802847d2 100644 --- a/tests/tools/test_computer_use_capture_routing.py +++ b/tests/tools/test_computer_use_capture_routing.py @@ -1,4 +1,4 @@ -"""End-to-end regression for #24015 — capture routing via auxiliary.vision. +"""End-to-end regression for #24015 -- capture routing via auxiliary.vision. When ``computer_use(action='capture', mode='som'|'vision')`` returns a screenshot, ``_capture_response`` previously always returned a @@ -33,13 +33,13 @@ import pytest # Fixtures / helpers # --------------------------------------------------------------------------- -# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly. +# 8×8 PNG (transparent) -- minimal provider-acceptable bytes that decode cleanly. _PNG_B64 = ( "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG" "NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg==" ) -# 1×1 JPEG — used to verify mime detection works for either stream type. +# 1×1 JPEG -- used to verify mime detection works for either stream type. _JPEG_B64 = ( "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB" "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/" @@ -140,7 +140,7 @@ class TestCaptureResponseDefaultPath: return_value=True) as routing: resp = cu_tool._capture_response(cap) - # ax never even consults the routing helper — short-circuited above + # ax never even consults the routing helper -- short-circuited above # the image branch. routing.assert_not_called() assert isinstance(resp, str) @@ -298,15 +298,17 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - # Aux failure → fall back to multimodal envelope (so the user still - # gets *something* useful even if vision is broken). - assert isinstance(resp, dict) - assert resp.get("_multimodal") is True + # Aux failure with routing requested (text-only main model) degrades + # to the AX/SOM text payload — a multimodal envelope would hand a + # screenshot to a model that cannot consume images. + assert isinstance(resp, str) + body = json.loads(resp) + assert body.get("vision_unavailable") is True # Temp file must still be cleaned up. assert observed_path["path"] assert not os.path.exists(observed_path["path"]) - def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir): + def test_empty_aux_analysis_degrades_to_text_payload(self, tmp_cache_dir): from tools.computer_use import tool as cu_tool cap = _make_capture(mode="som") @@ -323,12 +325,15 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - # Empty analysis is treated as failure — we'd rather show pixels - # than embed an empty 'vision_analysis' string into the result. - assert isinstance(resp, dict) - assert resp.get("_multimodal") is True + # Empty analysis is treated as failure; with routing requested the + # capture degrades to the AX/SOM text payload (elements stay usable) + # rather than embedding an empty 'vision_analysis' string. + assert isinstance(resp, str) + body = json.loads(resp) + assert body.get("vision_unavailable") is True + assert body.get("elements") is not None - def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir): + def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir): from tools.computer_use import tool as cu_tool cap = _make_capture(mode="som") @@ -345,8 +350,9 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - assert isinstance(resp, dict) - assert resp.get("_multimodal") is True + assert isinstance(resp, str) + body = json.loads(resp) + assert body.get("vision_unavailable") is True # --------------------------------------------------------------------------- @@ -398,7 +404,7 @@ class TestRoutingDecisionWiring: with patch("hermes_cli.config.load_config", side_effect=RuntimeError("config.yaml unreadable")): - # No exception should bubble up — fail open by returning False + # No exception should bubble up -- fail open by returning False # so the legacy multimodal envelope continues to work. assert cu_tool._should_route_through_aux_vision() is False @@ -417,14 +423,14 @@ class TestRoutingDecisionWiring: # --------------------------------------------------------------------------- -# Bug reproduction marker — proves the fix is needed. +# Bug reproduction marker -- proves the fix is needed. # --------------------------------------------------------------------------- class TestBugReproductionAnchor: """Without the fix, this test would assert the wrong thing. On upstream/main HEAD prior to this branch, _capture_response returns a - multimodal envelope unconditionally — so when a non-vision main model + multimodal envelope unconditionally -- so when a non-vision main model is configured, the captured PNG is delivered to the main provider as image_url content and the request is rejected with HTTP 404. We don't have a live provider here, but we can pin the contract: with routing @@ -455,7 +461,7 @@ class TestBugReproductionAnchor: # Must be a string (text-only result). assert isinstance(resp, str) - # Must NOT contain a base64 image URL anywhere — that's what tripped + # Must NOT contain a base64 image URL anywhere -- that's what tripped # 'No endpoints found that support image input' on the reporter's # main provider in #24015. assert "data:image" not in resp diff --git a/tests/tools/test_computer_use_windows.py b/tests/tools/test_computer_use_windows.py index f1d54cc9cf75..5d9a80262b55 100644 --- a/tests/tools/test_computer_use_windows.py +++ b/tests/tools/test_computer_use_windows.py @@ -291,3 +291,85 @@ class TestShrinkCaptureForVision: from tools.computer_use.tool import _shrink_capture_for_vision raw = b"not an image at all" assert _shrink_capture_for_vision(raw, ".png") is raw + + +# --------------------------------------------------------------------------- +# Hardening: vision-down fallback, stale-coordinate translation, idle guard +# --------------------------------------------------------------------------- + +class TestVisionDownFallback: + def test_capture_degrades_to_text_when_aux_vision_fails(self, monkeypatch): + """Routing requested (text-only main) + vision down => AX text payload, + never a multimodal envelope a text model can't consume.""" + from tools.computer_use import tool + from tools.computer_use.backend import CaptureResult, UIElement + monkeypatch.setattr(tool, "_should_route_through_aux_vision", lambda: True) + monkeypatch.setattr(tool, "_route_capture_through_aux_vision", + lambda cap, summary: None) + # Must be >= 8x8 or _capture_response's provider-minimum check skips + # the vision branch entirely before the fallback we're testing. + import base64 + import io + pil = pytest.importorskip("PIL.Image") + buf = io.BytesIO() + pil.new("RGB", (16, 16), (40, 40, 40)).save(buf, format="PNG") + png_b64 = base64.b64encode(buf.getvalue()).decode("ascii") + cap = CaptureResult( + mode="som", width=800, height=600, png_b64=png_b64, + elements=[UIElement(index=1, role="Button", label="OK", + bounds=(1, 2, 3, 4))], + app="x.exe", window_title="W") + resp = tool._capture_response(cap) + assert isinstance(resp, str), "must be a text payload, not multimodal" + body = json.loads(resp) + assert body["vision_unavailable"] is True + assert body["elements"][0]["index"] == 1 + assert "Element-index actions still work" in body["summary"] + + +class TestStaleCoordinateTranslation: + def _backend_with_element(self, monkeypatch, new_rect): + from tools.computer_use import windows_backend as wb + from tools.computer_use.backend import UIElement + b = wb.WindowsUIABackend() + b._elements[1] = UIElement(index=1, role="Button", label="OK", + bounds=(110, 220, 100, 50), window_id=777) + b._capture_rect = (100, 200, 640, 480) + monkeypatch.setattr(wb, "win32gui", + types.SimpleNamespace(IsWindow=lambda h: True), + raising=False) + monkeypatch.setattr(wb, "_window_rect", lambda h: new_rect) + return b + + def test_window_moved_translates_click_point(self, monkeypatch): + b = self._backend_with_element(monkeypatch, (130, 250, 640, 480)) + x, y, what = b._resolve_point(1, None, None) + assert (x, y) == (160 + 30, 245 + 50) # center (160,245) + delta (30,50) + assert "window moved" in what + + def test_window_unmoved_uses_cached_center(self, monkeypatch): + b = self._backend_with_element(monkeypatch, (100, 200, 640, 480)) + x, y, _ = b._resolve_point(1, None, None) + assert (x, y) == (160, 245) + + def test_window_resized_demands_recapture(self, monkeypatch): + b = self._backend_with_element(monkeypatch, (100, 200, 800, 480)) + res = b.click(element=1) + assert not res.ok + assert "resized" in res.message + + +class TestIdleGuard: + def test_zero_threshold_disables_guard(self, monkeypatch): + from tools.computer_use import windows_backend as wb + monkeypatch.setenv("HERMES_COMPUTER_USE_IDLE_WAIT", "0") + wb._wait_for_user_idle() # must return immediately, touch no win32 + + def test_returns_once_user_is_idle(self, monkeypatch): + from tools.computer_use import windows_backend as wb + monkeypatch.setenv("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5") + monkeypatch.setattr(wb, "_seconds_since_user_input", lambda: 99.0) + import time as _t + t0 = _t.monotonic() + wb._wait_for_user_idle() + assert _t.monotonic() - t0 < 1.0 diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 84371fe8400d..1143afbe7837 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -582,10 +582,39 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME routed = _route_capture_through_aux_vision(cap, summary) if routed is not None: return routed - # Aux routing was requested but failed (no vision client, aux - # call raised, etc.). Fall through to the multimodal envelope — - # better to surface a tool-result error from the main model - # than to silently drop the screenshot entirely. + # Aux routing was requested but failed (vision node down, aux + # call raised, etc.). Routing being *requested* means the main + # model cannot consume images — falling through to the + # multimodal envelope would put a screenshot in front of a + # text-only model and break the capture with a provider error. + # Degrade to the AX/SOM text payload instead: the element index + # still supports element-targeted actions, so the agent can + # keep driving blind until vision comes back. + summary_lines.append( + " (vision unavailable: the auxiliary vision model could not " + "be reached; screenshot omitted. Element-index actions still " + "work — drive via the element list above.)" + ) + if truncated_elements: + summary_lines.append( + f" (response truncated to {len(visible_elements)} of " + f"{total_elements} elements; raise max_elements or pass " + "app= to narrow)" + ) + payload = { + "mode": cap.mode, + "width": response_width, + "height": response_height, + "app": cap.app, + "window_title": cap.window_title, + "elements": [_element_to_dict(e) for e in visible_elements], + "total_elements": total_elements, + "summary": "\n".join(summary_lines), + "vision_unavailable": True, + } + if truncated_elements: + payload["truncated_elements"] = truncated_elements + return json.dumps(payload) # Detect actual image format from base64 magic bytes so the MIME type # matches what the data contains (cua-driver may return JPEG or PNG). diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index 184dea598e46..9512187f6577 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -223,6 +223,44 @@ _EXTENDED_VKS = frozenset({0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, _MODIFIER_VKS = frozenset({0x10, 0x11, 0x12, 0x5B}) +class _LASTINPUTINFO(ctypes.Structure): + _fields_ = [("cbSize", ctypes.wintypes.UINT), ("dwTime", ctypes.wintypes.DWORD)] + + +def _seconds_since_user_input() -> float: + """Seconds since the user's last keyboard/mouse input (session-wide).""" + info = _LASTINPUTINFO() + info.cbSize = ctypes.sizeof(_LASTINPUTINFO) + if not ctypes.windll.user32.GetLastInputInfo(ctypes.byref(info)): + return float("inf") + # GetTickCount wraps at 49.7 days; the unsigned subtraction below stays + # correct across a single wrap. + delta = (ctypes.windll.kernel32.GetTickCount() - info.dwTime) & 0xFFFFFFFF + return delta / 1000.0 + + +def _wait_for_user_idle() -> None: + """Hold injected input briefly while the user is actively typing/mousing. + + Synthetic input lands in whatever has focus; colliding with a human + mid-keystroke sprays input across both parties' targets. Wait for + HERMES_COMPUTER_USE_IDLE_WAIT seconds (default 1.5, 0 disables) of user + idle, but never longer than ~8s total — the agent should yield, not + deadlock behind a user who is working. + """ + try: + threshold = float(os.environ.get("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5")) + except ValueError: + threshold = 1.5 + if threshold <= 0: + return + deadline = time.monotonic() + 8.0 + while time.monotonic() < deadline: + if _seconds_since_user_input() >= threshold: + return + time.sleep(0.2) + + def _vk_for_key(name: str) -> Optional[int]: """Map a key name to a virtual-key code; None when unknown.""" name = name.strip().lower() @@ -420,6 +458,11 @@ class WindowsUIABackend(ComputerUseBackend): self._last_app: Optional[str] = None self._target_hwnd: Optional[int] = None self._target_pid: Optional[int] = None + # Window rect at the time of the last capture — element bounds are + # absolute screen coords, so if the window moves between capture and + # click we translate by the origin delta instead of clicking stale + # pixels (see _element_offset). + self._capture_rect: Optional[Tuple[int, int, int, int]] = None self._started = False self._overlay = _OverlayClient() @@ -526,6 +569,10 @@ class WindowsUIABackend(ComputerUseBackend): return False def _ensure_target_foreground(self) -> None: + # Called exactly once at the top of every input-injecting action — + # the idle guard lives here so all of click/drag/scroll/type/key + # yield to an actively-working user before touching the desktop. + _wait_for_user_idle() if self._target_hwnd and win32gui.IsWindow(self._target_hwnd): self._bring_to_foreground(self._target_hwnd) @@ -563,6 +610,7 @@ class WindowsUIABackend(ComputerUseBackend): return CaptureResult(mode=mode, width=0, height=0) x, y, w, h = _window_rect(hwnd) + self._capture_rect = (x, y, w, h) window_title = win32gui.GetWindowText(hwnd) img = None @@ -690,6 +738,27 @@ class WindowsUIABackend(ComputerUseBackend): draw.text((ix + 3, iy + 1), text, fill=(255, 255, 255)) # ── Pointer actions ───────────────────────────────────────────── + def _element_offset(self, el: UIElement) -> Tuple[int, int]: + """Origin shift to apply to cached element bounds. + + If the element's window moved since the capture, the cached absolute + coords are stale by exactly the window-origin delta — translate. + A resize invalidates interior layout, so that demands a re-capture. + """ + hwnd = el.window_id + if not (hwnd and self._capture_rect and win32gui.IsWindow(hwnd)): + return 0, 0 + try: + cx, cy, cw, ch = self._capture_rect + nx, ny, nw, nh = _window_rect(hwnd) + except Exception: + return 0, 0 + if (nw, nh) != (cw, ch): + raise ValueError( + "the target window was resized since the last capture — " + "re-run capture(mode='som') for fresh element positions") + return nx - cx, ny - cy + def _resolve_point(self, element: Optional[int], x: Optional[int], y: Optional[int]) -> Tuple[int, int, str]: """Return (x, y, what) for an action target; raises ValueError.""" @@ -699,8 +768,12 @@ class WindowsUIABackend(ComputerUseBackend): raise ValueError( f"element #{element} is not in the last capture — re-run " "capture(mode='som') and use a fresh index") + dx, dy = self._element_offset(el) cx, cy = el.center() - return cx, cy, f"element #{element} ({el.role} {el.label!r})" + what = f"element #{element} ({el.role} {el.label!r})" + if dx or dy: + what += f" [window moved {dx:+d},{dy:+d} since capture]" + return cx + dx, cy + dy, what if x is None or y is None: raise ValueError("requires element= or coordinate [x, y]") return int(x), int(y), f"({x}, {y})"