From 17b35a7d95683cbdc2ec1fe34ba5fad143277ab5 Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 14 Jun 2026 12:14:58 -0400 Subject: [PATCH] fix(tools): release stuck input and prevent UIA index drift on Windows Addresses review feedback on the Windows computer_use backend (#43927). 1. A failed click/drag/scroll left modifier keys - and, for drag, the mouse button - synthetically held down: the release ran after the action inside the same try block, so an injection error skipped it. Move the release into a finally so Ctrl/Alt/Shift and the button are always released and the cursor restored even when an injection raises. 2. capture (_walk_elements) and set_value (_control_at_index) each reimplemented the same BFS + interactability filter; if the two ever diverged, set_value would resolve an index to a different control than the capture advertised. Both now consume one _iter_interactable generator, so element #N is the same control in both paths. 3. That shared walk uses collections.deque.popleft() instead of the O(n) list.pop(0). 7 new dependency-free tests (modifier/button release on failure, capture/set_value index agreement, BFS order, Text-pattern filter); they pass off-Windows. Full computer_use suites: 137 passed on Windows 11. Thanks to @Icather for spotting all three and proposing the fixes (originally raised in #45976). Suggested-by: ChengLong Han <97326386+Icather@users.noreply.github.com> Co-authored-by: ChengLong Han <97326386+Icather@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- tests/tools/test_computer_use_windows.py | 185 +++++++++++++++++ tools/computer_use/windows_backend.py | 247 ++++++++++++----------- 2 files changed, 315 insertions(+), 117 deletions(-) diff --git a/tests/tools/test_computer_use_windows.py b/tests/tools/test_computer_use_windows.py index 5d9a80262b55..59d801612d5b 100644 --- a/tests/tools/test_computer_use_windows.py +++ b/tests/tools/test_computer_use_windows.py @@ -373,3 +373,188 @@ class TestIdleGuard: t0 = _t.monotonic() wb._wait_for_user_idle() assert _t.monotonic() - t0 < 1.0 + + +# --------------------------------------------------------------------------- +# Input-state safety: a failed pointer action must never strand modifiers +# (or, for drag, the mouse button) in the held-down state. Regression guard +# for the try/finally release in click/drag/scroll. +# --------------------------------------------------------------------------- + +def _raise(*_a, **_k): + raise RuntimeError("synthetic injection failure") + + +class TestInputStateReleasedOnFailure: + def _backend(self, monkeypatch): + from tools.computer_use import windows_backend as wb + b = wb.WindowsUIABackend() + b._elements[1] = UIElement(index=1, role="Button", label="OK", + bounds=(110, 220, 100, 50), window_id=777) + b._capture_rect = (100, 200, 640, 480) + # Window present and unmoved -> _resolve_point yields the cached center. + monkeypatch.setattr(wb, "win32gui", + types.SimpleNamespace(IsWindow=lambda h: True), + raising=False) + monkeypatch.setattr(wb, "_window_rect", lambda h: (100, 200, 640, 480), + raising=False) + monkeypatch.setattr(wb, "win32api", + types.SimpleNamespace(GetCursorPos=lambda: (5, 5)), + raising=False) + monkeypatch.setattr(b, "_overlay", + types.SimpleNamespace(send=lambda *a, **k: None, pid=0)) + monkeypatch.setattr(b, "_ensure_target_foreground", lambda: None) + # Tag the modifier down/up batches so we can assert both were sent. + monkeypatch.setattr(b, "_with_modifiers", + lambda modifiers=None: (["DOWN"], ["UP"])) + return wb, b + + def test_click_releases_modifiers_when_action_fails(self, monkeypatch): + wb, b = self._backend(monkeypatch) + sent = [] + monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch)) + monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None) + monkeypatch.setattr(wb, "_mouse_button", _raise) + res = b.click(element=1, modifiers=["ctrl"]) + assert not res.ok + assert sent == [["DOWN"], ["UP"]], "mods_up must run in the finally" + + def test_click_releases_modifiers_on_success(self, monkeypatch): + wb, b = self._backend(monkeypatch) + sent = [] + monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch)) + monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None) + monkeypatch.setattr(wb, "_mouse_button", lambda *a, **k: None) + res = b.click(element=1, modifiers=["ctrl"]) + assert res.ok + assert sent == [["DOWN"], ["UP"]] + + def test_drag_releases_button_and_modifiers_midway(self, monkeypatch): + wb, b = self._backend(monkeypatch) + sent, buttons = [], [] + calls = {"moves": 0} + + def _mv(*_a, **_k): + calls["moves"] += 1 + if calls["moves"] == 2: # 1 = move to start, 2 = first drag step + raise RuntimeError("boom mid-drag") + + monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch)) + monkeypatch.setattr(wb, "_mouse_move", _mv) + monkeypatch.setattr(wb, "_mouse_button", + lambda button, down: buttons.append((button, down))) + res = b.drag(from_element=1, to_xy=(300, 400), modifiers=["alt"]) + assert not res.ok + # Primary button was pressed, then released in the finally; mods released. + assert ("left", True) in buttons + assert buttons[-1] == ("left", False) + assert sent == [["DOWN"], ["UP"]] + + def test_scroll_releases_modifiers_when_wheel_fails(self, monkeypatch): + wb, b = self._backend(monkeypatch) + sent = [] + monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch)) + monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None) + monkeypatch.setattr(wb, "_mouse_wheel", _raise) + res = b.scroll(direction="down", element=1, modifiers=["shift"]) + assert not res.ok + assert sent == [["DOWN"], ["UP"]] + + +# --------------------------------------------------------------------------- +# Shared tree-walk: capture (_walk_elements) and set_value (_control_at_index) +# consume one generator (_iter_interactable), so an element index resolves to +# the same control in both, discovery is breadth-first, and the filter is +# applied identically. Guards findings #2 (deque) and #3 (single walk). +# --------------------------------------------------------------------------- + +class _FakeRect: + def __init__(self, left, top, right, bottom): + self.left, self.top, self.right, self.bottom = left, top, right, bottom + + +class _FakeCtrl: + def __init__(self, role, name="", rect=(0, 0, 10, 10), enabled=True, + offscreen=False, children=None, automation_id="", patterns=None): + self.ControlTypeName = role + self.Name = name + self.AutomationId = automation_id + self.IsEnabled = enabled + self.IsOffscreen = offscreen + self.BoundingRectangle = _FakeRect(*rect) + self._children = list(children or []) + self._patterns = patterns or {} + + def GetChildren(self): + return list(self._children) + + def GetPattern(self, pid): + return self._patterns.get(pid) + + +class _FakeInitializer: + def __enter__(self): + return self + + def __exit__(self, *_a): + return False + + +def _fake_auto(root): + return types.SimpleNamespace( + ControlFromHandle=lambda hwnd: root, + UIAutomationInitializerInThread=_FakeInitializer, + PatternId=types.SimpleNamespace(ValuePattern=1, InvokePattern=2), + ) + + +_WIDE = (0, 0, 1000, 1000) + + +class TestSharedTreeWalk: + def _install(self, monkeypatch, root): + from tools.computer_use import windows_backend as wb + monkeypatch.setattr(wb, "_auto", _fake_auto(root), raising=False) + monkeypatch.setattr(wb, "_window_rect", lambda hwnd: _WIDE, raising=False) + return wb, wb.WindowsUIABackend() + + def test_capture_and_set_value_resolve_same_control(self, monkeypatch): + beta = _FakeCtrl("EditControl", name="Beta", rect=(10, 40, 60, 70)) + gamma = _FakeCtrl("ButtonControl", name="Gamma", offscreen=True) + mid = _FakeCtrl("PaneControl", children=[gamma, beta]) + alpha = _FakeCtrl("ButtonControl", name="Alpha", rect=(10, 10, 60, 30)) + delta = _FakeCtrl("ButtonControl", name="Delta", enabled=False) + root = _FakeCtrl("PaneControl", children=[alpha, mid, delta]) + wb, b = self._install(monkeypatch, root) + + els = b._walk_elements(123, _WIDE) + # Offscreen Gamma + disabled Delta filtered; BFS order Alpha then Beta. + assert [e.label for e in els] == ["Alpha", "Beta"] + assert [e.index for e in els] == [1, 2] + # Every advertised index re-resolves to the SAME control. + for e in els: + ctrl = b._control_at_index(123, e.index) + assert ctrl is not None and ctrl.Name == e.label + assert b._control_at_index(123, 99) is None + + def test_discovery_order_is_breadth_first(self, monkeypatch): + # BFS yields the shallow button before the nested one; a LIFO queue or + # DFS would invert these, so this pins deque.popleft() ordering. + deep = _FakeCtrl("ButtonControl", name="Second") + sub = _FakeCtrl("PaneControl", children=[deep]) + first = _FakeCtrl("ButtonControl", name="First", rect=(20, 20, 40, 40)) + root = _FakeCtrl("PaneControl", children=[first, sub]) + wb, b = self._install(monkeypatch, root) + assert [e.label for e in b._walk_elements(1, _WIDE)] == ["First", "Second"] + + def test_text_node_counts_only_with_value_pattern(self, monkeypatch): + # A Text control is non-interactable unless it exposes Value/Invoke — + # the special case must apply identically in the shared walk. + plain = _FakeCtrl("TextControl", name="label") + editable = _FakeCtrl("TextControl", name="field", rect=(0, 20, 30, 40), + patterns={1: object()}) # PatternId.ValuePattern + root = _FakeCtrl("PaneControl", children=[plain, editable]) + wb, b = self._install(monkeypatch, root) + els = b._walk_elements(1, _WIDE) + assert [e.label for e in els] == ["field"] + assert b._control_at_index(1, 1).Name == "field" diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index 9512187f6577..8f7506d8d4cd 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -26,6 +26,7 @@ import os import socket import sys import time +from collections import deque from typing import Any, Dict, List, Optional, Tuple from tools.computer_use.backend import ( @@ -655,6 +656,63 @@ class WindowsUIABackend(ComputerUseBackend): png_bytes_len=png_bytes_len, ) + def _iter_interactable( + self, + hwnd: int, + win_rect: Tuple[int, int, int, int], + max_nodes: int = 1500, + max_depth: int = 60, + time_budget: float = 4.0, + ): + """Yield (ctrl, role, label, rect) for interactable controls under + `hwnd`, in breadth-first discovery order. + + This is the single source of element discovery order *and* the + interactability/visibility filter. Both the capture walk + (`_walk_elements`) and set_value's re-find (`_control_at_index`) + consume it, so element #N resolves to the same control in both — if + the two ever drifted, set_value would act on a different control than + the capture advertised under that index. The caller owns the + UIAutomation COM apartment (the live `ctrl`/`rect` are only valid for + the duration of the iteration). + """ + wx, wy, ww, wh = win_rect + root = _auto.ControlFromHandle(hwnd) + if root is None: + return + deadline = time.monotonic() + time_budget + queue: deque = deque([(root, 0)]) + yielded = 0 + while queue and yielded < max_nodes and time.monotonic() < deadline: + ctrl, depth = queue.popleft() + try: + role = ctrl.ControlTypeName + if role.endswith("Control"): + role = role[: -len("Control")] + interactable = role in _INTERACTABLE_TYPES + if not interactable and role == "Text": + interactable = any( + ctrl.GetPattern(pid) is not None + for pid in (_auto.PatternId.ValuePattern, + _auto.PatternId.InvokePattern)) + if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen: + r = ctrl.BoundingRectangle + if (r.right > r.left and r.bottom > r.top + and r.left < wx + ww and r.right > wx + and r.top < wy + wh and r.bottom > wy): + label = (ctrl.Name or ctrl.AutomationId or "") + if len(label) > 120: + label = label[:120] + yielded += 1 + yield ctrl, role, label, r + except Exception: + pass + if depth < max_depth: + try: + queue.extend((c, depth + 1) for c in ctrl.GetChildren()) + except Exception: + pass + def _walk_elements( self, hwnd: int, @@ -663,62 +721,27 @@ class WindowsUIABackend(ComputerUseBackend): max_depth: int = 60, time_budget: float = 4.0, ) -> List[UIElement]: - """BFS the UIA tree under `hwnd`, collecting interactable elements. + """Collect interactable elements under `hwnd` for a capture. - Deterministic discovery order — set_value relies on a re-walk - producing the same indices as the capture that advertised them. + Indices follow `_iter_interactable`'s discovery order, which + set_value re-walks (via `_control_at_index`) to resolve an element + back to a live control. """ - wx, wy, ww, wh = win_rect elements: List[UIElement] = [] try: with _auto.UIAutomationInitializerInThread(): - root = _auto.ControlFromHandle(hwnd) - if root is None: - return elements - deadline = time.monotonic() + time_budget - queue: List[Tuple[Any, int]] = [(root, 0)] - visited = 0 - while queue and len(elements) < max_nodes and time.monotonic() < deadline: - ctrl, depth = queue.pop(0) - visited += 1 - try: - role = ctrl.ControlTypeName - if role.endswith("Control"): - role = role[: -len("Control")] - interactable = role in _INTERACTABLE_TYPES - if not interactable and role == "Text": - interactable = any( - ctrl.GetPattern(pid) is not None - for pid in (_auto.PatternId.ValuePattern, - _auto.PatternId.InvokePattern)) - if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen: - r = ctrl.BoundingRectangle - bw, bh = r.right - r.left, r.bottom - r.top - if (bw > 0 and bh > 0 - and r.left < wx + ww and r.right > wx - and r.top < wy + wh and r.bottom > wy): - label = (ctrl.Name or ctrl.AutomationId or "") - if len(label) > 120: - label = label[:120] - elements.append(UIElement( - index=len(elements) + 1, - role=role, - label=label, - bounds=(r.left, r.top, bw, bh), - app=self._last_app or "", - pid=self._target_pid or 0, - window_id=hwnd, - )) - except Exception: - pass - if depth < max_depth: - try: - queue.extend((c, depth + 1) for c in ctrl.GetChildren()) - except Exception: - pass - if queue: - logger.debug("element walk stopped early: %d visited, %d queued", - visited, len(queue)) + for ctrl, role, label, r in self._iter_interactable( + hwnd, win_rect, max_nodes=max_nodes, + max_depth=max_depth, time_budget=time_budget): + elements.append(UIElement( + index=len(elements) + 1, + role=role, + label=label, + bounds=(r.left, r.top, r.right - r.left, r.bottom - r.top), + app=self._last_app or "", + pid=self._target_pid or 0, + window_id=hwnd, + )) except Exception as e: logger.warning("UIA element walk failed: %s", e) return elements @@ -813,16 +836,21 @@ class WindowsUIABackend(ComputerUseBackend): self._overlay.send({"cmd": "flash", "text": f"click · {what}", "ttl": 1.5}) self._ensure_target_foreground() old_pos = win32api.GetCursorPos() - _send_inputs(mods_down) - _mouse_move(px, py) - time.sleep(0.03) - for i in range(max(1, click_count)): - _mouse_button(button, True) - _mouse_button(button, False) - if i + 1 < click_count: - time.sleep(0.05) - _send_inputs(mods_up) - _mouse_move(*old_pos) + try: + _send_inputs(mods_down) + _mouse_move(px, py) + time.sleep(0.03) + for i in range(max(1, click_count)): + _mouse_button(button, True) + _mouse_button(button, False) + if i + 1 < click_count: + time.sleep(0.05) + finally: + # Release modifiers + restore the cursor even if an injection + # above raised — otherwise a failed click strands Ctrl/Alt/Shift + # in the held-down state for the user at the keyboard. + _send_inputs(mods_up) + _mouse_move(*old_pos) return ActionResult(ok=True, action="click", message=f"{button}-clicked {what}" + (f" x{click_count}" if click_count > 1 else "")) @@ -855,17 +883,23 @@ class WindowsUIABackend(ComputerUseBackend): self._overlay.send({"cmd": "flash", "text": f"drag · {src} → {dst}", "ttl": 1.5}) self._ensure_target_foreground() old_pos = win32api.GetCursorPos() - _send_inputs(mods_down) - _mouse_move(fx, fy) - time.sleep(0.05) - _mouse_button(button, True) - steps = 12 - for i in range(1, steps + 1): - _mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps) - time.sleep(0.01) - _mouse_button(button, False) - _send_inputs(mods_up) - _mouse_move(*old_pos) + try: + _send_inputs(mods_down) + _mouse_move(fx, fy) + time.sleep(0.05) + _mouse_button(button, True) + steps = 12 + for i in range(1, steps + 1): + _mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps) + time.sleep(0.01) + _mouse_button(button, False) + finally: + # A mid-drag failure must not strand the button or modifiers + # held down — a stuck primary button turns every later move + # into a drag-select. The button-up is idempotent on success. + _mouse_button(button, False) + _send_inputs(mods_up) + _mouse_move(*old_pos) return ActionResult(ok=True, action="drag", message=f"dragged {src} -> {dst}") except Exception as e: @@ -903,16 +937,20 @@ class WindowsUIABackend(ComputerUseBackend): "text": f"scroll {direction} x{amount}", "ttl": 1.2}) self._ensure_target_foreground() old_pos = win32api.GetCursorPos() - _send_inputs(mods_down) - _mouse_move(px, py) - time.sleep(0.03) - if direction in ("up", "down"): - _mouse_wheel(amount if direction == "up" else -amount) - else: - _mouse_wheel(amount if direction == "right" else -amount, - horizontal=True) - _send_inputs(mods_up) - _mouse_move(*old_pos) + try: + _send_inputs(mods_down) + _mouse_move(px, py) + time.sleep(0.03) + if direction in ("up", "down"): + _mouse_wheel(amount if direction == "up" else -amount) + else: + _mouse_wheel(amount if direction == "right" else -amount, + horizontal=True) + finally: + # Release modifiers + restore the cursor even if the wheel + # injection raised, so a failed scroll can't strand a modifier. + _send_inputs(mods_up) + _mouse_move(*old_pos) return ActionResult(ok=True, action="scroll", message=f"scrolled {direction} x{amount} at {what}") except Exception as e: @@ -1005,41 +1043,16 @@ class WindowsUIABackend(ComputerUseBackend): return ctrl def _control_at_index(self, hwnd: int, index: int) -> Optional[Any]: - """Return the live control at 1-based `index` in discovery order.""" - count = 0 - root = _auto.ControlFromHandle(hwnd) - if root is None: - return None - wx, wy, ww, wh = _window_rect(hwnd) - deadline = time.monotonic() + 4.0 - queue: List[Tuple[Any, int]] = [(root, 0)] - while queue and time.monotonic() < deadline: - ctrl, depth = queue.pop(0) - try: - role = ctrl.ControlTypeName - if role.endswith("Control"): - role = role[: -len("Control")] - interactable = role in _INTERACTABLE_TYPES - if not interactable and role == "Text": - interactable = any( - ctrl.GetPattern(pid) is not None - for pid in (_auto.PatternId.ValuePattern, - _auto.PatternId.InvokePattern)) - if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen: - r = ctrl.BoundingRectangle - if (r.right > r.left and r.bottom > r.top - and r.left < wx + ww and r.right > wx - and r.top < wy + wh and r.bottom > wy): - count += 1 - if count == index: - return ctrl - except Exception: - pass - if depth < 60: - try: - queue.extend((c, depth + 1) for c in ctrl.GetChildren()) - except Exception: - pass + """Return the live control at 1-based `index` in capture discovery order. + + Re-walks via the same `_iter_interactable` traversal the capture used, + so `index` resolves to the control capture advertised under that + number. The caller owns the UIAutomation COM apartment. + """ + for pos, (ctrl, _role, _label, _rect) in enumerate( + self._iter_interactable(hwnd, _window_rect(hwnd)), start=1): + if pos == index: + return ctrl return None @staticmethod