fix(tools): release stuck input and prevent UIA index drift on Windows

Addresses review feedback on the Windows computer_use backend (#43927).

1. A failed click/drag/scroll left modifier keys - and, for drag, the
   mouse button - synthetically held down: the release ran after the
   action inside the same try block, so an injection error skipped it.
   Move the release into a finally so Ctrl/Alt/Shift and the button are
   always released and the cursor restored even when an injection raises.

2. capture (_walk_elements) and set_value (_control_at_index) each
   reimplemented the same BFS + interactability filter; if the two ever
   diverged, set_value would resolve an index to a different control than
   the capture advertised. Both now consume one _iter_interactable
   generator, so element #N is the same control in both paths.

3. That shared walk uses collections.deque.popleft() instead of the
   O(n) list.pop(0).

7 new dependency-free tests (modifier/button release on failure,
capture/set_value index agreement, BFS order, Text-pattern filter); they
pass off-Windows. Full computer_use suites: 137 passed on Windows 11.

Thanks to @Icather for spotting all three and proposing the fixes
(originally raised in #45976).

Suggested-by: ChengLong Han <97326386+Icather@users.noreply.github.com>
Co-authored-by: ChengLong Han <97326386+Icather@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jeff 2026-06-14 12:14:58 -04:00 • committed by Teknium
parent b062a5d4b9
commit 17b35a7d95
No known key found for this signature in database
2 changed files with 315 additions and 117 deletions

View file

@ -373,3 +373,188 @@ class TestIdleGuard:
t0 = _t.monotonic()
wb._wait_for_user_idle()
assert _t.monotonic() - t0 < 1.0
# ---------------------------------------------------------------------------
# Input-state safety: a failed pointer action must never strand modifiers
# (or, for drag, the mouse button) in the held-down state. Regression guard
# for the try/finally release in click/drag/scroll.
# ---------------------------------------------------------------------------
def _raise(*_a, **_k):
raise RuntimeError("synthetic injection failure")
class TestInputStateReleasedOnFailure:
def _backend(self, monkeypatch):
from tools.computer_use import windows_backend as wb
b = wb.WindowsUIABackend()
b._elements[1] = UIElement(index=1, role="Button", label="OK",
bounds=(110, 220, 100, 50), window_id=777)
b._capture_rect = (100, 200, 640, 480)
# Window present and unmoved -> _resolve_point yields the cached center.
monkeypatch.setattr(wb, "win32gui",
types.SimpleNamespace(IsWindow=lambda h: True),
raising=False)
monkeypatch.setattr(wb, "_window_rect", lambda h: (100, 200, 640, 480),
raising=False)
monkeypatch.setattr(wb, "win32api",
types.SimpleNamespace(GetCursorPos=lambda: (5, 5)),
raising=False)
monkeypatch.setattr(b, "_overlay",
types.SimpleNamespace(send=lambda *a, **k: None, pid=0))
monkeypatch.setattr(b, "_ensure_target_foreground", lambda: None)
# Tag the modifier down/up batches so we can assert both were sent.
monkeypatch.setattr(b, "_with_modifiers",
lambda modifiers=None: (["DOWN"], ["UP"]))
return wb, b
def test_click_releases_modifiers_when_action_fails(self, monkeypatch):
wb, b = self._backend(monkeypatch)
sent = []
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None)
monkeypatch.setattr(wb, "_mouse_button", _raise)
res = b.click(element=1, modifiers=["ctrl"])
assert not res.ok
assert sent == [["DOWN"], ["UP"]], "mods_up must run in the finally"
def test_click_releases_modifiers_on_success(self, monkeypatch):
wb, b = self._backend(monkeypatch)
sent = []
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None)
monkeypatch.setattr(wb, "_mouse_button", lambda *a, **k: None)
res = b.click(element=1, modifiers=["ctrl"])
assert res.ok
assert sent == [["DOWN"], ["UP"]]
def test_drag_releases_button_and_modifiers_midway(self, monkeypatch):
wb, b = self._backend(monkeypatch)
sent, buttons = [], []
calls = {"moves": 0}
def _mv(*_a, **_k):
calls["moves"] += 1
if calls["moves"] == 2: # 1 = move to start, 2 = first drag step
raise RuntimeError("boom mid-drag")
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
monkeypatch.setattr(wb, "_mouse_move", _mv)
monkeypatch.setattr(wb, "_mouse_button",
lambda button, down: buttons.append((button, down)))
res = b.drag(from_element=1, to_xy=(300, 400), modifiers=["alt"])
assert not res.ok
# Primary button was pressed, then released in the finally; mods released.
assert ("left", True) in buttons
assert buttons[-1] == ("left", False)
assert sent == [["DOWN"], ["UP"]]
def test_scroll_releases_modifiers_when_wheel_fails(self, monkeypatch):
wb, b = self._backend(monkeypatch)
sent = []
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None)
monkeypatch.setattr(wb, "_mouse_wheel", _raise)
res = b.scroll(direction="down", element=1, modifiers=["shift"])
assert not res.ok
assert sent == [["DOWN"], ["UP"]]
# ---------------------------------------------------------------------------
# Shared tree-walk: capture (_walk_elements) and set_value (_control_at_index)
# consume one generator (_iter_interactable), so an element index resolves to
# the same control in both, discovery is breadth-first, and the filter is
# applied identically. Guards findings #2 (deque) and #3 (single walk).
# ---------------------------------------------------------------------------
class _FakeRect:
def __init__(self, left, top, right, bottom):
self.left, self.top, self.right, self.bottom = left, top, right, bottom
class _FakeCtrl:
def __init__(self, role, name="", rect=(0, 0, 10, 10), enabled=True,
offscreen=False, children=None, automation_id="", patterns=None):
self.ControlTypeName = role
self.Name = name
self.AutomationId = automation_id
self.IsEnabled = enabled
self.IsOffscreen = offscreen
self.BoundingRectangle = _FakeRect(*rect)
self._children = list(children or [])
self._patterns = patterns or {}
def GetChildren(self):
return list(self._children)
def GetPattern(self, pid):
return self._patterns.get(pid)
class _FakeInitializer:
def __enter__(self):
return self
def __exit__(self, *_a):
return False
def _fake_auto(root):
return types.SimpleNamespace(
ControlFromHandle=lambda hwnd: root,
UIAutomationInitializerInThread=_FakeInitializer,
PatternId=types.SimpleNamespace(ValuePattern=1, InvokePattern=2),
)
_WIDE = (0, 0, 1000, 1000)
class TestSharedTreeWalk:
def _install(self, monkeypatch, root):
from tools.computer_use import windows_backend as wb
monkeypatch.setattr(wb, "_auto", _fake_auto(root), raising=False)
monkeypatch.setattr(wb, "_window_rect", lambda hwnd: _WIDE, raising=False)
return wb, wb.WindowsUIABackend()
def test_capture_and_set_value_resolve_same_control(self, monkeypatch):
beta = _FakeCtrl("EditControl", name="Beta", rect=(10, 40, 60, 70))
gamma = _FakeCtrl("ButtonControl", name="Gamma", offscreen=True)
mid = _FakeCtrl("PaneControl", children=[gamma, beta])
alpha = _FakeCtrl("ButtonControl", name="Alpha", rect=(10, 10, 60, 30))
delta = _FakeCtrl("ButtonControl", name="Delta", enabled=False)
root = _FakeCtrl("PaneControl", children=[alpha, mid, delta])
wb, b = self._install(monkeypatch, root)
els = b._walk_elements(123, _WIDE)
# Offscreen Gamma + disabled Delta filtered; BFS order Alpha then Beta.
assert [e.label for e in els] == ["Alpha", "Beta"]
assert [e.index for e in els] == [1, 2]
# Every advertised index re-resolves to the SAME control.
for e in els:
ctrl = b._control_at_index(123, e.index)
assert ctrl is not None and ctrl.Name == e.label
assert b._control_at_index(123, 99) is None
def test_discovery_order_is_breadth_first(self, monkeypatch):
# BFS yields the shallow button before the nested one; a LIFO queue or
# DFS would invert these, so this pins deque.popleft() ordering.
deep = _FakeCtrl("ButtonControl", name="Second")
sub = _FakeCtrl("PaneControl", children=[deep])
first = _FakeCtrl("ButtonControl", name="First", rect=(20, 20, 40, 40))
root = _FakeCtrl("PaneControl", children=[first, sub])
wb, b = self._install(monkeypatch, root)
assert [e.label for e in b._walk_elements(1, _WIDE)] == ["First", "Second"]
def test_text_node_counts_only_with_value_pattern(self, monkeypatch):
# A Text control is non-interactable unless it exposes Value/Invoke —
# the special case must apply identically in the shared walk.
plain = _FakeCtrl("TextControl", name="label")
editable = _FakeCtrl("TextControl", name="field", rect=(0, 20, 30, 40),
patterns={1: object()}) # PatternId.ValuePattern
root = _FakeCtrl("PaneControl", children=[plain, editable])
wb, b = self._install(monkeypatch, root)
els = b._walk_elements(1, _WIDE)
assert [e.label for e in els] == ["field"]
assert b._control_at_index(1, 1).Name == "field"

View file

@ -26,6 +26,7 @@ import os
import socket
import sys
import time
from collections import deque
from typing import Any, Dict, List, Optional, Tuple
from tools.computer_use.backend import (
@ -655,6 +656,63 @@ class WindowsUIABackend(ComputerUseBackend):
png_bytes_len=png_bytes_len,
)
def _iter_interactable(
self,
hwnd: int,
win_rect: Tuple[int, int, int, int],
max_nodes: int = 1500,
max_depth: int = 60,
time_budget: float = 4.0,
):
"""Yield (ctrl, role, label, rect) for interactable controls under
`hwnd`, in breadth-first discovery order.
This is the single source of element discovery order *and* the
interactability/visibility filter. Both the capture walk
(`_walk_elements`) and set_value's re-find (`_control_at_index`)
consume it, so element #N resolves to the same control in both — if
the two ever drifted, set_value would act on a different control than
the capture advertised under that index. The caller owns the
UIAutomation COM apartment (the live `ctrl`/`rect` are only valid for
the duration of the iteration).
"""
wx, wy, ww, wh = win_rect
root = _auto.ControlFromHandle(hwnd)
if root is None:
return
deadline = time.monotonic() + time_budget
queue: deque = deque([(root, 0)])
yielded = 0
while queue and yielded < max_nodes and time.monotonic() < deadline:
ctrl, depth = queue.popleft()
try:
role = ctrl.ControlTypeName
if role.endswith("Control"):
role = role[: -len("Control")]
interactable = role in _INTERACTABLE_TYPES
if not interactable and role == "Text":
interactable = any(
ctrl.GetPattern(pid) is not None
for pid in (_auto.PatternId.ValuePattern,
_auto.PatternId.InvokePattern))
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
r = ctrl.BoundingRectangle
if (r.right > r.left and r.bottom > r.top
and r.left < wx + ww and r.right > wx
and r.top < wy + wh and r.bottom > wy):
label = (ctrl.Name or ctrl.AutomationId or "")
if len(label) > 120:
label = label[:120]
yielded += 1
yield ctrl, role, label, r
except Exception:
pass
if depth < max_depth:
try:
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
except Exception:
pass
def _walk_elements(
self,
hwnd: int,
@ -663,62 +721,27 @@ class WindowsUIABackend(ComputerUseBackend):
max_depth: int = 60,
time_budget: float = 4.0,
) -> List[UIElement]:
"""BFS the UIA tree under `hwnd`, collecting interactable elements.
"""Collect interactable elements under `hwnd` for a capture.
Deterministic discovery order — set_value relies on a re-walk
producing the same indices as the capture that advertised them.
Indices follow `_iter_interactable`'s discovery order, which
set_value re-walks (via `_control_at_index`) to resolve an element
back to a live control.
"""
wx, wy, ww, wh = win_rect
elements: List[UIElement] = []
try:
with _auto.UIAutomationInitializerInThread():
root = _auto.ControlFromHandle(hwnd)
if root is None:
return elements
deadline = time.monotonic() + time_budget
queue: List[Tuple[Any, int]] = [(root, 0)]
visited = 0
while queue and len(elements) < max_nodes and time.monotonic() < deadline:
ctrl, depth = queue.pop(0)
visited += 1
try:
role = ctrl.ControlTypeName
if role.endswith("Control"):
role = role[: -len("Control")]
interactable = role in _INTERACTABLE_TYPES
if not interactable and role == "Text":
interactable = any(
ctrl.GetPattern(pid) is not None
for pid in (_auto.PatternId.ValuePattern,
_auto.PatternId.InvokePattern))
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
r = ctrl.BoundingRectangle
bw, bh = r.right - r.left, r.bottom - r.top
if (bw > 0 and bh > 0
and r.left < wx + ww and r.right > wx
and r.top < wy + wh and r.bottom > wy):
label = (ctrl.Name or ctrl.AutomationId or "")
if len(label) > 120:
label = label[:120]
elements.append(UIElement(
index=len(elements) + 1,
role=role,
label=label,
bounds=(r.left, r.top, bw, bh),
app=self._last_app or "",
pid=self._target_pid or 0,
window_id=hwnd,
))
except Exception:
pass
if depth < max_depth:
try:
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
except Exception:
pass
if queue:
logger.debug("element walk stopped early: %d visited, %d queued",
visited, len(queue))
for ctrl, role, label, r in self._iter_interactable(
hwnd, win_rect, max_nodes=max_nodes,
max_depth=max_depth, time_budget=time_budget):
elements.append(UIElement(
index=len(elements) + 1,
role=role,
label=label,
bounds=(r.left, r.top, r.right - r.left, r.bottom - r.top),
app=self._last_app or "",
pid=self._target_pid or 0,
window_id=hwnd,
))
except Exception as e:
logger.warning("UIA element walk failed: %s", e)
return elements
@ -813,16 +836,21 @@ class WindowsUIABackend(ComputerUseBackend):
self._overlay.send({"cmd": "flash", "text": f"click · {what}", "ttl": 1.5})
self._ensure_target_foreground()
old_pos = win32api.GetCursorPos()
_send_inputs(mods_down)
_mouse_move(px, py)
time.sleep(0.03)
for i in range(max(1, click_count)):
_mouse_button(button, True)
_mouse_button(button, False)
if i + 1 < click_count:
time.sleep(0.05)
_send_inputs(mods_up)
_mouse_move(*old_pos)
try:
_send_inputs(mods_down)
_mouse_move(px, py)
time.sleep(0.03)
for i in range(max(1, click_count)):
_mouse_button(button, True)
_mouse_button(button, False)
if i + 1 < click_count:
time.sleep(0.05)
finally:
# Release modifiers + restore the cursor even if an injection
# above raised — otherwise a failed click strands Ctrl/Alt/Shift
# in the held-down state for the user at the keyboard.
_send_inputs(mods_up)
_mouse_move(*old_pos)
return ActionResult(ok=True, action="click",
message=f"{button}-clicked {what}"
+ (f" x{click_count}" if click_count > 1 else ""))
@ -855,17 +883,23 @@ class WindowsUIABackend(ComputerUseBackend):
self._overlay.send({"cmd": "flash", "text": f"drag · {src} → {dst}", "ttl": 1.5})
self._ensure_target_foreground()
old_pos = win32api.GetCursorPos()
_send_inputs(mods_down)
_mouse_move(fx, fy)
time.sleep(0.05)
_mouse_button(button, True)
steps = 12
for i in range(1, steps + 1):
_mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps)
time.sleep(0.01)
_mouse_button(button, False)
_send_inputs(mods_up)
_mouse_move(*old_pos)
try:
_send_inputs(mods_down)
_mouse_move(fx, fy)
time.sleep(0.05)
_mouse_button(button, True)
steps = 12
for i in range(1, steps + 1):
_mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps)
time.sleep(0.01)
_mouse_button(button, False)
finally:
# A mid-drag failure must not strand the button or modifiers
# held down — a stuck primary button turns every later move
# into a drag-select. The button-up is idempotent on success.
_mouse_button(button, False)
_send_inputs(mods_up)
_mouse_move(*old_pos)
return ActionResult(ok=True, action="drag",
message=f"dragged {src} -> {dst}")
except Exception as e:
@ -903,16 +937,20 @@ class WindowsUIABackend(ComputerUseBackend):
"text": f"scroll {direction} x{amount}", "ttl": 1.2})
self._ensure_target_foreground()
old_pos = win32api.GetCursorPos()
_send_inputs(mods_down)
_mouse_move(px, py)
time.sleep(0.03)
if direction in ("up", "down"):
_mouse_wheel(amount if direction == "up" else -amount)
else:
_mouse_wheel(amount if direction == "right" else -amount,
horizontal=True)
_send_inputs(mods_up)
_mouse_move(*old_pos)
try:
_send_inputs(mods_down)
_mouse_move(px, py)
time.sleep(0.03)
if direction in ("up", "down"):
_mouse_wheel(amount if direction == "up" else -amount)
else:
_mouse_wheel(amount if direction == "right" else -amount,
horizontal=True)
finally:
# Release modifiers + restore the cursor even if the wheel
# injection raised, so a failed scroll can't strand a modifier.
_send_inputs(mods_up)
_mouse_move(*old_pos)
return ActionResult(ok=True, action="scroll",
message=f"scrolled {direction} x{amount} at {what}")
except Exception as e:
@ -1005,41 +1043,16 @@ class WindowsUIABackend(ComputerUseBackend):
return ctrl
def _control_at_index(self, hwnd: int, index: int) -> Optional[Any]:
"""Return the live control at 1-based `index` in discovery order."""
count = 0
root = _auto.ControlFromHandle(hwnd)
if root is None:
return None
wx, wy, ww, wh = _window_rect(hwnd)
deadline = time.monotonic() + 4.0
queue: List[Tuple[Any, int]] = [(root, 0)]
while queue and time.monotonic() < deadline:
ctrl, depth = queue.pop(0)
try:
role = ctrl.ControlTypeName
if role.endswith("Control"):
role = role[: -len("Control")]
interactable = role in _INTERACTABLE_TYPES
if not interactable and role == "Text":
interactable = any(
ctrl.GetPattern(pid) is not None
for pid in (_auto.PatternId.ValuePattern,
_auto.PatternId.InvokePattern))
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
r = ctrl.BoundingRectangle
if (r.right > r.left and r.bottom > r.top
and r.left < wx + ww and r.right > wx
and r.top < wy + wh and r.bottom > wy):
count += 1
if count == index:
return ctrl
except Exception:
pass
if depth < 60:
try:
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
except Exception:
pass
"""Return the live control at 1-based `index` in capture discovery order.
Re-walks via the same `_iter_interactable` traversal the capture used,
so `index` resolves to the control capture advertised under that
number. The caller owns the UIAutomation COM apartment.
"""
for pos, (ctrl, _role, _label, _rect) in enumerate(
self._iter_interactable(hwnd, _window_rect(hwnd)), start=1):
if pos == index:
return ctrl
return None
@staticmethod