mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-21 07:55:08 +00:00
fix(tools): release stuck input and prevent UIA index drift on Windows
Addresses review feedback on the Windows computer_use backend (#43927). 1. A failed click/drag/scroll left modifier keys - and, for drag, the mouse button - synthetically held down: the release ran after the action inside the same try block, so an injection error skipped it. Move the release into a finally so Ctrl/Alt/Shift and the button are always released and the cursor restored even when an injection raises. 2. capture (_walk_elements) and set_value (_control_at_index) each reimplemented the same BFS + interactability filter; if the two ever diverged, set_value would resolve an index to a different control than the capture advertised. Both now consume one _iter_interactable generator, so element #N is the same control in both paths. 3. That shared walk uses collections.deque.popleft() instead of the O(n) list.pop(0). 7 new dependency-free tests (modifier/button release on failure, capture/set_value index agreement, BFS order, Text-pattern filter); they pass off-Windows. Full computer_use suites: 137 passed on Windows 11. Thanks to @Icather for spotting all three and proposing the fixes (originally raised in #45976). Suggested-by: ChengLong Han <97326386+Icather@users.noreply.github.com> Co-authored-by: ChengLong Han <97326386+Icather@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b062a5d4b9
commit
17b35a7d95
2 changed files with 315 additions and 117 deletions
|
|
@ -373,3 +373,188 @@ class TestIdleGuard:
|
|||
t0 = _t.monotonic()
|
||||
wb._wait_for_user_idle()
|
||||
assert _t.monotonic() - t0 < 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input-state safety: a failed pointer action must never strand modifiers
|
||||
# (or, for drag, the mouse button) in the held-down state. Regression guard
|
||||
# for the try/finally release in click/drag/scroll.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _raise(*_a, **_k):
|
||||
raise RuntimeError("synthetic injection failure")
|
||||
|
||||
|
||||
class TestInputStateReleasedOnFailure:
|
||||
def _backend(self, monkeypatch):
|
||||
from tools.computer_use import windows_backend as wb
|
||||
b = wb.WindowsUIABackend()
|
||||
b._elements[1] = UIElement(index=1, role="Button", label="OK",
|
||||
bounds=(110, 220, 100, 50), window_id=777)
|
||||
b._capture_rect = (100, 200, 640, 480)
|
||||
# Window present and unmoved -> _resolve_point yields the cached center.
|
||||
monkeypatch.setattr(wb, "win32gui",
|
||||
types.SimpleNamespace(IsWindow=lambda h: True),
|
||||
raising=False)
|
||||
monkeypatch.setattr(wb, "_window_rect", lambda h: (100, 200, 640, 480),
|
||||
raising=False)
|
||||
monkeypatch.setattr(wb, "win32api",
|
||||
types.SimpleNamespace(GetCursorPos=lambda: (5, 5)),
|
||||
raising=False)
|
||||
monkeypatch.setattr(b, "_overlay",
|
||||
types.SimpleNamespace(send=lambda *a, **k: None, pid=0))
|
||||
monkeypatch.setattr(b, "_ensure_target_foreground", lambda: None)
|
||||
# Tag the modifier down/up batches so we can assert both were sent.
|
||||
monkeypatch.setattr(b, "_with_modifiers",
|
||||
lambda modifiers=None: (["DOWN"], ["UP"]))
|
||||
return wb, b
|
||||
|
||||
def test_click_releases_modifiers_when_action_fails(self, monkeypatch):
|
||||
wb, b = self._backend(monkeypatch)
|
||||
sent = []
|
||||
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
|
||||
monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None)
|
||||
monkeypatch.setattr(wb, "_mouse_button", _raise)
|
||||
res = b.click(element=1, modifiers=["ctrl"])
|
||||
assert not res.ok
|
||||
assert sent == [["DOWN"], ["UP"]], "mods_up must run in the finally"
|
||||
|
||||
def test_click_releases_modifiers_on_success(self, monkeypatch):
|
||||
wb, b = self._backend(monkeypatch)
|
||||
sent = []
|
||||
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
|
||||
monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None)
|
||||
monkeypatch.setattr(wb, "_mouse_button", lambda *a, **k: None)
|
||||
res = b.click(element=1, modifiers=["ctrl"])
|
||||
assert res.ok
|
||||
assert sent == [["DOWN"], ["UP"]]
|
||||
|
||||
def test_drag_releases_button_and_modifiers_midway(self, monkeypatch):
|
||||
wb, b = self._backend(monkeypatch)
|
||||
sent, buttons = [], []
|
||||
calls = {"moves": 0}
|
||||
|
||||
def _mv(*_a, **_k):
|
||||
calls["moves"] += 1
|
||||
if calls["moves"] == 2: # 1 = move to start, 2 = first drag step
|
||||
raise RuntimeError("boom mid-drag")
|
||||
|
||||
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
|
||||
monkeypatch.setattr(wb, "_mouse_move", _mv)
|
||||
monkeypatch.setattr(wb, "_mouse_button",
|
||||
lambda button, down: buttons.append((button, down)))
|
||||
res = b.drag(from_element=1, to_xy=(300, 400), modifiers=["alt"])
|
||||
assert not res.ok
|
||||
# Primary button was pressed, then released in the finally; mods released.
|
||||
assert ("left", True) in buttons
|
||||
assert buttons[-1] == ("left", False)
|
||||
assert sent == [["DOWN"], ["UP"]]
|
||||
|
||||
def test_scroll_releases_modifiers_when_wheel_fails(self, monkeypatch):
|
||||
wb, b = self._backend(monkeypatch)
|
||||
sent = []
|
||||
monkeypatch.setattr(wb, "_send_inputs", lambda batch: sent.append(batch))
|
||||
monkeypatch.setattr(wb, "_mouse_move", lambda *a, **k: None)
|
||||
monkeypatch.setattr(wb, "_mouse_wheel", _raise)
|
||||
res = b.scroll(direction="down", element=1, modifiers=["shift"])
|
||||
assert not res.ok
|
||||
assert sent == [["DOWN"], ["UP"]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared tree-walk: capture (_walk_elements) and set_value (_control_at_index)
|
||||
# consume one generator (_iter_interactable), so an element index resolves to
|
||||
# the same control in both, discovery is breadth-first, and the filter is
|
||||
# applied identically. Guards findings #2 (deque) and #3 (single walk).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _FakeRect:
|
||||
def __init__(self, left, top, right, bottom):
|
||||
self.left, self.top, self.right, self.bottom = left, top, right, bottom
|
||||
|
||||
|
||||
class _FakeCtrl:
|
||||
def __init__(self, role, name="", rect=(0, 0, 10, 10), enabled=True,
|
||||
offscreen=False, children=None, automation_id="", patterns=None):
|
||||
self.ControlTypeName = role
|
||||
self.Name = name
|
||||
self.AutomationId = automation_id
|
||||
self.IsEnabled = enabled
|
||||
self.IsOffscreen = offscreen
|
||||
self.BoundingRectangle = _FakeRect(*rect)
|
||||
self._children = list(children or [])
|
||||
self._patterns = patterns or {}
|
||||
|
||||
def GetChildren(self):
|
||||
return list(self._children)
|
||||
|
||||
def GetPattern(self, pid):
|
||||
return self._patterns.get(pid)
|
||||
|
||||
|
||||
class _FakeInitializer:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_a):
|
||||
return False
|
||||
|
||||
|
||||
def _fake_auto(root):
|
||||
return types.SimpleNamespace(
|
||||
ControlFromHandle=lambda hwnd: root,
|
||||
UIAutomationInitializerInThread=_FakeInitializer,
|
||||
PatternId=types.SimpleNamespace(ValuePattern=1, InvokePattern=2),
|
||||
)
|
||||
|
||||
|
||||
_WIDE = (0, 0, 1000, 1000)
|
||||
|
||||
|
||||
class TestSharedTreeWalk:
|
||||
def _install(self, monkeypatch, root):
|
||||
from tools.computer_use import windows_backend as wb
|
||||
monkeypatch.setattr(wb, "_auto", _fake_auto(root), raising=False)
|
||||
monkeypatch.setattr(wb, "_window_rect", lambda hwnd: _WIDE, raising=False)
|
||||
return wb, wb.WindowsUIABackend()
|
||||
|
||||
def test_capture_and_set_value_resolve_same_control(self, monkeypatch):
|
||||
beta = _FakeCtrl("EditControl", name="Beta", rect=(10, 40, 60, 70))
|
||||
gamma = _FakeCtrl("ButtonControl", name="Gamma", offscreen=True)
|
||||
mid = _FakeCtrl("PaneControl", children=[gamma, beta])
|
||||
alpha = _FakeCtrl("ButtonControl", name="Alpha", rect=(10, 10, 60, 30))
|
||||
delta = _FakeCtrl("ButtonControl", name="Delta", enabled=False)
|
||||
root = _FakeCtrl("PaneControl", children=[alpha, mid, delta])
|
||||
wb, b = self._install(monkeypatch, root)
|
||||
|
||||
els = b._walk_elements(123, _WIDE)
|
||||
# Offscreen Gamma + disabled Delta filtered; BFS order Alpha then Beta.
|
||||
assert [e.label for e in els] == ["Alpha", "Beta"]
|
||||
assert [e.index for e in els] == [1, 2]
|
||||
# Every advertised index re-resolves to the SAME control.
|
||||
for e in els:
|
||||
ctrl = b._control_at_index(123, e.index)
|
||||
assert ctrl is not None and ctrl.Name == e.label
|
||||
assert b._control_at_index(123, 99) is None
|
||||
|
||||
def test_discovery_order_is_breadth_first(self, monkeypatch):
|
||||
# BFS yields the shallow button before the nested one; a LIFO queue or
|
||||
# DFS would invert these, so this pins deque.popleft() ordering.
|
||||
deep = _FakeCtrl("ButtonControl", name="Second")
|
||||
sub = _FakeCtrl("PaneControl", children=[deep])
|
||||
first = _FakeCtrl("ButtonControl", name="First", rect=(20, 20, 40, 40))
|
||||
root = _FakeCtrl("PaneControl", children=[first, sub])
|
||||
wb, b = self._install(monkeypatch, root)
|
||||
assert [e.label for e in b._walk_elements(1, _WIDE)] == ["First", "Second"]
|
||||
|
||||
def test_text_node_counts_only_with_value_pattern(self, monkeypatch):
|
||||
# A Text control is non-interactable unless it exposes Value/Invoke —
|
||||
# the special case must apply identically in the shared walk.
|
||||
plain = _FakeCtrl("TextControl", name="label")
|
||||
editable = _FakeCtrl("TextControl", name="field", rect=(0, 20, 30, 40),
|
||||
patterns={1: object()}) # PatternId.ValuePattern
|
||||
root = _FakeCtrl("PaneControl", children=[plain, editable])
|
||||
wb, b = self._install(monkeypatch, root)
|
||||
els = b._walk_elements(1, _WIDE)
|
||||
assert [e.label for e in els] == ["field"]
|
||||
assert b._control_at_index(1, 1).Name == "field"
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import os
|
|||
import socket
|
||||
import sys
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from tools.computer_use.backend import (
|
||||
|
|
@ -655,6 +656,63 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
png_bytes_len=png_bytes_len,
|
||||
)
|
||||
|
||||
def _iter_interactable(
|
||||
self,
|
||||
hwnd: int,
|
||||
win_rect: Tuple[int, int, int, int],
|
||||
max_nodes: int = 1500,
|
||||
max_depth: int = 60,
|
||||
time_budget: float = 4.0,
|
||||
):
|
||||
"""Yield (ctrl, role, label, rect) for interactable controls under
|
||||
`hwnd`, in breadth-first discovery order.
|
||||
|
||||
This is the single source of element discovery order *and* the
|
||||
interactability/visibility filter. Both the capture walk
|
||||
(`_walk_elements`) and set_value's re-find (`_control_at_index`)
|
||||
consume it, so element #N resolves to the same control in both — if
|
||||
the two ever drifted, set_value would act on a different control than
|
||||
the capture advertised under that index. The caller owns the
|
||||
UIAutomation COM apartment (the live `ctrl`/`rect` are only valid for
|
||||
the duration of the iteration).
|
||||
"""
|
||||
wx, wy, ww, wh = win_rect
|
||||
root = _auto.ControlFromHandle(hwnd)
|
||||
if root is None:
|
||||
return
|
||||
deadline = time.monotonic() + time_budget
|
||||
queue: deque = deque([(root, 0)])
|
||||
yielded = 0
|
||||
while queue and yielded < max_nodes and time.monotonic() < deadline:
|
||||
ctrl, depth = queue.popleft()
|
||||
try:
|
||||
role = ctrl.ControlTypeName
|
||||
if role.endswith("Control"):
|
||||
role = role[: -len("Control")]
|
||||
interactable = role in _INTERACTABLE_TYPES
|
||||
if not interactable and role == "Text":
|
||||
interactable = any(
|
||||
ctrl.GetPattern(pid) is not None
|
||||
for pid in (_auto.PatternId.ValuePattern,
|
||||
_auto.PatternId.InvokePattern))
|
||||
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
|
||||
r = ctrl.BoundingRectangle
|
||||
if (r.right > r.left and r.bottom > r.top
|
||||
and r.left < wx + ww and r.right > wx
|
||||
and r.top < wy + wh and r.bottom > wy):
|
||||
label = (ctrl.Name or ctrl.AutomationId or "")
|
||||
if len(label) > 120:
|
||||
label = label[:120]
|
||||
yielded += 1
|
||||
yield ctrl, role, label, r
|
||||
except Exception:
|
||||
pass
|
||||
if depth < max_depth:
|
||||
try:
|
||||
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _walk_elements(
|
||||
self,
|
||||
hwnd: int,
|
||||
|
|
@ -663,62 +721,27 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
max_depth: int = 60,
|
||||
time_budget: float = 4.0,
|
||||
) -> List[UIElement]:
|
||||
"""BFS the UIA tree under `hwnd`, collecting interactable elements.
|
||||
"""Collect interactable elements under `hwnd` for a capture.
|
||||
|
||||
Deterministic discovery order — set_value relies on a re-walk
|
||||
producing the same indices as the capture that advertised them.
|
||||
Indices follow `_iter_interactable`'s discovery order, which
|
||||
set_value re-walks (via `_control_at_index`) to resolve an element
|
||||
back to a live control.
|
||||
"""
|
||||
wx, wy, ww, wh = win_rect
|
||||
elements: List[UIElement] = []
|
||||
try:
|
||||
with _auto.UIAutomationInitializerInThread():
|
||||
root = _auto.ControlFromHandle(hwnd)
|
||||
if root is None:
|
||||
return elements
|
||||
deadline = time.monotonic() + time_budget
|
||||
queue: List[Tuple[Any, int]] = [(root, 0)]
|
||||
visited = 0
|
||||
while queue and len(elements) < max_nodes and time.monotonic() < deadline:
|
||||
ctrl, depth = queue.pop(0)
|
||||
visited += 1
|
||||
try:
|
||||
role = ctrl.ControlTypeName
|
||||
if role.endswith("Control"):
|
||||
role = role[: -len("Control")]
|
||||
interactable = role in _INTERACTABLE_TYPES
|
||||
if not interactable and role == "Text":
|
||||
interactable = any(
|
||||
ctrl.GetPattern(pid) is not None
|
||||
for pid in (_auto.PatternId.ValuePattern,
|
||||
_auto.PatternId.InvokePattern))
|
||||
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
|
||||
r = ctrl.BoundingRectangle
|
||||
bw, bh = r.right - r.left, r.bottom - r.top
|
||||
if (bw > 0 and bh > 0
|
||||
and r.left < wx + ww and r.right > wx
|
||||
and r.top < wy + wh and r.bottom > wy):
|
||||
label = (ctrl.Name or ctrl.AutomationId or "")
|
||||
if len(label) > 120:
|
||||
label = label[:120]
|
||||
elements.append(UIElement(
|
||||
index=len(elements) + 1,
|
||||
role=role,
|
||||
label=label,
|
||||
bounds=(r.left, r.top, bw, bh),
|
||||
app=self._last_app or "",
|
||||
pid=self._target_pid or 0,
|
||||
window_id=hwnd,
|
||||
))
|
||||
except Exception:
|
||||
pass
|
||||
if depth < max_depth:
|
||||
try:
|
||||
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
|
||||
except Exception:
|
||||
pass
|
||||
if queue:
|
||||
logger.debug("element walk stopped early: %d visited, %d queued",
|
||||
visited, len(queue))
|
||||
for ctrl, role, label, r in self._iter_interactable(
|
||||
hwnd, win_rect, max_nodes=max_nodes,
|
||||
max_depth=max_depth, time_budget=time_budget):
|
||||
elements.append(UIElement(
|
||||
index=len(elements) + 1,
|
||||
role=role,
|
||||
label=label,
|
||||
bounds=(r.left, r.top, r.right - r.left, r.bottom - r.top),
|
||||
app=self._last_app or "",
|
||||
pid=self._target_pid or 0,
|
||||
window_id=hwnd,
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning("UIA element walk failed: %s", e)
|
||||
return elements
|
||||
|
|
@ -813,16 +836,21 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
self._overlay.send({"cmd": "flash", "text": f"click · {what}", "ttl": 1.5})
|
||||
self._ensure_target_foreground()
|
||||
old_pos = win32api.GetCursorPos()
|
||||
_send_inputs(mods_down)
|
||||
_mouse_move(px, py)
|
||||
time.sleep(0.03)
|
||||
for i in range(max(1, click_count)):
|
||||
_mouse_button(button, True)
|
||||
_mouse_button(button, False)
|
||||
if i + 1 < click_count:
|
||||
time.sleep(0.05)
|
||||
_send_inputs(mods_up)
|
||||
_mouse_move(*old_pos)
|
||||
try:
|
||||
_send_inputs(mods_down)
|
||||
_mouse_move(px, py)
|
||||
time.sleep(0.03)
|
||||
for i in range(max(1, click_count)):
|
||||
_mouse_button(button, True)
|
||||
_mouse_button(button, False)
|
||||
if i + 1 < click_count:
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
# Release modifiers + restore the cursor even if an injection
|
||||
# above raised — otherwise a failed click strands Ctrl/Alt/Shift
|
||||
# in the held-down state for the user at the keyboard.
|
||||
_send_inputs(mods_up)
|
||||
_mouse_move(*old_pos)
|
||||
return ActionResult(ok=True, action="click",
|
||||
message=f"{button}-clicked {what}"
|
||||
+ (f" x{click_count}" if click_count > 1 else ""))
|
||||
|
|
@ -855,17 +883,23 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
self._overlay.send({"cmd": "flash", "text": f"drag · {src} → {dst}", "ttl": 1.5})
|
||||
self._ensure_target_foreground()
|
||||
old_pos = win32api.GetCursorPos()
|
||||
_send_inputs(mods_down)
|
||||
_mouse_move(fx, fy)
|
||||
time.sleep(0.05)
|
||||
_mouse_button(button, True)
|
||||
steps = 12
|
||||
for i in range(1, steps + 1):
|
||||
_mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps)
|
||||
time.sleep(0.01)
|
||||
_mouse_button(button, False)
|
||||
_send_inputs(mods_up)
|
||||
_mouse_move(*old_pos)
|
||||
try:
|
||||
_send_inputs(mods_down)
|
||||
_mouse_move(fx, fy)
|
||||
time.sleep(0.05)
|
||||
_mouse_button(button, True)
|
||||
steps = 12
|
||||
for i in range(1, steps + 1):
|
||||
_mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps)
|
||||
time.sleep(0.01)
|
||||
_mouse_button(button, False)
|
||||
finally:
|
||||
# A mid-drag failure must not strand the button or modifiers
|
||||
# held down — a stuck primary button turns every later move
|
||||
# into a drag-select. The button-up is idempotent on success.
|
||||
_mouse_button(button, False)
|
||||
_send_inputs(mods_up)
|
||||
_mouse_move(*old_pos)
|
||||
return ActionResult(ok=True, action="drag",
|
||||
message=f"dragged {src} -> {dst}")
|
||||
except Exception as e:
|
||||
|
|
@ -903,16 +937,20 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
"text": f"scroll {direction} x{amount}", "ttl": 1.2})
|
||||
self._ensure_target_foreground()
|
||||
old_pos = win32api.GetCursorPos()
|
||||
_send_inputs(mods_down)
|
||||
_mouse_move(px, py)
|
||||
time.sleep(0.03)
|
||||
if direction in ("up", "down"):
|
||||
_mouse_wheel(amount if direction == "up" else -amount)
|
||||
else:
|
||||
_mouse_wheel(amount if direction == "right" else -amount,
|
||||
horizontal=True)
|
||||
_send_inputs(mods_up)
|
||||
_mouse_move(*old_pos)
|
||||
try:
|
||||
_send_inputs(mods_down)
|
||||
_mouse_move(px, py)
|
||||
time.sleep(0.03)
|
||||
if direction in ("up", "down"):
|
||||
_mouse_wheel(amount if direction == "up" else -amount)
|
||||
else:
|
||||
_mouse_wheel(amount if direction == "right" else -amount,
|
||||
horizontal=True)
|
||||
finally:
|
||||
# Release modifiers + restore the cursor even if the wheel
|
||||
# injection raised, so a failed scroll can't strand a modifier.
|
||||
_send_inputs(mods_up)
|
||||
_mouse_move(*old_pos)
|
||||
return ActionResult(ok=True, action="scroll",
|
||||
message=f"scrolled {direction} x{amount} at {what}")
|
||||
except Exception as e:
|
||||
|
|
@ -1005,41 +1043,16 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
return ctrl
|
||||
|
||||
def _control_at_index(self, hwnd: int, index: int) -> Optional[Any]:
|
||||
"""Return the live control at 1-based `index` in discovery order."""
|
||||
count = 0
|
||||
root = _auto.ControlFromHandle(hwnd)
|
||||
if root is None:
|
||||
return None
|
||||
wx, wy, ww, wh = _window_rect(hwnd)
|
||||
deadline = time.monotonic() + 4.0
|
||||
queue: List[Tuple[Any, int]] = [(root, 0)]
|
||||
while queue and time.monotonic() < deadline:
|
||||
ctrl, depth = queue.pop(0)
|
||||
try:
|
||||
role = ctrl.ControlTypeName
|
||||
if role.endswith("Control"):
|
||||
role = role[: -len("Control")]
|
||||
interactable = role in _INTERACTABLE_TYPES
|
||||
if not interactable and role == "Text":
|
||||
interactable = any(
|
||||
ctrl.GetPattern(pid) is not None
|
||||
for pid in (_auto.PatternId.ValuePattern,
|
||||
_auto.PatternId.InvokePattern))
|
||||
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
|
||||
r = ctrl.BoundingRectangle
|
||||
if (r.right > r.left and r.bottom > r.top
|
||||
and r.left < wx + ww and r.right > wx
|
||||
and r.top < wy + wh and r.bottom > wy):
|
||||
count += 1
|
||||
if count == index:
|
||||
return ctrl
|
||||
except Exception:
|
||||
pass
|
||||
if depth < 60:
|
||||
try:
|
||||
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
|
||||
except Exception:
|
||||
pass
|
||||
"""Return the live control at 1-based `index` in capture discovery order.
|
||||
|
||||
Re-walks via the same `_iter_interactable` traversal the capture used,
|
||||
so `index` resolves to the control capture advertised under that
|
||||
number. The caller owns the UIAutomation COM apartment.
|
||||
"""
|
||||
for pos, (ctrl, _role, _label, _rect) in enumerate(
|
||||
self._iter_interactable(hwnd, _window_rect(hwnd)), start=1):
|
||||
if pos == index:
|
||||
return ctrl
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue