mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-10-20 05:15:31 +00:00
1196 lines
48 KiB
Python
1196 lines
48 KiB
Python
"""Windows implementation of ComputerUseBackend — UI Automation + SendInput.
|
|
|
|
The Windows analogue of cua_backend.py. Element discovery and set_value go
|
|
through UI Automation (the Windows counterpart of the macOS AX tree, via the
|
|
`uiautomation` package); screenshots through PIL.ImageGrab; mouse/keyboard
|
|
through SendInput.
|
|
|
|
One behavioural difference from the macOS backend: Windows has no supported
|
|
way to post input to a background window, so pointer/keyboard actions bring
|
|
the target window to the foreground first. `set_value` is the exception — it
|
|
mutates element values through UIA patterns and works without focus.
|
|
|
|
All coordinates are physical pixels; start() opts the process into
|
|
per-monitor DPI awareness so UIA bounds, ImageGrab and SendInput agree.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import ctypes
|
|
import ctypes.wintypes
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import socket
|
|
import sys
|
|
import time
|
|
from collections import deque
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from tools.computer_use.backend import (
|
|
ActionResult,
|
|
CaptureResult,
|
|
ComputerUseBackend,
|
|
UIElement,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_IMPORT_ERROR: Optional[Exception] = None
|
|
try:
|
|
import uiautomation as _auto
|
|
import win32api
|
|
import win32con
|
|
import win32gui
|
|
import win32process
|
|
from PIL import Image, ImageDraw, ImageGrab
|
|
except Exception as _e: # pragma: no cover - exercised via availability check
|
|
_IMPORT_ERROR = _e
|
|
|
|
|
|
def windows_backend_available() -> bool:
|
|
"""True iff this host can run the Windows UIA backend."""
|
|
return sys.platform == "win32" and _IMPORT_ERROR is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DPI awareness
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = ctypes.c_void_p(-4)
|
|
|
|
|
|
def _set_dpi_awareness() -> None:
|
|
"""Opt into per-monitor-v2 DPI awareness, best-effort with fallbacks."""
|
|
user32 = ctypes.windll.user32
|
|
try:
|
|
if user32.SetProcessDpiAwarenessContext(_DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2):
|
|
return
|
|
except Exception:
|
|
pass
|
|
try:
|
|
ctypes.windll.shcore.SetProcessDpiAwareness(2) # PROCESS_PER_MONITOR_DPI_AWARE
|
|
return
|
|
except Exception:
|
|
pass
|
|
try:
|
|
user32.SetProcessDPIAware()
|
|
except Exception:
|
|
logger.warning("could not set DPI awareness; coordinates may be scaled")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SendInput layer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_INPUT_MOUSE = 0
|
|
_INPUT_KEYBOARD = 1
|
|
_MOUSEEVENTF_MOVE = 0x0001
|
|
_MOUSEEVENTF_ABSOLUTE = 0x8000
|
|
_MOUSEEVENTF_VIRTUALDESK = 0x4000
|
|
_MOUSEEVENTF_WHEEL = 0x0800
|
|
_MOUSEEVENTF_HWHEEL = 0x1000
|
|
_KEYEVENTF_KEYUP = 0x0002
|
|
_KEYEVENTF_UNICODE = 0x0004
|
|
_KEYEVENTF_EXTENDEDKEY = 0x0001
|
|
_WHEEL_DELTA = 120
|
|
|
|
_BUTTON_FLAGS = {
|
|
"left": (0x0002, 0x0004), # MOUSEEVENTF_LEFTDOWN / LEFTUP
|
|
"right": (0x0008, 0x0010), # RIGHTDOWN / RIGHTUP
|
|
"middle": (0x0020, 0x0040), # MIDDLEDOWN / MIDDLEUP
|
|
}
|
|
|
|
_ULONG_PTR = ctypes.c_size_t
|
|
|
|
|
|
class _MOUSEINPUT(ctypes.Structure):
|
|
_fields_ = [
|
|
("dx", ctypes.wintypes.LONG),
|
|
("dy", ctypes.wintypes.LONG),
|
|
("mouseData", ctypes.wintypes.DWORD),
|
|
("dwFlags", ctypes.wintypes.DWORD),
|
|
("time", ctypes.wintypes.DWORD),
|
|
("dwExtraInfo", _ULONG_PTR),
|
|
]
|
|
|
|
|
|
class _KEYBDINPUT(ctypes.Structure):
|
|
_fields_ = [
|
|
("wVk", ctypes.wintypes.WORD),
|
|
("wScan", ctypes.wintypes.WORD),
|
|
("dwFlags", ctypes.wintypes.DWORD),
|
|
("time", ctypes.wintypes.DWORD),
|
|
("dwExtraInfo", _ULONG_PTR),
|
|
]
|
|
|
|
|
|
class _INPUTUNION(ctypes.Union):
|
|
_fields_ = [("mi", _MOUSEINPUT), ("ki", _KEYBDINPUT)]
|
|
|
|
|
|
class _INPUT(ctypes.Structure):
|
|
_anonymous_ = ("u",)
|
|
_fields_ = [("type", ctypes.wintypes.DWORD), ("u", _INPUTUNION)]
|
|
|
|
|
|
def _send_inputs(inputs: List[_INPUT]) -> int:
|
|
if not inputs:
|
|
return 0
|
|
arr = (_INPUT * len(inputs))(*inputs)
|
|
sent = ctypes.windll.user32.SendInput(len(inputs), arr, ctypes.sizeof(_INPUT))
|
|
if sent != len(inputs):
|
|
raise OSError(f"SendInput injected {sent}/{len(inputs)} events "
|
|
f"(error {ctypes.get_last_error()})")
|
|
return sent
|
|
|
|
|
|
def _mouse_input(dx: int = 0, dy: int = 0, data: int = 0, flags: int = 0) -> _INPUT:
|
|
inp = _INPUT(type=_INPUT_MOUSE)
|
|
inp.mi = _MOUSEINPUT(dx=dx, dy=dy, mouseData=data & 0xFFFFFFFF, dwFlags=flags,
|
|
time=0, dwExtraInfo=0)
|
|
return inp
|
|
|
|
|
|
def _key_input(vk: int = 0, scan: int = 0, flags: int = 0) -> _INPUT:
|
|
inp = _INPUT(type=_INPUT_KEYBOARD)
|
|
inp.ki = _KEYBDINPUT(wVk=vk, wScan=scan, dwFlags=flags, time=0, dwExtraInfo=0)
|
|
return inp
|
|
|
|
|
|
def _abs_coords(x: int, y: int) -> Tuple[int, int]:
|
|
"""Normalize physical screen coords to 0..65535 across the virtual desktop."""
|
|
user32 = ctypes.windll.user32
|
|
vx = user32.GetSystemMetrics(76) # SM_XVIRTUALSCREEN
|
|
vy = user32.GetSystemMetrics(77) # SM_YVIRTUALSCREEN
|
|
vw = user32.GetSystemMetrics(78) # SM_CXVIRTUALSCREEN
|
|
vh = user32.GetSystemMetrics(79) # SM_CYVIRTUALSCREEN
|
|
nx = round((x - vx) * 65535 / max(1, vw - 1))
|
|
ny = round((y - vy) * 65535 / max(1, vh - 1))
|
|
return max(0, min(65535, nx)), max(0, min(65535, ny))
|
|
|
|
|
|
def _mouse_move(x: int, y: int) -> None:
|
|
nx, ny = _abs_coords(x, y)
|
|
_send_inputs([_mouse_input(
|
|
dx=nx, dy=ny,
|
|
flags=_MOUSEEVENTF_MOVE | _MOUSEEVENTF_ABSOLUTE | _MOUSEEVENTF_VIRTUALDESK)])
|
|
|
|
|
|
def _mouse_button(button: str, down: bool) -> None:
|
|
flags = _BUTTON_FLAGS.get(button)
|
|
if flags is None:
|
|
raise ValueError(f"unknown button {button!r}")
|
|
_send_inputs([_mouse_input(flags=flags[0] if down else flags[1])])
|
|
|
|
|
|
def _mouse_wheel(ticks: int, horizontal: bool = False) -> None:
|
|
flag = _MOUSEEVENTF_HWHEEL if horizontal else _MOUSEEVENTF_WHEEL
|
|
_send_inputs([_mouse_input(data=ticks * _WHEEL_DELTA, flags=flag)])
|
|
|
|
|
|
# Virtual-key map. 'cmd' deliberately aliases to CTRL: models carry macOS
|
|
# habits ("cmd+s" = save) and Ctrl is the Windows equivalent. The Windows key
|
|
# is reachable as 'win' (tool.py also aliases windows/super/meta to it).
|
|
_VK_MAP: Dict[str, int] = {
|
|
"ctrl": 0x11, "control": 0x11, "cmd": 0x11, "command": 0x11,
|
|
"alt": 0x12, "option": 0x12,
|
|
"shift": 0x10,
|
|
"win": 0x5B, "windows": 0x5B, "super": 0x5B, "meta": 0x5B,
|
|
"enter": 0x0D, "return": 0x0D,
|
|
"esc": 0x1B, "escape": 0x1B,
|
|
"tab": 0x09, "space": 0x20,
|
|
"backspace": 0x08,
|
|
"delete": 0x2E, "del": 0x2E,
|
|
"insert": 0x2D,
|
|
"home": 0x24, "end": 0x23,
|
|
"pageup": 0x21, "pgup": 0x21, "pagedown": 0x22, "pgdn": 0x22,
|
|
"left": 0x25, "up": 0x26, "right": 0x27, "down": 0x28,
|
|
"arrowleft": 0x25, "arrowup": 0x26, "arrowright": 0x27, "arrowdown": 0x28,
|
|
"capslock": 0x14, "numlock": 0x90, "printscreen": 0x2C,
|
|
"apps": 0x5D, "menu": 0x5D,
|
|
}
|
|
for _i in range(1, 25):
|
|
_VK_MAP[f"f{_i}"] = 0x70 + _i - 1
|
|
for _c in "abcdefghijklmnopqrstuvwxyz0123456789":
|
|
_VK_MAP[_c] = ord(_c.upper())
|
|
|
|
# Keys that need KEYEVENTF_EXTENDEDKEY for correct scan codes.
|
|
_EXTENDED_VKS = frozenset({0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
|
|
0x2C, 0x2D, 0x2E, 0x5B, 0x5D, 0x90})
|
|
|
|
_MODIFIER_VKS = frozenset({0x10, 0x11, 0x12, 0x5B})
|
|
|
|
|
|
class _LASTINPUTINFO(ctypes.Structure):
|
|
_fields_ = [("cbSize", ctypes.wintypes.UINT), ("dwTime", ctypes.wintypes.DWORD)]
|
|
|
|
|
|
def _seconds_since_user_input() -> float:
|
|
"""Seconds since the user's last keyboard/mouse input (session-wide)."""
|
|
info = _LASTINPUTINFO()
|
|
info.cbSize = ctypes.sizeof(_LASTINPUTINFO)
|
|
if not ctypes.windll.user32.GetLastInputInfo(ctypes.byref(info)):
|
|
return float("inf")
|
|
# GetTickCount wraps at 49.7 days; the unsigned subtraction below stays
|
|
# correct across a single wrap.
|
|
delta = (ctypes.windll.kernel32.GetTickCount() - info.dwTime) & 0xFFFFFFFF
|
|
return delta / 1000.0
|
|
|
|
|
|
def _wait_for_user_idle() -> None:
|
|
"""Hold injected input briefly while the user is actively typing/mousing.
|
|
|
|
Synthetic input lands in whatever has focus; colliding with a human
|
|
mid-keystroke sprays input across both parties' targets. Wait for
|
|
``computer_use.idle_wait_seconds`` seconds (default 1.5, 0 disables) of
|
|
user idle, but never longer than ~8s total — the agent should yield, not
|
|
deadlock behind a user who is working.
|
|
"""
|
|
try:
|
|
from tools.computer_use.tool import _computer_use_config
|
|
threshold = float(_computer_use_config().get("idle_wait_seconds", 1.5))
|
|
except (TypeError, ValueError):
|
|
threshold = 1.5
|
|
except Exception:
|
|
threshold = 1.5
|
|
env_threshold = os.environ.get("HERMES_COMPUTER_USE_IDLE_WAIT")
|
|
if env_threshold is not None:
|
|
try:
|
|
threshold = float(env_threshold)
|
|
except ValueError:
|
|
pass
|
|
if threshold <= 0:
|
|
return
|
|
deadline = time.monotonic() + 8.0
|
|
while time.monotonic() < deadline:
|
|
if _seconds_since_user_input() >= threshold:
|
|
return
|
|
time.sleep(0.2)
|
|
|
|
|
|
def _vk_for_key(name: str) -> Optional[int]:
|
|
"""Map a key name to a virtual-key code; None when unknown."""
|
|
name = name.strip().lower()
|
|
if not name:
|
|
return None
|
|
vk = _VK_MAP.get(name)
|
|
if vk is not None:
|
|
return vk
|
|
if len(name) == 1:
|
|
# Punctuation etc. — layout-dependent lookup.
|
|
res = ctypes.windll.user32.VkKeyScanW(ord(name))
|
|
if res != -1:
|
|
return res & 0xFF
|
|
return None
|
|
|
|
|
|
def _key_event(vk: int, down: bool) -> _INPUT:
|
|
flags = 0 if down else _KEYEVENTF_KEYUP
|
|
if vk in _EXTENDED_VKS:
|
|
flags |= _KEYEVENTF_EXTENDEDKEY
|
|
scan = ctypes.windll.user32.MapVirtualKeyW(vk, 0) # MAPVK_VK_TO_VSC
|
|
return _key_input(vk=vk, scan=scan, flags=flags)
|
|
|
|
|
|
def _press_combo(vks: List[int]) -> None:
|
|
"""Hold all but the last code as modifiers, tap the last, release."""
|
|
mods, tap = vks[:-1], vks[-1]
|
|
seq = [_key_event(vk, True) for vk in mods]
|
|
seq += [_key_event(tap, True), _key_event(tap, False)]
|
|
seq += [_key_event(vk, False) for vk in reversed(mods)]
|
|
_send_inputs(seq)
|
|
|
|
|
|
def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool:
|
|
"""Switch virtual desktop via Ctrl+Win+Left/Right SendInput.
|
|
|
|
The overlay subprocess (full-screen tkinter window) is killed by any
|
|
virtual-desktop transition, so we stop it before switching and restart
|
|
it on the new desktop.
|
|
"""
|
|
if direction not in ("left", "right"):
|
|
return False
|
|
|
|
VK_CONTROL = 0x11
|
|
VK_LWIN = 0x5B
|
|
VK_LEFT = 0x25
|
|
VK_RIGHT = 0x27
|
|
|
|
vk_dir = VK_LEFT if direction == "left" else VK_RIGHT
|
|
|
|
# Single-batch SendInput matching _press_combo semantics:
|
|
# hold modifiers → tap arrow → release, so the system input thread
|
|
# sees the same event order as a physical keyboard.
|
|
seq = [_key_event(VK_CONTROL, True),
|
|
_key_event(VK_LWIN, True),
|
|
_key_event(vk_dir, True),
|
|
_key_event(vk_dir, False),
|
|
_key_event(VK_LWIN, False),
|
|
_key_event(VK_CONTROL, False)]
|
|
|
|
try:
|
|
overlay_client.stop()
|
|
_send_inputs(seq)
|
|
# The virtual-desktop transition is asynchronous — a brief delay
|
|
# before restarting the overlay avoids racing the DWM compositor.
|
|
time.sleep(0.15)
|
|
overlay_client._dead = False
|
|
overlay_client.start()
|
|
return True
|
|
except Exception as e:
|
|
logger.warning("switch_desktop SendInput failed: %s", e)
|
|
try:
|
|
overlay_client._dead = False
|
|
overlay_client.start()
|
|
except Exception as e2:
|
|
logger.warning("switch_desktop overlay restart also failed: %s", e2)
|
|
return False
|
|
|
|
|
|
def _type_unicode(text: str) -> None:
|
|
"""Type text via KEYEVENTF_UNICODE; newlines become Return taps."""
|
|
batch: List[_INPUT] = []
|
|
for ch in text.replace("\r\n", "\n"):
|
|
if ch in ("\n", "\r"):
|
|
batch.append(_key_event(0x0D, True))
|
|
batch.append(_key_event(0x0D, False))
|
|
continue
|
|
units = ch.encode("utf-16-le")
|
|
for i in range(0, len(units), 2):
|
|
code = units[i] | (units[i + 1] << 8)
|
|
batch.append(_key_input(scan=code, flags=_KEYEVENTF_UNICODE))
|
|
batch.append(_key_input(scan=code, flags=_KEYEVENTF_UNICODE | _KEYEVENTF_KEYUP))
|
|
if len(batch) >= 100:
|
|
_send_inputs(batch)
|
|
batch = []
|
|
time.sleep(0.01)
|
|
_send_inputs(batch)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Window / UIA helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# On-screen overlay (visible "PC use mode") — optional, best-effort
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _OverlayClient:
|
|
"""Drives the overlay subprocess (tools/computer_use/overlay.py).
|
|
|
|
Strictly fire-and-forget: every failure disables the overlay silently;
|
|
desktop-control actions must never be affected by overlay problems.
|
|
Disable with ``computer_use.overlay: false``. ``HERMES_COMPUTER_USE_OVERLAY``
|
|
remains as a test/escape hatch.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self._proc = None
|
|
self._sock: Optional[socket.socket] = None
|
|
self._addr: Optional[Tuple[str, int]] = None
|
|
overlay_enabled = True
|
|
try:
|
|
from tools.computer_use.tool import _computer_use_config
|
|
raw_overlay = _computer_use_config().get("overlay", True)
|
|
if isinstance(raw_overlay, str):
|
|
overlay_enabled = raw_overlay.strip().lower() not in {"0", "false", "no", "off"}
|
|
else:
|
|
overlay_enabled = bool(raw_overlay)
|
|
except Exception:
|
|
overlay_enabled = True
|
|
env_overlay = os.environ.get("HERMES_COMPUTER_USE_OVERLAY")
|
|
if env_overlay is not None:
|
|
overlay_enabled = env_overlay.strip().lower() not in {"0", "false", "no", "off"}
|
|
self._dead = not overlay_enabled
|
|
|
|
@property
|
|
def pid(self) -> Optional[int]:
|
|
return self._proc.pid if self._proc is not None else None
|
|
|
|
def start(self) -> None:
|
|
if self._dead or self._proc is not None:
|
|
return
|
|
try:
|
|
import subprocess
|
|
overlay_py = os.path.join(os.path.dirname(__file__), "overlay.py")
|
|
self._proc = subprocess.Popen(
|
|
[sys.executable, "-u", overlay_py],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
|
)
|
|
line = ""
|
|
deadline = time.monotonic() + 8.0
|
|
while time.monotonic() < deadline:
|
|
line = (self._proc.stdout.readline() or b"").decode("utf-8", "ignore").strip()
|
|
if line.startswith("PORT "):
|
|
break
|
|
if not line.startswith("PORT "):
|
|
raise RuntimeError(f"overlay did not report a port (got {line!r})")
|
|
self._addr = ("127.0.0.1", int(line.split()[1]))
|
|
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
self.send({"cmd": "banner", "text": "HERMES — DESKTOP CONTROL",
|
|
"state": "active"})
|
|
except Exception as e:
|
|
logger.warning("computer_use overlay unavailable: %s", e)
|
|
self._shutdown()
|
|
self._dead = True
|
|
|
|
def send(self, msg: Dict[str, Any]) -> None:
|
|
if self._dead or self._sock is None or self._addr is None:
|
|
return
|
|
try:
|
|
self._sock.sendto(json.dumps(msg).encode("utf-8"), self._addr)
|
|
except Exception:
|
|
self._dead = True
|
|
self._shutdown()
|
|
|
|
def stop(self) -> None:
|
|
self.send({"cmd": "bye"})
|
|
self._shutdown()
|
|
|
|
def _shutdown(self) -> None:
|
|
try:
|
|
if self._sock is not None:
|
|
self._sock.close()
|
|
except Exception:
|
|
pass
|
|
self._sock = None
|
|
try:
|
|
if self._proc is not None:
|
|
try:
|
|
self._proc.stdin.close() # stdin EOF → overlay exits
|
|
except Exception:
|
|
pass
|
|
self._proc.terminate()
|
|
except Exception:
|
|
pass
|
|
self._proc = None
|
|
|
|
|
|
_DWMWA_CLOAKED = 14
|
|
_DWMWA_EXTENDED_FRAME_BOUNDS = 9
|
|
|
|
# Control types we surface as interactable. Mirrors the macOS AX role list.
|
|
_INTERACTABLE_TYPES = frozenset({
|
|
"Button", "CheckBox", "ComboBox", "Edit", "Hyperlink", "ListItem",
|
|
"MenuItem", "RadioButton", "Slider", "Spinner", "SplitButton",
|
|
"TabItem", "TreeItem", "DataItem", "Document",
|
|
})
|
|
|
|
|
|
def _window_rect(hwnd: int) -> Tuple[int, int, int, int]:
|
|
"""(x, y, w, h) of the window, preferring DWM extended frame bounds."""
|
|
rect = ctypes.wintypes.RECT()
|
|
try:
|
|
hr = ctypes.windll.dwmapi.DwmGetWindowAttribute(
|
|
ctypes.wintypes.HWND(hwnd), _DWMWA_EXTENDED_FRAME_BOUNDS,
|
|
ctypes.byref(rect), ctypes.sizeof(rect))
|
|
if hr == 0 and rect.right > rect.left and rect.bottom > rect.top:
|
|
return rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top
|
|
except Exception:
|
|
pass
|
|
left, top, right, bottom = win32gui.GetWindowRect(hwnd)
|
|
return left, top, right - left, bottom - top
|
|
|
|
|
|
def _is_cloaked(hwnd: int) -> bool:
|
|
cloaked = ctypes.wintypes.DWORD(0)
|
|
try:
|
|
ctypes.windll.dwmapi.DwmGetWindowAttribute(
|
|
ctypes.wintypes.HWND(hwnd), _DWMWA_CLOAKED,
|
|
ctypes.byref(cloaked), ctypes.sizeof(cloaked))
|
|
except Exception:
|
|
return False
|
|
return cloaked.value != 0
|
|
|
|
|
|
def _exe_for_pid(pid: int) -> str:
|
|
try:
|
|
handle = win32api.OpenProcess(
|
|
win32con.PROCESS_QUERY_INFORMATION | win32con.PROCESS_VM_READ, False, pid)
|
|
try:
|
|
return os.path.basename(win32process.GetModuleFileNameEx(handle, 0))
|
|
finally:
|
|
handle.close()
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
class WindowsUIABackend(ComputerUseBackend):
|
|
"""Desktop control through UI Automation + SendInput."""
|
|
|
|
def __init__(self) -> None:
|
|
self._elements: Dict[int, UIElement] = {}
|
|
self._last_app: Optional[str] = None
|
|
self._target_hwnd: Optional[int] = None
|
|
self._target_pid: Optional[int] = None
|
|
# Window rect at the time of the last capture — element bounds are
|
|
# absolute screen coords, so if the window moves between capture and
|
|
# click we translate by the origin delta instead of clicking stale
|
|
# pixels (see _element_offset).
|
|
self._capture_rect: Optional[Tuple[int, int, int, int]] = None
|
|
self._started = False
|
|
self._overlay = _OverlayClient()
|
|
|
|
# ── Lifecycle ──────────────────────────────────────────────────
|
|
def start(self) -> None:
|
|
if self._started:
|
|
return
|
|
if not windows_backend_available():
|
|
raise RuntimeError(f"Windows backend unavailable: {_IMPORT_ERROR}")
|
|
_set_dpi_awareness()
|
|
self._overlay.start()
|
|
self._started = True
|
|
|
|
def stop(self) -> None:
|
|
self._elements.clear()
|
|
self._overlay.stop()
|
|
self._started = False
|
|
|
|
def is_available(self) -> bool:
|
|
return windows_backend_available()
|
|
|
|
# ── Window enumeration ─────────────────────────────────────────
|
|
def _enum_top_windows(self) -> List[Dict[str, Any]]:
|
|
"""Visible, titled, non-cloaked, non-tool top-level windows."""
|
|
windows: List[Dict[str, Any]] = []
|
|
|
|
def handler(hwnd: int, _arg: Any) -> bool:
|
|
try:
|
|
if not win32gui.IsWindowVisible(hwnd):
|
|
return True
|
|
if win32gui.GetWindowLong(hwnd, win32con.GWL_EXSTYLE) & win32con.WS_EX_TOOLWINDOW:
|
|
return True
|
|
title = win32gui.GetWindowText(hwnd)
|
|
if not title or _is_cloaked(hwnd):
|
|
return True
|
|
_tid, pid = win32process.GetWindowThreadProcessId(hwnd)
|
|
windows.append({"hwnd": hwnd, "title": title, "pid": pid,
|
|
"exe": _exe_for_pid(pid)})
|
|
except Exception:
|
|
pass
|
|
return True
|
|
|
|
win32gui.EnumWindows(handler, None)
|
|
return windows
|
|
|
|
def _find_window(self, app: str) -> Optional[Dict[str, Any]]:
|
|
needle = app.lower()
|
|
for w in self._enum_top_windows():
|
|
if needle in w["exe"].lower() or needle in w["title"].lower():
|
|
return w
|
|
return None
|
|
|
|
def list_apps(self) -> List[Dict[str, Any]]:
|
|
apps: Dict[str, Dict[str, Any]] = {}
|
|
for w in self._enum_top_windows():
|
|
entry = apps.setdefault(w["exe"], {
|
|
"app": w["exe"], "pid": w["pid"], "windows": [], "window_count": 0,
|
|
})
|
|
entry["windows"].append(w["title"])
|
|
entry["window_count"] += 1
|
|
return list(apps.values())
|
|
|
|
def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:
|
|
target = self._find_window(app)
|
|
if target is None:
|
|
return ActionResult(ok=False, action="focus_app",
|
|
message=f"No window matching {app!r}. "
|
|
f"Use list_apps to see what is running.")
|
|
self._target_hwnd = target["hwnd"]
|
|
self._target_pid = target["pid"]
|
|
self._last_app = target["exe"]
|
|
if raise_window:
|
|
ok = self._bring_to_foreground(target["hwnd"])
|
|
return ActionResult(
|
|
ok=ok, action="focus_app",
|
|
message=(f"Raised {target['exe']} ({target['title']!r})." if ok
|
|
else f"Targeted {target['exe']} but could not raise it."),
|
|
meta={"hwnd": target["hwnd"], "pid": target["pid"]})
|
|
return ActionResult(
|
|
ok=True, action="focus_app",
|
|
message=(f"Targeted {target['exe']} ({target['title']!r}). Note: on "
|
|
"Windows, pointer/keyboard actions raise the target window "
|
|
"when they run."),
|
|
meta={"hwnd": target["hwnd"], "pid": target["pid"]})
|
|
|
|
@staticmethod
|
|
def _bring_to_foreground(hwnd: int) -> bool:
|
|
try:
|
|
if win32gui.IsIconic(hwnd):
|
|
win32gui.ShowWindow(hwnd, win32con.SW_RESTORE)
|
|
if win32gui.GetForegroundWindow() == hwnd:
|
|
return True
|
|
try:
|
|
win32gui.SetForegroundWindow(hwnd)
|
|
except Exception:
|
|
# Foreground-lock workaround: an injected no-op ALT tap makes
|
|
# our process the "last input" owner, unlocking the call.
|
|
_send_inputs([_key_event(0x12, True), _key_event(0x12, False)])
|
|
win32gui.SetForegroundWindow(hwnd)
|
|
time.sleep(0.15)
|
|
return win32gui.GetForegroundWindow() == hwnd
|
|
except Exception as e:
|
|
logger.warning("SetForegroundWindow(%s) failed: %s", hwnd, e)
|
|
return False
|
|
|
|
def _ensure_target_foreground(self) -> None:
|
|
# Called exactly once at the top of every input-injecting action —
|
|
# the idle guard lives here so all of click/drag/scroll/type/key
|
|
# yield to an actively-working user before touching the desktop.
|
|
_wait_for_user_idle()
|
|
if self._target_hwnd and win32gui.IsWindow(self._target_hwnd):
|
|
self._bring_to_foreground(self._target_hwnd)
|
|
|
|
# ── Capture ─────────────────────────────────────────────────────
|
|
def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult:
|
|
hwnd: Optional[int] = None
|
|
if app:
|
|
target = self._find_window(app)
|
|
if target is None:
|
|
return CaptureResult(mode=mode, width=0, height=0,
|
|
app=app, window_title="(no matching window)")
|
|
hwnd = target["hwnd"]
|
|
self._target_hwnd = hwnd
|
|
self._target_pid = target["pid"]
|
|
self._last_app = target["exe"]
|
|
elif self._target_hwnd and win32gui.IsWindow(self._target_hwnd):
|
|
hwnd = self._target_hwnd
|
|
else:
|
|
hwnd = win32gui.GetForegroundWindow()
|
|
if hwnd:
|
|
_tid, pid = win32process.GetWindowThreadProcessId(hwnd)
|
|
if pid == self._overlay.pid:
|
|
# Never capture our own overlay; fall back to the first
|
|
# real top-level window.
|
|
wins = self._enum_top_windows()
|
|
if wins:
|
|
hwnd, pid = wins[0]["hwnd"], wins[0]["pid"]
|
|
else:
|
|
hwnd = None
|
|
if hwnd:
|
|
self._target_hwnd, self._target_pid = hwnd, pid
|
|
self._last_app = _exe_for_pid(pid)
|
|
|
|
if not hwnd:
|
|
return CaptureResult(mode=mode, width=0, height=0)
|
|
|
|
x, y, w, h = _window_rect(hwnd)
|
|
self._capture_rect = (x, y, w, h)
|
|
window_title = win32gui.GetWindowText(hwnd)
|
|
|
|
img = None
|
|
png_b64: Optional[str] = None
|
|
png_bytes_len = 0
|
|
if mode != "ax":
|
|
try:
|
|
img = ImageGrab.grab(bbox=(x, y, x + w, y + h), all_screens=True)
|
|
except Exception as e:
|
|
logger.warning("screenshot failed: %s", e)
|
|
|
|
elements = self._walk_elements(hwnd, (x, y, w, h)) if mode in ("som", "ax") else []
|
|
self._elements = {e.index: e for e in elements}
|
|
|
|
if img is not None and mode == "som" and elements:
|
|
self._draw_som_overlay(img, elements, origin=(x, y))
|
|
if img is not None:
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="PNG")
|
|
png_bytes = buf.getvalue()
|
|
png_b64 = base64.b64encode(png_bytes).decode("ascii")
|
|
png_bytes_len = len(png_bytes)
|
|
|
|
# Mirror what Hermes sees onto the user's screen (sent AFTER the
|
|
# grab, so the boxes are never part of the screenshot itself).
|
|
self._overlay.send({
|
|
"cmd": "elements",
|
|
"items": [{"index": e.index, "bounds": list(e.bounds)} for e in elements],
|
|
"ttl": 4.0,
|
|
})
|
|
self._overlay.send({"cmd": "flash",
|
|
"text": f"capture · {len(elements)} elements", "ttl": 2.0})
|
|
|
|
return CaptureResult(
|
|
mode=mode,
|
|
width=img.width if img is not None else w,
|
|
height=img.height if img is not None else h,
|
|
png_b64=png_b64,
|
|
elements=elements,
|
|
app=self._last_app or "",
|
|
window_title=window_title,
|
|
png_bytes_len=png_bytes_len,
|
|
)
|
|
|
|
def _iter_interactable(
|
|
self,
|
|
hwnd: int,
|
|
win_rect: Tuple[int, int, int, int],
|
|
max_nodes: int = 1500,
|
|
max_depth: int = 60,
|
|
time_budget: float = 4.0,
|
|
):
|
|
"""Yield (ctrl, role, label, rect) for interactable controls under
|
|
`hwnd`, in breadth-first discovery order.
|
|
|
|
This is the single source of element discovery order *and* the
|
|
interactability/visibility filter. Both the capture walk
|
|
(`_walk_elements`) and set_value's re-find (`_control_at_index`)
|
|
consume it, so element #N resolves to the same control in both — if
|
|
the two ever drifted, set_value would act on a different control than
|
|
the capture advertised under that index. The caller owns the
|
|
UIAutomation COM apartment (the live `ctrl`/`rect` are only valid for
|
|
the duration of the iteration).
|
|
"""
|
|
wx, wy, ww, wh = win_rect
|
|
root = _auto.ControlFromHandle(hwnd)
|
|
if root is None:
|
|
return
|
|
deadline = time.monotonic() + time_budget
|
|
queue: deque = deque([(root, 0)])
|
|
yielded = 0
|
|
while queue and yielded < max_nodes and time.monotonic() < deadline:
|
|
ctrl, depth = queue.popleft()
|
|
try:
|
|
role = ctrl.ControlTypeName
|
|
if role.endswith("Control"):
|
|
role = role[: -len("Control")]
|
|
interactable = role in _INTERACTABLE_TYPES
|
|
if not interactable and role == "Text":
|
|
interactable = any(
|
|
ctrl.GetPattern(pid) is not None
|
|
for pid in (_auto.PatternId.ValuePattern,
|
|
_auto.PatternId.InvokePattern))
|
|
if interactable and ctrl.IsEnabled and not ctrl.IsOffscreen:
|
|
r = ctrl.BoundingRectangle
|
|
if (r.right > r.left and r.bottom > r.top
|
|
and r.left < wx + ww and r.right > wx
|
|
and r.top < wy + wh and r.bottom > wy):
|
|
label = (ctrl.Name or ctrl.AutomationId or "")
|
|
if len(label) > 120:
|
|
label = label[:120]
|
|
yielded += 1
|
|
yield ctrl, role, label, r
|
|
except Exception:
|
|
pass
|
|
if depth < max_depth:
|
|
try:
|
|
queue.extend((c, depth + 1) for c in ctrl.GetChildren())
|
|
except Exception:
|
|
pass
|
|
|
|
def _walk_elements(
|
|
self,
|
|
hwnd: int,
|
|
win_rect: Tuple[int, int, int, int],
|
|
max_nodes: int = 1500,
|
|
max_depth: int = 60,
|
|
time_budget: float = 4.0,
|
|
) -> List[UIElement]:
|
|
"""Collect interactable elements under `hwnd` for a capture.
|
|
|
|
Indices follow `_iter_interactable`'s discovery order, which
|
|
set_value re-walks (via `_control_at_index`) to resolve an element
|
|
back to a live control.
|
|
"""
|
|
elements: List[UIElement] = []
|
|
try:
|
|
with _auto.UIAutomationInitializerInThread():
|
|
for ctrl, role, label, r in self._iter_interactable(
|
|
hwnd, win_rect, max_nodes=max_nodes,
|
|
max_depth=max_depth, time_budget=time_budget):
|
|
elements.append(UIElement(
|
|
index=len(elements) + 1,
|
|
role=role,
|
|
label=label,
|
|
bounds=(r.left, r.top, r.right - r.left, r.bottom - r.top),
|
|
app=self._last_app or "",
|
|
pid=self._target_pid or 0,
|
|
window_id=hwnd,
|
|
))
|
|
except Exception as e:
|
|
logger.warning("UIA element walk failed: %s", e)
|
|
return elements
|
|
|
|
@staticmethod
|
|
def _draw_som_overlay(img: "Image.Image", elements: List[UIElement],
|
|
origin: Tuple[int, int]) -> None:
|
|
ox, oy = origin
|
|
draw = ImageDraw.Draw(img)
|
|
for e in elements:
|
|
ex, ey, ew, eh = e.bounds
|
|
ix, iy = ex - ox, ey - oy
|
|
draw.rectangle([ix, iy, ix + ew, iy + eh], outline=(255, 0, 0), width=2)
|
|
text = str(e.index)
|
|
badge_w = 7 * len(text) + 6
|
|
draw.rectangle([ix, iy, ix + badge_w, iy + 14], fill=(255, 0, 0))
|
|
draw.text((ix + 3, iy + 1), text, fill=(255, 255, 255))
|
|
|
|
# ── Pointer actions ─────────────────────────────────────────────
|
|
def _element_offset(self, el: UIElement) -> Tuple[int, int]:
|
|
"""Origin shift to apply to cached element bounds.
|
|
|
|
If the element's window moved since the capture, the cached absolute
|
|
coords are stale by exactly the window-origin delta — translate.
|
|
A resize invalidates interior layout, so that demands a re-capture.
|
|
"""
|
|
hwnd = el.window_id
|
|
if not (hwnd and self._capture_rect and win32gui.IsWindow(hwnd)):
|
|
return 0, 0
|
|
try:
|
|
cx, cy, cw, ch = self._capture_rect
|
|
nx, ny, nw, nh = _window_rect(hwnd)
|
|
except Exception:
|
|
return 0, 0
|
|
if (nw, nh) != (cw, ch):
|
|
raise ValueError(
|
|
"the target window was resized since the last capture — "
|
|
"re-run capture(mode='som') for fresh element positions")
|
|
return nx - cx, ny - cy
|
|
|
|
def _resolve_point(self, element: Optional[int], x: Optional[int],
|
|
y: Optional[int]) -> Tuple[int, int, str]:
|
|
"""Return (x, y, what) for an action target; raises ValueError."""
|
|
if element is not None:
|
|
el = self._elements.get(element)
|
|
if el is None:
|
|
raise ValueError(
|
|
f"element #{element} is not in the last capture — re-run "
|
|
"capture(mode='som') and use a fresh index")
|
|
dx, dy = self._element_offset(el)
|
|
cx, cy = el.center()
|
|
what = f"element #{element} ({el.role} {el.label!r})"
|
|
if dx or dy:
|
|
what += f" [window moved {dx:+d},{dy:+d} since capture]"
|
|
return cx + dx, cy + dy, what
|
|
if x is None or y is None:
|
|
raise ValueError("requires element= or coordinate [x, y]")
|
|
return int(x), int(y), f"({x}, {y})"
|
|
|
|
def _with_modifiers(self, modifiers: Optional[List[str]]):
|
|
"""Return (down_inputs, up_inputs) for a modifier list."""
|
|
vks: List[int] = []
|
|
for m in modifiers or []:
|
|
vk = _vk_for_key(m)
|
|
if vk is None or vk not in _MODIFIER_VKS:
|
|
raise ValueError(f"unknown modifier {m!r}")
|
|
vks.append(vk)
|
|
down = [_key_event(vk, True) for vk in vks]
|
|
up = [_key_event(vk, False) for vk in reversed(vks)]
|
|
return down, up
|
|
|
|
def click(
|
|
self,
|
|
*,
|
|
element: Optional[int] = None,
|
|
x: Optional[int] = None,
|
|
y: Optional[int] = None,
|
|
button: str = "left",
|
|
click_count: int = 1,
|
|
modifiers: Optional[List[str]] = None,
|
|
) -> ActionResult:
|
|
try:
|
|
px, py, what = self._resolve_point(element, x, y)
|
|
mods_down, mods_up = self._with_modifiers(modifiers)
|
|
except ValueError as e:
|
|
return ActionResult(ok=False, action="click", message=str(e))
|
|
if button not in _BUTTON_FLAGS:
|
|
return ActionResult(ok=False, action="click",
|
|
message=f"unknown button {button!r}")
|
|
try:
|
|
self._overlay.send({"cmd": "click", "x": px, "y": py})
|
|
self._overlay.send({"cmd": "flash", "text": f"click · {what}", "ttl": 1.5})
|
|
self._ensure_target_foreground()
|
|
old_pos = win32api.GetCursorPos()
|
|
try:
|
|
_send_inputs(mods_down)
|
|
_mouse_move(px, py)
|
|
time.sleep(0.03)
|
|
for i in range(max(1, click_count)):
|
|
_mouse_button(button, True)
|
|
_mouse_button(button, False)
|
|
if i + 1 < click_count:
|
|
time.sleep(0.05)
|
|
finally:
|
|
# Release modifiers + restore the cursor even if an injection
|
|
# above raised — otherwise a failed click strands Ctrl/Alt/Shift
|
|
# in the held-down state for the user at the keyboard.
|
|
_send_inputs(mods_up)
|
|
_mouse_move(*old_pos)
|
|
return ActionResult(ok=True, action="click",
|
|
message=f"{button}-clicked {what}"
|
|
+ (f" x{click_count}" if click_count > 1 else ""))
|
|
except Exception as e:
|
|
return ActionResult(ok=False, action="click", message=f"click failed: {e}")
|
|
|
|
def drag(
|
|
self,
|
|
*,
|
|
from_element: Optional[int] = None,
|
|
to_element: Optional[int] = None,
|
|
from_xy: Optional[Tuple[int, int]] = None,
|
|
to_xy: Optional[Tuple[int, int]] = None,
|
|
button: str = "left",
|
|
modifiers: Optional[List[str]] = None,
|
|
) -> ActionResult:
|
|
try:
|
|
fx, fy, src = self._resolve_point(
|
|
from_element, *(from_xy or (None, None)))
|
|
tx, ty, dst = self._resolve_point(
|
|
to_element, *(to_xy or (None, None)))
|
|
mods_down, mods_up = self._with_modifiers(modifiers)
|
|
except ValueError as e:
|
|
return ActionResult(ok=False, action="drag", message=str(e))
|
|
if button not in _BUTTON_FLAGS:
|
|
return ActionResult(ok=False, action="drag",
|
|
message=f"unknown button {button!r}")
|
|
try:
|
|
self._overlay.send({"cmd": "drag", "from": [fx, fy], "to": [tx, ty]})
|
|
self._overlay.send({"cmd": "flash", "text": f"drag · {src} → {dst}", "ttl": 1.5})
|
|
self._ensure_target_foreground()
|
|
old_pos = win32api.GetCursorPos()
|
|
try:
|
|
_send_inputs(mods_down)
|
|
_mouse_move(fx, fy)
|
|
time.sleep(0.05)
|
|
_mouse_button(button, True)
|
|
steps = 12
|
|
for i in range(1, steps + 1):
|
|
_mouse_move(fx + (tx - fx) * i // steps, fy + (ty - fy) * i // steps)
|
|
time.sleep(0.01)
|
|
_mouse_button(button, False)
|
|
finally:
|
|
# A mid-drag failure must not strand the button or modifiers
|
|
# held down — a stuck primary button turns every later move
|
|
# into a drag-select. The button-up is idempotent on success.
|
|
_mouse_button(button, False)
|
|
_send_inputs(mods_up)
|
|
_mouse_move(*old_pos)
|
|
return ActionResult(ok=True, action="drag",
|
|
message=f"dragged {src} -> {dst}")
|
|
except Exception as e:
|
|
return ActionResult(ok=False, action="drag", message=f"drag failed: {e}")
|
|
|
|
def scroll(
|
|
self,
|
|
*,
|
|
direction: str,
|
|
amount: int = 3,
|
|
element: Optional[int] = None,
|
|
x: Optional[int] = None,
|
|
y: Optional[int] = None,
|
|
modifiers: Optional[List[str]] = None,
|
|
) -> ActionResult:
|
|
if direction not in {"up", "down", "left", "right"}:
|
|
return ActionResult(ok=False, action="scroll",
|
|
message=f"bad direction {direction!r}")
|
|
amount = max(1, min(50, int(amount)))
|
|
try:
|
|
if element is not None or (x is not None and y is not None):
|
|
px, py, what = self._resolve_point(element, x, y)
|
|
elif self._target_hwnd and win32gui.IsWindow(self._target_hwnd):
|
|
wx, wy, ww, wh = _window_rect(self._target_hwnd)
|
|
px, py, what = wx + ww // 2, wy + wh // 2, "window center"
|
|
else:
|
|
return ActionResult(ok=False, action="scroll",
|
|
message="no target — pass element/coordinate "
|
|
"or capture first")
|
|
mods_down, mods_up = self._with_modifiers(modifiers)
|
|
except ValueError as e:
|
|
return ActionResult(ok=False, action="scroll", message=str(e))
|
|
try:
|
|
self._overlay.send({"cmd": "flash",
|
|
"text": f"scroll {direction} x{amount}", "ttl": 1.2})
|
|
self._ensure_target_foreground()
|
|
old_pos = win32api.GetCursorPos()
|
|
try:
|
|
_send_inputs(mods_down)
|
|
_mouse_move(px, py)
|
|
time.sleep(0.03)
|
|
if direction in ("up", "down"):
|
|
_mouse_wheel(amount if direction == "up" else -amount)
|
|
else:
|
|
_mouse_wheel(amount if direction == "right" else -amount,
|
|
horizontal=True)
|
|
finally:
|
|
# Release modifiers + restore the cursor even if the wheel
|
|
# injection raised, so a failed scroll can't strand a modifier.
|
|
_send_inputs(mods_up)
|
|
_mouse_move(*old_pos)
|
|
return ActionResult(ok=True, action="scroll",
|
|
message=f"scrolled {direction} x{amount} at {what}")
|
|
except Exception as e:
|
|
return ActionResult(ok=False, action="scroll", message=f"scroll failed: {e}")
|
|
|
|
# ── Keyboard ────────────────────────────────────────────────────
|
|
def type_text(self, text: str) -> ActionResult:
|
|
if len(text) > 20000:
|
|
return ActionResult(ok=False, action="type",
|
|
message=f"text too long ({len(text)} chars; max 20000)")
|
|
try:
|
|
self._overlay.send({"cmd": "flash",
|
|
"text": f"typing · {len(text)} chars", "ttl": 2.0})
|
|
self._ensure_target_foreground()
|
|
_type_unicode(text)
|
|
return ActionResult(ok=True, action="type",
|
|
message=f"typed {len(text)} characters")
|
|
except Exception as e:
|
|
return ActionResult(ok=False, action="type", message=f"type failed: {e}")
|
|
|
|
def key(self, keys: str) -> ActionResult:
|
|
parts = [p.strip() for p in keys.split("+") if p.strip()]
|
|
if not parts:
|
|
return ActionResult(ok=False, action="key", message="empty key combo")
|
|
vks: List[int] = []
|
|
for part in parts:
|
|
vk = _vk_for_key(part)
|
|
if vk is None:
|
|
return ActionResult(ok=False, action="key",
|
|
message=f"unknown key {part!r} in {keys!r}")
|
|
vks.append(vk)
|
|
try:
|
|
self._overlay.send({"cmd": "flash", "text": f"key · {keys}", "ttl": 1.5})
|
|
self._ensure_target_foreground()
|
|
_press_combo(vks)
|
|
return ActionResult(ok=True, action="key", message=f"pressed {keys}")
|
|
except Exception as e:
|
|
return ActionResult(ok=False, action="key", message=f"key failed: {e}")
|
|
|
|
def switch_desktop(self, direction: str) -> ActionResult:
|
|
"""Switch to adjacent virtual desktop.
|
|
|
|
Temporarily stops the overlay subprocess before switching and
|
|
restarts it on the new desktop, because any SendInput-based
|
|
virtual-desktop transition kills the full-screen tkinter window.
|
|
"""
|
|
if direction not in ("left", "right"):
|
|
return ActionResult(ok=False, action="switch_desktop",
|
|
message=f"unknown direction {direction!r}")
|
|
self._overlay.send({"cmd": "flash", "text": f"switch desktop · {direction}", "ttl": 1.0})
|
|
ok = _switch_desktop_via_keybd(direction, self._overlay)
|
|
return ActionResult(
|
|
ok=ok, action="switch_desktop",
|
|
message=(f"switched to {direction} virtual desktop"
|
|
if ok else "virtual desktop switch failed"))
|
|
|
|
# ── Native-value mutation ───────────────────────────────────────
|
|
def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
|
|
if element is None:
|
|
return ActionResult(ok=False, action="set_value",
|
|
message="set_value requires element=")
|
|
cached = self._elements.get(element)
|
|
if cached is None:
|
|
return ActionResult(ok=False, action="set_value",
|
|
message=f"element #{element} is not in the last "
|
|
"capture — re-run capture first")
|
|
hwnd = cached.window_id
|
|
if not (hwnd and win32gui.IsWindow(hwnd)):
|
|
return ActionResult(ok=False, action="set_value",
|
|
message="target window is gone — re-run capture")
|
|
self._overlay.send({"cmd": "elements",
|
|
"items": [{"index": cached.index, "bounds": list(cached.bounds)}],
|
|
"ttl": 2.0})
|
|
self._overlay.send({"cmd": "flash",
|
|
"text": f"set_value · #{cached.index}", "ttl": 1.5})
|
|
try:
|
|
with _auto.UIAutomationInitializerInThread():
|
|
ctrl = self._refind_control(hwnd, cached)
|
|
if ctrl is None:
|
|
return ActionResult(ok=False, action="set_value",
|
|
message=f"element #{element} no longer "
|
|
"matches the UI — re-run capture")
|
|
return self._apply_value(ctrl, cached, value)
|
|
except Exception as e:
|
|
return ActionResult(ok=False, action="set_value",
|
|
message=f"set_value failed: {e}")
|
|
|
|
def _refind_control(self, hwnd: int, cached: UIElement) -> Optional[Any]:
|
|
"""Re-walk the tree (same traversal as capture) to the cached index.
|
|
|
|
Live COM pointers cannot be safely reused across tool calls (the COM
|
|
apartment is torn down when each call's initializer exits), so we
|
|
re-discover the control and verify role+label still match.
|
|
"""
|
|
ctrl = self._control_at_index(hwnd, cached.index)
|
|
if ctrl is None:
|
|
return None
|
|
try:
|
|
role = ctrl.ControlTypeName
|
|
if role.endswith("Control"):
|
|
role = role[: -len("Control")]
|
|
label = (ctrl.Name or ctrl.AutomationId or "")[:120]
|
|
except Exception:
|
|
return None
|
|
if role != cached.role or label != cached.label:
|
|
return None
|
|
return ctrl
|
|
|
|
def _control_at_index(self, hwnd: int, index: int) -> Optional[Any]:
|
|
"""Return the live control at 1-based `index` in capture discovery order.
|
|
|
|
Re-walks via the same `_iter_interactable` traversal the capture used,
|
|
so `index` resolves to the control capture advertised under that
|
|
number. The caller owns the UIAutomation COM apartment.
|
|
"""
|
|
for pos, (ctrl, _role, _label, _rect) in enumerate(
|
|
self._iter_interactable(hwnd, _window_rect(hwnd)), start=1):
|
|
if pos == index:
|
|
return ctrl
|
|
return None
|
|
|
|
@staticmethod
|
|
def _apply_value(ctrl: Any, cached: UIElement, value: str) -> ActionResult:
|
|
# 1. ValuePattern — text fields and many custom controls.
|
|
try:
|
|
pattern = ctrl.GetPattern(_auto.PatternId.ValuePattern)
|
|
if pattern is not None:
|
|
pattern.SetValue(value)
|
|
return ActionResult(ok=True, action="set_value",
|
|
message=f"set value via ValuePattern on "
|
|
f"#{cached.index} ({cached.label!r})")
|
|
except Exception:
|
|
pass
|
|
# 2. ComboBox — expand, select matching item, collapse.
|
|
if cached.role == "ComboBox":
|
|
try:
|
|
expand = ctrl.GetPattern(_auto.PatternId.ExpandCollapsePattern)
|
|
if expand is not None:
|
|
expand.Expand()
|
|
time.sleep(0.2)
|
|
item = ctrl.ListItemControl(Name=value)
|
|
if item.Exists(1, 0.1):
|
|
sel = item.GetPattern(_auto.PatternId.SelectionItemPattern)
|
|
if sel is not None:
|
|
sel.Select()
|
|
if expand is not None:
|
|
try:
|
|
expand.Collapse()
|
|
except Exception:
|
|
pass
|
|
return ActionResult(ok=True, action="set_value",
|
|
message=f"selected {value!r} in combo "
|
|
f"#{cached.index}")
|
|
if expand is not None:
|
|
try:
|
|
expand.Collapse()
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
# 3. RangeValuePattern — sliders, spinners.
|
|
try:
|
|
rng = ctrl.GetPattern(_auto.PatternId.RangeValuePattern)
|
|
if rng is not None:
|
|
rng.SetValue(float(value))
|
|
return ActionResult(ok=True, action="set_value",
|
|
message=f"set range value {value} on "
|
|
f"#{cached.index}")
|
|
except Exception:
|
|
pass
|
|
return ActionResult(ok=False, action="set_value",
|
|
message=f"element #{cached.index} ({cached.role}) does "
|
|
"not accept a value via UIA patterns — try "
|
|
"click + type instead")
|