diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 0a6b23f35c26..8b2b38178458 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -463,6 +463,8 @@ _WINDOWS_COMPUTER_USE_GUIDANCE = ( "`action='key', keys='ctrl+s'` โ€” note `cmd` is accepted and maps to " "Ctrl; use `win` for the Windows key. For scrolling " "`action='scroll', direction='down', amount=3`.\n" + "Use `action='switch_desktop', direction='left'|'right'` only when the " + "task explicitly needs another Windows virtual desktop.\n" "4. Use `action='set_value'` with an `element` to set a text field, " "dropdown, or slider directly through UI Automation โ€” this is the ONE " "action that works WITHOUT foregrounding the window, so prefer it for " diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 7ee1f8690c6f..e971073d2760 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -811,6 +811,17 @@ DEFAULT_CONFIG = { "fallback_providers": [], "credential_pool_strategies": {}, "toolsets": ["hermes-cli"], + "computer_use": { + # auto = cua-driver on macOS, Windows UIA on Windows. Explicit values: + # "cua" / "windows" / "noop" (tests only). + "backend": "auto", + # Windows UIA backend: wait this long for the user to stop typing or + # moving the mouse before injecting input. 0 disables the guard. + "idle_wait_seconds": 1.5, + # Windows UIA backend: visible click/element overlay for shared desktop + # awareness. Best-effort; automation still works if it cannot start. + "overlay": True, + }, # Global active chat session cap across CLI, TUI/dashboard, and messaging. # None/0 = unbounded. "max_concurrent_sessions": None, diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index f76c56e667f3..dbd9f389c39c 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -79,7 +79,7 @@ CONFIGURABLE_TOOLSETS = [ ("discord", "๐Ÿ’ฌ Discord (read/participate)", "fetch messages, search members, create thread"), ("discord_admin", "๐Ÿ›ก๏ธ Discord Server Admin", "list channels/roles, pin, assign roles"), ("yuanbao", "๐Ÿค– Yuanbao", "group info, member queries, DM"), - ("computer_use", "๐Ÿ–ฑ๏ธ Computer Use (macOS)", "background desktop control via cua-driver"), + ("computer_use", "๐Ÿ–ฑ๏ธ Computer Use", "desktop control via cua-driver or Windows UIA"), ] @@ -517,9 +517,8 @@ TOOL_CATEGORIES = { ], }, "computer_use": { - "name": "Computer Use (macOS)", + "name": "Computer Use", "icon": "๐Ÿ–ฑ๏ธ", - "platform_gate": "darwin", "providers": [ { "name": "cua-driver (background)", @@ -535,6 +534,15 @@ TOOL_CATEGORIES = { ], "post_setup": "cua_driver", }, + { + "name": "Windows UIA + SendInput", + "badge": "free ยท local ยท Windows", + "tag": ( + "Native Windows UI Automation element tree plus SendInput. " + "Actions briefly foreground the target window." + ), + "env_vars": [], + }, ], }, "langfuse": { diff --git a/pyproject.toml b/pyproject.toml index bb26a48f1283..3df3c2846a97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,9 @@ dependencies = [ # (tools/computer_use/windows_backend.py). Pure-python over comtypes; # win32-only. The backend degrades to unavailable if the import fails, # so this never affects non-Windows installs. - "uiautomation>=2.0.29,<3; sys_platform == 'win32'", + "uiautomation==2.0.29; sys_platform == 'win32'", + # Win32 window enumeration / foreground management for Windows computer_use. + "pywin32==311; sys_platform == 'win32'", # Image resize recovery for the vision tools. Pillow shrinks oversized images # (>5 MB or >8000px) at embed time; without it the byte AND pixel-dimension # shrink paths no-op, so an oversized image bakes into immutable history and diff --git a/tests/tools/test_computer_use_capture_routing.py b/tests/tools/test_computer_use_capture_routing.py index 5f5e802847d2..70aeaa1dcc2c 100644 --- a/tests/tools/test_computer_use_capture_routing.py +++ b/tests/tools/test_computer_use_capture_routing.py @@ -1,4 +1,4 @@ -๏ปฟ"""End-to-end regression for #24015 -- capture routing via auxiliary.vision. +"""End-to-end regression for #24015 -- capture routing via auxiliary.vision. When ``computer_use(action='capture', mode='som'|'vision')`` returns a screenshot, ``_capture_response`` previously always returned a @@ -15,7 +15,7 @@ deterministic stubs for: * ``vision_analyze_tool`` (the aux LLM call) * ``hermes_constants.get_hermes_dir`` (cache path) -รขโ‚ฌยฆso the full code path is covered without a live cua-driver, a real +...so the full code path is covered without a live cua-driver, a real auxiliary client, or network access. """ @@ -33,13 +33,13 @@ import pytest # Fixtures / helpers # --------------------------------------------------------------------------- -# 8รƒโ€”8 PNG (transparent) -- minimal provider-acceptable bytes that decode cleanly. +# 8x8 PNG (transparent) -- minimal provider-acceptable bytes that decode cleanly. _PNG_B64 = ( "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG" "NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg==" ) -# 1รƒโ€”1 JPEG -- used to verify mime detection works for either stream type. +# 1x1 JPEG -- used to verify mime detection works for either stream type. _JPEG_B64 = ( "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB" "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/" @@ -65,7 +65,7 @@ def _make_capture( mode: str = "som", elements=None, app: str = "Safari", - window_title: str = "GitHub รขโ‚ฌโ€œ Issue #24015", + window_title: str = "GitHub - Issue #24015", width: int = 1280, height: int = 800, ): @@ -195,7 +195,7 @@ class TestCaptureResponseRoutedToAuxVision: # The original AX-only metadata (window title, element index, app) # is preserved alongside the new vision analysis so the agent loses # no context vs the multimodal path. - assert body["window_title"] == "GitHub รขโ‚ฌโ€œ Issue #24015" + assert body["window_title"] == "GitHub - Issue #24015" assert len(body["elements"]) == 2 assert captured_calls.get("called") is True diff --git a/tests/tools/test_computer_use_windows.py b/tests/tools/test_computer_use_windows.py index 59d801612d5b..760e013d5df1 100644 --- a/tests/tools/test_computer_use_windows.py +++ b/tests/tools/test_computer_use_windows.py @@ -155,6 +155,11 @@ class TestWiring: assert isinstance(backend, _FakeWindowsBackend) assert backend.started + def test_empty_env_uses_auto_backend(self): + from tools.computer_use.tool import _configured_backend_name + with patch.dict(os.environ, {"HERMES_COMPUTER_USE_BACKEND": ""}): + assert _configured_backend_name() == "auto" + def test_default_backend_is_windows_on_win32(self, monkeypatch): from tools.computer_use.tool import _default_backend_name monkeypatch.setattr(sys, "platform", "win32") @@ -207,6 +212,20 @@ class TestWindowsBlockedCombos: assert payload.get("ok") is True +class TestSwitchDesktopWiring: + def test_switch_desktop_requires_approval(self): + from tools.computer_use.tool import _DESTRUCTIVE_ACTIONS + assert "switch_desktop" in _DESTRUCTIVE_ACTIONS + + def test_schema_keeps_scroll_directions_and_switch_desktop(self): + from tools.computer_use.schema import COMPUTER_USE_SCHEMA + props = COMPUTER_USE_SCHEMA["parameters"]["properties"] + assert set(props["direction"]["enum"]) == {"up", "down", "left", "right"} + actions = set(props["action"]["enum"]) + assert "scroll" in actions + assert "switch_desktop" in actions + + # --------------------------------------------------------------------------- # Live (Windows only) โ€” no input injection, read-only against the real OS # --------------------------------------------------------------------------- diff --git a/tools/computer_use/backend.py b/tools/computer_use/backend.py index c9686e41b040..2bc2e8f97740 100644 --- a/tools/computer_use/backend.py +++ b/tools/computer_use/backend.py @@ -150,6 +150,14 @@ class ComputerUseBackend(ABC): `element` is the 1-based SOM index returned by a prior capture call. """ + def switch_desktop(self, direction: str) -> ActionResult: + """Switch to an adjacent virtual desktop when the backend supports it.""" + return ActionResult( + ok=False, + action="switch_desktop", + message="switch_desktop is not supported by this backend", + ) + # โ”€โ”€ Timing โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ def wait(self, seconds: float) -> ActionResult: """Default implementation: time.sleep.""" diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index 22259377ca7c..74ec7cafdbcf 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -86,9 +86,10 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "type": "string", "description": ( "Optional. Limit capture/action to a specific app " - "(by name, e.g. 'Safari', or bundle ID, " + "(by name, e.g. 'Safari' or 'Notepad', executable " + "name on Windows, or bundle ID on macOS such as " "'com.apple.Safari'). If omitted, operates on the " - "frontmost app's window or the whole screen." + "frontmost app/window." ), }, "max_elements": { @@ -142,7 +143,10 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "type": "array", "items": { "type": "string", - "enum": ["cmd", "shift", "option", "alt", "ctrl", "fn"], + "enum": [ + "cmd", "shift", "option", "alt", "ctrl", "fn", + "win", "windows", "super", "meta", + ], }, "description": "Modifier keys held during the action.", }, @@ -167,7 +171,11 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "direction": { "type": "string", "enum": ["up", "down", "left", "right"], - "description": "Scroll direction.", + "description": ( + "Scroll direction for action='scroll'. For " + "action='switch_desktop', use 'left' or 'right' to move " + "to the adjacent Windows virtual desktop." + ), }, "amount": { "type": "integer", @@ -205,18 +213,9 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "description": ( "Only for action='focus_app'. If true, brings the " "window to front (DISRUPTS the user). Default false " - "โ€” input is routed to the app without raising, " - "matching the background co-work model." - ), - }, - # โ”€โ”€ switch_desktop โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - "direction": { - "type": "string", - "enum": ["left", "right"], - "description": ( - "Only for action='switch_desktop'. Switches to the " - "adjacent virtual desktop. Requires Windows 10+ with " - "multiple virtual desktops." + "only records the target. macOS can route later input " + "without raising; Windows pointer/keyboard actions still " + "foreground the target when they run." ), }, # โ”€โ”€ return shape โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index c83183b62230..2edd806dfeae 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -77,6 +77,7 @@ _SAFE_ACTIONS = frozenset({"capture", "wait", "list_apps"}) _DESTRUCTIVE_ACTIONS = frozenset({ "click", "double_click", "right_click", "middle_click", "drag", "scroll", "type", "key", "set_value", "focus_app", + "switch_desktop", }) # Hard-blocked key combinations. Mirrored from #4562 โ€” these are destructive @@ -143,23 +144,46 @@ def _default_backend_name() -> str: return "windows" if sys.platform == "win32" else "cua" +def _computer_use_config() -> Dict[str, Any]: + """Return the non-secret computer_use config block from config.yaml.""" + try: + from hermes_cli.config import load_config + cfg = load_config() or {} + section = cfg.get("computer_use") + return section if isinstance(section, dict) else {} + except Exception: + return {} + + +def _configured_backend_name() -> str: + """Return the requested backend, honoring env only as a test/escape hatch.""" + env_backend = os.environ.get("HERMES_COMPUTER_USE_BACKEND") + if env_backend is not None: + return env_backend.strip().lower() or "auto" + cfg_backend = str(_computer_use_config().get("backend") or "auto").strip().lower() + return cfg_backend or "auto" + + def _get_backend() -> ComputerUseBackend: global _backend with _backend_lock: if _backend is None: - backend_name = os.environ.get("HERMES_COMPUTER_USE_BACKEND", "").lower() - if not backend_name: + backend_name = _configured_backend_name() + if backend_name == "auto": backend_name = _default_backend_name() if backend_name in {"cua", "cua-driver"}: from tools.computer_use.cua_backend import CuaDriverBackend _backend = CuaDriverBackend() - elif backend_name == "windows": + elif backend_name in {"windows", "win", "uia", "windows-uia"}: from tools.computer_use.windows_backend import WindowsUIABackend _backend = WindowsUIABackend() elif backend_name == "noop": # pragma: no cover _backend = _NoopBackend() else: - raise RuntimeError(f"Unknown HERMES_COMPUTER_USE_BACKEND={backend_name!r}") + raise RuntimeError( + "Unknown computer_use backend " + f"{backend_name!r}; use auto, cua, windows, or noop" + ) _backend.start() return _backend @@ -273,7 +297,10 @@ def handle_computer_use(args: Dict[str, Any], **kwargs) -> Any: except Exception as e: return json.dumps({ "error": f"computer_use backend unavailable: {e}", - "hint": "Run `hermes tools` and enable Computer Use to install cua-driver.", + "hint": ( + "Run `hermes tools` and enable Computer Use. macOS requires " + "cua-driver; Windows requires pywin32, uiautomation, and Pillow." + ), }) try: @@ -332,6 +359,8 @@ def _summarize_action(action: str, args: Dict[str, Any]) -> str: return f"key {args.get('keys', '')!r}" if action == "focus_app": return f"focus {args.get('app', '')!r}" + (" (raise)" if args.get("raise_window") else "") + if action == "switch_desktop": + return f"switch desktop {args.get('direction', '')!r}" return action @@ -428,8 +457,6 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> if action == "switch_desktop": direction = args.get("direction", "") - if not hasattr(backend, "switch_desktop"): - return json.dumps({"error": "switch_desktop not supported by current backend"}) res = backend.switch_desktop(str(direction)) return _maybe_follow_capture(backend, res, capture_after) @@ -797,7 +824,7 @@ def _route_capture_through_aux_vision( except Exception as exc: logger.warning( "computer_use: auxiliary.vision pre-analysis failed (%s); " - "falling back to native multimodal envelope", + "returning to caller without aux analysis", exc, ) return None @@ -899,18 +926,23 @@ def _element_to_dict(e: UIElement) -> Dict[str, Any]: def check_computer_use_requirements() -> bool: """Return True iff computer_use can run on this host. - macOS: cua-driver binary installed (or override via env). - Windows: uiautomation + Pillow importable (see windows_backend). + macOS: cua-driver binary installed. Windows: UIA backend dependencies + import cleanly. Other platforms stay hidden. """ - if sys.platform == "darwin": + backend_name = _configured_backend_name() + if backend_name == "auto": + backend_name = _default_backend_name() + if sys.platform == "darwin" and backend_name in {"cua", "cua-driver"}: from tools.computer_use.cua_backend import cua_driver_binary_available return cua_driver_binary_available() - if sys.platform == "win32": + if sys.platform == "win32" and backend_name in {"windows", "win", "uia", "windows-uia"}: try: from tools.computer_use.windows_backend import windows_backend_available return windows_backend_available() except Exception: return False + if backend_name == "noop": + return True return False diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index f9e878f29182..a9fc1c39e7fc 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -245,14 +245,23 @@ def _wait_for_user_idle() -> None: Synthetic input lands in whatever has focus; colliding with a human mid-keystroke sprays input across both parties' targets. Wait for - HERMES_COMPUTER_USE_IDLE_WAIT seconds (default 1.5, 0 disables) of user - idle, but never longer than ~8s total โ€” the agent should yield, not + ``computer_use.idle_wait_seconds`` seconds (default 1.5, 0 disables) of + user idle, but never longer than ~8s total โ€” the agent should yield, not deadlock behind a user who is working. """ try: - threshold = float(os.environ.get("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5")) - except ValueError: + from tools.computer_use.tool import _computer_use_config + threshold = float(_computer_use_config().get("idle_wait_seconds", 1.5)) + except (TypeError, ValueError): threshold = 1.5 + except Exception: + threshold = 1.5 + env_threshold = os.environ.get("HERMES_COMPUTER_USE_IDLE_WAIT") + if env_threshold is not None: + try: + threshold = float(env_threshold) + except ValueError: + pass if threshold <= 0: return deadline = time.monotonic() + 8.0 @@ -374,14 +383,28 @@ class _OverlayClient: Strictly fire-and-forget: every failure disables the overlay silently; desktop-control actions must never be affected by overlay problems. - Disable entirely with HERMES_COMPUTER_USE_OVERLAY=0. + Disable with ``computer_use.overlay: false``. ``HERMES_COMPUTER_USE_OVERLAY`` + remains as a test/escape hatch. """ def __init__(self) -> None: self._proc = None self._sock: Optional[socket.socket] = None self._addr: Optional[Tuple[str, int]] = None - self._dead = os.environ.get("HERMES_COMPUTER_USE_OVERLAY", "1") == "0" + overlay_enabled = True + try: + from tools.computer_use.tool import _computer_use_config + raw_overlay = _computer_use_config().get("overlay", True) + if isinstance(raw_overlay, str): + overlay_enabled = raw_overlay.strip().lower() not in {"0", "false", "no", "off"} + else: + overlay_enabled = bool(raw_overlay) + except Exception: + overlay_enabled = True + env_overlay = os.environ.get("HERMES_COMPUTER_USE_OVERLAY") + if env_overlay is not None: + overlay_enabled = env_overlay.strip().lower() not in {"0", "false", "no", "off"} + self._dead = not overlay_enabled @property def pid(self) -> Optional[int]: diff --git a/uv.lock b/uv.lock index 1e9f92b199ee..ba3283877d26 100644 --- a/uv.lock +++ b/uv.lock @@ -1419,6 +1419,7 @@ dependencies = [ { name = "pydantic" }, { name = "pyjwt", extra = ["crypto"] }, { name = "python-dotenv" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, { name = "pywinpty", marker = "sys_platform == 'win32'" }, { name = "pyyaml" }, { name = "requests" }, @@ -1677,6 +1678,7 @@ requires-dist = [ { name = "python-dotenv", specifier = "==1.2.2" }, { name = "python-telegram-bot", extras = ["webhooks"], marker = "extra == 'messaging'", specifier = "==22.6" }, { name = "python-telegram-bot", extras = ["webhooks"], marker = "extra == 'termux'", specifier = "==22.6" }, + { name = "pywin32", marker = "sys_platform == 'win32'", specifier = "==311" }, { name = "pywinpty", marker = "sys_platform == 'win32'", specifier = ">=2.0.0,<3" }, { name = "pyyaml", specifier = "==6.0.3" }, { name = "qrcode", marker = "extra == 'dingtalk'", specifier = "==7.4.2" }, @@ -1700,7 +1702,7 @@ requires-dist = [ { name = "tenacity", specifier = "==9.1.4" }, { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.21" }, { name = "tzdata", marker = "sys_platform == 'win32'", specifier = "==2025.3" }, - { name = "uiautomation", marker = "sys_platform == 'win32'", specifier = ">=2.0.29,<3" }, + { name = "uiautomation", marker = "sys_platform == 'win32'", specifier = "==2.0.29" }, { name = "urllib3", specifier = ">=2.7.0,<3" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.24.0,<1" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'web'", specifier = "==0.41.0" }, diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index f951c6cc5841..ca8f83ba3ea1 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -3,21 +3,21 @@ title: Computer Use sidebar_position: 16 --- -# Computer Use (macOS) - -Hermes Agent can drive your Mac's desktop โ€” clicking, typing, scrolling, -dragging โ€” in the **background**. Your cursor doesn't move, keyboard focus -doesn't change, and macOS doesn't switch Spaces on you. You and the agent -co-work on the same machine. +# Computer Use +Hermes Agent can drive your desktop โ€” clicking, typing, scrolling, and +dragging โ€” through one model-agnostic `computer_use` tool. On macOS it uses +cua-driver for background control. On Windows it uses UI Automation for the +element tree and SendInput for mouse/keyboard actions. Unlike most computer-use integrations, this works with **any tool-capable model** โ€” Claude, GPT, Gemini, or an open model on a local vLLM endpoint. There's no Anthropic-native schema to worry about. ## How it works -The `computer_use` toolset speaks MCP over stdio to [`cua-driver`](https://github.com/trycua/cua), -a macOS driver that uses SkyLight private SPIs (`SLEventPostToPid`, +On macOS, the `computer_use` toolset speaks MCP over stdio to +[`cua-driver`](https://github.com/trycua/cua), a driver that uses SkyLight +private SPIs (`SLEventPostToPid`, `SLPSPostEventRecordTo`) and the `_AXObserverAddNotificationAndCheckRemote` accessibility SPI to: @@ -30,9 +30,20 @@ accessibility SPI to: That combination is what OpenAI's Codex "background computer-use" ships. cua-driver is the open-source equivalent. +On Windows, Hermes uses the `uiautomation` package to enumerate controls and +set native values, Pillow for screenshots, and pywin32/SendInput for window +focus and mouse/keyboard injection. Windows cannot post input to background +windows, so pointer and keyboard actions briefly foreground the target window. +`set_value` is the exception: when the target control exposes the right UIA +pattern, Hermes can set it without moving focus. + ## Enabling -Pick whichever path is most convenient โ€” both run the same upstream installer: +On Windows, install Hermes normally and enable `Computer Use` from +`hermes tools`; the Python dependencies are included in the Windows install. + +On macOS, pick whichever path is most convenient โ€” both run the same upstream +installer: **Option 1: dedicated CLI command (most direct).** @@ -46,7 +57,7 @@ Use `hermes computer-use status` to verify the install. **Option 2: enable the toolset interactively.** -1. Run `hermes tools`, pick `๐Ÿ–ฑ๏ธ Computer Use (macOS)` โ†’ `cua-driver (background)`. +1. Run `hermes tools`, pick `๐Ÿ–ฑ๏ธ Computer Use` โ†’ `cua-driver (background)`. 2. The setup runs the upstream installer (same as Option 1). After installing, regardless of which path you took: @@ -95,8 +106,9 @@ The agent's plan: and get the new screenshot. 5. Click the top result, read the body, summarise. -During all of this, your cursor stays wherever you left it and Mail never -comes to front. +On macOS, your cursor stays wherever you left it and Mail never comes to +front. On Windows, the target window is foregrounded while pointer/keyboard +actions run; prefer `set_value` for form fields and dropdowns when possible. ## Provider compatibility @@ -149,12 +161,15 @@ of screenshot context, not ~600K. ## Limitations -- **macOS only.** cua-driver uses private Apple SPIs that don't exist on - Linux or Windows. For cross-platform GUI automation, use the `browser` - toolset. +- **Platform scope.** Desktop computer-use currently supports macOS via + cua-driver and Windows via UI Automation. Linux desktop automation is not + enabled yet. For cross-platform web tasks, prefer the `browser` toolset. - **Private SPI risk.** Apple can change SkyLight's symbol surface in any OS update. Pin the driver version with the `HERMES_CUA_DRIVER_VERSION` env var if you want reproducibility across a macOS bump. +- **Windows foregrounding.** Windows pointer/keyboard actions move the real + cursor and foreground the target window. Hermes waits briefly for user idle + before injecting input, but you should still avoid fighting an active user. - **Performance.** Background mode is slower than foreground โ€” SkyLight-routed events take ~5-20ms vs direct HID posting. Not noticeable for agent-speed clicking; noticeable if you try to record a @@ -177,12 +192,25 @@ Swap the backend entirely (for testing): HERMES_COMPUTER_USE_BACKEND=noop # records calls, no side effects ``` +Non-secret runtime settings live in `config.yaml`: + +```yaml +computer_use: + backend: auto # auto | cua | windows | noop + idle_wait_seconds: 1.5 # Windows user-idle guard; 0 disables + overlay: true # Windows visible element/click overlay +``` + ## Troubleshooting **`computer_use backend unavailable: cua-driver is not installed`** โ€” Run `hermes computer-use install` to fetch the cua-driver binary, or run `hermes tools` and enable the Computer Use toolset. +**`computer_use backend unavailable` on Windows** โ€” Re-run the current Hermes +installer/update so the Windows-only dependencies (`pywin32`, `uiautomation`, +Pillow) are present, then enable Computer Use in `hermes tools`. + **Clicks seem to have no effect** โ€” Capture and verify. A modal you didn't see may be blocking input. Dismiss it with `escape` or the close button.