mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-06-23 10:42:00 +00:00
feat(computer-use): add whole-screen/desktop capture target
capture(app='screen'|'desktop') now resolves to the OS shell/desktop window (Windows Progman/WorkerW desktop or Shell_TrayWnd taskbar, macOS Finder/Dock) so 'show me my screen' and 'click the taskbar' work. Previously capture() only matched application windows, and the schema advertised 'or the whole screen' without any code path delivering it. cua-driver is window-oriented (no virtual-desktop or per-monitor MCP tool), so a single image still cannot span multiple monitors — the schema now states this and the no-desktop-window path returns a clear message instead of silently grabbing the frontmost app.
This commit is contained in:
parent
5250335863
commit
30e5d0092d
3 changed files with 136 additions and 4 deletions
|
|
@ -2183,6 +2183,74 @@ class TestStructuredElementsConsumption:
|
|||
assert cap.width == 1
|
||||
assert cap.height == 1
|
||||
|
||||
def test_capture_app_screen_targets_desktop_window(self):
|
||||
"""capture(app='screen') resolves to the OS shell/desktop window
|
||||
(Windows Progman) rather than an application window, so 'show me my
|
||||
screen' works on cua-driver's window-oriented capture surface."""
|
||||
from tools.computer_use.cua_backend import CuaDriverBackend
|
||||
|
||||
backend = CuaDriverBackend()
|
||||
backend._session = MagicMock()
|
||||
|
||||
windows_payload = {
|
||||
"windows": [
|
||||
{"app_name": "Code", "pid": 11, "window_id": 1,
|
||||
"is_on_screen": True, "title": "editor", "z_index": 0},
|
||||
{"app_name": "Progman", "pid": 4, "window_id": 99,
|
||||
"is_on_screen": True, "title": "Program Manager", "z_index": 5},
|
||||
{"app_name": "Shell_TrayWnd", "pid": 4, "window_id": 50,
|
||||
"is_on_screen": True, "title": "Taskbar", "z_index": 4},
|
||||
],
|
||||
}
|
||||
|
||||
def fake_call_tool(name, args):
|
||||
if name == "list_windows":
|
||||
return {"data": "", "images": [], "image_mime_types": [],
|
||||
"structuredContent": windows_payload, "isError": False}
|
||||
if name == "get_window_state":
|
||||
# Should be invoked against the desktop backdrop, not Code.
|
||||
assert args["window_id"] == 99
|
||||
return {"data": "✅ Desktop — 0 elements", "images": [],
|
||||
"image_mime_types": [], "structuredContent": None,
|
||||
"isError": False}
|
||||
return {"data": "", "images": [], "image_mime_types": [],
|
||||
"structuredContent": None, "isError": False}
|
||||
|
||||
backend._session.call_tool.side_effect = fake_call_tool
|
||||
cap = backend.capture(mode="ax", app="screen")
|
||||
|
||||
assert backend._active_window_id == 99
|
||||
assert cap.app == "Progman"
|
||||
|
||||
def test_capture_app_screen_no_desktop_window_surfaces_limitation(self):
|
||||
"""When no desktop/shell window is present, capture(app='screen')
|
||||
returns a clear message about cua-driver's per-window capture limit
|
||||
instead of silently grabbing the frontmost app."""
|
||||
from tools.computer_use.cua_backend import CuaDriverBackend
|
||||
|
||||
backend = CuaDriverBackend()
|
||||
backend._session = MagicMock()
|
||||
|
||||
windows_payload = {
|
||||
"windows": [
|
||||
{"app_name": "Code", "pid": 11, "window_id": 1,
|
||||
"is_on_screen": True, "title": "editor", "z_index": 0},
|
||||
],
|
||||
}
|
||||
|
||||
def fake_call_tool(name, args):
|
||||
if name == "list_windows":
|
||||
return {"data": "", "images": [], "image_mime_types": [],
|
||||
"structuredContent": windows_payload, "isError": False}
|
||||
raise AssertionError(f"unexpected tool {name} — should short-circuit")
|
||||
|
||||
backend._session.call_tool.side_effect = fake_call_tool
|
||||
cap = backend.capture(mode="vision", app="desktop")
|
||||
|
||||
assert cap.width == 0 and cap.height == 0
|
||||
assert cap.png_b64 is None
|
||||
assert "captures one window at a time" in cap.window_title
|
||||
|
||||
|
||||
class TestCapabilityDiscovery:
|
||||
"""Surface 4 (NousResearch/hermes-agent#47072): the wrapper learns
|
||||
|
|
|
|||
|
|
@ -78,6 +78,29 @@ _CUA_DRIVER_ARGS = ["mcp"] # stdio MCP transport (fallback when the
|
|||
# driver doesn't expose `manifest` — see
|
||||
# `_resolve_mcp_invocation` below)
|
||||
|
||||
# Whole-screen / desktop capture. cua-driver is a window-oriented driver —
|
||||
# its `get_window_state` / `screenshot` tools capture a single window (by
|
||||
# pid + window_id), and there is no MCP tool that captures the entire virtual
|
||||
# desktop or an arbitrary monitor as one image. But the OS shell surfaces
|
||||
# themselves (the desktop backdrop and the taskbar/menu-bar) are real windows
|
||||
# that show up in `list_windows`, so "show me my screen" / "click the taskbar"
|
||||
# is reachable by targeting those windows. When `app` is one of these
|
||||
# sentinels, capture() resolves to the desktop/shell window instead of an
|
||||
# application window.
|
||||
_SCREEN_CAPTURE_SENTINELS = {"screen", "desktop", "fullscreen", "full screen", "all"}
|
||||
|
||||
# Known shell/desktop window identifiers across platforms. Matched
|
||||
# case-insensitively as a substring against both the window's app_name and
|
||||
# its title (cua-driver surfaces the Win32 class name / app name here).
|
||||
# Windows: Progman / WorkerW back the desktop; Shell_TrayWnd is the taskbar.
|
||||
# macOS: Finder owns the desktop; the menu bar / Dock are the shell.
|
||||
_DESKTOP_WINDOW_NAMES = (
|
||||
"progman", "workerw", "program manager", # Windows desktop
|
||||
"shell_traywnd", "taskbar", # Windows taskbar
|
||||
"finder", "desktop", "dock", # macOS desktop / shell
|
||||
)
|
||||
|
||||
|
||||
# Env var cua-driver reads to gate its anonymous usage telemetry (PostHog).
|
||||
# Setting it to "0" disables telemetry; absence => the binary's own default
|
||||
# (telemetry ON upstream).
|
||||
|
|
@ -968,7 +991,43 @@ class CuaDriverBackend(ComputerUseBackend):
|
|||
# returned by list_windows is the localized name (e.g. "計算機"), so
|
||||
# `app="Calculator"` legitimately matches no windows on a non-English
|
||||
# system and the caller needs to retry with the localized name.
|
||||
if app:
|
||||
if app and app.strip().lower() in _SCREEN_CAPTURE_SENTINELS:
|
||||
# Whole-screen / desktop request. cua-driver has no virtual-desktop
|
||||
# capture tool, so resolve to the OS shell/desktop window (the
|
||||
# desktop backdrop or the taskbar/menu-bar), which list_windows
|
||||
# does surface. This makes "show me my screen" and "click the
|
||||
# taskbar" work; a single image still can't span multiple monitors
|
||||
# — that's a driver limitation, not a wrapper one.
|
||||
def _is_desktop_window(w: Dict[str, Any]) -> bool:
|
||||
haystack = f"{w.get('app_name', '')} {w.get('title', '')}".lower()
|
||||
return any(name in haystack for name in _DESKTOP_WINDOW_NAMES)
|
||||
|
||||
desktop = [w for w in windows if _is_desktop_window(w)]
|
||||
if not desktop:
|
||||
return CaptureResult(
|
||||
mode=mode, width=0, height=0, png_b64=None,
|
||||
elements=[], app="",
|
||||
window_title=(
|
||||
f"<no desktop/shell window found for app={app!r}; "
|
||||
f"cua-driver captures one window at a time and exposes "
|
||||
f"no whole-virtual-desktop or per-monitor capture. "
|
||||
f"Call list_apps / capture(app='<AppName>') to target a "
|
||||
f"specific window instead. On Windows the taskbar is "
|
||||
f"'Shell_TrayWnd' and the desktop is 'Progman'.>"
|
||||
),
|
||||
png_bytes_len=0,
|
||||
)
|
||||
# Prefer the desktop backdrop (Progman/WorkerW/Finder) over the
|
||||
# taskbar when both are present, so a bare "screen" capture shows
|
||||
# the full desktop rather than just the task strip.
|
||||
windows = sorted(
|
||||
desktop,
|
||||
key=lambda w: 0 if any(
|
||||
n in f"{w.get('app_name', '')} {w.get('title', '')}".lower()
|
||||
for n in ("progman", "workerw", "program manager", "finder", "desktop")
|
||||
) else 1,
|
||||
)
|
||||
elif app:
|
||||
app_lower = app.lower()
|
||||
filtered = [w for w in windows if app_lower in w["app_name"].lower()]
|
||||
if not filtered:
|
||||
|
|
|
|||
|
|
@ -71,9 +71,14 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = {
|
|||
"type": "string",
|
||||
"description": (
|
||||
"Optional. Limit capture/action to a specific app "
|
||||
"(by name, e.g. 'Safari' or 'Notepad', or bundle ID "
|
||||
"where the platform supports it). If omitted, operates "
|
||||
"on the frontmost app's window or the whole screen."
|
||||
"(by name, e.g. 'Safari', or bundle ID, "
|
||||
"'com.apple.Safari'). If omitted, operates on the "
|
||||
"frontmost app's window. Pass app='screen' (or "
|
||||
"'desktop') to capture the OS desktop/shell surface — "
|
||||
"e.g. to see the wallpaper or click the taskbar. Note: "
|
||||
"capture is per-window; a single image cannot span "
|
||||
"multiple monitors, so on a multi-screen setup capture "
|
||||
"one window or display at a time."
|
||||
),
|
||||
},
|
||||
"max_elements": {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue