mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-24 16:54:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix(computer-use): handle Linux cua window metadata

Browse source 
Treat cua-driver's Linux `is_on_screen: null` as unknown instead of
off-screen, and skip GNOME Shell desktop/backdrop helper windows
(ding "Desktop Icons", @!x,y;BDHF) when selecting the default capture
target — they are targetable X11 windows but capture as empty.

Reconciled with the _NET_ACTIVE_WINDOW fallback from #58030: helper
windows are filtered out of the candidate pool first, then the tied
z-order active-window probe runs on the remaining real app windows.
Also falls back to the requested app name for _last_app when Linux
windows carry no app_name.

Salvaged from #54173 by @dnth.
This commit is contained in:
Dickson Neoh 2026-07-23 07:51:38 -07:00 • committed by Teknium
parent ec5835ab8b
commit 08298dabbd
4 changed files with 184 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

1
contributors/emails/dickson.neoh@gmail.com Normal file
View file

@ -0,0 +1 @@
dnth

52
tests/tools/test_computer_use.py
View file

@ -1886,6 +1886,43 @@ class TestCaptureAppFilterNoMatch:
assert backend._active_pid == 200
assert backend._active_window_id == 2
def test_linux_empty_app_name_matches_window_title(self):
windows = [
{"app_name": "", "pid": 100, "window_id": 1,
"is_on_screen": None, "title": "@!1921,0;BDHF", "z_index": 0},
{"app_name": "", "pid": 200, "window_id": 2,
"is_on_screen": None,
"title": "Guides — OMC Docs - Google Chrome", "z_index": 0},
]
backend = _make_cua_backend_with_windows_and_apps(windows, [])
backend.capture(mode="ax", app="Chrome")
assert backend._active_pid == 200
assert backend._active_window_id == 2
assert backend._last_app == "Chrome"
def test_linux_default_capture_skips_gnome_shell_helper(self):
windows = [
{"app_name": "", "pid": 100, "window_id": 1,
"is_on_screen": None, "title": "@!1921,0;BDHF", "z_index": 0},
{"app_name": "", "pid": 200, "window_id": 2,
"is_on_screen": None,
"title": "Guides — OMC Docs - Google Chrome", "z_index": 0},
]
backend = _make_cua_backend_with_windows(windows)
backend._session.call_tool.side_effect = [
{"data": "", "images": [], "isError": False,
"structuredContent": {"windows": windows}},
{"data": "✅ Chrome — 0 elements\n", "images": [], "isError": False,
"structuredContent": None},
]
backend.capture(mode="ax")
assert backend._active_pid == 200
assert backend._active_window_id == 2
def test_app_filter_falls_back_to_list_apps_metadata(self):
windows = [
{"app_name": "Qt6Application", "pid": 7675, "window_id": 42,
@ -2169,6 +2206,21 @@ class TestFocusAppFilterNoMatch:
assert backend._active_pid == 200
assert backend._active_window_id == 2
def test_linux_empty_app_name_matches_window_title(self):
windows = [
{"app_name": "", "pid": 200, "window_id": 2,
"is_on_screen": None,
"title": "Guides — OMC Docs - Google Chrome", "z_index": 0},
]
backend = _make_cua_backend_with_windows(windows)
res = backend.focus_app("Chrome")
assert res.ok is True
assert backend._active_pid == 200
assert backend._active_window_id == 2
assert backend._last_app == "Chrome"
def test_focus_app_falls_back_to_list_apps_metadata(self):
windows = [
{"app_name": "Qt6Application", "pid": 7675, "window_id": 42,

92
tests/tools/test_computer_use_cua_backend_linux.py
View file

@ -1,4 +1,4 @@
"""Regression tests for Linux/X11 capture target selection (#58026)."""
"""Regression tests for Linux/X11 capture target selection (#58026, #54173)."""
from __future__ import annotations
@ -40,6 +40,35 @@ ISSUE_58026_WINDOWS = [
},
]
# Linux metadata-quirk fixture from #54173 (null is_on_screen, GNOME Shell
# @!x,y;BDHF backdrop helper ahead of real app windows).
LINUX_LIST_WINDOWS = [
{
"app_name": "",
"pid": 2951331,
"window_id": 98566147,
"title": "@!1921,0;BDHF",
"is_on_screen": None,
"z_index": 0,
},
{
"app_name": "",
"pid": 11715,
"window_id": 81790890,
"title": "Guides — OMC Docs - Google Chrome",
"is_on_screen": None,
"z_index": 0,
},
{
"app_name": "",
"pid": 11433,
"window_id": 41943052,
"title": "README.md - hermes-agent - Visual Studio Code",
"is_on_screen": False,
"z_index": 0,
},
]
def _normalized_windows(raw=ISSUE_58026_WINDOWS):
from tools.computer_use.cua_backend import _ingest_windows
@ -83,7 +112,8 @@ def test_default_capture_prefers_x11_active_window_when_z_index_tied():
assert target["window_id"] == 84043449
def test_default_capture_falls_back_to_list_order_when_active_window_unknown():
def test_default_capture_skips_desktop_helper_when_active_window_unknown():
"""Even without _NET_ACTIVE_WINDOW, ding/Desktop helpers must not win (#54173)."""
from tools.computer_use.cua_backend import _select_capture_target
windows = _normalized_windows()
@ -94,10 +124,10 @@ def test_default_capture_falls_back_to_list_order_when_active_window_unknown():
):
target = _select_capture_target(windows, app_requested=False)
# Without informative z-order or active-window, keep list order (caller
# sorts higher-z frontmost; here all tied so first on-screen wins).
assert target["window_id"] == 33554439
assert target["title"] == "Desktop Icons 1"
# "Desktop Icons 1" is a shell helper window that captures as empty; with
# the active window unknown, the first REAL app window wins list order.
assert target["window_id"] == 60817412
assert target["title"] == "zcode"
def test_default_capture_keeps_higher_z_index_when_ordering_informative():
@ -196,3 +226,53 @@ def test_exact_pid_window_capture_does_not_probe_x11_active_window():
assert backend._active_window_id == 60817412
active.assert_not_called()
assert all(c.args[0] != "list_windows" for c in session.call_tool.call_args_list)
def test_linux_null_is_on_screen_is_treated_as_unknown_not_offscreen():
"""cua-driver 0.6.x may return JSON null for Linux is_on_screen (#54173)."""
windows = _normalized_windows(LINUX_LIST_WINDOWS)
assert windows[0]["off_screen"] is False
assert windows[1]["off_screen"] is False
assert windows[2]["off_screen"] is True
def test_default_capture_skips_gnome_shell_background_window():
"""GNOME Shell @!x,y;BDHF windows appear before app windows but screenshot empty."""
from tools.computer_use.cua_backend import _select_capture_target
windows = _normalized_windows(LINUX_LIST_WINDOWS)
with patch("tools.computer_use.cua_backend.sys.platform", "linux"), patch(
"tools.computer_use.cua_backend._linux_x11_active_window_id",
return_value=None,
):
target = _select_capture_target(windows, app_requested=False)
assert target["pid"] == 11715
assert target["window_id"] == 81790890
assert "Google Chrome" in target["title"]
def test_default_capture_prefers_active_window_over_gnome_helper_skip_order():
"""Helper skip and _NET_ACTIVE_WINDOW compose: probe runs on the real-app pool."""
from tools.computer_use.cua_backend import _select_capture_target
windows = _normalized_windows(LINUX_LIST_WINDOWS)
with patch("tools.computer_use.cua_backend.sys.platform", "linux"), patch(
"tools.computer_use.cua_backend._linux_x11_active_window_id",
return_value=81790890,
):
target = _select_capture_target(windows, app_requested=False)
assert target["window_id"] == 81790890
def test_explicit_app_capture_preserves_filtered_target_order():
"""When the caller filters first, target selection should not skip the match."""
from tools.computer_use.cua_backend import _select_capture_target
chrome = _normalized_windows(LINUX_LIST_WINDOWS)[1]
assert _select_capture_target([chrome], app_requested=True) == chrome

66
tools/computer_use/cua_backend.py
View file

@ -166,6 +166,17 @@ _DESKTOP_WINDOW_NAMES = (
"finder", "desktop", "dock", # macOS desktop / shell
)
# Linux/X11 can surface GNOME Shell / desktop backdrop windows before real app
# windows and cua-driver 0.6.x currently does not assign a useful z-order for
# them. These windows are targetable X11 windows but do not produce screenshots
# through get_window_state, so default app capture must skip them.
_NON_APP_WINDOW_TITLE_PREFIXES = (
"@!", # GNOME Shell background/monitor helper windows
"Desktop",
"gnome-shell",
"GNOME Shell",
)
# Env var cua-driver reads to gate its anonymous usage telemetry (PostHog).
# Setting it to "0" disables telemetry; absence => the binary's own default
@ -295,6 +306,15 @@ def _linux_x11_active_window_id() -> Optional[int]:
return _parse_xprop_net_active_window(proc.stdout or "")
def _is_real_app_window(w: Dict[str, Any]) -> bool:
"""Return False for desktop/shell helper windows that capture as empty."""
title = w.get("title", "")
return not any(
title.startswith(p) or title.lower().startswith(p.lower())
for p in _NON_APP_WINDOW_TITLE_PREFIXES
)
def _select_capture_target(
windows: List[Dict[str, Any]],
*,
@ -305,25 +325,26 @@ def _select_capture_target(
Callers pass windows already sorted by ``z_index`` descending (higher =
frontmost). When ordering is informative, keep that frontmost contract.
On Linux/X11, for unqualified default captures only (no app filter and no
exact pid/window_id), when every on-screen candidate shares the same
``z_index`` (tied or unknown), prefer ``_NET_ACTIVE_WINDOW`` over list
order (#58026). Exact-target captures must not pay for an ``xprop`` probe.
For unqualified default captures (no app filter and no exact
pid/window_id) on Linux, desktop/shell helper windows (GNOME ``ding``
"Desktop Icons", ``@!x,y;BDHF`` backdrop helpers) are skipped first —
they are targetable X11 windows but capture as empty. Then, when every
remaining candidate shares the same ``z_index`` (tied or unknown, the
common Linux/X11 case), prefer ``_NET_ACTIVE_WINDOW`` over list order
(#58026). Exact-target captures must not pay for an ``xprop`` probe.
"""
candidates = [w for w in windows if not w["off_screen"]]
pool = candidates
if (
not exact_target
and not app_requested
and pool
and sys.platform == "linux"
and _z_index_uninformative(pool)
):
active_id = _linux_x11_active_window_id()
if active_id is not None:
for w in pool:
if w.get("window_id") == active_id:
return w
if not exact_target and not app_requested and sys.platform == "linux":
real_apps = [w for w in candidates if _is_real_app_window(w)]
if real_apps:
pool = real_apps
if pool and _z_index_uninformative(pool):
active_id = _linux_x11_active_window_id()
if active_id is not None:
for w in pool:
if w.get("window_id") == active_id:
return w
if pool:
return pool[0]
return windows[0]
@ -1497,7 +1518,9 @@ def _ingest_windows(raw_windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"app_name": w.get("app_name", ""),
"pid": pid_int,
"window_id": window_id_int,
"off_screen": not w.get("is_on_screen", True),
# cua-driver 0.6.x on Linux may return JSON null here.
# Only explicit False means off-screen; null means unknown.
"off_screen": w.get("is_on_screen") is False,
"title": w.get("title", ""),
"z_index": z_index,
})
@ -1893,8 +1916,9 @@ class CuaDriverBackend(ComputerUseBackend):
windows = filtered
# Pick first on-screen window (sorted by z_index / z-order above).
# On Linux/X11, unqualified default captures with tied/unknown z_index
# may additionally consult _NET_ACTIVE_WINDOW (#58026).
# On Linux, unqualified default captures skip desktop/shell helper
# windows and, with tied/unknown z_index, may additionally consult
# _NET_ACTIVE_WINDOW (#58026).
target = _select_capture_target(
windows,
app_requested=bool(app),
@ -1909,7 +1933,7 @@ class CuaDriverBackend(ComputerUseBackend):
# Record the resolved app name so capture_after= follow-ups can re-target
# the same app rather than falling back to the frontmost window.
if app or not self._last_app:
self._last_app = app_name
self._last_app = app_name or app or ""
self._last_target = {
"pid": self._active_pid,
"window_id": self._active_window_id,
@ -2423,7 +2447,7 @@ class CuaDriverBackend(ComputerUseBackend):
self._active_pid = target["pid"]
self._active_window_id = target["window_id"]
self._snapshot_tokens = {}
self._last_app = target["app_name"] # retained for back-compat diagnostics
self._last_app = target["app_name"] or app # retained for back-compat diagnostics
self._last_target = {
"pid": self._active_pid,
"window_id": self._active_window_id,