mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix(computer-use): handle Linux cua window metadata

Browse source 
Treat cua-driver's Linux `is_on_screen: null` as unknown instead of
off-screen, and skip GNOME Shell desktop/backdrop helper windows
(ding "Desktop Icons", @!x,y;BDHF) when selecting the default capture
target — they are targetable X11 windows but capture as empty.

Reconciled with the _NET_ACTIVE_WINDOW fallback from #58030: helper
windows are filtered out of the candidate pool first, then the tied
z-order active-window probe runs on the remaining real app windows.
Also falls back to the requested app name for _last_app when Linux
windows carry no app_name.

Salvaged from #54173 by @dnth.
This commit is contained in:
Dickson Neoh 2026-07-23 07:51:38 -07:00 • committed by Teknium
parent ec5835ab8b
commit 08298dabbd
4 changed files with 184 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

66
tools/computer_use/cua_backend.py
View file

@ -166,6 +166,17 @@ _DESKTOP_WINDOW_NAMES = (
"finder", "desktop", "dock", # macOS desktop / shell
)
# Linux/X11 can surface GNOME Shell / desktop backdrop windows before real app
# windows and cua-driver 0.6.x currently does not assign a useful z-order for
# them. These windows are targetable X11 windows but do not produce screenshots
# through get_window_state, so default app capture must skip them.
_NON_APP_WINDOW_TITLE_PREFIXES = (
"@!", # GNOME Shell background/monitor helper windows
"Desktop",
"gnome-shell",
"GNOME Shell",
)
# Env var cua-driver reads to gate its anonymous usage telemetry (PostHog).
# Setting it to "0" disables telemetry; absence => the binary's own default
@ -295,6 +306,15 @@ def _linux_x11_active_window_id() -> Optional[int]:
return _parse_xprop_net_active_window(proc.stdout or "")
def _is_real_app_window(w: Dict[str, Any]) -> bool:
"""Return False for desktop/shell helper windows that capture as empty."""
title = w.get("title", "")
return not any(
title.startswith(p) or title.lower().startswith(p.lower())
for p in _NON_APP_WINDOW_TITLE_PREFIXES
)
def _select_capture_target(
windows: List[Dict[str, Any]],
*,
@ -305,25 +325,26 @@ def _select_capture_target(
Callers pass windows already sorted by ``z_index`` descending (higher =
frontmost). When ordering is informative, keep that frontmost contract.
On Linux/X11, for unqualified default captures only (no app filter and no
exact pid/window_id), when every on-screen candidate shares the same
``z_index`` (tied or unknown), prefer ``_NET_ACTIVE_WINDOW`` over list
order (#58026). Exact-target captures must not pay for an ``xprop`` probe.
For unqualified default captures (no app filter and no exact
pid/window_id) on Linux, desktop/shell helper windows (GNOME ``ding``
"Desktop Icons", ``@!x,y;BDHF`` backdrop helpers) are skipped first —
they are targetable X11 windows but capture as empty. Then, when every
remaining candidate shares the same ``z_index`` (tied or unknown, the
common Linux/X11 case), prefer ``_NET_ACTIVE_WINDOW`` over list order
(#58026). Exact-target captures must not pay for an ``xprop`` probe.
"""
candidates = [w for w in windows if not w["off_screen"]]
pool = candidates
if (
not exact_target
and not app_requested
and pool
and sys.platform == "linux"
and _z_index_uninformative(pool)
):
active_id = _linux_x11_active_window_id()
if active_id is not None:
for w in pool:
if w.get("window_id") == active_id:
return w
if not exact_target and not app_requested and sys.platform == "linux":
real_apps = [w for w in candidates if _is_real_app_window(w)]
if real_apps:
pool = real_apps
if pool and _z_index_uninformative(pool):
active_id = _linux_x11_active_window_id()
if active_id is not None:
for w in pool:
if w.get("window_id") == active_id:
return w
if pool:
return pool[0]
return windows[0]
@ -1497,7 +1518,9 @@ def _ingest_windows(raw_windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"app_name": w.get("app_name", ""),
"pid": pid_int,
"window_id": window_id_int,
"off_screen": not w.get("is_on_screen", True),
# cua-driver 0.6.x on Linux may return JSON null here.
# Only explicit False means off-screen; null means unknown.
"off_screen": w.get("is_on_screen") is False,
"title": w.get("title", ""),
"z_index": z_index,
})
@ -1893,8 +1916,9 @@ class CuaDriverBackend(ComputerUseBackend):
windows = filtered
# Pick first on-screen window (sorted by z_index / z-order above).
# On Linux/X11, unqualified default captures with tied/unknown z_index
# may additionally consult _NET_ACTIVE_WINDOW (#58026).
# On Linux, unqualified default captures skip desktop/shell helper
# windows and, with tied/unknown z_index, may additionally consult
# _NET_ACTIVE_WINDOW (#58026).
target = _select_capture_target(
windows,
app_requested=bool(app),
@ -1909,7 +1933,7 @@ class CuaDriverBackend(ComputerUseBackend):
# Record the resolved app name so capture_after= follow-ups can re-target
# the same app rather than falling back to the frontmost window.
if app or not self._last_app:
self._last_app = app_name
self._last_app = app_name or app or ""
self._last_target = {
"pid": self._active_pid,
"window_id": self._active_window_id,
@ -2423,7 +2447,7 @@ class CuaDriverBackend(ComputerUseBackend):
self._active_pid = target["pid"]
self._active_window_id = target["window_id"]
self._snapshot_tokens = {}
self._last_app = target["app_name"] # retained for back-compat diagnostics
self._last_app = target["app_name"] or app # retained for back-compat diagnostics
self._last_target = {
"pid": self._active_pid,
"window_id": self._active_window_id,