diff --git a/contributors/emails/dickson.neoh@gmail.com b/contributors/emails/dickson.neoh@gmail.com new file mode 100644 index 000000000000..ca9f6b5c6054 --- /dev/null +++ b/contributors/emails/dickson.neoh@gmail.com @@ -0,0 +1 @@ +dnth diff --git a/tests/tools/test_computer_use.py b/tests/tools/test_computer_use.py index a9ec592524b9..d25bfde38c68 100644 --- a/tests/tools/test_computer_use.py +++ b/tests/tools/test_computer_use.py @@ -1886,6 +1886,43 @@ class TestCaptureAppFilterNoMatch: assert backend._active_pid == 200 assert backend._active_window_id == 2 + def test_linux_empty_app_name_matches_window_title(self): + windows = [ + {"app_name": "", "pid": 100, "window_id": 1, + "is_on_screen": None, "title": "@!1921,0;BDHF", "z_index": 0}, + {"app_name": "", "pid": 200, "window_id": 2, + "is_on_screen": None, + "title": "Guides — OMC Docs - Google Chrome", "z_index": 0}, + ] + backend = _make_cua_backend_with_windows_and_apps(windows, []) + + backend.capture(mode="ax", app="Chrome") + + assert backend._active_pid == 200 + assert backend._active_window_id == 2 + assert backend._last_app == "Chrome" + + def test_linux_default_capture_skips_gnome_shell_helper(self): + windows = [ + {"app_name": "", "pid": 100, "window_id": 1, + "is_on_screen": None, "title": "@!1921,0;BDHF", "z_index": 0}, + {"app_name": "", "pid": 200, "window_id": 2, + "is_on_screen": None, + "title": "Guides — OMC Docs - Google Chrome", "z_index": 0}, + ] + backend = _make_cua_backend_with_windows(windows) + backend._session.call_tool.side_effect = [ + {"data": "", "images": [], "isError": False, + "structuredContent": {"windows": windows}}, + {"data": "✅ Chrome — 0 elements\n", "images": [], "isError": False, + "structuredContent": None}, + ] + + backend.capture(mode="ax") + + assert backend._active_pid == 200 + assert backend._active_window_id == 2 + def test_app_filter_falls_back_to_list_apps_metadata(self): windows = [ {"app_name": "Qt6Application", "pid": 7675, "window_id": 42, @@ -2169,6 +2206,21 @@ class TestFocusAppFilterNoMatch: assert backend._active_pid == 200 assert backend._active_window_id == 2 + def test_linux_empty_app_name_matches_window_title(self): + windows = [ + {"app_name": "", "pid": 200, "window_id": 2, + "is_on_screen": None, + "title": "Guides — OMC Docs - Google Chrome", "z_index": 0}, + ] + backend = _make_cua_backend_with_windows(windows) + + res = backend.focus_app("Chrome") + + assert res.ok is True + assert backend._active_pid == 200 + assert backend._active_window_id == 2 + assert backend._last_app == "Chrome" + def test_focus_app_falls_back_to_list_apps_metadata(self): windows = [ {"app_name": "Qt6Application", "pid": 7675, "window_id": 42, diff --git a/tests/tools/test_computer_use_cua_backend_linux.py b/tests/tools/test_computer_use_cua_backend_linux.py index 5ceab6796458..f402096c43df 100644 --- a/tests/tools/test_computer_use_cua_backend_linux.py +++ b/tests/tools/test_computer_use_cua_backend_linux.py @@ -1,4 +1,4 @@ -"""Regression tests for Linux/X11 capture target selection (#58026).""" +"""Regression tests for Linux/X11 capture target selection (#58026, #54173).""" from __future__ import annotations @@ -40,6 +40,35 @@ ISSUE_58026_WINDOWS = [ }, ] +# Linux metadata-quirk fixture from #54173 (null is_on_screen, GNOME Shell +# @!x,y;BDHF backdrop helper ahead of real app windows). +LINUX_LIST_WINDOWS = [ + { + "app_name": "", + "pid": 2951331, + "window_id": 98566147, + "title": "@!1921,0;BDHF", + "is_on_screen": None, + "z_index": 0, + }, + { + "app_name": "", + "pid": 11715, + "window_id": 81790890, + "title": "Guides — OMC Docs - Google Chrome", + "is_on_screen": None, + "z_index": 0, + }, + { + "app_name": "", + "pid": 11433, + "window_id": 41943052, + "title": "README.md - hermes-agent - Visual Studio Code", + "is_on_screen": False, + "z_index": 0, + }, +] + def _normalized_windows(raw=ISSUE_58026_WINDOWS): from tools.computer_use.cua_backend import _ingest_windows @@ -83,7 +112,8 @@ def test_default_capture_prefers_x11_active_window_when_z_index_tied(): assert target["window_id"] == 84043449 -def test_default_capture_falls_back_to_list_order_when_active_window_unknown(): +def test_default_capture_skips_desktop_helper_when_active_window_unknown(): + """Even without _NET_ACTIVE_WINDOW, ding/Desktop helpers must not win (#54173).""" from tools.computer_use.cua_backend import _select_capture_target windows = _normalized_windows() @@ -94,10 +124,10 @@ def test_default_capture_falls_back_to_list_order_when_active_window_unknown(): ): target = _select_capture_target(windows, app_requested=False) - # Without informative z-order or active-window, keep list order (caller - # sorts higher-z frontmost; here all tied so first on-screen wins). - assert target["window_id"] == 33554439 - assert target["title"] == "Desktop Icons 1" + # "Desktop Icons 1" is a shell helper window that captures as empty; with + # the active window unknown, the first REAL app window wins list order. + assert target["window_id"] == 60817412 + assert target["title"] == "zcode" def test_default_capture_keeps_higher_z_index_when_ordering_informative(): @@ -196,3 +226,53 @@ def test_exact_pid_window_capture_does_not_probe_x11_active_window(): assert backend._active_window_id == 60817412 active.assert_not_called() assert all(c.args[0] != "list_windows" for c in session.call_tool.call_args_list) + + +def test_linux_null_is_on_screen_is_treated_as_unknown_not_offscreen(): + """cua-driver 0.6.x may return JSON null for Linux is_on_screen (#54173).""" + windows = _normalized_windows(LINUX_LIST_WINDOWS) + + assert windows[0]["off_screen"] is False + assert windows[1]["off_screen"] is False + assert windows[2]["off_screen"] is True + + +def test_default_capture_skips_gnome_shell_background_window(): + """GNOME Shell @!x,y;BDHF windows appear before app windows but screenshot empty.""" + from tools.computer_use.cua_backend import _select_capture_target + + windows = _normalized_windows(LINUX_LIST_WINDOWS) + + with patch("tools.computer_use.cua_backend.sys.platform", "linux"), patch( + "tools.computer_use.cua_backend._linux_x11_active_window_id", + return_value=None, + ): + target = _select_capture_target(windows, app_requested=False) + + assert target["pid"] == 11715 + assert target["window_id"] == 81790890 + assert "Google Chrome" in target["title"] + + +def test_default_capture_prefers_active_window_over_gnome_helper_skip_order(): + """Helper skip and _NET_ACTIVE_WINDOW compose: probe runs on the real-app pool.""" + from tools.computer_use.cua_backend import _select_capture_target + + windows = _normalized_windows(LINUX_LIST_WINDOWS) + + with patch("tools.computer_use.cua_backend.sys.platform", "linux"), patch( + "tools.computer_use.cua_backend._linux_x11_active_window_id", + return_value=81790890, + ): + target = _select_capture_target(windows, app_requested=False) + + assert target["window_id"] == 81790890 + + +def test_explicit_app_capture_preserves_filtered_target_order(): + """When the caller filters first, target selection should not skip the match.""" + from tools.computer_use.cua_backend import _select_capture_target + + chrome = _normalized_windows(LINUX_LIST_WINDOWS)[1] + + assert _select_capture_target([chrome], app_requested=True) == chrome diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 1d18b4dbf8f2..94b456dd67f4 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -166,6 +166,17 @@ _DESKTOP_WINDOW_NAMES = ( "finder", "desktop", "dock", # macOS desktop / shell ) +# Linux/X11 can surface GNOME Shell / desktop backdrop windows before real app +# windows and cua-driver 0.6.x currently does not assign a useful z-order for +# them. These windows are targetable X11 windows but do not produce screenshots +# through get_window_state, so default app capture must skip them. +_NON_APP_WINDOW_TITLE_PREFIXES = ( + "@!", # GNOME Shell background/monitor helper windows + "Desktop", + "gnome-shell", + "GNOME Shell", +) + # Env var cua-driver reads to gate its anonymous usage telemetry (PostHog). # Setting it to "0" disables telemetry; absence => the binary's own default @@ -295,6 +306,15 @@ def _linux_x11_active_window_id() -> Optional[int]: return _parse_xprop_net_active_window(proc.stdout or "") +def _is_real_app_window(w: Dict[str, Any]) -> bool: + """Return False for desktop/shell helper windows that capture as empty.""" + title = w.get("title", "") + return not any( + title.startswith(p) or title.lower().startswith(p.lower()) + for p in _NON_APP_WINDOW_TITLE_PREFIXES + ) + + def _select_capture_target( windows: List[Dict[str, Any]], *, @@ -305,25 +325,26 @@ def _select_capture_target( Callers pass windows already sorted by ``z_index`` descending (higher = frontmost). When ordering is informative, keep that frontmost contract. - On Linux/X11, for unqualified default captures only (no app filter and no - exact pid/window_id), when every on-screen candidate shares the same - ``z_index`` (tied or unknown), prefer ``_NET_ACTIVE_WINDOW`` over list - order (#58026). Exact-target captures must not pay for an ``xprop`` probe. + For unqualified default captures (no app filter and no exact + pid/window_id) on Linux, desktop/shell helper windows (GNOME ``ding`` + "Desktop Icons", ``@!x,y;BDHF`` backdrop helpers) are skipped first — + they are targetable X11 windows but capture as empty. Then, when every + remaining candidate shares the same ``z_index`` (tied or unknown, the + common Linux/X11 case), prefer ``_NET_ACTIVE_WINDOW`` over list order + (#58026). Exact-target captures must not pay for an ``xprop`` probe. """ candidates = [w for w in windows if not w["off_screen"]] pool = candidates - if ( - not exact_target - and not app_requested - and pool - and sys.platform == "linux" - and _z_index_uninformative(pool) - ): - active_id = _linux_x11_active_window_id() - if active_id is not None: - for w in pool: - if w.get("window_id") == active_id: - return w + if not exact_target and not app_requested and sys.platform == "linux": + real_apps = [w for w in candidates if _is_real_app_window(w)] + if real_apps: + pool = real_apps + if pool and _z_index_uninformative(pool): + active_id = _linux_x11_active_window_id() + if active_id is not None: + for w in pool: + if w.get("window_id") == active_id: + return w if pool: return pool[0] return windows[0] @@ -1497,7 +1518,9 @@ def _ingest_windows(raw_windows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: "app_name": w.get("app_name", ""), "pid": pid_int, "window_id": window_id_int, - "off_screen": not w.get("is_on_screen", True), + # cua-driver 0.6.x on Linux may return JSON null here. + # Only explicit False means off-screen; null means unknown. + "off_screen": w.get("is_on_screen") is False, "title": w.get("title", ""), "z_index": z_index, }) @@ -1893,8 +1916,9 @@ class CuaDriverBackend(ComputerUseBackend): windows = filtered # Pick first on-screen window (sorted by z_index / z-order above). - # On Linux/X11, unqualified default captures with tied/unknown z_index - # may additionally consult _NET_ACTIVE_WINDOW (#58026). + # On Linux, unqualified default captures skip desktop/shell helper + # windows and, with tied/unknown z_index, may additionally consult + # _NET_ACTIVE_WINDOW (#58026). target = _select_capture_target( windows, app_requested=bool(app), @@ -1909,7 +1933,7 @@ class CuaDriverBackend(ComputerUseBackend): # Record the resolved app name so capture_after= follow-ups can re-target # the same app rather than falling back to the frontmost window. if app or not self._last_app: - self._last_app = app_name + self._last_app = app_name or app or "" self._last_target = { "pid": self._active_pid, "window_id": self._active_window_id, @@ -2423,7 +2447,7 @@ class CuaDriverBackend(ComputerUseBackend): self._active_pid = target["pid"] self._active_window_id = target["window_id"] self._snapshot_tokens = {} - self._last_app = target["app_name"] # retained for back-compat diagnostics + self._last_app = target["app_name"] or app # retained for back-compat diagnostics self._last_target = { "pid": self._active_pid, "window_id": self._active_window_id,