From 519ec7b3b327fa55fd5f64709ed8114fec6b0ecf Mon Sep 17 00:00:00 2001 From: Adrian Lastra Date: Sun, 7 Jun 2026 12:41:13 -0400 Subject: [PATCH] fix(computer_use): parse (label) and = "value" AX element label forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SOM/AX element list dropped labels for two extremely common cua-driver render forms, leaving the model unable to target elements by name: - [79] AXButton (Dark) -> parenthesised label - [4] AXStaticText = "Wi-Fi" -> = "value" form - [92] AXPopUpButton = "Automatic" -> = "value" form The old regex only matched quoted "label" and id=Label, so System Settings buttons/text/popups all surfaced with empty labels. That's why selecting the macOS Appearance 'Dark' button by element index required guessing — the labels weren't available to aim with. Fix: extend _ELEMENT_LINE_RE to capture all four label forms (= "value", "quoted", (parenthesised), id=Label), skipping a pure-digit (N) order number in favour of the id= label. Verified live against System Settings: the Appearance buttons now surface as Auto/Light/Dark. Adds a regression test covering all label forms. Full suite: 84 passed. --- tests/tools/test_computer_use.py | 30 +++++++++++++++++++++ tools/computer_use/cua_backend.py | 43 +++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/tests/tools/test_computer_use.py b/tests/tools/test_computer_use.py index 9d4c8ec1925..9f4a6a5ca05 100644 --- a/tests/tools/test_computer_use.py +++ b/tests/tools/test_computer_use.py @@ -1132,6 +1132,36 @@ class TestElementLabelParsing: assert labels[14] == "One" assert labels[15] == "Search" + def test_parenthesised_and_value_label_formats(self): + """Real cua-driver System Settings format: `(label)` and `= "value"`. + + Regression for the bug where AXButton (Dark), AXStaticText = "Wi-Fi", + and AXPopUpButton = "Automatic" all came back with EMPTY labels because + the regex only matched the quoted and id= forms. A pure-digit (N) is an + order number, not a label, and must be skipped in favour of id=. + """ + from tools.computer_use.cua_backend import _parse_elements_from_tree + tree = ( + '- [77] AXButton (Auto) [help="..." actions=[press]]\n' + '- [78] AXButton (Light) [help="..." actions=[press]]\n' + '- [79] AXButton (Dark) [help="Use a dark appearance..." actions=[press]]\n' + ' - [4] AXStaticText = "Wi\u2011Fi" [id=com.apple.wifi actions=[showmenu]]\n' + '- [92] AXPopUpButton = "Automatic" [id=HighlightColorPicker actions=[press]]\n' + '- [100] AXRadioButton (Always) [actions=[press]]\n' + '[200] AXButton (5) id=RealLabel\n' # (5) is order number -> label from id= + '[201] AXButton (7)\n' # order number only, no real label + ) + els = _parse_elements_from_tree(tree) + labels = {e.index: e.label for e in els} + assert labels[77] == "Auto" + assert labels[78] == "Light" + assert labels[79] == "Dark" # the exact case that broke theme-switching + assert labels[4] == "Wi\u2011Fi" + assert labels[92] == "Automatic" + assert labels[100] == "Always" + assert labels[200] == "RealLabel" # (5) order skipped, id= used + assert labels[201] == "" # pure order number, no label + class TestUpdateCheck: """cua_driver_update_check() / _nudge(): native `check-update --json`. diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index ca14c53adb8..67f2add21a4 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -191,16 +191,31 @@ def _resolve_mcp_invocation( # Regex to parse element lines from get_window_state AX tree markdown. # -# Handles two output formats from different cua-driver versions: -# Classic: " - [N] AXRole \"label\"" -# New: "[N] AXRole (order) id=Label" +# cua-driver renders each actionable node as one of: +# - [N] AXRole "label" (quoted label, classic) +# - [N] AXRole = "value" (value form, e.g. AXStaticText/AXPopUpButton) +# - [N] AXRole (label) (parenthesised label, e.g. AXButton (Dark)) +# - [N] AXRole (order) id=Label (order number + id= label, newer builds) +# - [N] AXRole id=Label (id= label only) +# - [N] AXRole (no label) +# followed by trailing metadata like [help="..." actions=[...]]. # -# Group 1: element index -# Group 2: AX role -# Group 3: quoted label (classic format) -# Group 4: id= label (new format) +# Earlier the regex only matched the quoted and id= forms, so the very common +# `(label)` and `= "value"` forms (System Settings buttons, static text, popups) +# came back with an empty label — which made label-driven clicking impossible. +# A parenthesised group that is purely digits is an ORDER index, not a label, so +# it is excluded and we fall through to the id= label. +# +# Group 1: element index Group 2: AX role +# Groups 3-6: the label in value / quoted / paren / id= form (whichever matched) _ELEMENT_LINE_RE = re.compile( - r'^\s*(?:-\s+)?\[(\d+)\]\s+(\w+)(?:\s+"([^"]*)"|(?:\s+\(\d+\))?\s+id=([^\s\[\]]*))?' , + r'^\s*(?:-\s+)?\[(\d+)\]\s+(\w+)' + r'(?:' + r'\s*=\s*"([^"]*)"' # = "value" + r'|\s+"([^"]*)"' # "value" + r'|\s+\((?!\d+\))([^)]*)\)' # (value) but not a pure-digit (order) number + r')?' + r'(?:\s+(?:\(\d+\)\s+)?id=([^\s\[\]]+))?', # optional id=value (after an optional (order)) re.MULTILINE, ) @@ -323,15 +338,17 @@ def _parse_elements_from_tree(markdown: str) -> List[UIElement]: ``_parse_elements_from_structured`` — Surface 2 of #47072 prefers that path). - Handles both the classic ``"label"``-quoted format and the newer - ``id=Label`` format introduced in cua-driver v0.1.6. Bounds always + Captures the label whichever form cua-driver used: ``= "value"``, + ``"quoted"``, ``(parenthesised)``, or ``id=Label``. Bounds always come back ``(0, 0, 0, 0)`` because the markdown surface doesn't - carry them — yet another reason to prefer the structured path. + carry them — yet another reason to prefer the structured path; + element-index clicks don't need them (the driver resolves the index + to a frame internally). """ elements = [] for m in _ELEMENT_LINE_RE.finditer(markdown): - # group(3) = quoted label (classic); group(4) = id= label (new) - label = m.group(3) or m.group(4) or "" + # groups 3-6: value / quoted / paren / id= label (first non-None wins) + label = m.group(3) or m.group(4) or m.group(5) or m.group(6) or "" elements.append(UIElement( index=int(m.group(1)), role=m.group(2),