fix(tools): harden Windows computer_use for daily co-located use

Three failure modes found preparing for real daytime use on a shared
desktop:

1. Vision-node outage broke captures outright. When aux-vision routing
   is requested (the main model cannot consume images) and the aux call
   fails, the old fallthrough returned the multimodal envelope - putting
   a screenshot in front of a text-only model and erroring the capture.
   Degrade to the AX/SOM text payload instead (vision_unavailable flag
   set): element-index actions keep working blind until vision returns.

2. Stale coordinates after a window move. Element bounds are absolute
   screen coords frozen at capture time; dragging the window between
   capture and click landed clicks on whatever sat at the old position.
   Track the captured window rect and translate element centers by the
   origin delta; a resize (interior layout changed) fails with an
   explicit re-capture message instead of guessing.

3. Input collisions with an active user. Synthetic input lands in
   whatever has focus; injecting mid-keystroke sprays input across both
   parties' targets. All input actions now wait for a short user-idle
   window (HERMES_COMPUTER_USE_IDLE_WAIT, default 1.5s, 0 disables),
   capped at 8s so the agent yields but never deadlocks.

Routing tests updated for the new degradation contract; new tests cover
all three behaviors and run dependency-free off-Windows.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jeff 2026-06-11 07:25:39 -04:00 • committed by Teknium
parent 9480721f1d
commit b062a5d4b9
No known key found for this signature in database
4 changed files with 215 additions and 25 deletions

View file

@ -1,4 +1,4 @@
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
"""End-to-end regression for #24015 -- capture routing via auxiliary.vision.
When ``computer_use(action='capture', mode='som'|'vision')`` returns a
screenshot, ``_capture_response`` previously always returned a
@ -33,13 +33,13 @@ import pytest
# Fixtures / helpers
# ---------------------------------------------------------------------------
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
# 8×8 PNG (transparent) -- minimal provider-acceptable bytes that decode cleanly.
_PNG_B64 = (
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
"NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
)
# 1×1 JPEG — used to verify mime detection works for either stream type.
# 1×1 JPEG -- used to verify mime detection works for either stream type.
_JPEG_B64 = (
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB"
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/"
@ -140,7 +140,7 @@ class TestCaptureResponseDefaultPath:
return_value=True) as routing:
resp = cu_tool._capture_response(cap)
# ax never even consults the routing helper — short-circuited above
# ax never even consults the routing helper -- short-circuited above
# the image branch.
routing.assert_not_called()
assert isinstance(resp, str)
@ -298,15 +298,17 @@ class TestCaptureResponseRoutedToAuxVision:
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
# Aux failure → fall back to multimodal envelope (so the user still
# gets *something* useful even if vision is broken).
assert isinstance(resp, dict)
assert resp.get("_multimodal") is True
# Aux failure with routing requested (text-only main model) degrades
# to the AX/SOM text payload — a multimodal envelope would hand a
# screenshot to a model that cannot consume images.
assert isinstance(resp, str)
body = json.loads(resp)
assert body.get("vision_unavailable") is True
# Temp file must still be cleaned up.
assert observed_path["path"]
assert not os.path.exists(observed_path["path"])
def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir):
def test_empty_aux_analysis_degrades_to_text_payload(self, tmp_cache_dir):
from tools.computer_use import tool as cu_tool
cap = _make_capture(mode="som")
@ -323,12 +325,15 @@ class TestCaptureResponseRoutedToAuxVision:
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
# Empty analysis is treated as failure — we'd rather show pixels
# than embed an empty 'vision_analysis' string into the result.
assert isinstance(resp, dict)
assert resp.get("_multimodal") is True
# Empty analysis is treated as failure; with routing requested the
# capture degrades to the AX/SOM text payload (elements stay usable)
# rather than embedding an empty 'vision_analysis' string.
assert isinstance(resp, str)
body = json.loads(resp)
assert body.get("vision_unavailable") is True
assert body.get("elements") is not None
def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir):
def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir):
from tools.computer_use import tool as cu_tool
cap = _make_capture(mode="som")
@ -345,8 +350,9 @@ class TestCaptureResponseRoutedToAuxVision:
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
assert isinstance(resp, dict)
assert resp.get("_multimodal") is True
assert isinstance(resp, str)
body = json.loads(resp)
assert body.get("vision_unavailable") is True
# ---------------------------------------------------------------------------
@ -398,7 +404,7 @@ class TestRoutingDecisionWiring:
with patch("hermes_cli.config.load_config",
side_effect=RuntimeError("config.yaml unreadable")):
# No exception should bubble up — fail open by returning False
# No exception should bubble up -- fail open by returning False
# so the legacy multimodal envelope continues to work.
assert cu_tool._should_route_through_aux_vision() is False
@ -417,14 +423,14 @@ class TestRoutingDecisionWiring:
# ---------------------------------------------------------------------------
# Bug reproduction marker — proves the fix is needed.
# Bug reproduction marker -- proves the fix is needed.
# ---------------------------------------------------------------------------
class TestBugReproductionAnchor:
"""Without the fix, this test would assert the wrong thing.
On upstream/main HEAD prior to this branch, _capture_response returns a
multimodal envelope unconditionally — so when a non-vision main model
multimodal envelope unconditionally -- so when a non-vision main model
is configured, the captured PNG is delivered to the main provider as
image_url content and the request is rejected with HTTP 404. We don't
have a live provider here, but we can pin the contract: with routing
@ -455,7 +461,7 @@ class TestBugReproductionAnchor:
# Must be a string (text-only result).
assert isinstance(resp, str)
# Must NOT contain a base64 image URL anywhere — that's what tripped
# Must NOT contain a base64 image URL anywhere -- that's what tripped
# 'No endpoints found that support image input' on the reporter's
# main provider in #24015.
assert "data:image" not in resp

View file

@ -291,3 +291,85 @@ class TestShrinkCaptureForVision:
from tools.computer_use.tool import _shrink_capture_for_vision
raw = b"not an image at all"
assert _shrink_capture_for_vision(raw, ".png") is raw
# ---------------------------------------------------------------------------
# Hardening: vision-down fallback, stale-coordinate translation, idle guard
# ---------------------------------------------------------------------------
class TestVisionDownFallback:
def test_capture_degrades_to_text_when_aux_vision_fails(self, monkeypatch):
"""Routing requested (text-only main) + vision down => AX text payload,
never a multimodal envelope a text model can't consume."""
from tools.computer_use import tool
from tools.computer_use.backend import CaptureResult, UIElement
monkeypatch.setattr(tool, "_should_route_through_aux_vision", lambda: True)
monkeypatch.setattr(tool, "_route_capture_through_aux_vision",
lambda cap, summary: None)
# Must be >= 8x8 or _capture_response's provider-minimum check skips
# the vision branch entirely before the fallback we're testing.
import base64
import io
pil = pytest.importorskip("PIL.Image")
buf = io.BytesIO()
pil.new("RGB", (16, 16), (40, 40, 40)).save(buf, format="PNG")
png_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
cap = CaptureResult(
mode="som", width=800, height=600, png_b64=png_b64,
elements=[UIElement(index=1, role="Button", label="OK",
bounds=(1, 2, 3, 4))],
app="x.exe", window_title="W")
resp = tool._capture_response(cap)
assert isinstance(resp, str), "must be a text payload, not multimodal"
body = json.loads(resp)
assert body["vision_unavailable"] is True
assert body["elements"][0]["index"] == 1
assert "Element-index actions still work" in body["summary"]
class TestStaleCoordinateTranslation:
def _backend_with_element(self, monkeypatch, new_rect):
from tools.computer_use import windows_backend as wb
from tools.computer_use.backend import UIElement
b = wb.WindowsUIABackend()
b._elements[1] = UIElement(index=1, role="Button", label="OK",
bounds=(110, 220, 100, 50), window_id=777)
b._capture_rect = (100, 200, 640, 480)
monkeypatch.setattr(wb, "win32gui",
types.SimpleNamespace(IsWindow=lambda h: True),
raising=False)
monkeypatch.setattr(wb, "_window_rect", lambda h: new_rect)
return b
def test_window_moved_translates_click_point(self, monkeypatch):
b = self._backend_with_element(monkeypatch, (130, 250, 640, 480))
x, y, what = b._resolve_point(1, None, None)
assert (x, y) == (160 + 30, 245 + 50) # center (160,245) + delta (30,50)
assert "window moved" in what
def test_window_unmoved_uses_cached_center(self, monkeypatch):
b = self._backend_with_element(monkeypatch, (100, 200, 640, 480))
x, y, _ = b._resolve_point(1, None, None)
assert (x, y) == (160, 245)
def test_window_resized_demands_recapture(self, monkeypatch):
b = self._backend_with_element(monkeypatch, (100, 200, 800, 480))
res = b.click(element=1)
assert not res.ok
assert "resized" in res.message
class TestIdleGuard:
def test_zero_threshold_disables_guard(self, monkeypatch):
from tools.computer_use import windows_backend as wb
monkeypatch.setenv("HERMES_COMPUTER_USE_IDLE_WAIT", "0")
wb._wait_for_user_idle() # must return immediately, touch no win32
def test_returns_once_user_is_idle(self, monkeypatch):
from tools.computer_use import windows_backend as wb
monkeypatch.setenv("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5")
monkeypatch.setattr(wb, "_seconds_since_user_input", lambda: 99.0)
import time as _t
t0 = _t.monotonic()
wb._wait_for_user_idle()
assert _t.monotonic() - t0 < 1.0

View file

@ -582,10 +582,39 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
routed = _route_capture_through_aux_vision(cap, summary)
if routed is not None:
return routed
# Aux routing was requested but failed (no vision client, aux
# call raised, etc.). Fall through to the multimodal envelope —
# better to surface a tool-result error from the main model
# than to silently drop the screenshot entirely.
# Aux routing was requested but failed (vision node down, aux
# call raised, etc.). Routing being *requested* means the main
# model cannot consume images — falling through to the
# multimodal envelope would put a screenshot in front of a
# text-only model and break the capture with a provider error.
# Degrade to the AX/SOM text payload instead: the element index
# still supports element-targeted actions, so the agent can
# keep driving blind until vision comes back.
summary_lines.append(
" (vision unavailable: the auxiliary vision model could not "
"be reached; screenshot omitted. Element-index actions still "
"work — drive via the element list above.)"
)
if truncated_elements:
summary_lines.append(
f" (response truncated to {len(visible_elements)} of "
f"{total_elements} elements; raise max_elements or pass "
"app= to narrow)"
)
payload = {
"mode": cap.mode,
"width": response_width,
"height": response_height,
"app": cap.app,
"window_title": cap.window_title,
"elements": [_element_to_dict(e) for e in visible_elements],
"total_elements": total_elements,
"summary": "\n".join(summary_lines),
"vision_unavailable": True,
}
if truncated_elements:
payload["truncated_elements"] = truncated_elements
return json.dumps(payload)
# Detect actual image format from base64 magic bytes so the MIME type
# matches what the data contains (cua-driver may return JPEG or PNG).

View file

@ -223,6 +223,44 @@ _EXTENDED_VKS = frozenset({0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
_MODIFIER_VKS = frozenset({0x10, 0x11, 0x12, 0x5B})
class _LASTINPUTINFO(ctypes.Structure):
_fields_ = [("cbSize", ctypes.wintypes.UINT), ("dwTime", ctypes.wintypes.DWORD)]
def _seconds_since_user_input() -> float:
"""Seconds since the user's last keyboard/mouse input (session-wide)."""
info = _LASTINPUTINFO()
info.cbSize = ctypes.sizeof(_LASTINPUTINFO)
if not ctypes.windll.user32.GetLastInputInfo(ctypes.byref(info)):
return float("inf")
# GetTickCount wraps at 49.7 days; the unsigned subtraction below stays
# correct across a single wrap.
delta = (ctypes.windll.kernel32.GetTickCount() - info.dwTime) & 0xFFFFFFFF
return delta / 1000.0
def _wait_for_user_idle() -> None:
"""Hold injected input briefly while the user is actively typing/mousing.
Synthetic input lands in whatever has focus; colliding with a human
mid-keystroke sprays input across both parties' targets. Wait for
HERMES_COMPUTER_USE_IDLE_WAIT seconds (default 1.5, 0 disables) of user
idle, but never longer than ~8s total — the agent should yield, not
deadlock behind a user who is working.
"""
try:
threshold = float(os.environ.get("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5"))
except ValueError:
threshold = 1.5
if threshold <= 0:
return
deadline = time.monotonic() + 8.0
while time.monotonic() < deadline:
if _seconds_since_user_input() >= threshold:
return
time.sleep(0.2)
def _vk_for_key(name: str) -> Optional[int]:
"""Map a key name to a virtual-key code; None when unknown."""
name = name.strip().lower()
@ -420,6 +458,11 @@ class WindowsUIABackend(ComputerUseBackend):
self._last_app: Optional[str] = None
self._target_hwnd: Optional[int] = None
self._target_pid: Optional[int] = None
# Window rect at the time of the last capture — element bounds are
# absolute screen coords, so if the window moves between capture and
# click we translate by the origin delta instead of clicking stale
# pixels (see _element_offset).
self._capture_rect: Optional[Tuple[int, int, int, int]] = None
self._started = False
self._overlay = _OverlayClient()
@ -526,6 +569,10 @@ class WindowsUIABackend(ComputerUseBackend):
return False
def _ensure_target_foreground(self) -> None:
# Called exactly once at the top of every input-injecting action —
# the idle guard lives here so all of click/drag/scroll/type/key
# yield to an actively-working user before touching the desktop.
_wait_for_user_idle()
if self._target_hwnd and win32gui.IsWindow(self._target_hwnd):
self._bring_to_foreground(self._target_hwnd)
@ -563,6 +610,7 @@ class WindowsUIABackend(ComputerUseBackend):
return CaptureResult(mode=mode, width=0, height=0)
x, y, w, h = _window_rect(hwnd)
self._capture_rect = (x, y, w, h)
window_title = win32gui.GetWindowText(hwnd)
img = None
@ -690,6 +738,27 @@ class WindowsUIABackend(ComputerUseBackend):
draw.text((ix + 3, iy + 1), text, fill=(255, 255, 255))
# ── Pointer actions ─────────────────────────────────────────────
def _element_offset(self, el: UIElement) -> Tuple[int, int]:
"""Origin shift to apply to cached element bounds.
If the element's window moved since the capture, the cached absolute
coords are stale by exactly the window-origin delta — translate.
A resize invalidates interior layout, so that demands a re-capture.
"""
hwnd = el.window_id
if not (hwnd and self._capture_rect and win32gui.IsWindow(hwnd)):
return 0, 0
try:
cx, cy, cw, ch = self._capture_rect
nx, ny, nw, nh = _window_rect(hwnd)
except Exception:
return 0, 0
if (nw, nh) != (cw, ch):
raise ValueError(
"the target window was resized since the last capture — "
"re-run capture(mode='som') for fresh element positions")
return nx - cx, ny - cy
def _resolve_point(self, element: Optional[int], x: Optional[int],
y: Optional[int]) -> Tuple[int, int, str]:
"""Return (x, y, what) for an action target; raises ValueError."""
@ -699,8 +768,12 @@ class WindowsUIABackend(ComputerUseBackend):
raise ValueError(
f"element #{element} is not in the last capture — re-run "
"capture(mode='som') and use a fresh index")
dx, dy = self._element_offset(el)
cx, cy = el.center()
return cx, cy, f"element #{element} ({el.role} {el.label!r})"
what = f"element #{element} ({el.role} {el.label!r})"
if dx or dy:
what += f" [window moved {dx:+d},{dy:+d} since capture]"
return cx + dx, cy + dy, what
if x is None or y is None:
raise ValueError("requires element= or coordinate [x, y]")
return int(x), int(y), f"({x}, {y})"