mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-21 07:55:08 +00:00
fix(tools): harden Windows computer_use for daily co-located use
Three failure modes found preparing for real daytime use on a shared desktop: 1. Vision-node outage broke captures outright. When aux-vision routing is requested (the main model cannot consume images) and the aux call fails, the old fallthrough returned the multimodal envelope - putting a screenshot in front of a text-only model and erroring the capture. Degrade to the AX/SOM text payload instead (vision_unavailable flag set): element-index actions keep working blind until vision returns. 2. Stale coordinates after a window move. Element bounds are absolute screen coords frozen at capture time; dragging the window between capture and click landed clicks on whatever sat at the old position. Track the captured window rect and translate element centers by the origin delta; a resize (interior layout changed) fails with an explicit re-capture message instead of guessing. 3. Input collisions with an active user. Synthetic input lands in whatever has focus; injecting mid-keystroke sprays input across both parties' targets. All input actions now wait for a short user-idle window (HERMES_COMPUTER_USE_IDLE_WAIT, default 1.5s, 0 disables), capped at 8s so the agent yields but never deadlocks. Routing tests updated for the new degradation contract; new tests cover all three behaviors and run dependency-free off-Windows. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
9480721f1d
commit
b062a5d4b9
4 changed files with 215 additions and 25 deletions
|
|
@ -1,4 +1,4 @@
|
|||
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
|
||||
"""End-to-end regression for #24015 -- capture routing via auxiliary.vision.
|
||||
|
||||
When ``computer_use(action='capture', mode='som'|'vision')`` returns a
|
||||
screenshot, ``_capture_response`` previously always returned a
|
||||
|
|
@ -33,13 +33,13 @@ import pytest
|
|||
# Fixtures / helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
|
||||
# 8×8 PNG (transparent) -- minimal provider-acceptable bytes that decode cleanly.
|
||||
_PNG_B64 = (
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
|
||||
"NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
|
||||
)
|
||||
|
||||
# 1×1 JPEG — used to verify mime detection works for either stream type.
|
||||
# 1×1 JPEG -- used to verify mime detection works for either stream type.
|
||||
_JPEG_B64 = (
|
||||
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB"
|
||||
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/"
|
||||
|
|
@ -140,7 +140,7 @@ class TestCaptureResponseDefaultPath:
|
|||
return_value=True) as routing:
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
# ax never even consults the routing helper — short-circuited above
|
||||
# ax never even consults the routing helper -- short-circuited above
|
||||
# the image branch.
|
||||
routing.assert_not_called()
|
||||
assert isinstance(resp, str)
|
||||
|
|
@ -298,15 +298,17 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
new_callable=lambda: fake_vat):
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
# Aux failure → fall back to multimodal envelope (so the user still
|
||||
# gets *something* useful even if vision is broken).
|
||||
assert isinstance(resp, dict)
|
||||
assert resp.get("_multimodal") is True
|
||||
# Aux failure with routing requested (text-only main model) degrades
|
||||
# to the AX/SOM text payload — a multimodal envelope would hand a
|
||||
# screenshot to a model that cannot consume images.
|
||||
assert isinstance(resp, str)
|
||||
body = json.loads(resp)
|
||||
assert body.get("vision_unavailable") is True
|
||||
# Temp file must still be cleaned up.
|
||||
assert observed_path["path"]
|
||||
assert not os.path.exists(observed_path["path"])
|
||||
|
||||
def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir):
|
||||
def test_empty_aux_analysis_degrades_to_text_payload(self, tmp_cache_dir):
|
||||
from tools.computer_use import tool as cu_tool
|
||||
|
||||
cap = _make_capture(mode="som")
|
||||
|
|
@ -323,12 +325,15 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
new_callable=lambda: fake_vat):
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
# Empty analysis is treated as failure — we'd rather show pixels
|
||||
# than embed an empty 'vision_analysis' string into the result.
|
||||
assert isinstance(resp, dict)
|
||||
assert resp.get("_multimodal") is True
|
||||
# Empty analysis is treated as failure; with routing requested the
|
||||
# capture degrades to the AX/SOM text payload (elements stay usable)
|
||||
# rather than embedding an empty 'vision_analysis' string.
|
||||
assert isinstance(resp, str)
|
||||
body = json.loads(resp)
|
||||
assert body.get("vision_unavailable") is True
|
||||
assert body.get("elements") is not None
|
||||
|
||||
def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir):
|
||||
def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir):
|
||||
from tools.computer_use import tool as cu_tool
|
||||
|
||||
cap = _make_capture(mode="som")
|
||||
|
|
@ -345,8 +350,9 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
new_callable=lambda: fake_vat):
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
assert isinstance(resp, dict)
|
||||
assert resp.get("_multimodal") is True
|
||||
assert isinstance(resp, str)
|
||||
body = json.loads(resp)
|
||||
assert body.get("vision_unavailable") is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -398,7 +404,7 @@ class TestRoutingDecisionWiring:
|
|||
|
||||
with patch("hermes_cli.config.load_config",
|
||||
side_effect=RuntimeError("config.yaml unreadable")):
|
||||
# No exception should bubble up — fail open by returning False
|
||||
# No exception should bubble up -- fail open by returning False
|
||||
# so the legacy multimodal envelope continues to work.
|
||||
assert cu_tool._should_route_through_aux_vision() is False
|
||||
|
||||
|
|
@ -417,14 +423,14 @@ class TestRoutingDecisionWiring:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bug reproduction marker — proves the fix is needed.
|
||||
# Bug reproduction marker -- proves the fix is needed.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBugReproductionAnchor:
|
||||
"""Without the fix, this test would assert the wrong thing.
|
||||
|
||||
On upstream/main HEAD prior to this branch, _capture_response returns a
|
||||
multimodal envelope unconditionally — so when a non-vision main model
|
||||
multimodal envelope unconditionally -- so when a non-vision main model
|
||||
is configured, the captured PNG is delivered to the main provider as
|
||||
image_url content and the request is rejected with HTTP 404. We don't
|
||||
have a live provider here, but we can pin the contract: with routing
|
||||
|
|
@ -455,7 +461,7 @@ class TestBugReproductionAnchor:
|
|||
|
||||
# Must be a string (text-only result).
|
||||
assert isinstance(resp, str)
|
||||
# Must NOT contain a base64 image URL anywhere — that's what tripped
|
||||
# Must NOT contain a base64 image URL anywhere -- that's what tripped
|
||||
# 'No endpoints found that support image input' on the reporter's
|
||||
# main provider in #24015.
|
||||
assert "data:image" not in resp
|
||||
|
|
|
|||
|
|
@ -291,3 +291,85 @@ class TestShrinkCaptureForVision:
|
|||
from tools.computer_use.tool import _shrink_capture_for_vision
|
||||
raw = b"not an image at all"
|
||||
assert _shrink_capture_for_vision(raw, ".png") is raw
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hardening: vision-down fallback, stale-coordinate translation, idle guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestVisionDownFallback:
|
||||
def test_capture_degrades_to_text_when_aux_vision_fails(self, monkeypatch):
|
||||
"""Routing requested (text-only main) + vision down => AX text payload,
|
||||
never a multimodal envelope a text model can't consume."""
|
||||
from tools.computer_use import tool
|
||||
from tools.computer_use.backend import CaptureResult, UIElement
|
||||
monkeypatch.setattr(tool, "_should_route_through_aux_vision", lambda: True)
|
||||
monkeypatch.setattr(tool, "_route_capture_through_aux_vision",
|
||||
lambda cap, summary: None)
|
||||
# Must be >= 8x8 or _capture_response's provider-minimum check skips
|
||||
# the vision branch entirely before the fallback we're testing.
|
||||
import base64
|
||||
import io
|
||||
pil = pytest.importorskip("PIL.Image")
|
||||
buf = io.BytesIO()
|
||||
pil.new("RGB", (16, 16), (40, 40, 40)).save(buf, format="PNG")
|
||||
png_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
cap = CaptureResult(
|
||||
mode="som", width=800, height=600, png_b64=png_b64,
|
||||
elements=[UIElement(index=1, role="Button", label="OK",
|
||||
bounds=(1, 2, 3, 4))],
|
||||
app="x.exe", window_title="W")
|
||||
resp = tool._capture_response(cap)
|
||||
assert isinstance(resp, str), "must be a text payload, not multimodal"
|
||||
body = json.loads(resp)
|
||||
assert body["vision_unavailable"] is True
|
||||
assert body["elements"][0]["index"] == 1
|
||||
assert "Element-index actions still work" in body["summary"]
|
||||
|
||||
|
||||
class TestStaleCoordinateTranslation:
|
||||
def _backend_with_element(self, monkeypatch, new_rect):
|
||||
from tools.computer_use import windows_backend as wb
|
||||
from tools.computer_use.backend import UIElement
|
||||
b = wb.WindowsUIABackend()
|
||||
b._elements[1] = UIElement(index=1, role="Button", label="OK",
|
||||
bounds=(110, 220, 100, 50), window_id=777)
|
||||
b._capture_rect = (100, 200, 640, 480)
|
||||
monkeypatch.setattr(wb, "win32gui",
|
||||
types.SimpleNamespace(IsWindow=lambda h: True),
|
||||
raising=False)
|
||||
monkeypatch.setattr(wb, "_window_rect", lambda h: new_rect)
|
||||
return b
|
||||
|
||||
def test_window_moved_translates_click_point(self, monkeypatch):
|
||||
b = self._backend_with_element(monkeypatch, (130, 250, 640, 480))
|
||||
x, y, what = b._resolve_point(1, None, None)
|
||||
assert (x, y) == (160 + 30, 245 + 50) # center (160,245) + delta (30,50)
|
||||
assert "window moved" in what
|
||||
|
||||
def test_window_unmoved_uses_cached_center(self, monkeypatch):
|
||||
b = self._backend_with_element(monkeypatch, (100, 200, 640, 480))
|
||||
x, y, _ = b._resolve_point(1, None, None)
|
||||
assert (x, y) == (160, 245)
|
||||
|
||||
def test_window_resized_demands_recapture(self, monkeypatch):
|
||||
b = self._backend_with_element(monkeypatch, (100, 200, 800, 480))
|
||||
res = b.click(element=1)
|
||||
assert not res.ok
|
||||
assert "resized" in res.message
|
||||
|
||||
|
||||
class TestIdleGuard:
|
||||
def test_zero_threshold_disables_guard(self, monkeypatch):
|
||||
from tools.computer_use import windows_backend as wb
|
||||
monkeypatch.setenv("HERMES_COMPUTER_USE_IDLE_WAIT", "0")
|
||||
wb._wait_for_user_idle() # must return immediately, touch no win32
|
||||
|
||||
def test_returns_once_user_is_idle(self, monkeypatch):
|
||||
from tools.computer_use import windows_backend as wb
|
||||
monkeypatch.setenv("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5")
|
||||
monkeypatch.setattr(wb, "_seconds_since_user_input", lambda: 99.0)
|
||||
import time as _t
|
||||
t0 = _t.monotonic()
|
||||
wb._wait_for_user_idle()
|
||||
assert _t.monotonic() - t0 < 1.0
|
||||
|
|
|
|||
|
|
@ -582,10 +582,39 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
|
|||
routed = _route_capture_through_aux_vision(cap, summary)
|
||||
if routed is not None:
|
||||
return routed
|
||||
# Aux routing was requested but failed (no vision client, aux
|
||||
# call raised, etc.). Fall through to the multimodal envelope —
|
||||
# better to surface a tool-result error from the main model
|
||||
# than to silently drop the screenshot entirely.
|
||||
# Aux routing was requested but failed (vision node down, aux
|
||||
# call raised, etc.). Routing being *requested* means the main
|
||||
# model cannot consume images — falling through to the
|
||||
# multimodal envelope would put a screenshot in front of a
|
||||
# text-only model and break the capture with a provider error.
|
||||
# Degrade to the AX/SOM text payload instead: the element index
|
||||
# still supports element-targeted actions, so the agent can
|
||||
# keep driving blind until vision comes back.
|
||||
summary_lines.append(
|
||||
" (vision unavailable: the auxiliary vision model could not "
|
||||
"be reached; screenshot omitted. Element-index actions still "
|
||||
"work — drive via the element list above.)"
|
||||
)
|
||||
if truncated_elements:
|
||||
summary_lines.append(
|
||||
f" (response truncated to {len(visible_elements)} of "
|
||||
f"{total_elements} elements; raise max_elements or pass "
|
||||
"app= to narrow)"
|
||||
)
|
||||
payload = {
|
||||
"mode": cap.mode,
|
||||
"width": response_width,
|
||||
"height": response_height,
|
||||
"app": cap.app,
|
||||
"window_title": cap.window_title,
|
||||
"elements": [_element_to_dict(e) for e in visible_elements],
|
||||
"total_elements": total_elements,
|
||||
"summary": "\n".join(summary_lines),
|
||||
"vision_unavailable": True,
|
||||
}
|
||||
if truncated_elements:
|
||||
payload["truncated_elements"] = truncated_elements
|
||||
return json.dumps(payload)
|
||||
|
||||
# Detect actual image format from base64 magic bytes so the MIME type
|
||||
# matches what the data contains (cua-driver may return JPEG or PNG).
|
||||
|
|
|
|||
|
|
@ -223,6 +223,44 @@ _EXTENDED_VKS = frozenset({0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
|
|||
_MODIFIER_VKS = frozenset({0x10, 0x11, 0x12, 0x5B})
|
||||
|
||||
|
||||
class _LASTINPUTINFO(ctypes.Structure):
|
||||
_fields_ = [("cbSize", ctypes.wintypes.UINT), ("dwTime", ctypes.wintypes.DWORD)]
|
||||
|
||||
|
||||
def _seconds_since_user_input() -> float:
|
||||
"""Seconds since the user's last keyboard/mouse input (session-wide)."""
|
||||
info = _LASTINPUTINFO()
|
||||
info.cbSize = ctypes.sizeof(_LASTINPUTINFO)
|
||||
if not ctypes.windll.user32.GetLastInputInfo(ctypes.byref(info)):
|
||||
return float("inf")
|
||||
# GetTickCount wraps at 49.7 days; the unsigned subtraction below stays
|
||||
# correct across a single wrap.
|
||||
delta = (ctypes.windll.kernel32.GetTickCount() - info.dwTime) & 0xFFFFFFFF
|
||||
return delta / 1000.0
|
||||
|
||||
|
||||
def _wait_for_user_idle() -> None:
|
||||
"""Hold injected input briefly while the user is actively typing/mousing.
|
||||
|
||||
Synthetic input lands in whatever has focus; colliding with a human
|
||||
mid-keystroke sprays input across both parties' targets. Wait for
|
||||
HERMES_COMPUTER_USE_IDLE_WAIT seconds (default 1.5, 0 disables) of user
|
||||
idle, but never longer than ~8s total — the agent should yield, not
|
||||
deadlock behind a user who is working.
|
||||
"""
|
||||
try:
|
||||
threshold = float(os.environ.get("HERMES_COMPUTER_USE_IDLE_WAIT", "1.5"))
|
||||
except ValueError:
|
||||
threshold = 1.5
|
||||
if threshold <= 0:
|
||||
return
|
||||
deadline = time.monotonic() + 8.0
|
||||
while time.monotonic() < deadline:
|
||||
if _seconds_since_user_input() >= threshold:
|
||||
return
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def _vk_for_key(name: str) -> Optional[int]:
|
||||
"""Map a key name to a virtual-key code; None when unknown."""
|
||||
name = name.strip().lower()
|
||||
|
|
@ -420,6 +458,11 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
self._last_app: Optional[str] = None
|
||||
self._target_hwnd: Optional[int] = None
|
||||
self._target_pid: Optional[int] = None
|
||||
# Window rect at the time of the last capture — element bounds are
|
||||
# absolute screen coords, so if the window moves between capture and
|
||||
# click we translate by the origin delta instead of clicking stale
|
||||
# pixels (see _element_offset).
|
||||
self._capture_rect: Optional[Tuple[int, int, int, int]] = None
|
||||
self._started = False
|
||||
self._overlay = _OverlayClient()
|
||||
|
||||
|
|
@ -526,6 +569,10 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
return False
|
||||
|
||||
def _ensure_target_foreground(self) -> None:
|
||||
# Called exactly once at the top of every input-injecting action —
|
||||
# the idle guard lives here so all of click/drag/scroll/type/key
|
||||
# yield to an actively-working user before touching the desktop.
|
||||
_wait_for_user_idle()
|
||||
if self._target_hwnd and win32gui.IsWindow(self._target_hwnd):
|
||||
self._bring_to_foreground(self._target_hwnd)
|
||||
|
||||
|
|
@ -563,6 +610,7 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
return CaptureResult(mode=mode, width=0, height=0)
|
||||
|
||||
x, y, w, h = _window_rect(hwnd)
|
||||
self._capture_rect = (x, y, w, h)
|
||||
window_title = win32gui.GetWindowText(hwnd)
|
||||
|
||||
img = None
|
||||
|
|
@ -690,6 +738,27 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
draw.text((ix + 3, iy + 1), text, fill=(255, 255, 255))
|
||||
|
||||
# ── Pointer actions ─────────────────────────────────────────────
|
||||
def _element_offset(self, el: UIElement) -> Tuple[int, int]:
|
||||
"""Origin shift to apply to cached element bounds.
|
||||
|
||||
If the element's window moved since the capture, the cached absolute
|
||||
coords are stale by exactly the window-origin delta — translate.
|
||||
A resize invalidates interior layout, so that demands a re-capture.
|
||||
"""
|
||||
hwnd = el.window_id
|
||||
if not (hwnd and self._capture_rect and win32gui.IsWindow(hwnd)):
|
||||
return 0, 0
|
||||
try:
|
||||
cx, cy, cw, ch = self._capture_rect
|
||||
nx, ny, nw, nh = _window_rect(hwnd)
|
||||
except Exception:
|
||||
return 0, 0
|
||||
if (nw, nh) != (cw, ch):
|
||||
raise ValueError(
|
||||
"the target window was resized since the last capture — "
|
||||
"re-run capture(mode='som') for fresh element positions")
|
||||
return nx - cx, ny - cy
|
||||
|
||||
def _resolve_point(self, element: Optional[int], x: Optional[int],
|
||||
y: Optional[int]) -> Tuple[int, int, str]:
|
||||
"""Return (x, y, what) for an action target; raises ValueError."""
|
||||
|
|
@ -699,8 +768,12 @@ class WindowsUIABackend(ComputerUseBackend):
|
|||
raise ValueError(
|
||||
f"element #{element} is not in the last capture — re-run "
|
||||
"capture(mode='som') and use a fresh index")
|
||||
dx, dy = self._element_offset(el)
|
||||
cx, cy = el.center()
|
||||
return cx, cy, f"element #{element} ({el.role} {el.label!r})"
|
||||
what = f"element #{element} ({el.role} {el.label!r})"
|
||||
if dx or dy:
|
||||
what += f" [window moved {dx:+d},{dy:+d} since capture]"
|
||||
return cx + dx, cy + dy, what
|
||||
if x is None or y is None:
|
||||
raise ValueError("requires element= or coordinate [x, y]")
|
||||
return int(x), int(y), f"({x}, {y})"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue