diff --git a/tests/tools/test_computer_use_capture_routing.py b/tests/tools/test_computer_use_capture_routing.py index c4ccd2e889f7..b2c5c81bf412 100644 --- a/tests/tools/test_computer_use_capture_routing.py +++ b/tests/tools/test_computer_use_capture_routing.py @@ -1,4 +1,4 @@ -"""End-to-end regression for #24015 — capture routing via auxiliary.vision. +"""End-to-end regression for #24015 — capture routing via auxiliary.vision. When ``computer_use(action='capture', mode='som'|'vision')`` returns a screenshot, ``_capture_response`` previously always returned a @@ -15,7 +15,7 @@ deterministic stubs for: * ``vision_analyze_tool`` (the aux LLM call) * ``hermes_constants.get_hermes_dir`` (cache path) -…so the full code path is covered without a live cua-driver, a real +…so the full code path is covered without a live cua-driver, a real auxiliary client, or network access. """ @@ -33,13 +33,13 @@ import pytest # Fixtures / helpers # --------------------------------------------------------------------------- -# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly. +# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly. _PNG_B64 = ( "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG" "NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg==" ) -# 1×1 JPEG — used to verify mime detection works for either stream type. +# 1×1 JPEG — used to verify mime detection works for either stream type. _JPEG_B64 = ( "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB" "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/" @@ -65,7 +65,7 @@ def _make_capture( mode: str = "som", elements=None, app: str = "Safari", - window_title: str = "GitHub – Issue #24015", + window_title: str = "GitHub – Issue #24015", width: int = 1280, height: int = 800, ): @@ -140,7 +140,7 @@ class TestCaptureResponseDefaultPath: return_value=True) as routing: resp = cu_tool._capture_response(cap) - # ax never even consults the routing helper — short-circuited above + # ax never even consults the routing helper — short-circuited above # the image branch. routing.assert_not_called() assert isinstance(resp, str) @@ -195,7 +195,7 @@ class TestCaptureResponseRoutedToAuxVision: # The original AX-only metadata (window title, element index, app) # is preserved alongside the new vision analysis so the agent loses # no context vs the multimodal path. - assert body["window_title"] == "GitHub – Issue #24015" + assert body["window_title"] == "GitHub – Issue #24015" assert len(body["elements"]) == 2 assert captured_calls.get("called") is True @@ -204,7 +204,7 @@ class TestCaptureResponseRoutedToAuxVision: args, _kwargs = fake_vat.call_args path_arg, prompt_arg = args[0], args[1] assert str(tmp_cache_dir) in path_arg - assert "macOS application screenshot" in prompt_arg + assert "application screenshot" in prompt_arg # AX summary is included so the aux model can ground its description # against the same set-of-mark index the agent will see. assert "Sign in" in prompt_arg @@ -298,7 +298,7 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - # Aux failure → fall back to multimodal envelope (so the user still + # Aux failure → fall back to multimodal envelope (so the user still # gets *something* useful even if vision is broken). assert isinstance(resp, dict) assert resp.get("_multimodal") is True @@ -323,7 +323,7 @@ class TestCaptureResponseRoutedToAuxVision: new_callable=lambda: fake_vat): resp = cu_tool._capture_response(cap) - # Empty analysis is treated as failure — we'd rather show pixels + # Empty analysis is treated as failure — we'd rather show pixels # than embed an empty 'vision_analysis' string into the result. assert isinstance(resp, dict) assert resp.get("_multimodal") is True @@ -398,7 +398,7 @@ class TestRoutingDecisionWiring: with patch("hermes_cli.config.load_config", side_effect=RuntimeError("config.yaml unreadable")): - # No exception should bubble up — fail open by returning False + # No exception should bubble up — fail open by returning False # so the legacy multimodal envelope continues to work. assert cu_tool._should_route_through_aux_vision() is False @@ -417,14 +417,14 @@ class TestRoutingDecisionWiring: # --------------------------------------------------------------------------- -# Bug reproduction marker — proves the fix is needed. +# Bug reproduction marker — proves the fix is needed. # --------------------------------------------------------------------------- class TestBugReproductionAnchor: """Without the fix, this test would assert the wrong thing. On upstream/main HEAD prior to this branch, _capture_response returns a - multimodal envelope unconditionally — so when a non-vision main model + multimodal envelope unconditionally — so when a non-vision main model is configured, the captured PNG is delivered to the main provider as image_url content and the request is rejected with HTTP 404. We don't have a live provider here, but we can pin the contract: with routing @@ -455,7 +455,7 @@ class TestBugReproductionAnchor: # Must be a string (text-only result). assert isinstance(resp, str) - # Must NOT contain a base64 image URL anywhere — that's what tripped + # Must NOT contain a base64 image URL anywhere — that's what tripped # 'No endpoints found that support image input' on the reporter's # main provider in #24015. assert "data:image" not in resp diff --git a/tests/tools/test_computer_use_windows.py b/tests/tools/test_computer_use_windows.py index 9f0eeca1c7a4..f1d54cc9cf75 100644 --- a/tests/tools/test_computer_use_windows.py +++ b/tests/tools/test_computer_use_windows.py @@ -227,3 +227,67 @@ class TestLiveReadOnly: cap = b.capture(mode="ax") assert cap.mode == "ax" assert cap.png_b64 is None + + +# --------------------------------------------------------------------------- +# Overlay client — gating and fail-safety (any platform) +# --------------------------------------------------------------------------- + +class TestOverlayClient: + def test_env_kill_switch_disables_overlay(self, monkeypatch): + monkeypatch.setenv("HERMES_COMPUTER_USE_OVERLAY", "0") + from tools.computer_use.windows_backend import _OverlayClient + client = _OverlayClient() + client.start() # must not spawn anything + assert client._proc is None + assert client.pid is None + client.send({"cmd": "flash"}) # must be a silent no-op + client.stop() + + def test_send_before_start_is_noop(self): + from tools.computer_use.windows_backend import _OverlayClient + client = _OverlayClient() + client.send({"cmd": "click", "x": 1, "y": 2}) # no socket yet — no raise + assert client.pid is None + + def test_backend_constructs_overlay_client(self): + backend = _fresh_backend() + assert hasattr(backend, "_overlay") + # Overlay failures must never surface through backend actions: a dead + # client swallows sends. + backend._overlay._dead = True + backend._overlay.send({"cmd": "flash"}) + + +# --------------------------------------------------------------------------- +# Vision downscale helper (any platform; needs Pillow, a core dependency) +# --------------------------------------------------------------------------- + +class TestShrinkCaptureForVision: + @staticmethod + def _png_bytes(w, h): + pil = pytest.importorskip("PIL.Image") + import io + buf = io.BytesIO() + pil.new("RGB", (w, h), (10, 20, 30)).save(buf, format="PNG") + return buf.getvalue() + + def test_oversized_image_is_downscaled(self): + from PIL import Image + import io + from tools.computer_use.tool import _shrink_capture_for_vision + raw = self._png_bytes(1920, 1080) + out = _shrink_capture_for_vision(raw, ".png", max_dim=1456) + img = Image.open(io.BytesIO(out)) + assert max(img.size) == 1456 + assert img.size == (1456, 819) # aspect ratio preserved + + def test_small_image_passes_through_unchanged(self): + from tools.computer_use.tool import _shrink_capture_for_vision + raw = self._png_bytes(800, 600) + assert _shrink_capture_for_vision(raw, ".png", max_dim=1456) is raw + + def test_garbage_bytes_return_unchanged(self): + from tools.computer_use.tool import _shrink_capture_for_vision + raw = b"not an image at all" + assert _shrink_capture_for_vision(raw, ".png") is raw diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 46addb34d7c7..84371fe8400d 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -633,6 +633,37 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME # auxiliary.vision routing for captured screenshots (#24015) # --------------------------------------------------------------------------- +# Longest image side handed to the aux vision model. Full-resolution desktop +# captures (e.g. 1920x1032) tokenize to thousands of vision tokens and +# overflow small local-model context windows ("the vision API rejected the +# image"); ~1456px keeps SOM badges legible while fitting comfortably and +# cutting per-capture vision latency roughly in half. +_MAX_VISION_DIM = 1456 + + +def _shrink_capture_for_vision(raw: bytes, ext: str, + max_dim: int = _MAX_VISION_DIM) -> bytes: + """Downscale encoded image bytes so the longest side is <= max_dim. + + Returns the original bytes unchanged when the image already fits or when + Pillow is unavailable/fails — the vision call then proceeds with the + full-size image, which is no worse than the pre-shrink behavior. + """ + try: + from io import BytesIO + from PIL import Image + img = Image.open(BytesIO(raw)) + if max(img.size) <= max_dim: + return raw + img.thumbnail((max_dim, max_dim)) + out = BytesIO() + img.save(out, format="JPEG" if ext == ".jpg" else "PNG") + return out.getvalue() + except Exception as exc: + logger.debug("computer_use: vision downscale skipped: %s", exc) + return raw + + def _should_route_through_aux_vision() -> bool: """Return True when ``_capture_response`` should hand the PNG to aux vision. @@ -711,24 +742,7 @@ def _route_capture_through_aux_vision( cache_dir.mkdir(parents=True, exist_ok=True) temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}" - # Downscale before handing to the aux vision model. Full-resolution - # desktop captures (e.g. 1920x1032) tokenize to thousands of vision - # tokens and overflow small local-model context windows ("the vision - # API rejected the image"); ~1456px keeps SOM badges legible while - # fitting comfortably and cutting latency. - _MAX_VISION_DIM = 1456 - try: - from io import BytesIO as _BytesIO - from PIL import Image as _Image - img = _Image.open(_BytesIO(raw)) - if max(img.size) > _MAX_VISION_DIM: - img.thumbnail((_MAX_VISION_DIM, _MAX_VISION_DIM)) - out = _BytesIO() - img.save(out, format="JPEG" if ext == ".jpg" else "PNG") - raw = out.getvalue() - except Exception as exc: - logger.debug("computer_use: vision downscale skipped: %s", exc) - + raw = _shrink_capture_for_vision(raw, ext) temp_image_path.write_bytes(raw) prompt = (