test(tools): cover overlay-client gating and vision downscale helper

Extract the capture downscale into _shrink_capture_for_vision so it is
unit-testable without the aux-vision plumbing, and add dependency-free
tests for it (oversize shrinks with aspect preserved, small and
non-image bytes pass through untouched) plus the overlay client's
fail-safe contract (env kill switch spawns nothing, sends before
start or after death are silent no-ops).

Also update the one capture-routing assertion that pinned the literal
"macOS application screenshot" prompt wording, which became
platform-neutral when Windows hosts started producing captures.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jeff 2026-06-10 22:20:47 -04:00 • committed by Teknium
parent 63316ae271
commit 9480721f1d
No known key found for this signature in database
3 changed files with 110 additions and 32 deletions

View file

@ -1,4 +1,4 @@
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
When ``computer_use(action='capture', mode='som'|'vision')`` returns a
screenshot, ``_capture_response`` previously always returned a
@ -15,7 +15,7 @@ deterministic stubs for:
* ``vision_analyze_tool`` (the aux LLM call)
* ``hermes_constants.get_hermes_dir`` (cache path)
…so the full code path is covered without a live cua-driver, a real
…so the full code path is covered without a live cua-driver, a real
auxiliary client, or network access.
"""
@ -33,13 +33,13 @@ import pytest
# Fixtures / helpers
# ---------------------------------------------------------------------------
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
_PNG_B64 = (
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
"NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
)
# 1×1 JPEG — used to verify mime detection works for either stream type.
# 1×1 JPEG — used to verify mime detection works for either stream type.
_JPEG_B64 = (
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB"
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/"
@ -65,7 +65,7 @@ def _make_capture(
mode: str = "som",
elements=None,
app: str = "Safari",
window_title: str = "GitHub – Issue #24015",
window_title: str = "GitHub – Issue #24015",
width: int = 1280,
height: int = 800,
):
@ -140,7 +140,7 @@ class TestCaptureResponseDefaultPath:
return_value=True) as routing:
resp = cu_tool._capture_response(cap)
# ax never even consults the routing helper — short-circuited above
# ax never even consults the routing helper — short-circuited above
# the image branch.
routing.assert_not_called()
assert isinstance(resp, str)
@ -195,7 +195,7 @@ class TestCaptureResponseRoutedToAuxVision:
# The original AX-only metadata (window title, element index, app)
# is preserved alongside the new vision analysis so the agent loses
# no context vs the multimodal path.
assert body["window_title"] == "GitHub – Issue #24015"
assert body["window_title"] == "GitHub – Issue #24015"
assert len(body["elements"]) == 2
assert captured_calls.get("called") is True
@ -204,7 +204,7 @@ class TestCaptureResponseRoutedToAuxVision:
args, _kwargs = fake_vat.call_args
path_arg, prompt_arg = args[0], args[1]
assert str(tmp_cache_dir) in path_arg
assert "macOS application screenshot" in prompt_arg
assert "application screenshot" in prompt_arg
# AX summary is included so the aux model can ground its description
# against the same set-of-mark index the agent will see.
assert "Sign in" in prompt_arg
@ -298,7 +298,7 @@ class TestCaptureResponseRoutedToAuxVision:
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
# Aux failure → fall back to multimodal envelope (so the user still
# Aux failure → fall back to multimodal envelope (so the user still
# gets *something* useful even if vision is broken).
assert isinstance(resp, dict)
assert resp.get("_multimodal") is True
@ -323,7 +323,7 @@ class TestCaptureResponseRoutedToAuxVision:
new_callable=lambda: fake_vat):
resp = cu_tool._capture_response(cap)
# Empty analysis is treated as failure — we'd rather show pixels
# Empty analysis is treated as failure — we'd rather show pixels
# than embed an empty 'vision_analysis' string into the result.
assert isinstance(resp, dict)
assert resp.get("_multimodal") is True
@ -398,7 +398,7 @@ class TestRoutingDecisionWiring:
with patch("hermes_cli.config.load_config",
side_effect=RuntimeError("config.yaml unreadable")):
# No exception should bubble up — fail open by returning False
# No exception should bubble up — fail open by returning False
# so the legacy multimodal envelope continues to work.
assert cu_tool._should_route_through_aux_vision() is False
@ -417,14 +417,14 @@ class TestRoutingDecisionWiring:
# ---------------------------------------------------------------------------
# Bug reproduction marker — proves the fix is needed.
# Bug reproduction marker — proves the fix is needed.
# ---------------------------------------------------------------------------
class TestBugReproductionAnchor:
"""Without the fix, this test would assert the wrong thing.
On upstream/main HEAD prior to this branch, _capture_response returns a
multimodal envelope unconditionally — so when a non-vision main model
multimodal envelope unconditionally — so when a non-vision main model
is configured, the captured PNG is delivered to the main provider as
image_url content and the request is rejected with HTTP 404. We don't
have a live provider here, but we can pin the contract: with routing
@ -455,7 +455,7 @@ class TestBugReproductionAnchor:
# Must be a string (text-only result).
assert isinstance(resp, str)
# Must NOT contain a base64 image URL anywhere — that's what tripped
# Must NOT contain a base64 image URL anywhere — that's what tripped
# 'No endpoints found that support image input' on the reporter's
# main provider in #24015.
assert "data:image" not in resp

View file

@ -227,3 +227,67 @@ class TestLiveReadOnly:
cap = b.capture(mode="ax")
assert cap.mode == "ax"
assert cap.png_b64 is None
# ---------------------------------------------------------------------------
# Overlay client — gating and fail-safety (any platform)
# ---------------------------------------------------------------------------
class TestOverlayClient:
def test_env_kill_switch_disables_overlay(self, monkeypatch):
monkeypatch.setenv("HERMES_COMPUTER_USE_OVERLAY", "0")
from tools.computer_use.windows_backend import _OverlayClient
client = _OverlayClient()
client.start() # must not spawn anything
assert client._proc is None
assert client.pid is None
client.send({"cmd": "flash"}) # must be a silent no-op
client.stop()
def test_send_before_start_is_noop(self):
from tools.computer_use.windows_backend import _OverlayClient
client = _OverlayClient()
client.send({"cmd": "click", "x": 1, "y": 2}) # no socket yet — no raise
assert client.pid is None
def test_backend_constructs_overlay_client(self):
backend = _fresh_backend()
assert hasattr(backend, "_overlay")
# Overlay failures must never surface through backend actions: a dead
# client swallows sends.
backend._overlay._dead = True
backend._overlay.send({"cmd": "flash"})
# ---------------------------------------------------------------------------
# Vision downscale helper (any platform; needs Pillow, a core dependency)
# ---------------------------------------------------------------------------
class TestShrinkCaptureForVision:
@staticmethod
def _png_bytes(w, h):
pil = pytest.importorskip("PIL.Image")
import io
buf = io.BytesIO()
pil.new("RGB", (w, h), (10, 20, 30)).save(buf, format="PNG")
return buf.getvalue()
def test_oversized_image_is_downscaled(self):
from PIL import Image
import io
from tools.computer_use.tool import _shrink_capture_for_vision
raw = self._png_bytes(1920, 1080)
out = _shrink_capture_for_vision(raw, ".png", max_dim=1456)
img = Image.open(io.BytesIO(out))
assert max(img.size) == 1456
assert img.size == (1456, 819) # aspect ratio preserved
def test_small_image_passes_through_unchanged(self):
from tools.computer_use.tool import _shrink_capture_for_vision
raw = self._png_bytes(800, 600)
assert _shrink_capture_for_vision(raw, ".png", max_dim=1456) is raw
def test_garbage_bytes_return_unchanged(self):
from tools.computer_use.tool import _shrink_capture_for_vision
raw = b"not an image at all"
assert _shrink_capture_for_vision(raw, ".png") is raw

View file

@ -633,6 +633,37 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
# auxiliary.vision routing for captured screenshots (#24015)
# ---------------------------------------------------------------------------
# Longest image side handed to the aux vision model. Full-resolution desktop
# captures (e.g. 1920x1032) tokenize to thousands of vision tokens and
# overflow small local-model context windows ("the vision API rejected the
# image"); ~1456px keeps SOM badges legible while fitting comfortably and
# cutting per-capture vision latency roughly in half.
_MAX_VISION_DIM = 1456
def _shrink_capture_for_vision(raw: bytes, ext: str,
max_dim: int = _MAX_VISION_DIM) -> bytes:
"""Downscale encoded image bytes so the longest side is <= max_dim.
Returns the original bytes unchanged when the image already fits or when
Pillow is unavailable/fails — the vision call then proceeds with the
full-size image, which is no worse than the pre-shrink behavior.
"""
try:
from io import BytesIO
from PIL import Image
img = Image.open(BytesIO(raw))
if max(img.size) <= max_dim:
return raw
img.thumbnail((max_dim, max_dim))
out = BytesIO()
img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
return out.getvalue()
except Exception as exc:
logger.debug("computer_use: vision downscale skipped: %s", exc)
return raw
def _should_route_through_aux_vision() -> bool:
"""Return True when ``_capture_response`` should hand the PNG to aux vision.
@ -711,24 +742,7 @@ def _route_capture_through_aux_vision(
cache_dir.mkdir(parents=True, exist_ok=True)
temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}"
# Downscale before handing to the aux vision model. Full-resolution
# desktop captures (e.g. 1920x1032) tokenize to thousands of vision
# tokens and overflow small local-model context windows ("the vision
# API rejected the image"); ~1456px keeps SOM badges legible while
# fitting comfortably and cutting latency.
_MAX_VISION_DIM = 1456
try:
from io import BytesIO as _BytesIO
from PIL import Image as _Image
img = _Image.open(_BytesIO(raw))
if max(img.size) > _MAX_VISION_DIM:
img.thumbnail((_MAX_VISION_DIM, _MAX_VISION_DIM))
out = _BytesIO()
img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
raw = out.getvalue()
except Exception as exc:
logger.debug("computer_use: vision downscale skipped: %s", exc)
raw = _shrink_capture_for_vision(raw, ext)
temp_image_path.write_bytes(raw)
prompt = (