mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-21 07:55:08 +00:00
test(tools): cover overlay-client gating and vision downscale helper
Extract the capture downscale into _shrink_capture_for_vision so it is unit-testable without the aux-vision plumbing, and add dependency-free tests for it (oversize shrinks with aspect preserved, small and non-image bytes pass through untouched) plus the overlay client's fail-safe contract (env kill switch spawns nothing, sends before start or after death are silent no-ops). Also update the one capture-routing assertion that pinned the literal "macOS application screenshot" prompt wording, which became platform-neutral when Windows hosts started producing captures. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
63316ae271
commit
9480721f1d
3 changed files with 110 additions and 32 deletions
|
|
@ -1,4 +1,4 @@
|
|||
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
|
||||
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
|
||||
|
||||
When ``computer_use(action='capture', mode='som'|'vision')`` returns a
|
||||
screenshot, ``_capture_response`` previously always returned a
|
||||
|
|
@ -15,7 +15,7 @@ deterministic stubs for:
|
|||
* ``vision_analyze_tool`` (the aux LLM call)
|
||||
* ``hermes_constants.get_hermes_dir`` (cache path)
|
||||
|
||||
…so the full code path is covered without a live cua-driver, a real
|
||||
…so the full code path is covered without a live cua-driver, a real
|
||||
auxiliary client, or network access.
|
||||
"""
|
||||
|
||||
|
|
@ -33,13 +33,13 @@ import pytest
|
|||
# Fixtures / helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
|
||||
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
|
||||
_PNG_B64 = (
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
|
||||
"NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
|
||||
)
|
||||
|
||||
# 1×1 JPEG — used to verify mime detection works for either stream type.
|
||||
# 1×1 JPEG — used to verify mime detection works for either stream type.
|
||||
_JPEG_B64 = (
|
||||
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB"
|
||||
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/"
|
||||
|
|
@ -65,7 +65,7 @@ def _make_capture(
|
|||
mode: str = "som",
|
||||
elements=None,
|
||||
app: str = "Safari",
|
||||
window_title: str = "GitHub – Issue #24015",
|
||||
window_title: str = "GitHub – Issue #24015",
|
||||
width: int = 1280,
|
||||
height: int = 800,
|
||||
):
|
||||
|
|
@ -140,7 +140,7 @@ class TestCaptureResponseDefaultPath:
|
|||
return_value=True) as routing:
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
# ax never even consults the routing helper — short-circuited above
|
||||
# ax never even consults the routing helper — short-circuited above
|
||||
# the image branch.
|
||||
routing.assert_not_called()
|
||||
assert isinstance(resp, str)
|
||||
|
|
@ -195,7 +195,7 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
# The original AX-only metadata (window title, element index, app)
|
||||
# is preserved alongside the new vision analysis so the agent loses
|
||||
# no context vs the multimodal path.
|
||||
assert body["window_title"] == "GitHub – Issue #24015"
|
||||
assert body["window_title"] == "GitHub – Issue #24015"
|
||||
assert len(body["elements"]) == 2
|
||||
|
||||
assert captured_calls.get("called") is True
|
||||
|
|
@ -204,7 +204,7 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
args, _kwargs = fake_vat.call_args
|
||||
path_arg, prompt_arg = args[0], args[1]
|
||||
assert str(tmp_cache_dir) in path_arg
|
||||
assert "macOS application screenshot" in prompt_arg
|
||||
assert "application screenshot" in prompt_arg
|
||||
# AX summary is included so the aux model can ground its description
|
||||
# against the same set-of-mark index the agent will see.
|
||||
assert "Sign in" in prompt_arg
|
||||
|
|
@ -298,7 +298,7 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
new_callable=lambda: fake_vat):
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
# Aux failure → fall back to multimodal envelope (so the user still
|
||||
# Aux failure → fall back to multimodal envelope (so the user still
|
||||
# gets *something* useful even if vision is broken).
|
||||
assert isinstance(resp, dict)
|
||||
assert resp.get("_multimodal") is True
|
||||
|
|
@ -323,7 +323,7 @@ class TestCaptureResponseRoutedToAuxVision:
|
|||
new_callable=lambda: fake_vat):
|
||||
resp = cu_tool._capture_response(cap)
|
||||
|
||||
# Empty analysis is treated as failure — we'd rather show pixels
|
||||
# Empty analysis is treated as failure — we'd rather show pixels
|
||||
# than embed an empty 'vision_analysis' string into the result.
|
||||
assert isinstance(resp, dict)
|
||||
assert resp.get("_multimodal") is True
|
||||
|
|
@ -398,7 +398,7 @@ class TestRoutingDecisionWiring:
|
|||
|
||||
with patch("hermes_cli.config.load_config",
|
||||
side_effect=RuntimeError("config.yaml unreadable")):
|
||||
# No exception should bubble up — fail open by returning False
|
||||
# No exception should bubble up — fail open by returning False
|
||||
# so the legacy multimodal envelope continues to work.
|
||||
assert cu_tool._should_route_through_aux_vision() is False
|
||||
|
||||
|
|
@ -417,14 +417,14 @@ class TestRoutingDecisionWiring:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bug reproduction marker — proves the fix is needed.
|
||||
# Bug reproduction marker — proves the fix is needed.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBugReproductionAnchor:
|
||||
"""Without the fix, this test would assert the wrong thing.
|
||||
|
||||
On upstream/main HEAD prior to this branch, _capture_response returns a
|
||||
multimodal envelope unconditionally — so when a non-vision main model
|
||||
multimodal envelope unconditionally — so when a non-vision main model
|
||||
is configured, the captured PNG is delivered to the main provider as
|
||||
image_url content and the request is rejected with HTTP 404. We don't
|
||||
have a live provider here, but we can pin the contract: with routing
|
||||
|
|
@ -455,7 +455,7 @@ class TestBugReproductionAnchor:
|
|||
|
||||
# Must be a string (text-only result).
|
||||
assert isinstance(resp, str)
|
||||
# Must NOT contain a base64 image URL anywhere — that's what tripped
|
||||
# Must NOT contain a base64 image URL anywhere — that's what tripped
|
||||
# 'No endpoints found that support image input' on the reporter's
|
||||
# main provider in #24015.
|
||||
assert "data:image" not in resp
|
||||
|
|
|
|||
|
|
@ -227,3 +227,67 @@ class TestLiveReadOnly:
|
|||
cap = b.capture(mode="ax")
|
||||
assert cap.mode == "ax"
|
||||
assert cap.png_b64 is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Overlay client — gating and fail-safety (any platform)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestOverlayClient:
|
||||
def test_env_kill_switch_disables_overlay(self, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_COMPUTER_USE_OVERLAY", "0")
|
||||
from tools.computer_use.windows_backend import _OverlayClient
|
||||
client = _OverlayClient()
|
||||
client.start() # must not spawn anything
|
||||
assert client._proc is None
|
||||
assert client.pid is None
|
||||
client.send({"cmd": "flash"}) # must be a silent no-op
|
||||
client.stop()
|
||||
|
||||
def test_send_before_start_is_noop(self):
|
||||
from tools.computer_use.windows_backend import _OverlayClient
|
||||
client = _OverlayClient()
|
||||
client.send({"cmd": "click", "x": 1, "y": 2}) # no socket yet — no raise
|
||||
assert client.pid is None
|
||||
|
||||
def test_backend_constructs_overlay_client(self):
|
||||
backend = _fresh_backend()
|
||||
assert hasattr(backend, "_overlay")
|
||||
# Overlay failures must never surface through backend actions: a dead
|
||||
# client swallows sends.
|
||||
backend._overlay._dead = True
|
||||
backend._overlay.send({"cmd": "flash"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vision downscale helper (any platform; needs Pillow, a core dependency)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestShrinkCaptureForVision:
|
||||
@staticmethod
|
||||
def _png_bytes(w, h):
|
||||
pil = pytest.importorskip("PIL.Image")
|
||||
import io
|
||||
buf = io.BytesIO()
|
||||
pil.new("RGB", (w, h), (10, 20, 30)).save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
def test_oversized_image_is_downscaled(self):
|
||||
from PIL import Image
|
||||
import io
|
||||
from tools.computer_use.tool import _shrink_capture_for_vision
|
||||
raw = self._png_bytes(1920, 1080)
|
||||
out = _shrink_capture_for_vision(raw, ".png", max_dim=1456)
|
||||
img = Image.open(io.BytesIO(out))
|
||||
assert max(img.size) == 1456
|
||||
assert img.size == (1456, 819) # aspect ratio preserved
|
||||
|
||||
def test_small_image_passes_through_unchanged(self):
|
||||
from tools.computer_use.tool import _shrink_capture_for_vision
|
||||
raw = self._png_bytes(800, 600)
|
||||
assert _shrink_capture_for_vision(raw, ".png", max_dim=1456) is raw
|
||||
|
||||
def test_garbage_bytes_return_unchanged(self):
|
||||
from tools.computer_use.tool import _shrink_capture_for_vision
|
||||
raw = b"not an image at all"
|
||||
assert _shrink_capture_for_vision(raw, ".png") is raw
|
||||
|
|
|
|||
|
|
@ -633,6 +633,37 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
|
|||
# auxiliary.vision routing for captured screenshots (#24015)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Longest image side handed to the aux vision model. Full-resolution desktop
|
||||
# captures (e.g. 1920x1032) tokenize to thousands of vision tokens and
|
||||
# overflow small local-model context windows ("the vision API rejected the
|
||||
# image"); ~1456px keeps SOM badges legible while fitting comfortably and
|
||||
# cutting per-capture vision latency roughly in half.
|
||||
_MAX_VISION_DIM = 1456
|
||||
|
||||
|
||||
def _shrink_capture_for_vision(raw: bytes, ext: str,
|
||||
max_dim: int = _MAX_VISION_DIM) -> bytes:
|
||||
"""Downscale encoded image bytes so the longest side is <= max_dim.
|
||||
|
||||
Returns the original bytes unchanged when the image already fits or when
|
||||
Pillow is unavailable/fails — the vision call then proceeds with the
|
||||
full-size image, which is no worse than the pre-shrink behavior.
|
||||
"""
|
||||
try:
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
img = Image.open(BytesIO(raw))
|
||||
if max(img.size) <= max_dim:
|
||||
return raw
|
||||
img.thumbnail((max_dim, max_dim))
|
||||
out = BytesIO()
|
||||
img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
|
||||
return out.getvalue()
|
||||
except Exception as exc:
|
||||
logger.debug("computer_use: vision downscale skipped: %s", exc)
|
||||
return raw
|
||||
|
||||
|
||||
def _should_route_through_aux_vision() -> bool:
|
||||
"""Return True when ``_capture_response`` should hand the PNG to aux vision.
|
||||
|
||||
|
|
@ -711,24 +742,7 @@ def _route_capture_through_aux_vision(
|
|||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
temp_image_path = cache_dir / f"computer_use_{_uuid.uuid4().hex}{ext}"
|
||||
|
||||
# Downscale before handing to the aux vision model. Full-resolution
|
||||
# desktop captures (e.g. 1920x1032) tokenize to thousands of vision
|
||||
# tokens and overflow small local-model context windows ("the vision
|
||||
# API rejected the image"); ~1456px keeps SOM badges legible while
|
||||
# fitting comfortably and cutting latency.
|
||||
_MAX_VISION_DIM = 1456
|
||||
try:
|
||||
from io import BytesIO as _BytesIO
|
||||
from PIL import Image as _Image
|
||||
img = _Image.open(_BytesIO(raw))
|
||||
if max(img.size) > _MAX_VISION_DIM:
|
||||
img.thumbnail((_MAX_VISION_DIM, _MAX_VISION_DIM))
|
||||
out = _BytesIO()
|
||||
img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
|
||||
raw = out.getvalue()
|
||||
except Exception as exc:
|
||||
logger.debug("computer_use: vision downscale skipped: %s", exc)
|
||||
|
||||
raw = _shrink_capture_for_vision(raw, ext)
|
||||
temp_image_path.write_bytes(raw)
|
||||
|
||||
prompt = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue