From ac94f2c8a1e4419c649c563006ae57a00c3d7959 Mon Sep 17 00:00:00 2001 From: Jonathan Almanzar Date: Thu, 25 Jun 2026 13:46:38 -0700 Subject: [PATCH] fix(vision): convert SVG/unsupported image formats to PNG before embedding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vision_analyze embedded SVG (and BMP/TIFF) tool-results into conversation history with media_type image/svg+xml. Anthropic only accepts jpeg/png/ gif/webp, so the request fails with a non-retryable 400. Because the image is baked into immutable history and re-sent every turn, the session is permanently wedged on resume — retries re-send the same bad bytes. Add _normalize_to_supported_image(): SVG is rasterized to PNG (best-effort via cairosvg/svglib/rsvg-convert/inkscape), other non-supported raster formats are re-encoded to PNG via Pillow, and if conversion is impossible the tool returns an actionable error instead of a session-wedging payload. Wired into both the native-vision fast path and the auxiliary-API path so the whole bug class is covered, not just the one call site. All 99 existing vision tests pass. --- tests/tools/test_image_source.py | 14 +-- tools/image_source.py | 10 +- tools/vision_tools.py | 158 ++++++++++++++++++++++++++++++- 3 files changed, 168 insertions(+), 14 deletions(-) diff --git a/tests/tools/test_image_source.py b/tests/tools/test_image_source.py index ca93abb64d7..b54f8103659 100644 --- a/tests/tools/test_image_source.py +++ b/tests/tools/test_image_source.py @@ -91,15 +91,17 @@ class TestLocalBackend: "ftp://example.com/pic.png", isrc.ResolveContext()) @pytest.mark.asyncio - async def test_svg_rejected_with_conversion_hint(self, tmp_path, monkeypatch): - """SVG has no raster magic bytes; the resolver rejects it with a - dedicated, actionable message (review note W3).""" + async def test_svg_passes_through_for_rasterization(self, tmp_path, monkeypatch): + """SVG has no raster magic bytes but is passed through with mime + image/svg+xml so the vision call sites can rasterize it to PNG.""" isrc = _reload(monkeypatch, tmp_path / "hermes") monkeypatch.setenv("TERMINAL_ENV", "local") svg = tmp_path / "art.svg" - svg.write_bytes(b'') - with pytest.raises(isrc.NotAnImage, match="SVG is not supported"): - await isrc.resolve_image_source(str(svg), isrc.ResolveContext()) + svg_bytes = b'' + svg.write_bytes(svg_bytes) + res = await isrc.resolve_image_source(str(svg), isrc.ResolveContext()) + assert res.mime == "image/svg+xml" + assert res.data == svg_bytes class TestNonLocalBackendConfinement: diff --git a/tools/image_source.py b/tools/image_source.py index 9046577359c..17b924b52b9 100644 --- a/tools/image_source.py +++ b/tools/image_source.py @@ -304,11 +304,9 @@ def _finalize(data: bytes, declared_mime: str, origin: str, src: str) -> Resolve sniffed = _detect_image_mime_type_from_bytes(data) if sniffed is None: if b" Optional[str]: """Magic-byte MIME sniff on raw bytes (authoritative; no extension trust). Returns ``None`` for anything without a recognized image header — including - SVG, which has no magic bytes and cannot be ingested as raster vision input - by any provider. The resolver surfaces a dedicated error for SVG sources. + SVG, which has no magic bytes. The resolver special-cases SVG (sniffs + `` Optional[str]: return None +# Media types the major vision providers (Anthropic in particular) accept for +# inline base64 images. Anything outside this set — SVG, BMP, TIFF, etc. — is +# rejected with a non-retryable 400. Because a vision tool-result is baked into +# immutable conversation history and re-sent every turn, embedding an +# unsupported media_type permanently wedges the session (retries re-send the +# same bad bytes). We MUST normalize to one of these before embedding. +_ANTHROPIC_SUPPORTED_MEDIA_TYPES = frozenset( + {"image/jpeg", "image/png", "image/gif", "image/webp"} +) + + +def _rasterize_svg_to_png(svg_path: Path, out_path: Path) -> bool: + """Best-effort SVG → PNG rasterization. Returns True on success. + + Tries, in order: cairosvg, svglib+reportlab, then system rasterizers + (rsvg-convert, inkscape). All are soft dependencies; if none is available + we return False and the caller rejects the image with an actionable error + rather than embedding an unsupported media_type that would wedge the + session. + """ + # 1) cairosvg (pure-python-ish, most common) + try: + import cairosvg # type: ignore + cairosvg.svg2png(url=str(svg_path), write_to=str(out_path)) + return out_path.exists() and out_path.stat().st_size > 0 + except Exception: + pass + # 2) svglib + reportlab + try: + from svglib.svglib import svg2rlg # type: ignore + from reportlab.graphics import renderPM # type: ignore + drawing = svg2rlg(str(svg_path)) + if drawing is not None: + renderPM.drawToFile(drawing, str(out_path), fmt="PNG") + return out_path.exists() and out_path.stat().st_size > 0 + except Exception: + pass + # 3) system rasterizers + import shutil as _shutil + import subprocess as _subprocess + for cmd in ( + ["rsvg-convert", "-o", str(out_path), str(svg_path)], + ["inkscape", str(svg_path), "--export-type=png", + f"--export-filename={out_path}"], + ): + if _shutil.which(cmd[0]): + try: + _subprocess.run(cmd, check=True, capture_output=True, timeout=30) + if out_path.exists() and out_path.stat().st_size > 0: + return True + except Exception: + continue + return False + + +def _normalize_to_supported_image( + image_path: Path, detected_mime: str +) -> tuple[Optional[Path], Optional[str], Optional[str]]: + """Ensure an image is in a vision-provider-supported format. + + Returns a 3-tuple ``(path, mime, error)``: + - If ``detected_mime`` is already supported: ``(image_path, detected_mime, None)``. + - If conversion succeeds: ``(new_png_path, "image/png", None)`` — the new + path is a temp file the CALLER must clean up. + - If conversion is impossible: ``(None, None, )``. + + SVG is rasterized to PNG (best-effort, soft deps). Other raster formats + Pillow can read (BMP, TIFF, etc.) are re-encoded to PNG. This runs BEFORE + the image is base64-embedded into conversation history, so an unsupported + media_type can never reach the provider and wedge the session. + """ + if detected_mime in _ANTHROPIC_SUPPORTED_MEDIA_TYPES: + return image_path, detected_mime, None + + out_path = ( + get_hermes_dir("cache/vision", "temp_vision_images") + / f"converted_{uuid.uuid4()}.png" + ) + + # SVG: needs a rasterizer (Pillow cannot render SVG). + if detected_mime == "image/svg+xml": + if _rasterize_svg_to_png(image_path, out_path): + return out_path, "image/png", None + return ( + None, + None, + "This is an SVG, which vision models cannot read directly, and no " + "SVG rasterizer is installed (tried cairosvg, svglib, rsvg-convert, " + "inkscape). Convert the SVG to PNG first — e.g. open it in a browser " + "and screenshot it, or install a rasterizer " + "(`pip install cairosvg`) — then re-run vision_analyze on the PNG.", + ) + + # Other non-supported raster formats (BMP, TIFF, ...): re-encode via Pillow. + try: + from PIL import Image as _PILImage + with _PILImage.open(image_path) as _img: + if _img.mode not in ("RGB", "RGBA", "L"): + _img = _img.convert("RGBA") + _img.save(out_path, format="PNG") + if out_path.exists() and out_path.stat().st_size > 0: + return out_path, "image/png", None + except Exception as _exc: + logger.warning("Failed to normalize %s image to PNG: %s", + detected_mime, _exc) + return ( + None, + None, + f"Image format {detected_mime!r} is not supported by the vision API " + f"and could not be converted to PNG (install Pillow for raster " + f"conversion). Convert it to PNG or JPEG and try again.", + ) + + def _is_retryable_download_error(error: Exception) -> bool: """Return True only for transient image-download failures worth retrying. @@ -863,6 +977,29 @@ async def _vision_analyze_native( await asyncio.to_thread(temp_image_path.write_bytes, resolved.data) should_cleanup = True + # Normalize unsupported formats (SVG, BMP, ...) to PNG BEFORE embedding. + # Anthropic only accepts jpeg/png/gif/webp; an unsupported media_type + # baked into immutable history wedges the session with a 400 on every + # resume. Convert here so it can never enter history. Offloaded — the + # rasterizers/Pillow are blocking. + normalized_path, detected_mime_type, _norm_err = await asyncio.to_thread( + _normalize_to_supported_image, temp_image_path, detected_mime_type, + ) + if _norm_err or normalized_path is None: + return tool_error( + _norm_err or "Image normalization failed.", success=False, + ) + if normalized_path != temp_image_path: + # We created a temp PNG — swap to it and ensure it's cleaned up. + if should_cleanup and temp_image_path.exists(): + try: + temp_image_path.unlink() + except Exception: + pass + temp_image_path = normalized_path + should_cleanup = True + image_size_bytes = temp_image_path.stat().st_size + image_data_url = await _run_encode_on_cpu_executor( _image_to_base64_data_url, temp_image_path, mime_type=detected_mime_type, @@ -1014,6 +1151,23 @@ async def vision_analyze_tool( image_size_bytes = len(resolved.data) image_size_kb = image_size_bytes / 1024 logger.info("Image ready (%.1f KB)", image_size_kb) + # Normalize unsupported formats (SVG, BMP, ...) to PNG. Vision providers + # reject these media types; convert before encoding. Offloaded — the + # rasterizers/Pillow are blocking. + normalized_path, detected_mime_type, _norm_err = await asyncio.to_thread( + _normalize_to_supported_image, temp_image_path, detected_mime_type, + ) + if _norm_err or normalized_path is None: + raise ValueError(_norm_err or "Image normalization failed.") + if normalized_path != temp_image_path: + if should_cleanup and temp_image_path.exists(): + try: + temp_image_path.unlink() + except Exception: + pass + temp_image_path = normalized_path + should_cleanup = True + # Convert image to base64 — send at full resolution first. # If the provider rejects it as too large, we auto-resize and retry. # Offloaded to the bounded vision CPU executor so a fan-out of encodes