From 2068754d6f7e03d576293a2357b53ce61eec4af5 Mon Sep 17 00:00:00 2001 From: Tarun Ravikumar Date: Thu, 2 Jul 2026 02:45:16 -0700 Subject: [PATCH] feat(api-server): inline MEDIA: image tags as base64 data URLs for remote frontends Salvage of the surviving piece of #2696 by @tarunravi. The PR's other two changes (tool progress streaming, SSE None-sentinel fix) were independently superseded on main by the structured hermes.tool.progress SSE events and the rewritten queue-drain loop. Remote OpenAI-compatible frontends can't read server-local file paths, so MEDIA: tags (browser screenshots, generated images) were dead text. _resolve_media_to_data_urls() now inlines small (<=5MB) local images as markdown data URLs across all four response surfaces: chat completions (non-streaming), session chat, session chat stream final event, and the Responses API. Non-image, missing, or oversized paths pass through untouched. --- gateway/platforms/api_server.py | 57 +++++++++++++- scripts/release.py | 1 + .../test_api_server_media_data_urls.py | 77 +++++++++++++++++++ 3 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 tests/gateway/test_api_server_media_data_urls.py diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 1e4ad4c3a9a..fa2f14267bb 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -572,6 +572,55 @@ else: cors_middleware = None # type: ignore[assignment] +_MEDIA_IMG_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"} +_MEDIA_MIME = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", +} +_MEDIA_TAG_RE = re.compile( + r"[`\"']?MEDIA:\s*(`[^`\n]+`|\"[^\"\n]+\"|'[^'\n]+'|\S+)[`\"']?" +) +_MEDIA_DATA_URL_MAX_BYTES = 5 * 1024 * 1024 # skip images larger than 5MB + + +def _resolve_media_to_data_urls(text: str) -> str: + """Replace ``MEDIA:`` image tags with inline base64 data URLs. + + Remote OpenAI-compatible frontends can't read local file paths, so + ``MEDIA:`` tags referencing images on the server are useless to them. + Inline small local images as markdown data URLs; non-image or unreadable + paths are left untouched. + """ + if not text or "MEDIA:" not in text: + return text + import base64 + + def _to_data_url(path_str: str) -> Optional[str]: + p = Path(path_str.strip().strip("`\"'")).expanduser() + suffix = p.suffix.lower() + if suffix not in _MEDIA_IMG_EXT: + return None + try: + if not p.is_file() or p.stat().st_size > _MEDIA_DATA_URL_MAX_BYTES: + return None + b64 = base64.b64encode(p.read_bytes()).decode() + except OSError: + return None + return f"![image](data:{_MEDIA_MIME[suffix]};base64,{b64})" + + def _repl(m: "re.Match[str]") -> str: + return _to_data_url(m.group(1)) or m.group(0) + + try: + return _MEDIA_TAG_RE.sub(_repl, text) + except Exception: + return text + + def _redact_api_error_text(value: Any, *, limit: int | None = None) -> str: """Redact API-bound error text before it crosses the HTTP boundary.""" redacted = redact_sensitive_text(str(value), force=True) @@ -1675,7 +1724,7 @@ class APIServerAdapter(BasePlatformAdapter): gateway_session_key=gateway_session_key, ) effective_session_id = result.get("session_id") if isinstance(result, dict) else session_id - final_response = result.get("final_response", "") if isinstance(result, dict) else "" + final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "") headers = {"X-Hermes-Session-Id": effective_session_id or session_id} if gateway_session_key: headers["X-Hermes-Session-Key"] = gateway_session_key @@ -1765,7 +1814,7 @@ class APIServerAdapter(BasePlatformAdapter): tool_progress_callback=_tool_progress, gateway_session_key=gateway_session_key, ) - final_response = result.get("final_response", "") if isinstance(result, dict) else "" + final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "") effective_session_id = result.get("session_id", session_id) if isinstance(result, dict) else session_id turn_messages = self._turn_transcript_messages(history, user_message, result) if isinstance(result, dict) else [] await queue.put(_event_payload("assistant.completed", { @@ -2087,7 +2136,7 @@ class APIServerAdapter(BasePlatformAdapter): status=500, ) - final_response = result.get("final_response") or "" + final_response = _resolve_media_to_data_urls(result.get("final_response") or "") is_partial = bool(result.get("partial")) is_failed = bool(result.get("failed")) completed = bool(result.get("completed", True)) @@ -3186,7 +3235,7 @@ class APIServerAdapter(BasePlatformAdapter): status=500, ) - final_response = result.get("final_response", "") + final_response = _resolve_media_to_data_urls(result.get("final_response", "")) if not final_response: final_response = _redact_api_error_text(result.get("error", "(No response generated)")) diff --git a/scripts/release.py b/scripts/release.py index cd0cda9081e..1a580983e73 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -45,6 +45,7 @@ ACP_REGISTRY_MANIFEST = REPO_ROOT / "acp_registry" / "agent.json" # Auto-extracted from noreply emails + manual overrides AUTHOR_MAP = { + "tarunravi@gmail.com": "tarunravi", # PR #2696 salvage (api-server: inline MEDIA: image tags as base64 data URLs in final responses so remote OpenAI-compatible frontends can render server-local screenshots; the PR's tool-progress-streaming and SSE-sentinel pieces were independently superseded on main) "aqdrgg19@gmail.com": "VolodymyrBg", # PR #2861 salvage (webhook: drop the unused full request payload from retained _delivery_info entries — up to ~1MB dead weight per delivery for the 1h idempotency TTL) "ohyes9711@gmail.com": "CharmingGroot", # PR #2794 salvage (email: guard msg_data[0][1] against malformed IMAP fetch structures so one bad response can't abort the batch and permanently lose seen-marked messages; Message-ID domain falls back to localhost when EMAIL_ADDRESS lacks '@') "sahibzada@fastino.ai": "sahibzada-allahyar", # PR #39227 salvage (desktop: configured terminal.cwd overrides a stale remembered workspace-cwd localStorage value when no session is active; #38855) diff --git a/tests/gateway/test_api_server_media_data_urls.py b/tests/gateway/test_api_server_media_data_urls.py new file mode 100644 index 00000000000..960f4b194f8 --- /dev/null +++ b/tests/gateway/test_api_server_media_data_urls.py @@ -0,0 +1,77 @@ +"""MEDIA: tag → base64 data-URL resolution for the API server (salvage of #2696). + +Remote OpenAI-compatible frontends can't read local file paths, so +``MEDIA:`` image tags in final responses are inlined as markdown +data URLs before crossing the HTTP boundary. +""" + +import base64 +import unittest + +import pytest + +pytest.importorskip("aiohttp") + +from gateway.platforms.api_server import _resolve_media_to_data_urls # noqa: E402 + +# 1x1 transparent PNG +_PNG_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQAB" + "h6FO1AAAAABJRU5ErkJggg==" +) + + +class TestResolveMediaToDataUrls(unittest.TestCase): + def _write_png(self, tmpdir_name="hermes_media_test"): + import tempfile + from pathlib import Path + + d = Path(tempfile.mkdtemp(prefix=tmpdir_name)) + p = d / "shot.png" + p.write_bytes(_PNG_BYTES) + return p + + def test_media_tag_inlined(self): + p = self._write_png() + out = _resolve_media_to_data_urls(f"Here you go: MEDIA:{p}") + self.assertIn("data:image/png;base64,", out) + self.assertNotIn("MEDIA:", out) + + def test_backtick_wrapped_tag(self): + p = self._write_png() + out = _resolve_media_to_data_urls(f"See `MEDIA:{p}` above") + self.assertIn("data:image/png;base64,", out) + + def test_missing_file_left_untouched(self): + text = "MEDIA:/nonexistent/path/shot.png" + self.assertEqual(_resolve_media_to_data_urls(text), text) + + def test_non_image_left_untouched(self): + text = "MEDIA:/tmp/archive.zip" + self.assertEqual(_resolve_media_to_data_urls(text), text) + + def test_text_without_media_passthrough(self): + self.assertEqual(_resolve_media_to_data_urls("plain text"), "plain text") + self.assertEqual(_resolve_media_to_data_urls(""), "") + + def test_oversized_image_skipped(self): + from gateway.platforms import api_server as mod + + p = self._write_png() + orig = mod._MEDIA_DATA_URL_MAX_BYTES + mod._MEDIA_DATA_URL_MAX_BYTES = 1 + try: + text = f"MEDIA:{p}" + self.assertEqual(_resolve_media_to_data_urls(text), text) + finally: + mod._MEDIA_DATA_URL_MAX_BYTES = orig + + def test_multiple_tags(self): + p1 = self._write_png() + p2 = self._write_png("hermes_media_test2") + out = _resolve_media_to_data_urls(f"MEDIA:{p1}\nand MEDIA:{p2}") + self.assertEqual(out.count("data:image/png;base64,"), 2) + + +if __name__ == "__main__": + unittest.main()