mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
Merge pull request #61415 from kshitijk4poor/fix/media-tag-caption
Some checks are pending
CI / Detect affected areas (push) Waiting to run
CI / Python tests (push) Blocked by required conditions
CI / Python lints (push) Blocked by required conditions
CI / TypeScript (push) Blocked by required conditions
CI / Docs Site (push) Blocked by required conditions
CI / Deny unrelated histories (push) Blocked by required conditions
CI / Check contributors (push) Blocked by required conditions
CI / Check uv.lock (push) Blocked by required conditions
CI / Lint Docker scripts (push) Blocked by required conditions
CI / Build&Test Docker image (push) Blocked by required conditions
CI / Supply-chain scan (push) Blocked by required conditions
CI / OSV scan (push) Waiting to run
CI / All required checks pass (push) Blocked by required conditions
CI / CI timing report (push) Blocked by required conditions
Deploy Site / deploy-vercel (push) Waiting to run
Deploy Site / deploy-docs (push) Waiting to run
Some checks are pending
CI / Detect affected areas (push) Waiting to run
CI / Python tests (push) Blocked by required conditions
CI / Python lints (push) Blocked by required conditions
CI / TypeScript (push) Blocked by required conditions
CI / Docs Site (push) Blocked by required conditions
CI / Deny unrelated histories (push) Blocked by required conditions
CI / Check contributors (push) Blocked by required conditions
CI / Check uv.lock (push) Blocked by required conditions
CI / Lint Docker scripts (push) Blocked by required conditions
CI / Build&Test Docker image (push) Blocked by required conditions
CI / Supply-chain scan (push) Blocked by required conditions
CI / OSV scan (push) Waiting to run
CI / All required checks pass (push) Blocked by required conditions
CI / CI timing report (push) Blocked by required conditions
Deploy Site / deploy-vercel (push) Waiting to run
Deploy Site / deploy-docs (push) Waiting to run
feat(gateway): attach MEDIA: caption to the media bubble on standalone sends
This commit is contained in:
commit
73b611ad19
8 changed files with 716 additions and 18 deletions
|
|
@ -7881,6 +7881,7 @@ async def _standalone_send(
|
|||
thread_id: Optional[str] = None,
|
||||
media_files: Optional[list] = None,
|
||||
force_document: bool = False,
|
||||
caption: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Send via Discord REST API without a live gateway adapter.
|
||||
|
||||
|
|
@ -7981,7 +7982,7 @@ async def _standalone_send(
|
|||
{"id": str(idx), "filename": os.path.basename(path)}
|
||||
for idx, path in enumerate(valid_media)
|
||||
]
|
||||
starter_message = {"content": message, "attachments": attachments_meta}
|
||||
starter_message = {"content": (caption or message), "attachments": attachments_meta}
|
||||
payload_json = json.dumps({"name": thread_name, "message": starter_message})
|
||||
|
||||
form = aiohttp.FormData()
|
||||
|
|
@ -8061,16 +8062,42 @@ async def _standalone_send(
|
|||
_DISCORD_STANDALONE_JSON_BODY_LIMIT_BYTES,
|
||||
)
|
||||
|
||||
# Send each media file as a separate multipart upload
|
||||
# Send each media file as a separate multipart upload. When a
|
||||
# MEDIA:<path> caption was supplied, ride it as the message content
|
||||
# on the attachment so it appears under the media bubble instead of
|
||||
# as a separate message. caption_pending tracks whether the caption
|
||||
# still needs delivering, so a missing file falls back to a plain
|
||||
# message rather than silently dropping the text.
|
||||
caption_pending = bool(caption)
|
||||
for media_path, _is_voice in media_files:
|
||||
if not os.path.exists(media_path):
|
||||
warning = f"Media file not found, skipping: {media_path}"
|
||||
logger.warning(warning)
|
||||
warnings.append(warning)
|
||||
if caption_pending:
|
||||
try:
|
||||
async with session.post(
|
||||
url, headers=json_headers,
|
||||
json={"content": caption}, **_req_kw,
|
||||
) as resp:
|
||||
if resp.status in {200, 201}:
|
||||
last_data = await _standalone_read_json_limited(
|
||||
resp, _DISCORD_STANDALONE_JSON_BODY_LIMIT_BYTES,
|
||||
)
|
||||
caption_pending = False
|
||||
except Exception:
|
||||
logger.warning("Discord caption-fallback send failed for missing media")
|
||||
continue
|
||||
try:
|
||||
form = aiohttp.FormData()
|
||||
filename = os.path.basename(media_path)
|
||||
if caption_pending:
|
||||
form.add_field(
|
||||
"payload_json",
|
||||
json.dumps({"content": caption}),
|
||||
content_type="application/json",
|
||||
)
|
||||
caption_pending = False
|
||||
with open(media_path, "rb") as f:
|
||||
form.add_field("files[0]", f, filename=filename)
|
||||
async with session.post(url, headers=auth_headers, data=form, **_req_kw) as resp:
|
||||
|
|
|
|||
|
|
@ -1569,12 +1569,18 @@ async def _standalone_send(
|
|||
thread_id=None,
|
||||
media_files=None,
|
||||
force_document=False,
|
||||
caption=None,
|
||||
):
|
||||
"""Out-of-process WhatsApp delivery via the local bridge HTTP API.
|
||||
|
||||
Implements the standalone_sender_fn contract so deliver=whatsapp cron jobs
|
||||
succeed when cron runs separately from the gateway. Replaces the legacy
|
||||
_send_whatsapp helper.
|
||||
|
||||
When ``caption`` is provided (single-file ``MEDIA:<path> caption`` send),
|
||||
the text rides on the media bubble's native caption via the bridge
|
||||
``/send-media`` ``caption`` field instead of being posted as a separate
|
||||
``/send`` message beforehand.
|
||||
"""
|
||||
extra = getattr(pconfig, "extra", {}) or {}
|
||||
try:
|
||||
|
|
@ -1586,10 +1592,14 @@ async def _standalone_send(
|
|||
normalized_chat_id = to_whatsapp_jid(chat_id)
|
||||
media = media_files or []
|
||||
text = message or ""
|
||||
# A caption only applies to a single media file; guard defensively so
|
||||
# a caption is never silently repeated across a multi-file send.
|
||||
media_caption = caption if (caption and len(media) == 1) else None
|
||||
last_message_id = None
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# 1) Text first (skip the /send call when this chunk is media-only).
|
||||
if text.strip():
|
||||
# 1) Text first (skip the /send call when this chunk is media-only
|
||||
# or when the text is delivered as the media caption instead).
|
||||
if text.strip() and not media_caption:
|
||||
async with session.post(
|
||||
f"http://localhost:{bridge_port}/send",
|
||||
json={"chatId": normalized_chat_id, "message": text},
|
||||
|
|
@ -1607,6 +1617,21 @@ async def _standalone_send(
|
|||
# bubble, and ogg/opus as a voice note — not a file/document.
|
||||
for media_path, is_voice in media:
|
||||
if not os.path.exists(media_path):
|
||||
# If the text was suppressed to ride as this file's caption
|
||||
# (caption mode), the words would otherwise be lost when the
|
||||
# file is missing — deliver the caption as a plain message
|
||||
# so nothing silently disappears.
|
||||
if media_caption:
|
||||
try:
|
||||
async with session.post(
|
||||
f"http://localhost:{bridge_port}/send",
|
||||
json={"chatId": normalized_chat_id, "message": media_caption},
|
||||
timeout=aiohttp.ClientTimeout(total=30),
|
||||
) as resp:
|
||||
if resp.status == 200:
|
||||
last_message_id = (await resp.json()).get("messageId")
|
||||
except Exception:
|
||||
logger.warning("WhatsApp caption-fallback send failed for missing media")
|
||||
return {"error": f"WhatsApp media file not found: {media_path}"}
|
||||
media_type = _bridge_media_type(media_path, is_voice, force_document)
|
||||
payload: Dict[str, Any] = {
|
||||
|
|
@ -1616,6 +1641,8 @@ async def _standalone_send(
|
|||
}
|
||||
if media_type == "document":
|
||||
payload["fileName"] = os.path.basename(media_path)
|
||||
if media_caption:
|
||||
payload["caption"] = media_caption
|
||||
async with session.post(
|
||||
f"http://localhost:{bridge_port}/send-media",
|
||||
json=payload,
|
||||
|
|
|
|||
133
tests/tools/test_discord_send_message_caption.py
Normal file
133
tests/tools/test_discord_send_message_caption.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Discord standalone MEDIA:<path> caption delivery.
|
||||
|
||||
When `hermes send --to discord "MEDIA:/x.png This Caption"` targets a normal
|
||||
(non-forum) channel, the caption must ride on the media message content rather
|
||||
than being posted as a separate message before the attachment. The Discord REST
|
||||
calls are mocked at the aiohttp.ClientSession boundary.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from plugins.platforms.discord.adapter import _remember_channel_is_forum, _standalone_send
|
||||
|
||||
|
||||
def _resp(status, json_data=None, text_data=None):
|
||||
r = AsyncMock()
|
||||
r.status = status
|
||||
body = json.dumps(json_data or {}).encode() if json_data is not None else (text_data or "").encode()
|
||||
r.json = AsyncMock(return_value=json_data or {})
|
||||
r.text = AsyncMock(return_value=text_data or "")
|
||||
# Discord's _standalone_read_*_limited helpers stream resp.content.read();
|
||||
# return the body once then EOF so the bounded reader terminates. AsyncMock
|
||||
# with a list side_effect yields each element on successive awaits.
|
||||
r.content = MagicMock()
|
||||
r.content.read = AsyncMock(side_effect=[body, b"", b""])
|
||||
# _standalone_response_encoding calls resp.get_encoding() expecting a str;
|
||||
# a bare AsyncMock would return a coroutine. Give it a plain callable.
|
||||
r.get_encoding = MagicMock(return_value="utf-8")
|
||||
return r
|
||||
|
||||
|
||||
def _session_with(responses):
|
||||
"""Mocked aiohttp.ClientSession recording every POST (url, json, data)."""
|
||||
calls = []
|
||||
idx = [0]
|
||||
|
||||
def _post(url, **kwargs):
|
||||
calls.append((url, kwargs.get("json"), kwargs.get("data")))
|
||||
r = responses[idx[0]] if idx[0] < len(responses) else responses[-1]
|
||||
idx[0] += 1
|
||||
ctx = MagicMock()
|
||||
ctx.__aenter__ = AsyncMock(return_value=r)
|
||||
ctx.__aexit__ = AsyncMock(return_value=False)
|
||||
return ctx
|
||||
|
||||
session = MagicMock()
|
||||
session.post = MagicMock(side_effect=_post)
|
||||
session_ctx = MagicMock()
|
||||
session_ctx.__aenter__ = AsyncMock(return_value=session)
|
||||
session_ctx.__aexit__ = AsyncMock(return_value=False)
|
||||
return session_ctx, calls
|
||||
|
||||
|
||||
def _pconfig():
|
||||
return SimpleNamespace(token="bot-token", extra={})
|
||||
|
||||
|
||||
def _tmpfile(suffix):
|
||||
f = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
|
||||
f.write(b"x")
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def _payload_json_content(form_data):
|
||||
"""Extract the 'content' from a FormData's payload_json field, if any."""
|
||||
for field in getattr(form_data, "_fields", []):
|
||||
# aiohttp FormData stores (type_options_dict, headers, value)
|
||||
try:
|
||||
type_opts = field[0]
|
||||
value = field[2]
|
||||
except (IndexError, TypeError):
|
||||
continue
|
||||
if type_opts.get("name") == "payload_json":
|
||||
return json.loads(value).get("content")
|
||||
return None
|
||||
|
||||
|
||||
def test_caption_rides_media_non_forum():
|
||||
chat_id = "999000111"
|
||||
_remember_channel_is_forum(chat_id, False) # avoid the live GET probe
|
||||
img = _tmpfile(".png")
|
||||
try:
|
||||
session_ctx, calls = _session_with([_resp(200, {"id": "m1"})])
|
||||
with patch("aiohttp.ClientSession", return_value=session_ctx):
|
||||
res = asyncio.run(
|
||||
_standalone_send(
|
||||
_pconfig(),
|
||||
chat_id,
|
||||
"",
|
||||
media_files=[(img, False)],
|
||||
caption="2-bedroom floor plan",
|
||||
)
|
||||
)
|
||||
assert res["success"] is True
|
||||
# Exactly one POST (the media upload) — no separate text message.
|
||||
assert len(calls) == 1
|
||||
url, _json, data = calls[0]
|
||||
assert url.endswith("/messages")
|
||||
assert _payload_json_content(data) == "2-bedroom floor plan"
|
||||
finally:
|
||||
os.unlink(img)
|
||||
|
||||
|
||||
def test_no_caption_non_forum_keeps_separate_text():
|
||||
"""Without a caption, text + media are two separate POSTs (unchanged)."""
|
||||
chat_id = "999000222"
|
||||
_remember_channel_is_forum(chat_id, False)
|
||||
img = _tmpfile(".png")
|
||||
try:
|
||||
session_ctx, calls = _session_with(
|
||||
[_resp(200, {"id": "t1"}), _resp(200, {"id": "m1"})]
|
||||
)
|
||||
with patch("aiohttp.ClientSession", return_value=session_ctx):
|
||||
res = asyncio.run(
|
||||
_standalone_send(
|
||||
_pconfig(),
|
||||
chat_id,
|
||||
"hello",
|
||||
media_files=[(img, False)],
|
||||
)
|
||||
)
|
||||
assert res["success"] is True
|
||||
# Two POSTs: the text content message, then the media upload.
|
||||
assert len(calls) == 2
|
||||
assert calls[0][1] == {"content": "hello"}
|
||||
assert calls[1][0].endswith("/messages")
|
||||
finally:
|
||||
os.unlink(img)
|
||||
115
tests/tools/test_media_caption_split.py
Normal file
115
tests/tools/test_media_caption_split.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
"""Guard test for the MEDIA:<path> caption chokepoint (_media_caption_split).
|
||||
|
||||
`hermes send` strips the MEDIA: tag and leaves the remaining prose as the
|
||||
accompanying text. Historically every standalone sender posted that text as a
|
||||
*separate* message before an uncaptioned media bubble, splitting
|
||||
``hermes send --to whatsapp "MEDIA:/x.png This Caption"`` into two parts.
|
||||
|
||||
`_media_caption_split` is the single enforced decision point that all standalone
|
||||
senders (WhatsApp, Telegram, Discord) consult to decide whether the text should
|
||||
ride on the media bubble as a native caption. This test pins that contract so
|
||||
the platforms can't diverge.
|
||||
"""
|
||||
|
||||
from tools.send_message_tool import (
|
||||
_DEFAULT_CAPTION_LIMIT,
|
||||
_TELEGRAM_CAPTION_LIMIT,
|
||||
_media_caption_split,
|
||||
)
|
||||
|
||||
|
||||
def test_single_image_short_text_becomes_caption():
|
||||
caption, body = _media_caption_split(
|
||||
"This Caption", [("/tmp/F22.png", False)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption == "This Caption"
|
||||
assert body == ""
|
||||
|
||||
|
||||
def test_single_video_short_text_becomes_caption():
|
||||
caption, body = _media_caption_split(
|
||||
"Model unit tour", [("/tmp/tour.mp4", False)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption == "Model unit tour"
|
||||
assert body == ""
|
||||
|
||||
|
||||
def test_single_document_short_text_becomes_caption():
|
||||
caption, body = _media_caption_split(
|
||||
"Q3 report", [("/tmp/report.pdf", False)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption == "Q3 report"
|
||||
assert body == ""
|
||||
|
||||
|
||||
def test_multi_file_keeps_separate_body():
|
||||
text = "two photos"
|
||||
caption, body = _media_caption_split(
|
||||
text,
|
||||
[("/tmp/a.png", False), ("/tmp/b.png", False)],
|
||||
max_caption_len=_DEFAULT_CAPTION_LIMIT,
|
||||
)
|
||||
assert caption is None
|
||||
assert body == text
|
||||
|
||||
|
||||
def test_voice_note_keeps_separate_body():
|
||||
text = "listen to this"
|
||||
caption, body = _media_caption_split(
|
||||
text, [("/tmp/note.ogg", True)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption is None
|
||||
assert body == text
|
||||
|
||||
|
||||
def test_empty_text_no_caption():
|
||||
caption, body = _media_caption_split(
|
||||
" ", [("/tmp/a.png", False)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption is None
|
||||
# body is returned unchanged (still whitespace) — sender's own guards drop it
|
||||
assert body == " "
|
||||
|
||||
|
||||
def test_no_media_no_caption():
|
||||
caption, body = _media_caption_split(
|
||||
"hello", [], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption is None
|
||||
assert body == "hello"
|
||||
|
||||
|
||||
def test_text_over_limit_stays_separate_body():
|
||||
long_text = "x" * (_TELEGRAM_CAPTION_LIMIT + 1)
|
||||
caption, body = _media_caption_split(
|
||||
long_text, [("/tmp/a.png", False)], max_caption_len=_TELEGRAM_CAPTION_LIMIT
|
||||
)
|
||||
assert caption is None
|
||||
assert body == long_text
|
||||
|
||||
|
||||
def test_text_at_limit_still_captions():
|
||||
text = "y" * _TELEGRAM_CAPTION_LIMIT
|
||||
caption, body = _media_caption_split(
|
||||
text, [("/tmp/a.png", False)], max_caption_len=_TELEGRAM_CAPTION_LIMIT
|
||||
)
|
||||
assert caption == text
|
||||
assert body == ""
|
||||
|
||||
|
||||
def test_caption_is_stripped():
|
||||
caption, body = _media_caption_split(
|
||||
" padded caption ", [("/tmp/a.png", False)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption == "padded caption"
|
||||
assert body == ""
|
||||
|
||||
|
||||
def test_unknown_extension_keeps_separate_body():
|
||||
# A non-captionable kind (e.g. an audio note that isn't flagged voice)
|
||||
text = "some audio"
|
||||
caption, body = _media_caption_split(
|
||||
text, [("/tmp/song.mp3", False)], max_caption_len=_DEFAULT_CAPTION_LIMIT
|
||||
)
|
||||
assert caption is None
|
||||
assert body == text
|
||||
|
|
@ -146,11 +146,14 @@ class _patch_discord_sender:
|
|||
self._entry = None
|
||||
self._original = None
|
||||
|
||||
async def _adapter(self, pconfig, chat_id, message, *, thread_id=None, media_files=None):
|
||||
async def _adapter(self, pconfig, chat_id, message, *, thread_id=None, media_files=None, caption=None):
|
||||
token = getattr(pconfig, "token", None)
|
||||
# Only forward caption= when set, so mocks written against the
|
||||
# pre-caption signature (no caption kwarg) keep working.
|
||||
extra = {"caption": caption} if caption is not None else {}
|
||||
return await self._mock(
|
||||
token, chat_id, message,
|
||||
thread_id=thread_id, media_files=media_files,
|
||||
thread_id=thread_id, media_files=media_files, **extra,
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
|
|
@ -595,7 +598,9 @@ class TestSendMessageTool:
|
|||
|
||||
|
||||
class TestSendTelegramMediaDelivery:
|
||||
def test_sends_text_then_photo_for_media_tag(self, tmp_path, monkeypatch):
|
||||
def test_sends_photo_with_caption_for_media_tag(self, tmp_path, monkeypatch):
|
||||
# A single captionable image + short text now rides as the photo's
|
||||
# native caption (MEDIA:<path> caption), not a separate text message.
|
||||
image_path = tmp_path / "photo.png"
|
||||
image_path.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 32)
|
||||
|
||||
|
|
@ -619,11 +624,10 @@ class TestSendTelegramMediaDelivery:
|
|||
|
||||
assert result["success"] is True
|
||||
assert result["message_id"] == "2"
|
||||
bot.send_message.assert_awaited_once()
|
||||
# No separate text send — the caption rides the photo bubble.
|
||||
bot.send_message.assert_not_awaited()
|
||||
bot.send_photo.assert_awaited_once()
|
||||
sent_text = bot.send_message.await_args.kwargs["text"]
|
||||
assert "MEDIA:" not in sent_text
|
||||
assert sent_text == "Hello there"
|
||||
assert bot.send_photo.await_args.kwargs.get("caption") == "Hello there"
|
||||
|
||||
def test_sends_voice_for_ogg_with_voice_directive(self, tmp_path, monkeypatch):
|
||||
voice_path = tmp_path / "voice.ogg"
|
||||
|
|
@ -2048,7 +2052,7 @@ class TestSendToPlatformDiscordMedia:
|
|||
assert call_log[1]["media_files"] == [("/fake/img.png", False)] # Last chunk: media attached
|
||||
|
||||
def test_single_chunk_gets_media(self):
|
||||
"""Short message (single chunk) gets media_files directly."""
|
||||
"""Short message + single image rides as the media caption."""
|
||||
send_mock = AsyncMock(return_value={"success": True, "message_id": "1"})
|
||||
|
||||
with _patch_discord_sender(send_mock):
|
||||
|
|
@ -2066,6 +2070,9 @@ class TestSendToPlatformDiscordMedia:
|
|||
send_mock.assert_awaited_once()
|
||||
call_kwargs = send_mock.await_args.kwargs
|
||||
assert call_kwargs["media_files"] == [("/fake/img.png", False)]
|
||||
# Text rides as the caption, not a separate positional message body.
|
||||
assert call_kwargs.get("caption") == "short message"
|
||||
assert send_mock.await_args.args[2] == ""
|
||||
|
||||
|
||||
class TestSendMatrixUrlEncoding:
|
||||
|
|
@ -3334,13 +3341,17 @@ class TestSendTelegramThreadNotFoundRetry:
|
|||
"retry should drop message_thread_id after thread-not-found"
|
||||
|
||||
def test_disable_web_page_preview_not_leaked_to_media_sends(self):
|
||||
"""disable_web_page_preview should only appear in text send, not media sends."""
|
||||
text_kwargs_seen = []
|
||||
"""disable_web_page_preview must never leak into a media send.
|
||||
|
||||
A single captionable file + short text now rides as the document's
|
||||
caption (no separate text send), so the invariant to protect is that
|
||||
the captioned send_document does not inherit disable_web_page_preview
|
||||
(valid only for send_message).
|
||||
"""
|
||||
media_kwargs_seen = []
|
||||
|
||||
class FakeBot:
|
||||
async def send_message(self, **kwargs):
|
||||
text_kwargs_seen.append(kwargs)
|
||||
return SimpleNamespace(message_id=1)
|
||||
|
||||
async def send_document(self, **kwargs):
|
||||
|
|
@ -3364,9 +3375,9 @@ class TestSendTelegramThreadNotFoundRetry:
|
|||
|
||||
result = asyncio.run(run_test())
|
||||
assert result["success"] is True
|
||||
# Text send should have disable_web_page_preview
|
||||
assert text_kwargs_seen[0].get("disable_web_page_preview") is True
|
||||
# Media send should NOT have disable_web_page_preview
|
||||
# Caption rides the document bubble.
|
||||
assert media_kwargs_seen[0].get("caption") == "check preview"
|
||||
# Media send must NOT carry disable_web_page_preview.
|
||||
assert "disable_web_page_preview" not in media_kwargs_seen[0], \
|
||||
"disable_web_page_preview leaked into send_document kwargs"
|
||||
finally:
|
||||
|
|
|
|||
141
tests/tools/test_telegram_send_message_caption.py
Normal file
141
tests/tools/test_telegram_send_message_caption.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
"""Standalone Telegram MEDIA:<path> caption delivery.
|
||||
|
||||
When `hermes send --to telegram "MEDIA:/x.png This Caption"` carries a single
|
||||
captionable file plus short text, the text must ride on the media bubble as the
|
||||
sendPhoto/sendVideo/sendDocument ``caption`` rather than being posted as a
|
||||
separate sendMessage beforehand. Longer text (> Telegram's 1024 caption cap)
|
||||
falls back to a separate message. The ``telegram`` package is stubbed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _install_telegram_mock(monkeypatch: pytest.MonkeyPatch, bot_factory: MagicMock) -> None:
|
||||
parse_mode = SimpleNamespace(MARKDOWN_V2="MarkdownV2", HTML="HTML")
|
||||
constants_mod = SimpleNamespace(ParseMode=parse_mode)
|
||||
_MessageEntity = lambda **_kw: SimpleNamespace(**_kw)
|
||||
telegram_mod = SimpleNamespace(
|
||||
Bot=bot_factory,
|
||||
MessageEntity=_MessageEntity,
|
||||
constants=constants_mod,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "telegram", telegram_mod)
|
||||
monkeypatch.setitem(sys.modules, "telegram.constants", constants_mod)
|
||||
|
||||
|
||||
def _make_bot() -> MagicMock:
|
||||
bot = MagicMock()
|
||||
bot.send_message = AsyncMock(return_value=SimpleNamespace(message_id=1))
|
||||
bot.send_photo = AsyncMock(return_value=SimpleNamespace(message_id=2))
|
||||
bot.send_video = AsyncMock(return_value=SimpleNamespace(message_id=3))
|
||||
bot.send_document = AsyncMock(return_value=SimpleNamespace(message_id=4))
|
||||
return bot
|
||||
|
||||
|
||||
def _no_proxy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for var in (
|
||||
"TELEGRAM_PROXY", "HTTPS_PROXY", "https_proxy", "HTTP_PROXY",
|
||||
"http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setattr("gateway.run._gateway_runner_ref", lambda: None, raising=False)
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
|
||||
|
||||
def _tmpfile(suffix: str) -> str:
|
||||
f = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
|
||||
f.write(b"x")
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def test_image_caption_rides_bubble_no_separate_text(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from tools.send_message_tool import _send_telegram
|
||||
|
||||
_no_proxy(monkeypatch)
|
||||
bot = _make_bot()
|
||||
_install_telegram_mock(monkeypatch, MagicMock(return_value=bot))
|
||||
img = _tmpfile(".png")
|
||||
try:
|
||||
res = asyncio.run(
|
||||
_send_telegram("tok", "123", "This Caption", media_files=[(img, False)])
|
||||
)
|
||||
assert res["success"] is True
|
||||
# No separate text message; caption rides the photo.
|
||||
bot.send_message.assert_not_awaited()
|
||||
bot.send_photo.assert_awaited_once()
|
||||
assert bot.send_photo.await_args.kwargs.get("caption") == "This Caption"
|
||||
finally:
|
||||
os.unlink(img)
|
||||
|
||||
|
||||
def test_video_caption_rides_bubble(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from tools.send_message_tool import _send_telegram
|
||||
|
||||
_no_proxy(monkeypatch)
|
||||
bot = _make_bot()
|
||||
_install_telegram_mock(monkeypatch, MagicMock(return_value=bot))
|
||||
vid = _tmpfile(".mp4")
|
||||
try:
|
||||
res = asyncio.run(
|
||||
_send_telegram("tok", "123", "Model unit tour", media_files=[(vid, False)])
|
||||
)
|
||||
assert res["success"] is True
|
||||
bot.send_message.assert_not_awaited()
|
||||
bot.send_video.assert_awaited_once()
|
||||
assert bot.send_video.await_args.kwargs.get("caption") == "Model unit tour"
|
||||
finally:
|
||||
os.unlink(vid)
|
||||
|
||||
|
||||
def test_long_text_falls_back_to_separate_message(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from tools.send_message_tool import _send_telegram
|
||||
|
||||
_no_proxy(monkeypatch)
|
||||
bot = _make_bot()
|
||||
_install_telegram_mock(monkeypatch, MagicMock(return_value=bot))
|
||||
img = _tmpfile(".png")
|
||||
long_text = "x" * 1100 # over Telegram's 1024 caption cap
|
||||
try:
|
||||
res = asyncio.run(
|
||||
_send_telegram("tok", "123", long_text, media_files=[(img, False)])
|
||||
)
|
||||
assert res["success"] is True
|
||||
# Text too long for a caption — sent as its own message, photo uncaptioned.
|
||||
bot.send_message.assert_awaited()
|
||||
bot.send_photo.assert_awaited_once()
|
||||
assert not bot.send_photo.await_args.kwargs.get("caption")
|
||||
finally:
|
||||
os.unlink(img)
|
||||
|
||||
|
||||
def test_multi_file_keeps_separate_text(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from tools.send_message_tool import _send_telegram
|
||||
|
||||
_no_proxy(monkeypatch)
|
||||
bot = _make_bot()
|
||||
_install_telegram_mock(monkeypatch, MagicMock(return_value=bot))
|
||||
img = _tmpfile(".png")
|
||||
img2 = _tmpfile(".jpg")
|
||||
try:
|
||||
res = asyncio.run(
|
||||
_send_telegram("tok", "123", "two pics", media_files=[(img, False), (img2, False)])
|
||||
)
|
||||
assert res["success"] is True
|
||||
# Ambiguous caption→file association: text stays a separate message.
|
||||
bot.send_message.assert_awaited()
|
||||
assert bot.send_photo.await_count == 2
|
||||
for call in bot.send_photo.await_args_list:
|
||||
assert not call.kwargs.get("caption")
|
||||
finally:
|
||||
os.unlink(img)
|
||||
os.unlink(img2)
|
||||
|
|
@ -219,3 +219,78 @@ def test_text_only_unchanged_behavior():
|
|||
"message_id": "t1",
|
||||
}
|
||||
assert len(calls) == 1 and calls[0][0].endswith("/send")
|
||||
|
||||
|
||||
def test_caption_rides_media_no_separate_text_send():
|
||||
"""MEDIA:<path> caption -> single /send-media with caption, no /send."""
|
||||
img = _tmpfile(".png")
|
||||
try:
|
||||
session_ctx, calls = _session_with([_resp(200, {"messageId": "m1"})])
|
||||
with patch("aiohttp.ClientSession", return_value=session_ctx):
|
||||
res = asyncio.run(
|
||||
_standalone_send(
|
||||
_pconfig(),
|
||||
"12345",
|
||||
"",
|
||||
media_files=[(img, False)],
|
||||
caption="2-bedroom floor plan",
|
||||
)
|
||||
)
|
||||
assert res["success"] is True
|
||||
# No separate /send — exactly one /send-media carrying the caption.
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0].endswith("/send-media")
|
||||
assert calls[0][1]["caption"] == "2-bedroom floor plan"
|
||||
assert calls[0][1]["mediaType"] == "image"
|
||||
finally:
|
||||
os.unlink(img)
|
||||
|
||||
|
||||
def test_caption_ignored_for_multi_file_send():
|
||||
"""A caption never rides a multi-file send (association is ambiguous)."""
|
||||
img = _tmpfile(".png")
|
||||
img2 = _tmpfile(".jpg")
|
||||
try:
|
||||
session_ctx, calls = _session_with(
|
||||
[_resp(200, {"messageId": "m1"}), _resp(200, {"messageId": "m2"})]
|
||||
)
|
||||
with patch("aiohttp.ClientSession", return_value=session_ctx):
|
||||
res = asyncio.run(
|
||||
_standalone_send(
|
||||
_pconfig(),
|
||||
"12345",
|
||||
"",
|
||||
media_files=[(img, False), (img2, False)],
|
||||
caption="should be ignored",
|
||||
)
|
||||
)
|
||||
assert res["success"] is True
|
||||
media_calls = [c for c in calls if c[0].endswith("/send-media")]
|
||||
assert len(media_calls) == 2
|
||||
assert all("caption" not in c[1] for c in media_calls)
|
||||
finally:
|
||||
os.unlink(img)
|
||||
os.unlink(img2)
|
||||
|
||||
|
||||
def test_missing_captioned_file_falls_back_to_text():
|
||||
"""If the single captioned file is missing, the caption is delivered as a
|
||||
plain /send message rather than being silently lost (W1)."""
|
||||
session_ctx, calls = _session_with([_resp(200, {"messageId": "t1"})])
|
||||
with patch("aiohttp.ClientSession", return_value=session_ctx):
|
||||
res = asyncio.run(
|
||||
_standalone_send(
|
||||
_pconfig(),
|
||||
"12345",
|
||||
"",
|
||||
media_files=[("/no/such/file.png", False)],
|
||||
caption="floor plan",
|
||||
)
|
||||
)
|
||||
# The send still surfaces the missing-file error...
|
||||
assert "error" in res
|
||||
assert "not found" in res["error"]
|
||||
# ...but the caption text was delivered on its own first.
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0].endswith("/send")
|
||||
assert calls[0][1]["message"] == "floor plan"
|
||||
|
|
|
|||
|
|
@ -65,6 +65,62 @@ _VOICE_EXTS = {".ogg", ".opus"}
|
|||
# formats either route through sendVoice (Opus/OGG) or fall back to
|
||||
# document delivery.
|
||||
_TELEGRAM_SEND_AUDIO_EXTS = {".mp3", ".m4a"}
|
||||
|
||||
# Extensions that carry a native caption on the media bubble itself
|
||||
# (photo/video/document). Voice/audio notes are excluded: a caption on a
|
||||
# voice note reads as a separate label rather than a bubble caption, and the
|
||||
# established convention is to keep the accompanying text as its own message.
|
||||
_CAPTIONABLE_EXTS = _IMAGE_EXTS | _VIDEO_EXTS | {
|
||||
".pdf", ".doc", ".docx", ".txt", ".md", ".csv", ".xlsx", ".zip",
|
||||
}
|
||||
|
||||
# Per-platform native caption length limits (characters). Text longer than
|
||||
# the limit can't ride on the media bubble and stays a separate body message.
|
||||
# Telegram's photo/video caption cap is 1024; WhatsApp and Discord are far
|
||||
# more generous, so a conservative shared ceiling keeps behavior predictable.
|
||||
_TELEGRAM_CAPTION_LIMIT = 1024
|
||||
_DEFAULT_CAPTION_LIMIT = 4096
|
||||
|
||||
|
||||
def _media_caption_split(text, media_files, *, max_caption_len):
|
||||
"""Decide whether the accompanying text should ride on the media bubble.
|
||||
|
||||
Single enforced chokepoint for the ``MEDIA:<path> caption`` behavior
|
||||
across every standalone sender. ``hermes send`` (and the send_message
|
||||
tool / cron) strips the ``MEDIA:`` tag and leaves the remaining prose as
|
||||
``text``; historically each platform sent that ``text`` as a *separate*
|
||||
message before an uncaptioned media bubble, splitting the reported case
|
||||
``hermes send --to whatsapp "MEDIA:/x.png This Caption"`` into two parts.
|
||||
|
||||
Returns ``(caption, body_text)``:
|
||||
|
||||
* ``(caption, "")`` — attach ``text`` to the media as its native caption
|
||||
and send *no* separate body message. Only when there is exactly one
|
||||
media file, it is a captionable kind (image/video/document, not a
|
||||
voice/audio note), and ``text`` fits ``max_caption_len``.
|
||||
* ``(None, text)`` — keep the historical behavior: ``text`` is a separate
|
||||
body message and the media carries no caption. Applies to multi-file
|
||||
sends (caption→file association is ambiguous), voice/audio notes, empty
|
||||
text, or text longer than the caption limit.
|
||||
"""
|
||||
stripped = (text or "").strip()
|
||||
media = media_files or []
|
||||
if not stripped or len(media) != 1:
|
||||
return None, text
|
||||
media_path, is_voice = media[0]
|
||||
if is_voice:
|
||||
return None, text
|
||||
ext = os.path.splitext(media_path)[1].lower()
|
||||
if ext not in _CAPTIONABLE_EXTS:
|
||||
return None, text
|
||||
# Measure the caption in Unicode codepoints — a portable upper bound that
|
||||
# never under-counts vs Telegram's UTF-16 units for BMP text, so an
|
||||
# over-count only fails safe (falls back to a separate message). The
|
||||
# Telegram call site additionally re-checks the *formatted* caption in
|
||||
# UTF-16 units, since MarkdownV2/HTML escaping can inflate the length.
|
||||
if len(stripped) > max_caption_len:
|
||||
return None, text
|
||||
return stripped, ""
|
||||
_URL_SECRET_QUERY_RE = re.compile(
|
||||
r"([?&](?:access_token|api[_-]?key|auth[_-]?token|token|signature|sig)=)([^&#\s]+)",
|
||||
re.IGNORECASE,
|
||||
|
|
@ -814,6 +870,26 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,
|
|||
entry = platform_registry.get("discord")
|
||||
if entry is None or entry.standalone_sender_fn is None:
|
||||
return {"error": "Discord plugin not registered or missing standalone_sender_fn"}
|
||||
# MEDIA:<path> caption: single captionable file + short text rides as
|
||||
# the media message content instead of a separate message before the
|
||||
# attachment (single enforced decision in _media_caption_split). Cap on
|
||||
# the platform's own message limit so the caption is always deliverable.
|
||||
_dc_caption, _ = _media_caption_split(
|
||||
message, media_files,
|
||||
max_caption_len=(max_len or _DEFAULT_CAPTION_LIMIT),
|
||||
)
|
||||
if _dc_caption is not None:
|
||||
result = await entry.standalone_sender_fn(
|
||||
pconfig,
|
||||
chat_id,
|
||||
"",
|
||||
thread_id=thread_id,
|
||||
media_files=media_files,
|
||||
caption=_dc_caption,
|
||||
)
|
||||
if isinstance(result, dict) and result.get("error"):
|
||||
return result
|
||||
return result
|
||||
last_result = None
|
||||
for i, chunk in enumerate(chunks):
|
||||
is_last = (i == len(chunks) - 1)
|
||||
|
|
@ -916,7 +992,30 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,
|
|||
_wa_entry = _pr_wa.get("whatsapp")
|
||||
if _wa_entry is None or _wa_entry.standalone_sender_fn is None:
|
||||
return {"error": "WhatsApp plugin not registered or missing standalone_sender_fn"}
|
||||
# MEDIA:<path> caption: a single captionable file + short text rides
|
||||
# as the media's native caption instead of a separate message before
|
||||
# the bubble (single enforced decision in _media_caption_split). Cap on
|
||||
# the platform's own message limit so the caption is always deliverable.
|
||||
_wa_caption, _ = _media_caption_split(
|
||||
message, media_files,
|
||||
max_caption_len=(max_len or _DEFAULT_CAPTION_LIMIT),
|
||||
)
|
||||
last_result = None
|
||||
if _wa_caption is not None:
|
||||
# Single-file captioned send: no separate text chunk, caption on
|
||||
# the media itself.
|
||||
result = await _wa_entry.standalone_sender_fn(
|
||||
pconfig,
|
||||
chat_id,
|
||||
"",
|
||||
media_files=media_files,
|
||||
thread_id=thread_id,
|
||||
force_document=force_document,
|
||||
caption=_wa_caption,
|
||||
)
|
||||
if isinstance(result, dict) and result.get("error"):
|
||||
return result
|
||||
return result
|
||||
for i, chunk in enumerate(chunks):
|
||||
is_last = (i == len(chunks) - 1)
|
||||
result = await _wa_entry.standalone_sender_fn(
|
||||
|
|
@ -1109,6 +1208,23 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No
|
|||
last_msg = None
|
||||
warnings = []
|
||||
|
||||
# MEDIA:<path> caption: when a single captionable file is accompanied
|
||||
# by short text, attach the text to the media bubble as its native
|
||||
# caption instead of sending it as a separate message beforehand
|
||||
# (single enforced decision in _media_caption_split). Caption with the
|
||||
# *formatted* text so MarkdownV2/HTML styling is preserved, but guard
|
||||
# the formatted length against Telegram's 1024 cap — formatting can
|
||||
# inflate a raw-<1024 string past it, in which case fall back to a
|
||||
# separate body message.
|
||||
_tg_caption = None
|
||||
from gateway.platforms.base import utf16_len as _utf16_len
|
||||
_cap, _ = _media_caption_split(
|
||||
message, media_files, max_caption_len=_TELEGRAM_CAPTION_LIMIT
|
||||
)
|
||||
if _cap is not None and _utf16_len(formatted) <= _TELEGRAM_CAPTION_LIMIT:
|
||||
_tg_caption = formatted
|
||||
formatted = "" # suppress the separate text send below
|
||||
|
||||
if formatted.strip():
|
||||
# Chunk *after* formatting: MarkdownV2/HTML escaping inflates the
|
||||
# text (each escaped char like `!`/`.`/`-` becomes `\!`/`\.`/`\-`),
|
||||
|
|
@ -1170,12 +1286,34 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No
|
|||
warning = f"Media file not found, skipping: {media_path}"
|
||||
logger.warning(warning)
|
||||
warnings.append(warning)
|
||||
# Caption mode suppressed the separate text send; if the file
|
||||
# it was meant to caption is gone, deliver the caption text on
|
||||
# its own so the words aren't silently lost.
|
||||
if _tg_caption is not None and last_msg is None:
|
||||
try:
|
||||
last_msg = await _send_telegram_message_with_retry(
|
||||
bot, chat_id=int_chat_id, text=_tg_caption,
|
||||
parse_mode=send_parse_mode, **text_kwargs
|
||||
)
|
||||
_tg_caption = None # delivered — don't re-caption a later file
|
||||
except Exception as _cap_err:
|
||||
logger.warning(
|
||||
"Telegram caption-fallback send failed for missing media: %s",
|
||||
_sanitize_error_text(_cap_err),
|
||||
)
|
||||
continue
|
||||
|
||||
ext = os.path.splitext(media_path)[1].lower()
|
||||
try:
|
||||
with open(media_path, "rb") as f:
|
||||
media_kwargs = dict(thread_kwargs)
|
||||
# Attach the MEDIA:<path> caption to the bubble itself for
|
||||
# captionable kinds (photo/video/document). _tg_caption is
|
||||
# only set for a single captionable file, so this never
|
||||
# double-captions a multi-file send or a voice note.
|
||||
if _tg_caption is not None and not (ext in _VOICE_EXTS and is_voice):
|
||||
media_kwargs["caption"] = _tg_caption
|
||||
media_kwargs["parse_mode"] = send_parse_mode
|
||||
try:
|
||||
if ext in _IMAGE_EXTS and not force_document:
|
||||
last_msg = await bot.send_photo(
|
||||
|
|
@ -1228,6 +1366,37 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No
|
|||
last_msg = await bot.send_document(
|
||||
chat_id=int_chat_id, document=f, **media_kwargs
|
||||
)
|
||||
elif media_kwargs.get("parse_mode") and (
|
||||
"parse" in str(media_err).lower()
|
||||
or "caption" in str(media_err).lower()
|
||||
):
|
||||
# Caption failed to parse as MarkdownV2/HTML —
|
||||
# retry with a plain-text caption so the media
|
||||
# (and its caption) still deliver.
|
||||
logger.warning(
|
||||
"Caption parse failed for media send, retrying plain: %s",
|
||||
_sanitize_error_text(media_err),
|
||||
)
|
||||
f.seek(0)
|
||||
media_kwargs.pop("parse_mode", None)
|
||||
if not _has_html and media_kwargs.get("caption"):
|
||||
try:
|
||||
from plugins.platforms.telegram.adapter import _strip_mdv2
|
||||
media_kwargs["caption"] = _strip_mdv2(media_kwargs["caption"])
|
||||
except Exception:
|
||||
pass
|
||||
if ext in _IMAGE_EXTS and not force_document:
|
||||
last_msg = await bot.send_photo(
|
||||
chat_id=int_chat_id, photo=f, **media_kwargs
|
||||
)
|
||||
elif ext in _VIDEO_EXTS:
|
||||
last_msg = await bot.send_video(
|
||||
chat_id=int_chat_id, video=f, **media_kwargs
|
||||
)
|
||||
else:
|
||||
last_msg = await bot.send_document(
|
||||
chat_id=int_chat_id, document=f, **media_kwargs
|
||||
)
|
||||
else:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue