diff --git a/gateway/platforms/whatsapp_common.py b/gateway/platforms/whatsapp_common.py index c5bf5055617..70f77b2973a 100644 --- a/gateway/platforms/whatsapp_common.py +++ b/gateway/platforms/whatsapp_common.py @@ -56,6 +56,24 @@ class WhatsAppBehaviorMixin: DEFAULT_REPLY_PREFIX: str = "⚕ *Hermes Agent*\n────────────\n" + _OUTBOUND_INVISIBLE_CHARS_RE = re.compile(r"[\u200b\u2060\u2063\ufeff]") + _OUTBOUND_ODD_SPACE_RE = re.compile(r"[\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000]") + + @classmethod + def _sanitize_outbound_text(cls, content: str) -> str: + """Remove invisible formatting chars that leak badly in WhatsApp. + + Some provider/gateway formatting paths can emit unicode like WORD + JOINER (U+2060) plus NARROW NO-BREAK SPACE (U+202F). WhatsApp may + render those as mojibake-looking prefixes (``⁠ text``) instead of + invisible spacing. Keep normal text and emoji joiners intact, but + strip known zero-width format chars and normalize odd unicode spaces. + """ + if not content: + return content + content = cls._OUTBOUND_INVISIBLE_CHARS_RE.sub("", content) + return cls._OUTBOUND_ODD_SPACE_RE.sub(" ", content) + @property def enforces_own_access_policy(self) -> bool: """WhatsApp gates DM/group access at intake via dm_policy/group_policy.""" @@ -375,6 +393,8 @@ class WhatsAppBehaviorMixin: if not content: return content + content = self._sanitize_outbound_text(content) + # --- 1. Protect fenced code blocks from formatting changes --- _FENCE_PH = "\x00FENCE" fences: list[str] = [] @@ -396,12 +416,19 @@ class WhatsAppBehaviorMixin: result = re.sub(r"`[^`\n]+`", _save_code, result) # --- 3. Convert markdown formatting to WhatsApp syntax --- + # Italic: standard Markdown *text* → WhatsApp _text_. Do this before + # bold conversion so **bold** does not become italic by accident. The + # lookarounds avoid list bullets and bold delimiters. + result = re.sub( + r"(? SendResult: + """Send a native WhatsApp poll via the Baileys bridge. + + This is a low-level transport primitive only. Gateway approval UX must + remain gateway-owned and add text fallback plus explicit confirmation + semantics before approval prompts are ever mapped onto polls. + """ + if not self._running or not self._http_session: + return SendResult(success=False, error="Not connected") + bridge_exit = await self._check_managed_bridge_exit() + if bridge_exit: + return SendResult(success=False, error=bridge_exit) + try: + import aiohttp + + payload: Dict[str, Any] = { + "chatId": to_whatsapp_jid(chat_id), + "question": question, + "options": list(options or []), + "selectableCount": selectable_count, + } + async with self._http_session.post( + f"http://127.0.0.1:{self._bridge_port}/send-poll", + json=payload, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + if resp.status == 200: + data = await resp.json() + return SendResult( + success=True, + message_id=data.get("messageId"), + raw_response=data, + ) + error = await resp.text() + return SendResult(success=False, error=error) + except Exception as e: + return SendResult(success=False, error=str(e)) + + async def send_clarify( + self, + chat_id: str, + question: str, + choices: Optional[list], + clarify_id: str, + session_key: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> SendResult: + """Render multiple-choice clarify as a native WhatsApp poll. + + The gateway registers the pending clarify before calling this method. + When Baileys later emits a poll_update with the selected option as + message text, the normal clarify text-intercept resolves the pending + question and the blocked agent continues. Open-ended clarifies use the + text fallback so the user's next typed message is captured. + """ + clean_choices = [str(choice).strip() for choice in (choices or []) if str(choice).strip()] + if 2 <= len(clean_choices) <= 12: + result = await self.send_poll( + chat_id, + str(question or "").strip(), + clean_choices, + selectable_count=1, + ) + if result.success: + return result + logger.warning( + "[%s] Native WhatsApp clarify poll failed; falling back to text: %s", + self.name, + result.error, + ) + return await super().send_clarify( + chat_id=chat_id, + question=question, + choices=choices, + clarify_id=clarify_id, + session_key=session_key, + metadata=metadata, + ) + + async def send_location( + self, + chat_id: str, + latitude: float, + longitude: float, + *, + name: Optional[str] = None, + address: Optional[str] = None, + reply_to: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> SendResult: + """Send a native WhatsApp location pin via the Baileys bridge.""" + if not self._running or not self._http_session: + return SendResult(success=False, error="Not connected") + bridge_exit = await self._check_managed_bridge_exit() + if bridge_exit: + return SendResult(success=False, error=bridge_exit) + try: + import aiohttp + + payload: Dict[str, Any] = { + "chatId": to_whatsapp_jid(chat_id), + "latitude": float(latitude), + "longitude": float(longitude), + } + if name: + payload["name"] = name + if address: + payload["address"] = address + async with self._http_session.post( + f"http://127.0.0.1:{self._bridge_port}/send-location", + json=payload, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + if resp.status == 200: + data = await resp.json() + return SendResult( + success=True, + message_id=data.get("messageId"), + raw_response=data, + ) + error = await resp.text() + return SendResult(success=False, error=error) + except Exception as e: + return SendResult(success=False, error=str(e)) + async def send_image( self, chat_id: str, @@ -1196,14 +1334,20 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): # Determine message type msg_type = MessageType.TEXT - if data.get("hasMedia"): - media_type = data.get("mediaType", "") + media_type = str(data.get("mediaType", "") or "") + if media_type in {"location", "live_location"}: + msg_type = MessageType.LOCATION + elif media_type == "sticker": + msg_type = MessageType.STICKER + elif data.get("hasMedia"): if "image" in media_type: msg_type = MessageType.PHOTO elif "video" in media_type: msg_type = MessageType.VIDEO - elif "audio" in media_type or "ptt" in media_type: # ptt = voice note + elif "ptt" in media_type: # ptt = WhatsApp voice note msg_type = MessageType.VOICE + elif "audio" in media_type: + msg_type = MessageType.AUDIO else: msg_type = MessageType.DOCUMENT @@ -1226,39 +1370,40 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): cached_urls = [] media_types = [] for url in raw_urls: + bridge_mime = str(data.get("mime") or "").strip() if msg_type == MessageType.PHOTO and url.startswith(("http://", "https://")): try: cached_path = await cache_image_from_url(url, ext=".jpg") cached_urls.append(cached_path) - media_types.append("image/jpeg") + media_types.append(bridge_mime or "image/jpeg") print(f"[{self.name}] Cached user image: {cached_path}", flush=True) except Exception as e: print(f"[{self.name}] Failed to cache image: {e}", flush=True) cached_urls.append(url) - media_types.append("image/jpeg") + media_types.append(bridge_mime or "image/jpeg") elif msg_type == MessageType.PHOTO and os.path.isabs(url): # Local file path — bridge already downloaded the image if _is_allowed_bridge_path(url): cached_urls.append(url) - media_types.append("image/jpeg") + media_types.append(bridge_mime or "image/jpeg") print(f"[{self.name}] Using bridge-cached image: {url}", flush=True) else: print(f"[{self.name}] Rejected bridge image path outside cache dir: {url}", flush=True) - elif msg_type == MessageType.VOICE and url.startswith(("http://", "https://")): + elif msg_type in {MessageType.VOICE, MessageType.AUDIO} and url.startswith(("http://", "https://")): try: cached_path = await cache_audio_from_url(url, ext=".ogg") cached_urls.append(cached_path) - media_types.append("audio/ogg") - print(f"[{self.name}] Cached user voice: {cached_path}", flush=True) + media_types.append(bridge_mime or ("audio/ogg" if msg_type == MessageType.VOICE else "audio/mpeg")) + print(f"[{self.name}] Cached user audio: {cached_path}", flush=True) except Exception as e: - print(f"[{self.name}] Failed to cache voice: {e}", flush=True) + print(f"[{self.name}] Failed to cache audio: {e}", flush=True) cached_urls.append(url) - media_types.append("audio/ogg") - elif msg_type == MessageType.VOICE and os.path.isabs(url): + media_types.append(bridge_mime or ("audio/ogg" if msg_type == MessageType.VOICE else "audio/mpeg")) + elif msg_type in {MessageType.VOICE, MessageType.AUDIO} and os.path.isabs(url): # Local file path — bridge already downloaded the audio if _is_allowed_bridge_path(url): cached_urls.append(url) - media_types.append("audio/ogg") + media_types.append(bridge_mime or ("audio/ogg" if msg_type == MessageType.VOICE else "audio/mpeg")) print(f"[{self.name}] Using bridge-cached audio: {url}", flush=True) else: print(f"[{self.name}] Rejected bridge audio path outside cache dir: {url}", flush=True) @@ -1267,7 +1412,7 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): if _is_allowed_bridge_path(url): cached_urls.append(url) ext = Path(url).suffix.lower() - mime = SUPPORTED_DOCUMENT_TYPES.get(ext, "application/octet-stream") + mime = bridge_mime or SUPPORTED_DOCUMENT_TYPES.get(ext, "application/octet-stream") media_types.append(mime) print(f"[{self.name}] Using bridge-cached document: {url}", flush=True) else: @@ -1275,7 +1420,7 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): elif msg_type == MessageType.VIDEO and os.path.isabs(url): if _is_allowed_bridge_path(url): cached_urls.append(url) - media_types.append("video/mp4") + media_types.append(bridge_mime or "video/mp4") print(f"[{self.name}] Using bridge-cached video: {url}", flush=True) else: print(f"[{self.name}] Rejected bridge video path outside cache dir: {url}", flush=True) @@ -1290,14 +1435,23 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): if data.get("isGroup"): body = self._clean_bot_mention_text(body, data) - # If this is a reply, include the quoted message text so the agent - # knows exactly what the user is responding to (fixes "approve" context issue) + # If this is a reply, keep the quoted message in structured fields + # only. GatewayRunner._prepare_inbound_message_text owns rendering + # the `[Replying to: ...]` pointer for every platform; pre-rendering + # it here makes WhatsApp replies show the quote twice. quoted_text = str(data.get("quotedText") or "").strip() - if quoted_text and data.get("hasQuotedMessage"): - # Truncate long quoted text to keep prompts reasonable - if len(quoted_text) > 300: - quoted_text = quoted_text[:297] + "..." - body = f"[Replying to: \"{quoted_text}\"]\n{body}" + reply_to_text = quoted_text or None + reply_to_message_id = None + reply_to_author_id = None + reply_to_is_own_message = False + if data.get("hasQuotedMessage"): + raw_reply_id = data.get("quotedMessageId") + if raw_reply_id is not None: + reply_to_message_id = str(raw_reply_id) + quoted_participant = self._normalize_whatsapp_id(data.get("quotedParticipant")) + if quoted_participant: + reply_to_author_id = quoted_participant + reply_to_is_own_message = self._message_is_reply_to_bot(data) MAX_TEXT_INJECT_BYTES = 100 * 1024 if msg_type == MessageType.DOCUMENT and cached_urls: for doc_path in cached_urls: @@ -1326,6 +1480,12 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): print(f"[{self.name}] Failed to read document text: {e}", flush=True) metadata: Dict[str, Any] = {} + native_type = str(data.get("nativeType") or "").strip() + native_metadata = data.get("nativeMetadata") + if native_type: + metadata["whatsapp_native_type"] = native_type + if isinstance(native_metadata, dict) and native_metadata: + metadata["whatsapp_native"] = native_metadata # The bridge sets ``fromOwner: true`` on inbound fromMe messages # that look owner-typed (linked-device send, not echoed from our # own /send). Surfaced under a platform-prefixed key so plugins @@ -1350,6 +1510,10 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): media_urls=cached_urls, media_types=media_types, metadata=metadata, + reply_to_message_id=reply_to_message_id, + reply_to_text=reply_to_text, + reply_to_author_id=reply_to_author_id, + reply_to_is_own_message=reply_to_is_own_message, ) except Exception as e: print(f"[{self.name}] Error building event: {e}") diff --git a/scripts/whatsapp-bridge/bridge.js b/scripts/whatsapp-bridge/bridge.js index 44faa1ab767..23db421f9f8 100644 --- a/scripts/whatsapp-bridge/bridge.js +++ b/scripts/whatsapp-bridge/bridge.js @@ -10,6 +10,7 @@ * POST /send - Send a message { chatId, message, replyTo? } * POST /edit - Edit a sent message { chatId, messageId, message } * POST /send-media - Send media natively { chatId, filePath, mediaType?, caption?, fileName? } + * POST /send-location - Send location pin { chatId, latitude, longitude, name?, address? } * POST /typing - Send typing indicator { chatId } * GET /chat/:id - Get chat info * GET /health - Health check @@ -18,20 +19,31 @@ * node bridge.js --port 3000 --session ~/.hermes/whatsapp/session */ -import { makeWASocket, useMultiFileAuthState, DisconnectReason, fetchLatestBaileysVersion, downloadMediaMessage } from '@whiskeysockets/baileys'; +import { makeWASocket, useMultiFileAuthState, DisconnectReason, fetchLatestBaileysVersion, downloadMediaMessage, getAggregateVotesInPollMessage, decryptPollVote, getKeyAuthor, jidNormalizedUser } from '@whiskeysockets/baileys'; import express from 'express'; import { Boom } from '@hapi/boom'; import pino from 'pino'; import path from 'path'; -import { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync, unlinkSync } from 'fs'; +import { mkdirSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'fs'; import { fileURLToPath } from 'url'; import { randomBytes, createHash } from 'crypto'; -import { execSync } from 'child_process'; +import { execFileSync } from 'child_process'; import { tmpdir } from 'os'; import qrcode from 'qrcode-terminal'; import { matchesAllowedUser, parseAllowedUsers } from './allowlist.js'; import { createOutboundIdTracker } from './outbound_ids.js'; import { classifyOwnerMessageGate } from './owner_message_gate.js'; +import { + buildPollPayload, + buildLocationPayload, + buildTextSendPayload, + createBoundedMessageStore, + extractBridgeEvent, + inferMediaType, + mediaPayloadForFile, + pollCreationMessageFromPayload, + pollUpdateForAggregation, +} from './bridge_helpers.js'; // Parse CLI args const args = process.argv.slice(2); @@ -119,7 +131,7 @@ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } -function sendWithTimeout(chatId, payload, timeoutMs = SEND_TIMEOUT_MS) { +function sendWithTimeout(chatId, payload, options = {}, timeoutMs = SEND_TIMEOUT_MS) { let timer; const timeoutPromise = new Promise((_, reject) => { timer = setTimeout( @@ -128,7 +140,7 @@ function sendWithTimeout(chatId, payload, timeoutMs = SEND_TIMEOUT_MS) { ); }); return enqueueSend(() => - Promise.race([sock.sendMessage(chatId, payload), timeoutPromise]) + Promise.race([sock.sendMessage(chatId, payload, options), timeoutPromise]) .finally(() => clearTimeout(timer)) ); } @@ -164,6 +176,18 @@ function splitLongMessage(message, maxLength = MAX_MESSAGE_LENGTH) { return chunks; } +function rememberSentMessage(sent, payload) { + if (!sent?.key?.id) return; + if (sent.message) { + messageStore.remember(sent); + return; + } + const syntheticMessage = pollCreationMessageFromPayload(payload); + if (syntheticMessage) { + messageStore.remember({ ...sent, message: syntheticMessage }); + } +} + function trackSentMessageId(sent) { rememberSentId(sent?.key?.id); } @@ -227,6 +251,98 @@ const MAX_QUEUE_SIZE = 100; // Capacity bounded (see outbound_ids.js) to keep memory flat under // sustained sending. const recentlySentIds = createOutboundIdTracker(512); +const recentlyProcessedPollUpdates = createOutboundIdTracker(512); +const messageStore = createBoundedMessageStore(512); + +function normalizePollUpdateOptions(aggregation, pollUpdateMessage, meId) { + const selected = []; + for (const option of aggregation || []) { + if ((option.voters || []).length > 0 && option.name && option.name !== 'Unknown') { + selected.push(option.name); + } + } + if (selected.length > 0) return selected; + + // Fallback for already-decrypted pollUpdateMessage payloads where Baileys did + // not have the creation message available. This may only yield hashes, but + // keeping them in metadata is still better than dropping the vote entirely. + const raw = pollUpdateMessage?.vote?.selectedOptions || []; + return raw.map(option => String(option)).filter(Boolean); +} + +function pollAggregationSummary(aggregation) { + return (aggregation || []).map(option => ({ + name: option?.name || '', + voterCount: (option?.voters || []).length, + })); +} + +function logPollUpdateDiagnostic({ sourcePath, pollId, pollCreation, pollUpdates, selectedOptions, aggregation }) { + const firstUpdate = pollUpdates?.[0] || {}; + try { + console.log(JSON.stringify({ + event: 'poll_update_decode', + sourcePath, + pollId: pollId || '', + pollCreationFound: !!pollCreation, + updateKeys: Object.keys(firstUpdate), + hasVote: !!firstUpdate.vote, + selectedOptionsLength: selectedOptions?.length || 0, + aggregation: pollAggregationSummary(aggregation), + })); + } catch {} +} + +function enqueuePollUpdateEvent({ key, update, selectedOptions, aggregation }) { + const chatId = normalizeWhatsAppId(key?.remoteJid || update?.pollUpdates?.[0]?.pollUpdateMessageKey?.remoteJid || ''); + const senderId = normalizeWhatsAppId( + key?.participant + || update?.pollUpdates?.[0]?.pollUpdateMessageKey?.participant + || chatId + ); + const pollId = key?.id + || update?.pollUpdates?.[0]?.pollCreationMessageKey?.id + || update?.pollUpdates?.[0]?.pollUpdateMessageKey?.id + || ''; + const chosenText = selectedOptions.length ? selectedOptions.join(', ') : `[Poll update${pollId ? `: ${pollId}` : ''}]`; + const dedupeId = `poll:${pollId}:${senderId}:${selectedOptions.join('|')}`; + if (recentlyProcessedPollUpdates.has(dedupeId)) return; + recentlyProcessedPollUpdates.remember(dedupeId); + const event = { + messageId: `${pollId || 'poll'}:update:${Date.now()}`, + chatId, + senderId, + senderName: senderId.replace(/@.*/, ''), + chatName: chatId.replace(/@.*/, ''), + isGroup: chatId.endsWith('@g.us'), + body: chosenText, + hasMedia: false, + mediaType: 'poll_update', + mime: '', + fileName: '', + nativeType: 'pollUpdateMessage', + nativeMetadata: { + pollUpdate: { + pollId, + selectedOptions, + aggregation, + }, + }, + mediaUrls: [], + mentionedIds: [], + quotedMessageId: pollId, + quotedParticipant: '', + quotedRemoteJid: chatId, + quotedText: '', + hasQuotedMessage: !!pollId, + botIds: [], + timestamp: Math.floor(Date.now() / 1000), + }; + messageQueue.push(event); + if (messageQueue.length > MAX_QUEUE_SIZE) { + messageQueue.shift(); + } +} function rememberSentId(id) { recentlySentIds.remember(id); @@ -294,6 +410,57 @@ async function startSocket() { } }); + sock.ev.on('messages.update', async (updates) => { + for (const { key, update } of updates || []) { + if (!update?.pollUpdates) continue; + const pollCreationId = key?.id || update.pollUpdates?.[0]?.pollCreationMessageKey?.id; + const pollCreation = messageStore.get(pollCreationId); + let aggregation = []; + let pollUpdates = update.pollUpdates; + try { + if (pollCreation) { + const meId = jidNormalizedUser(sock.user?.id || 'me'); + pollUpdates = update.pollUpdates.map(pollUpdate => ( + pollUpdateForAggregation({ + pollUpdateMessage: pollUpdate, + pollUpdateMessageKey: pollUpdate.pollUpdateMessageKey, + pollCreation, + decryptPollVote, + getKeyAuthor, + meId, + pollCreatorJids: [ + jidNormalizedUser(sock.user?.lid || ''), + jidNormalizedUser(sock.user?.id || ''), + getKeyAuthor(pollUpdate.pollCreationMessageKey || key, jidNormalizedUser(sock.user?.lid || '')), + getKeyAuthor(pollUpdate.pollCreationMessageKey || key, jidNormalizedUser(sock.user?.id || '')), + ], + voterJids: [ + normalizeWhatsAppId(pollUpdate.pollUpdateMessageKey?.participant || ''), + normalizeWhatsAppId(pollUpdate.pollUpdateMessageKey?.remoteJid || key?.remoteJid || ''), + ], + }) || pollUpdate + )); + aggregation = getAggregateVotesInPollMessage({ + message: pollCreation.message, + pollUpdates, + }); + } + } catch (err) { + console.warn('[bridge] failed to aggregate poll update:', err.message); + } + const selectedOptions = normalizePollUpdateOptions(aggregation, pollUpdates?.[0]); + logPollUpdateDiagnostic({ + sourcePath: 'messages.update', + pollId: pollCreationId, + pollCreation, + pollUpdates, + selectedOptions, + aggregation, + }); + enqueuePollUpdateEvent({ key, update: { ...update, pollUpdates }, selectedOptions, aggregation }); + } + }); + sock.ev.on('messages.upsert', async ({ messages, type }) => { // In self-chat mode, your own messages commonly arrive as 'append' rather // than 'notify'. Accept both and filter agent echo-backs below. @@ -405,93 +572,83 @@ async function startSocket() { } const messageContent = getMessageContent(msg); - const contextInfo = getContextInfo(messageContent); - const mentionedIds = Array.from(new Set((contextInfo?.mentionedJid || []).map(normalizeWhatsAppId).filter(Boolean))); - const quotedMessageId = contextInfo?.stanzaId || null; - const quotedParticipant = normalizeWhatsAppId(contextInfo?.participant || '') || null; - const quotedRemoteJid = normalizeWhatsAppId(contextInfo?.remoteJid || '') || null; - const hasQuotedMessage = !!contextInfo?.quotedMessage; - - // Extract message body - let body = ''; - let hasMedia = false; - let mediaType = ''; - const mediaUrls = []; - - if (messageContent.conversation) { - body = messageContent.conversation; - } else if (messageContent.extendedTextMessage?.text) { - body = messageContent.extendedTextMessage.text; - } else if (messageContent.imageMessage) { - body = messageContent.imageMessage.caption || ''; - hasMedia = true; - mediaType = 'image'; + if (messageContent.pollUpdateMessage) { + const pollUpdateMessage = messageContent.pollUpdateMessage; + const pollKey = pollUpdateMessage.pollCreationMessageKey || { + id: pollUpdateMessage.key?.id || msg.key.id, + remoteJid: chatId, + participant: senderId, + }; + const pollCreation = messageStore.get(pollKey.id); + let aggregation = []; + let pollUpdates = [pollUpdateMessage]; try { - const buf = await downloadMediaMessage(msg, 'buffer', {}, { logger, reuploadRequest: sock.updateMediaMessage }); - const mime = messageContent.imageMessage.mimetype || 'image/jpeg'; - const extMap = { 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp', 'image/gif': '.gif' }; - const ext = extMap[mime] || '.jpg'; - mkdirSync(IMAGE_CACHE_DIR, { recursive: true }); - const filePath = path.join(IMAGE_CACHE_DIR, `img_${randomBytes(6).toString('hex')}${ext}`); - writeFileSync(filePath, buf); - mediaUrls.push(filePath); + if (pollCreation) { + const meId = jidNormalizedUser(sock.user?.id || 'me'); + const pollUpdate = pollUpdateForAggregation({ + pollUpdateMessage, + pollUpdateMessageKey: msg.key, + pollCreation, + decryptPollVote, + getKeyAuthor, + meId, + pollCreatorJids: [ + jidNormalizedUser(sock.user?.lid || ''), + jidNormalizedUser(sock.user?.id || ''), + getKeyAuthor(pollUpdateMessage.pollCreationMessageKey || pollKey, jidNormalizedUser(sock.user?.lid || '')), + getKeyAuthor(pollUpdateMessage.pollCreationMessageKey || pollKey, jidNormalizedUser(sock.user?.id || '')), + ], + voterJids: [ + normalizeWhatsAppId(msg.key?.participant || ''), + normalizeWhatsAppId(msg.key?.remoteJid || chatId || ''), + normalizeWhatsAppId(senderId || ''), + ], + }); + if (pollUpdate) pollUpdates = [pollUpdate]; + aggregation = getAggregateVotesInPollMessage({ + message: pollCreation.message, + pollUpdates, + }); + } } catch (err) { - console.error('[bridge] Failed to download image:', err.message); - } - } else if (messageContent.videoMessage) { - body = messageContent.videoMessage.caption || ''; - hasMedia = true; - mediaType = 'video'; - try { - const buf = await downloadMediaMessage(msg, 'buffer', {}, { logger, reuploadRequest: sock.updateMediaMessage }); - const mime = messageContent.videoMessage.mimetype || 'video/mp4'; - const ext = mime.includes('mp4') ? '.mp4' : '.mkv'; - mkdirSync(DOCUMENT_CACHE_DIR, { recursive: true }); - const filePath = path.join(DOCUMENT_CACHE_DIR, `vid_${randomBytes(6).toString('hex')}${ext}`); - writeFileSync(filePath, buf); - mediaUrls.push(filePath); - } catch (err) { - console.error('[bridge] Failed to download video:', err.message); - } - } else if (messageContent.audioMessage || messageContent.pttMessage) { - hasMedia = true; - mediaType = messageContent.pttMessage ? 'ptt' : 'audio'; - try { - const audioMsg = messageContent.pttMessage || messageContent.audioMessage; - const buf = await downloadMediaMessage(msg, 'buffer', {}, { logger, reuploadRequest: sock.updateMediaMessage }); - const mime = audioMsg.mimetype || 'audio/ogg'; - const ext = mime.includes('ogg') ? '.ogg' : mime.includes('mp4') ? '.m4a' : '.ogg'; - mkdirSync(AUDIO_CACHE_DIR, { recursive: true }); - const filePath = path.join(AUDIO_CACHE_DIR, `aud_${randomBytes(6).toString('hex')}${ext}`); - writeFileSync(filePath, buf); - mediaUrls.push(filePath); - } catch (err) { - console.error('[bridge] Failed to download audio:', err.message); - } - } else if (messageContent.documentMessage) { - body = messageContent.documentMessage.caption || ''; - hasMedia = true; - mediaType = 'document'; - const fileName = messageContent.documentMessage.fileName || 'document'; - try { - const buf = await downloadMediaMessage(msg, 'buffer', {}, { logger, reuploadRequest: sock.updateMediaMessage }); - mkdirSync(DOCUMENT_CACHE_DIR, { recursive: true }); - const safeFileName = path.basename(fileName).replace(/[^a-zA-Z0-9._-]/g, '_'); - const filePath = path.join(DOCUMENT_CACHE_DIR, `doc_${randomBytes(6).toString('hex')}_${safeFileName}`); - writeFileSync(filePath, buf); - mediaUrls.push(filePath); - } catch (err) { - console.error('[bridge] Failed to download document:', err.message); + console.warn('[bridge] failed to aggregate poll upsert:', err.message); } + const selectedOptions = normalizePollUpdateOptions(aggregation, pollUpdates[0]); + logPollUpdateDiagnostic({ + sourcePath: 'messages.upsert', + pollId: pollKey.id, + pollCreation, + pollUpdates, + selectedOptions, + aggregation, + }); + enqueuePollUpdateEvent({ + key: { ...pollKey, remoteJid: pollKey.remoteJid || chatId, participant: pollKey.participant || senderId }, + update: { pollUpdates }, + selectedOptions, + aggregation, + }); + continue; } - // For media without caption, use a placeholder so the API message is never empty - if (hasMedia && !body) { - body = `[${mediaType} received]`; - } + const event = await extractBridgeEvent({ + msg, + chatId, + senderId, + senderNumber, + botIds, + isGroup, + downloadMedia: async (mediaMsg) => downloadMediaMessage(mediaMsg, 'buffer', {}, { logger, reuploadRequest: sock.updateMediaMessage }), + cacheDirs: { + image: IMAGE_CACHE_DIR, + document: DOCUMENT_CACHE_DIR, + audio: AUDIO_CACHE_DIR, + }, + }); + event.fromOwner = fromOwner; // Ignore Hermes' own reply messages in self-chat mode to avoid loops. - if (msg.key.fromMe && ((REPLY_PREFIX && body.startsWith(REPLY_PREFIX)) || recentlySentIds.has(msg.key.id))) { + if (msg.key.fromMe && ((REPLY_PREFIX && event.body.startsWith(REPLY_PREFIX)) || recentlySentIds.has(msg.key.id))) { if (WHATSAPP_DEBUG) { try { console.log(JSON.stringify({ event: 'ignored', reason: 'agent_echo', chatId, messageId: msg.key.id })); } catch {} } @@ -499,7 +656,7 @@ async function startSocket() { } // Skip empty messages - if (!body && !hasMedia) { + if (!event.body && !event.hasMedia) { if (WHATSAPP_DEBUG) { try { console.log(JSON.stringify({ event: 'ignored', reason: 'empty', chatId, messageKeys: Object.keys(msg.message || {}) })); @@ -510,34 +667,7 @@ async function startSocket() { continue; } - // Resolve LID → phone for the senderId so the gateway can match - // against phone-based allowlists (WHATSAPP_ALLOWED_USERS). - const resolvedSenderId = lidToPhone[senderNumber] - ? (lidToPhone[senderNumber] + '@s.whatsapp.net') - : senderId; - const resolvedSenderNumber = lidToPhone[senderNumber] || senderNumber; - - const event = { - messageId: msg.key.id, - chatId, - senderId: resolvedSenderId, - senderName: msg.pushName || resolvedSenderNumber, - chatName: isGroup ? (chatId.split('@')[0]) : (msg.pushName || resolvedSenderNumber), - isGroup, - body, - hasMedia, - mediaType, - mediaUrls, - mentionedIds, - quotedMessageId, - quotedParticipant, - quotedRemoteJid, - hasQuotedMessage, - botIds, - timestamp: msg.messageTimestamp, - fromOwner, - }; - + messageStore.remember(msg); messageQueue.push(event); if (messageQueue.length > MAX_QUEUE_SIZE) { messageQueue.shift(); @@ -602,8 +732,14 @@ app.post('/send', async (req, res) => { const chunks = splitLongMessage(formatOutgoingMessage(message)); const messageIds = []; for (let i = 0; i < chunks.length; i += 1) { - const sent = await sendWithTimeout(chatId, { text: chunks[i] }); + const { content: payload, options } = buildTextSendPayload(chunks[i], { + chatId, + replyTo: i === 0 ? replyTo : undefined, + messageStore, + }); + const sent = await sendWithTimeout(chatId, payload, options); trackSentMessageId(sent); + messageStore.remember(sent); if (sent?.key?.id) messageIds.push(sent.key.id); if (chunks.length > 1 && i < chunks.length - 1) { await sleep(CHUNK_DELAY_MS); @@ -654,25 +790,6 @@ app.post('/edit', async (req, res) => { } }); -// MIME type map and media type inference for /send-media -const MIME_MAP = { - jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', - webp: 'image/webp', gif: 'image/gif', - mp4: 'video/mp4', mov: 'video/quicktime', avi: 'video/x-msvideo', - mkv: 'video/x-matroska', '3gp': 'video/3gpp', - pdf: 'application/pdf', - doc: 'application/msword', - docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', -}; - -function inferMediaType(ext) { - if (['jpg', 'jpeg', 'png', 'webp', 'gif'].includes(ext)) return 'image'; - if (['mp4', 'mov', 'avi', 'mkv', '3gp'].includes(ext)) return 'video'; - if (['ogg', 'opus', 'mp3', 'wav', 'm4a'].includes(ext)) return 'audio'; - return 'document'; -} - // Send media (image, video, document) natively app.post('/send-media', async (req, res) => { if (!sock || connectionState !== 'connected') { @@ -696,10 +813,37 @@ app.post('/send-media', async (req, res) => { switch (type) { case 'image': - msgPayload = { image: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'image/jpeg' }; + if (ext === 'gif') { + // WhatsApp's native animated-GIF UX is an MP4 video payload with + // gifPlayback=true. Convert when ffmpeg is available; otherwise fall + // back to a truthful image/gif send instead of mislabeling GIF bytes + // as video/mp4. + let tmpGifMp4 = null; + try { + tmpGifMp4 = path.join(tmpdir(), `hermes_gif_${randomBytes(6).toString('hex')}.mp4`); + execFileSync( + 'ffmpeg', + ['-y', '-i', filePath, '-movflags', 'faststart', '-pix_fmt', 'yuv420p', '-vf', 'scale=trunc(iw/2)*2:trunc(ih/2)*2', tmpGifMp4], + { timeout: 30000, stdio: 'pipe' } + ); + msgPayload = { + video: readFileSync(tmpGifMp4), + caption: caption || undefined, + mimetype: 'video/mp4', + gifPlayback: true, + }; + } catch (gifErr) { + console.warn('[bridge] gif conversion failed, sending as image/gif:', gifErr.message); + msgPayload = mediaPayloadForFile({ buffer, filePath, mediaType: type, caption, fileName }); + } finally { + try { if (tmpGifMp4 && existsSync(tmpGifMp4)) unlinkSync(tmpGifMp4); } catch (_) {} + } + } else { + msgPayload = mediaPayloadForFile({ buffer, filePath, mediaType: type, caption, fileName }); + } break; case 'video': - msgPayload = { video: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'video/mp4' }; + msgPayload = mediaPayloadForFile({ buffer, filePath, mediaType: type, caption, fileName }); break; case 'audio': { // WhatsApp only renders a native voice bubble (ptt) when the file is ogg/opus. @@ -712,8 +856,9 @@ app.post('/send-media', async (req, res) => { if (needsConversion) { tmpPath = path.join(tmpdir(), `hermes_voice_${randomBytes(6).toString('hex')}.ogg`); try { - execSync( - `ffmpeg -y -i ${JSON.stringify(filePath)} -ar 48000 -ac 1 -c:a libopus ${JSON.stringify(tmpPath)}`, + execFileSync( + 'ffmpeg', + ['-y', '-i', filePath, '-ar', '48000', '-ac', '1', '-c:a', 'libopus', tmpPath], { timeout: 30000, stdio: 'pipe' } ); audioBuffer = readFileSync(tmpPath); @@ -731,23 +876,65 @@ app.post('/send-media', async (req, res) => { } case 'document': default: - msgPayload = { - document: buffer, - fileName: fileName || path.basename(filePath), - caption: caption || undefined, - mimetype: MIME_MAP[ext] || 'application/octet-stream', - }; + msgPayload = mediaPayloadForFile({ buffer, filePath, mediaType: 'document', caption, fileName }); break; } const sent = await sendWithTimeout(chatId, msgPayload); trackSentMessageId(sent); + messageStore.remember(sent); res.json({ success: true, messageId: sent?.key?.id }); } catch (err) { res.status(500).json({ error: err.message }); } }); +// Send poll primitive. Approval UX is intentionally not wired here; gateway +// approvals need text fallback and explicit confirmation semantics above this +// low-level transport helper. +app.post('/send-poll', async (req, res) => { + if (!sock || connectionState !== 'connected') { + return res.status(503).json({ error: 'Not connected to WhatsApp' }); + } + + const { chatId, question, options, selectableCount } = req.body; + if (!chatId || !question || !Array.isArray(options)) { + return res.status(400).json({ error: 'chatId, question, and options are required' }); + } + + try { + const payload = buildPollPayload({ question, options, selectableCount }); + const sent = await sendWithTimeout(chatId, payload); + trackSentMessageId(sent); + rememberSentMessage(sent, payload); + res.json({ success: true, messageId: sent?.key?.id }); + } catch (err) { + res.status(400).json({ error: err.message }); + } +}); + +// Send native WhatsApp location pin +app.post('/send-location', async (req, res) => { + if (!sock || connectionState !== 'connected') { + return res.status(503).json({ error: 'Not connected to WhatsApp' }); + } + + const { chatId, latitude, longitude, name, address } = req.body; + if (!chatId || latitude === undefined || longitude === undefined) { + return res.status(400).json({ error: 'chatId, latitude, and longitude are required' }); + } + + try { + const payload = buildLocationPayload({ latitude, longitude, name, address }); + const sent = await sendWithTimeout(chatId, payload); + trackSentMessageId(sent); + messageStore.remember(sent); + res.json({ success: true, messageId: sent?.key?.id }); + } catch (err) { + res.status(400).json({ error: err.message }); + } +}); + // Typing indicator app.post('/typing', async (req, res) => { if (!sock || connectionState !== 'connected') { diff --git a/scripts/whatsapp-bridge/bridge.native.test.mjs b/scripts/whatsapp-bridge/bridge.native.test.mjs new file mode 100644 index 00000000000..f9afdf0d240 --- /dev/null +++ b/scripts/whatsapp-bridge/bridge.native.test.mjs @@ -0,0 +1,327 @@ +/** + * Unit tests for WhatsApp-native bridge payload helpers. + * + * These tests avoid importing bridge.js because that file starts an HTTP + * server and Baileys socket at module load. Keep the helper module pure. + */ + +import { strict as assert } from 'node:assert'; +import { createHash } from 'node:crypto'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { getAggregateVotesInPollMessage } from '@whiskeysockets/baileys'; + +import { + buildPollPayload, + buildTextSendPayload, + createBoundedMessageStore, + extractBridgeEvent, + mediaPayloadForFile, + pollCreationMessageFromPayload, + pollUpdateForAggregation, +} from './bridge_helpers.js'; + +// -- quoted outbound text ------------------------------------------------- +{ + const store = createBoundedMessageStore(2); + store.remember({ + key: { + id: 'inbound-1', + remoteJid: '15551234567@s.whatsapp.net', + participant: '15550001111@s.whatsapp.net', + fromMe: false, + }, + message: { conversation: 'original text' }, + }); + + const { content, options } = buildTextSendPayload('reply text', { + chatId: '15551234567@s.whatsapp.net', + replyTo: 'inbound-1', + messageStore: store, + }); + + assert.deepEqual(content, { text: 'reply text' }); + assert.equal(options.quoted.key.id, 'inbound-1'); + assert.equal(options.quoted.message.conversation, 'original text'); + console.log(' ✓ text replies include Baileys quoted message when resolvable'); +} + +{ + const store = createBoundedMessageStore(2); + const { content, options } = buildTextSendPayload('plain text', { + chatId: '15551234567@s.whatsapp.net', + replyTo: 'missing-id', + messageStore: store, + }); + + assert.deepEqual(content, { text: 'plain text' }); + assert.deepEqual(options, {}); + console.log(' ✓ unresolved replyTo falls back to plain text'); +} + +// -- inbound quote/media/native metadata -------------------------------- +{ + const event = await extractBridgeEvent({ + msg: { + key: { + id: 'incoming-1', + remoteJid: '15551234567@s.whatsapp.net', + participant: '15550001111@s.whatsapp.net', + fromMe: false, + }, + pushName: 'Tester', + messageTimestamp: 123, + message: { + extendedTextMessage: { + text: 'approved', + contextInfo: { + stanzaId: 'outbound-1', + participant: '15559998888@s.whatsapp.net', + remoteJid: '15551234567@s.whatsapp.net', + quotedMessage: { conversation: 'approve deploy?' }, + }, + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + botIds: ['15559998888@s.whatsapp.net'], + downloadMedia: async () => Buffer.from(''), + }); + + assert.equal(event.quotedMessageId, 'outbound-1'); + assert.equal(event.quotedParticipant, '15559998888@s.whatsapp.net'); + assert.equal(event.quotedRemoteJid, '15551234567@s.whatsapp.net'); + assert.equal(event.quotedText, 'approve deploy?'); + assert.equal(event.hasQuotedMessage, true); + assert.equal(event.body, 'approved'); + console.log(' ✓ inbound quoted metadata includes quoted text'); +} + +{ + const event = await extractBridgeEvent({ + msg: { + key: { id: 'doc-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false }, + messageTimestamp: 123, + message: { + documentMessage: { + caption: 'see attached', + fileName: 'report.pdf', + mimetype: 'application/pdf', + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + downloadMedia: async () => Buffer.from('pdf'), + writeMediaFile: async () => '/tmp/report.pdf', + }); + + assert.equal(event.hasMedia, true); + assert.equal(event.mediaType, 'document'); + assert.equal(event.mime, 'application/pdf'); + assert.equal(event.fileName, 'report.pdf'); + assert.equal(event.nativeType, 'documentMessage'); + assert.deepEqual(event.mediaUrls, ['/tmp/report.pdf']); + console.log(' ✓ inbound document metadata preserves MIME and filename'); +} + +{ + const cacheDir = mkdtempSync(path.join(tmpdir(), 'hermes-wa-doc-')); + const event = await extractBridgeEvent({ + msg: { + key: { id: 'doc-2', remoteJid: '15551234567@s.whatsapp.net', fromMe: false }, + messageTimestamp: 123, + message: { + documentMessage: { + caption: 'see attached', + fileName: 'report', + mimetype: 'application/pdf', + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + downloadMedia: async () => Buffer.from('pdf'), + cacheDirs: { document: cacheDir }, + }); + + assert.equal(event.mediaUrls.length, 1); + assert.ok(event.mediaUrls[0].endsWith('_report.pdf'), event.mediaUrls[0]); + console.log(' ✓ MIME extension is preserved when document filename has none'); +} + +{ + const event = await extractBridgeEvent({ + msg: { + key: { id: 'loc-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false }, + messageTimestamp: 123, + message: { + locationMessage: { + name: 'HQ', + degreesLatitude: 41.015, + degreesLongitude: 28.979, + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + }); + + assert.equal(event.mediaType, 'location'); + assert.equal(event.body, '[Location: HQ 41.015,28.979]'); + assert.deepEqual(event.nativeMetadata.location, { + name: 'HQ', + address: '', + latitude: 41.015, + longitude: 28.979, + isLive: false, + }); + console.log(' ✓ native location messages get text fallback and metadata'); +} + +{ + const event = await extractBridgeEvent({ + msg: { + key: { id: 'poll-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false }, + messageTimestamp: 123, + message: { + pollCreationMessage: { + name: 'Approve deploy?', + options: [{ optionName: 'Approve' }, { optionName: 'Deny' }], + selectableOptionsCount: 1, + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + }); + + assert.equal(event.mediaType, 'poll'); + assert.equal(event.body, '[Poll: Approve deploy? Options: Approve, Deny]'); + assert.deepEqual(event.nativeMetadata.poll.options, ['Approve', 'Deny']); + console.log(' ✓ poll creation messages get text fallback and metadata'); +} + +// -- outbound media/poll helpers ----------------------------------------- +{ + const payload = mediaPayloadForFile({ + buffer: Buffer.from('gif89a'), + filePath: '/tmp/loop.gif', + mediaType: 'image', + caption: 'loop', + }); + + assert.ok(payload.image, 'pure helper fallback keeps raw GIF as image bytes'); + assert.equal(payload.gifPlayback, undefined); + assert.equal(payload.mimetype, 'image/gif'); + assert.equal(payload.caption, 'loop'); + console.log(' ✓ local GIF helper fallback stays truthful; live bridge converts to gifPlayback when possible'); +} + +{ + const payload = buildPollPayload({ + question: 'Proceed?', + options: ['Approve', 'Deny'], + selectableCount: 1, + }); + + assert.equal(payload.poll.name, 'Proceed?'); + assert.deepEqual(payload.poll.values, ['Approve', 'Deny']); + assert.equal(payload.poll.selectableCount, 1); + assert.equal(Buffer.isBuffer(payload.poll.messageSecret), true); + assert.equal(payload.poll.messageSecret.length, 32); + assert.deepEqual(pollCreationMessageFromPayload(payload), { + messageContextInfo: { + messageSecret: payload.poll.messageSecret, + }, + pollCreationMessageV3: { + name: 'Proceed?', + options: [{ optionName: 'Approve' }, { optionName: 'Deny' }], + selectableOptionsCount: 1, + }, + }); + console.log(' ✓ poll payload primitive carries a cacheable vote secret'); +} + +{ + const pollCreation = { + key: { + id: 'poll-creation', + remoteJid: '15551234567@s.whatsapp.net', + fromMe: true, + }, + message: { + messageContextInfo: { + messageSecret: Buffer.from('0123456789abcdef0123456789abcdef'), + }, + pollCreationMessageV3: { + name: 'Proceed?', + options: [{ optionName: 'Approve' }, { optionName: 'Deny' }], + selectableOptionsCount: 1, + }, + }, + }; + const voteKey = { + id: 'vote-message', + remoteJid: '15551234567@s.whatsapp.net', + participant: '15550001111@s.whatsapp.net', + fromMe: false, + }; + const encryptedVote = { + encPayload: Buffer.from('payload'), + encIv: Buffer.from('iv'), + }; + + const attempts = []; + const pollUpdate = pollUpdateForAggregation({ + pollUpdateMessage: { + pollCreationMessageKey: pollCreation.key, + vote: encryptedVote, + senderTimestampMs: 123, + }, + pollUpdateMessageKey: voteKey, + pollCreation, + decryptPollVote: (vote, ctx) => { + attempts.push({ pollCreatorJid: ctx.pollCreatorJid, voterJid: ctx.voterJid }); + assert.equal(vote, encryptedVote); + assert.equal(ctx.pollMsgId, 'poll-creation'); + assert.equal(ctx.pollEncKey, pollCreation.message.messageContextInfo.messageSecret); + if (ctx.pollCreatorJid !== 'creator-lid@lid') { + throw new Error('wrong creator jid'); + } + assert.equal(ctx.voterJid, '15550001111@s.whatsapp.net'); + return { + selectedOptions: [createHash('sha256').update(Buffer.from('Approve')).digest()], + }; + }, + getKeyAuthor: (key, meId = 'me') => (key?.fromMe ? meId : key?.participant || key?.remoteJid || ''), + meId: 'classic-me@s.whatsapp.net', + pollCreatorJids: ['classic-me@s.whatsapp.net', 'creator-lid@lid'], + }); + + assert.deepEqual(attempts.map(item => item.pollCreatorJid), ['classic-me@s.whatsapp.net', 'creator-lid@lid']); + + assert.equal(pollUpdate.pollUpdateMessageKey.id, 'vote-message'); + assert.equal(pollUpdate.senderTimestampMs, 123); + const aggregation = getAggregateVotesInPollMessage({ + message: pollCreation.message, + pollUpdates: [pollUpdate], + }); + assert.deepEqual( + aggregation.map(option => ({ name: option.name, voters: option.voters })), + [ + { name: 'Approve', voters: ['15550001111@s.whatsapp.net'] }, + { name: 'Deny', voters: [] }, + ], + ); + console.log(' ✓ encrypted poll upserts are wrapped into Baileys aggregation shape'); +} + +console.log('\n✅ All WhatsApp native bridge helper tests passed.'); diff --git a/scripts/whatsapp-bridge/bridge_helpers.js b/scripts/whatsapp-bridge/bridge_helpers.js new file mode 100644 index 00000000000..50a5584b5e9 --- /dev/null +++ b/scripts/whatsapp-bridge/bridge_helpers.js @@ -0,0 +1,525 @@ +import path from 'path'; +import { mkdirSync, writeFileSync } from 'fs'; +import { randomBytes } from 'crypto'; + +export const MIME_MAP = { + jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', + webp: 'image/webp', gif: 'image/gif', + mp4: 'video/mp4', mov: 'video/quicktime', avi: 'video/x-msvideo', + mkv: 'video/x-matroska', '3gp': 'video/3gpp', + pdf: 'application/pdf', + doc: 'application/msword', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +}; + +export function normalizeWhatsAppId(value) { + if (!value) return ''; + return String(value).replace(':', '@'); +} + +export function getMessageContent(msg) { + const content = msg?.message || {}; + if (content.ephemeralMessage?.message) return content.ephemeralMessage.message; + if (content.viewOnceMessage?.message) return content.viewOnceMessage.message; + if (content.viewOnceMessageV2?.message) return content.viewOnceMessageV2.message; + if (content.documentWithCaptionMessage?.message) return content.documentWithCaptionMessage.message; + if (content.templateMessage?.hydratedTemplate) return content.templateMessage.hydratedTemplate; + if (content.buttonsMessage) return content.buttonsMessage; + if (content.listMessage) return content.listMessage; + return content; +} + +export function getContextInfo(messageContent) { + if (!messageContent || typeof messageContent !== 'object') return {}; + for (const value of Object.values(messageContent)) { + if (value && typeof value === 'object' && value.contextInfo) { + return value.contextInfo; + } + } + return {}; +} + +export function createBoundedMessageStore(limit = 512) { + const byId = new Map(); + + function remember(msg) { + const id = msg?.key?.id; + if (!id) return; + byId.delete(id); + byId.set(id, msg); + while (byId.size > limit) { + const oldest = byId.keys().next().value; + byId.delete(oldest); + } + } + + function get(id) { + if (!id || !byId.has(id)) return null; + const msg = byId.get(id); + byId.delete(id); + byId.set(id, msg); + return msg; + } + + return { remember, get }; +} + +export function pollCreationMessageSecret(pollCreation) { + return pollCreation?.message?.messageContextInfo?.messageSecret + || pollCreation?.messageContextInfo?.messageSecret + || null; +} + +function uniqueStrings(values) { + const seen = new Set(); + const out = []; + for (const value of values || []) { + const text = String(value || '').trim(); + if (!text || seen.has(text)) continue; + seen.add(text); + out.push(text); + } + return out; +} + +export function pollUpdateForAggregation({ + pollUpdateMessage, + pollUpdateMessageKey, + pollCreation, + decryptPollVote, + getKeyAuthor, + meId = 'me', + pollCreatorJids = [], + voterJids = [], +}) { + if (!pollUpdateMessage) return null; + const updateKey = pollUpdateMessage.pollUpdateMessageKey + || pollUpdateMessageKey + || pollUpdateMessage.key; + if (!updateKey) return null; + + if (pollUpdateMessage.vote?.selectedOptions) { + return { + pollUpdateMessageKey: updateKey, + vote: pollUpdateMessage.vote, + senderTimestampMs: pollUpdateMessage.senderTimestampMs, + }; + } + + const creationKey = pollUpdateMessage.pollCreationMessageKey; + const secret = pollCreationMessageSecret(pollCreation); + if ( + !creationKey?.id + || !secret + || !pollUpdateMessage.vote?.encPayload + || !pollUpdateMessage.vote?.encIv + || typeof decryptPollVote !== 'function' + || typeof getKeyAuthor !== 'function' + ) { + return null; + } + + // Baileys poll decryption keys include both creator and voter JIDs. On + // WhatsApp LID chats, the poll creator can be the linked-device LID even + // when sock.user.id is the classic @s.whatsapp.net JID. Try the exact + // candidates the live bridge knows before falling back to the generic helper. + const creatorCandidates = uniqueStrings([ + ...pollCreatorJids, + getKeyAuthor(creationKey, meId), + ]); + const voterCandidates = uniqueStrings([ + ...voterJids, + getKeyAuthor(updateKey, meId), + ]); + + let lastError = null; + for (const pollCreatorJid of creatorCandidates) { + for (const voterJid of voterCandidates) { + try { + const vote = decryptPollVote(pollUpdateMessage.vote, { + pollCreatorJid, + pollMsgId: creationKey.id, + pollEncKey: secret, + voterJid, + }); + return { + pollUpdateMessageKey: updateKey, + vote, + senderTimestampMs: pollUpdateMessage.senderTimestampMs, + }; + } catch (err) { + lastError = err; + } + } + } + if (lastError) throw lastError; + return null; +} + +export function buildTextSendPayload(text, { replyTo, messageStore } = {}) { + const content = { text }; + const options = {}; + const quoted = messageStore?.get(replyTo); + if (quoted?.key && quoted?.message) { + // Baileys expects quoted messages as sendMessage options, not inside the + // message content payload. Keeping this split avoids silently sending a + // literal/ignored `quoted` field instead of a native WhatsApp reply. + options.quoted = quoted; + } + return { content, options }; +} + +export function buildLocationPayload({ latitude, longitude, name, address } = {}) { + const lat = Number(latitude); + const lon = Number(longitude); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) { + throw new Error('latitude and longitude must be numbers'); + } + if (lat < -90 || lat > 90 || lon < -180 || lon > 180) { + throw new Error('latitude/longitude out of range'); + } + + const location = { + degreesLatitude: lat, + degreesLongitude: lon, + }; + if (name) location.name = String(name); + if (address) location.address = String(address); + return { location }; +} + +function textFromQuotedMessage(quotedMessage) { + if (!quotedMessage) return ''; + if (quotedMessage.conversation) return quotedMessage.conversation; + if (quotedMessage.extendedTextMessage?.text) return quotedMessage.extendedTextMessage.text; + if (quotedMessage.imageMessage?.caption) return quotedMessage.imageMessage.caption; + if (quotedMessage.videoMessage?.caption) return quotedMessage.videoMessage.caption; + if (quotedMessage.documentMessage?.caption) return quotedMessage.documentMessage.caption; + if (quotedMessage.documentMessage?.fileName) return `[Document: ${quotedMessage.documentMessage.fileName}]`; + if (quotedMessage.locationMessage) return formatLocationText(quotedMessage.locationMessage, false); + if (quotedMessage.contactMessage) return formatContactText(quotedMessage.contactMessage); + if (quotedMessage.pollCreationMessage) return formatPollText(quotedMessage.pollCreationMessage); + return ''; +} + +function mediaExtForMime(mime, fallback) { + const normalized = String(mime || '').split(';', 1)[0].toLowerCase(); + const extMap = { + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/webp': '.webp', + 'image/gif': '.gif', + 'video/mp4': '.mp4', + 'video/quicktime': '.mov', + 'video/x-matroska': '.mkv', + 'audio/ogg': '.ogg', + 'audio/mp4': '.m4a', + 'audio/mpeg': '.mp3', + 'application/pdf': '.pdf', + }; + return extMap[normalized] || fallback; +} + +function defaultWriteMediaFile({ buffer, dir, prefix, ext, fileName }) { + mkdirSync(dir, { recursive: true }); + let safeName = fileName ? `_${path.basename(fileName).replace(/[^a-zA-Z0-9._-]/g, '_')}` : ''; + if (safeName && ext && !path.extname(safeName)) { + safeName = `${safeName}${ext}`; + } + const filePath = path.join(dir, `${prefix}_${randomBytes(6).toString('hex')}${safeName || ext}`); + writeFileSync(filePath, buffer); + return filePath; +} + +function formatLocationText(location, isLive) { + const name = location.name || location.address || ''; + const lat = location.degreesLatitude ?? location.latitude; + const lng = location.degreesLongitude ?? location.longitude; + const kind = isLive ? 'Live location' : 'Location'; + const coords = lat !== undefined && lng !== undefined ? `${lat},${lng}` : ''; + return `[${kind}: ${[name, coords].filter(Boolean).join(' ')}]`; +} + +function locationMetadata(location, isLive) { + return { + name: location.name || '', + address: location.address || '', + latitude: location.degreesLatitude ?? location.latitude ?? null, + longitude: location.degreesLongitude ?? location.longitude ?? null, + isLive, + }; +} + +function formatContactText(contact) { + const name = contact.displayName || contact.vcard?.match(/FN:(.+)/)?.[1] || 'unknown'; + const phone = contact.vcard?.match(/TEL[^:]*:(.+)/)?.[1] || ''; + return `[Contact: ${[name, phone].filter(Boolean).join(' ')}]`; +} + +function formatContactsText(contacts) { + const names = contacts.map(c => c.displayName).filter(Boolean); + return `[Contacts: ${names.join(', ') || contacts.length}]`; +} + +function formatReactionText(reaction) { + const emoji = reaction.text || ''; + const target = reaction.key?.id || ''; + return `[Reaction: ${emoji}${target ? ` to ${target}` : ''}]`; +} + +function pollOptions(poll) { + return (poll.options || []) + .map(option => option.optionName || option.name) + .filter(Boolean); +} + +function formatPollText(poll) { + const question = poll.name || poll.title || 'poll'; + const options = pollOptions(poll); + return `[Poll: ${question}${options.length ? ` Options: ${options.join(', ')}` : ''}]`; +} + +function formatPollUpdateText(update) { + const target = update.pollCreationMessageKey?.id || update.key?.id || ''; + return `[Poll update${target ? `: ${target}` : ''}]`; +} + +export async function extractBridgeEvent({ + msg, + chatId, + senderId, + senderNumber, + botIds = [], + isGroup = false, + downloadMedia, + writeMediaFile, + cacheDirs = {}, +}) { + const messageContent = getMessageContent(msg); + const contextInfo = getContextInfo(messageContent); + const mentionedIds = Array.from(new Set((contextInfo?.mentionedJid || []).map(normalizeWhatsAppId).filter(Boolean))); + const quotedMessageId = contextInfo?.stanzaId || null; + const quotedParticipant = normalizeWhatsAppId(contextInfo?.participant || '') || null; + const quotedRemoteJid = normalizeWhatsAppId(contextInfo?.remoteJid || '') || null; + const hasQuotedMessage = !!contextInfo?.quotedMessage; + const quotedText = textFromQuotedMessage(contextInfo?.quotedMessage); + + let body = ''; + let hasMedia = false; + let mediaType = ''; + let mime = ''; + let fileName = ''; + let nativeType = ''; + const mediaUrls = []; + const nativeMetadata = {}; + + const saveMedia = async ({ mediaMessage, dir, prefix, fallbackExt, fileName: name }) => { + if (!downloadMedia) return; + const buf = await downloadMedia(msg); + const ext = mediaExtForMime(mediaMessage?.mimetype, fallbackExt); + const writer = writeMediaFile || defaultWriteMediaFile; + const saved = await writer({ buffer: buf, dir, prefix, ext, fileName: name }); + if (saved) mediaUrls.push(saved); + }; + + if (messageContent.conversation) { + body = messageContent.conversation; + nativeType = 'conversation'; + } else if (messageContent.extendedTextMessage?.text) { + body = messageContent.extendedTextMessage.text; + nativeType = 'extendedTextMessage'; + } else if (messageContent.imageMessage) { + const item = messageContent.imageMessage; + body = item.caption || ''; + hasMedia = true; + mediaType = 'image'; + nativeType = 'imageMessage'; + mime = item.mimetype || 'image/jpeg'; + await saveMedia({ mediaMessage: item, dir: cacheDirs.image, prefix: 'img', fallbackExt: '.jpg' }); + } else if (messageContent.videoMessage) { + const item = messageContent.videoMessage; + body = item.caption || ''; + hasMedia = true; + mediaType = item.gifPlayback ? 'gif' : 'video'; + nativeType = 'videoMessage'; + mime = item.mimetype || 'video/mp4'; + nativeMetadata.video = { gifPlayback: !!item.gifPlayback }; + await saveMedia({ mediaMessage: item, dir: cacheDirs.document, prefix: 'vid', fallbackExt: '.mp4' }); + } else if (messageContent.audioMessage || messageContent.pttMessage) { + const item = messageContent.pttMessage || messageContent.audioMessage; + hasMedia = true; + mediaType = item.ptt || messageContent.pttMessage ? 'ptt' : 'audio'; + nativeType = messageContent.pttMessage ? 'pttMessage' : 'audioMessage'; + mime = item.mimetype || 'audio/ogg'; + nativeMetadata.audio = { ptt: mediaType === 'ptt' }; + await saveMedia({ mediaMessage: item, dir: cacheDirs.audio, prefix: 'aud', fallbackExt: '.ogg' }); + } else if (messageContent.documentMessage) { + const item = messageContent.documentMessage; + body = item.caption || ''; + hasMedia = true; + mediaType = 'document'; + nativeType = 'documentMessage'; + mime = item.mimetype || 'application/octet-stream'; + fileName = item.fileName || 'document'; + await saveMedia({ mediaMessage: item, dir: cacheDirs.document, prefix: 'doc', fallbackExt: '.bin', fileName }); + } else if (messageContent.stickerMessage) { + hasMedia = true; + mediaType = 'sticker'; + nativeType = 'stickerMessage'; + mime = messageContent.stickerMessage.mimetype || 'image/webp'; + body = '[Sticker]'; + nativeMetadata.sticker = { + animated: !!messageContent.stickerMessage.isAnimated, + mimetype: mime, + }; + await saveMedia({ mediaMessage: messageContent.stickerMessage, dir: cacheDirs.image, prefix: 'sticker', fallbackExt: '.webp' }); + } else if (messageContent.locationMessage || messageContent.liveLocationMessage) { + const isLive = !!messageContent.liveLocationMessage; + const item = messageContent.liveLocationMessage || messageContent.locationMessage; + mediaType = isLive ? 'live_location' : 'location'; + nativeType = isLive ? 'liveLocationMessage' : 'locationMessage'; + body = formatLocationText(item, isLive); + nativeMetadata.location = locationMetadata(item, isLive); + } else if (messageContent.contactMessage) { + mediaType = 'contact'; + nativeType = 'contactMessage'; + body = formatContactText(messageContent.contactMessage); + nativeMetadata.contact = { + displayName: messageContent.contactMessage.displayName || '', + vcard: messageContent.contactMessage.vcard || '', + }; + } else if (messageContent.contactsArrayMessage) { + const contacts = messageContent.contactsArrayMessage.contacts || []; + mediaType = 'contacts'; + nativeType = 'contactsArrayMessage'; + body = formatContactsText(contacts); + nativeMetadata.contacts = contacts.map(contact => ({ + displayName: contact.displayName || '', + vcard: contact.vcard || '', + })); + } else if (messageContent.reactionMessage) { + mediaType = 'reaction'; + nativeType = 'reactionMessage'; + body = formatReactionText(messageContent.reactionMessage); + nativeMetadata.reaction = { + text: messageContent.reactionMessage.text || '', + messageId: messageContent.reactionMessage.key?.id || '', + remoteJid: normalizeWhatsAppId(messageContent.reactionMessage.key?.remoteJid || ''), + participant: normalizeWhatsAppId(messageContent.reactionMessage.key?.participant || ''), + }; + } else if (messageContent.pollCreationMessage || messageContent.pollCreationMessageV2 || messageContent.pollCreationMessageV3) { + const item = messageContent.pollCreationMessage || messageContent.pollCreationMessageV2 || messageContent.pollCreationMessageV3; + mediaType = 'poll'; + nativeType = messageContent.pollCreationMessage ? 'pollCreationMessage' : messageContent.pollCreationMessageV2 ? 'pollCreationMessageV2' : 'pollCreationMessageV3'; + body = formatPollText(item); + nativeMetadata.poll = { + question: item.name || item.title || '', + options: pollOptions(item), + selectableCount: item.selectableOptionsCount || item.selectableCount || 1, + }; + } else if (messageContent.pollUpdateMessage) { + mediaType = 'poll_update'; + nativeType = 'pollUpdateMessage'; + body = formatPollUpdateText(messageContent.pollUpdateMessage); + nativeMetadata.pollUpdate = messageContent.pollUpdateMessage; + } + + if (hasMedia && !body) { + body = `[${mediaType} received]`; + } + + return { + messageId: msg.key.id, + chatId, + senderId, + senderName: msg.pushName || senderNumber, + chatName: isGroup ? (chatId.split('@')[0]) : (msg.pushName || senderNumber), + isGroup, + body, + hasMedia, + mediaType, + mime, + fileName, + nativeType, + nativeMetadata, + mediaUrls, + mentionedIds, + quotedMessageId, + quotedParticipant, + quotedRemoteJid, + quotedText, + hasQuotedMessage, + botIds, + timestamp: msg.messageTimestamp, + }; +} + +export function inferMediaType(ext) { + if (['jpg', 'jpeg', 'png', 'webp', 'gif'].includes(ext)) return 'image'; + if (['mp4', 'mov', 'avi', 'mkv', '3gp'].includes(ext)) return 'video'; + if (['ogg', 'opus', 'mp3', 'wav', 'm4a'].includes(ext)) return 'audio'; + return 'document'; +} + +export function mediaPayloadForFile({ buffer, filePath, mediaType, caption, fileName }) { + const ext = filePath.toLowerCase().split('.').pop(); + const type = mediaType || inferMediaType(ext); + if (type === 'image' && ext === 'gif') { + // Pure helper fallback: do not lie and label raw GIF bytes as mp4. + // The live bridge tries ffmpeg conversion to WhatsApp gifPlayback video + // before it falls back to this regular image payload. + return { image: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'image/gif' }; + } + switch (type) { + case 'image': + return { image: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'image/jpeg' }; + case 'video': + return { video: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'video/mp4' }; + case 'document': + return { + document: buffer, + fileName: fileName || path.basename(filePath), + caption: caption || undefined, + mimetype: MIME_MAP[ext] || 'application/octet-stream', + }; + default: + return null; + } +} + +export function buildPollPayload({ question, options, selectableCount = 1 }) { + const cleanQuestion = String(question || '').trim(); + const cleanOptions = (options || []).map(option => String(option || '').trim()).filter(Boolean); + if (!cleanQuestion) throw new Error('question is required'); + if (cleanOptions.length < 2) throw new Error('at least two poll options are required'); + if (cleanOptions.length > 12) throw new Error('at most 12 poll options are supported'); + const count = Math.max(1, Math.min(Number(selectableCount) || 1, cleanOptions.length)); + return { + poll: { + name: cleanQuestion, + values: cleanOptions, + selectableCount: count, + messageSecret: randomBytes(32), + }, + }; +} + +export function pollCreationMessageFromPayload(payload) { + const poll = payload?.poll; + if (!poll) return null; + const values = Array.isArray(poll.values) ? poll.values : []; + const options = values.map(value => String(value || '').trim()).filter(Boolean); + if (!poll.name || options.length < 2) return null; + const selectableOptionsCount = Math.max(1, Math.min(Number(poll.selectableCount) || 1, options.length)); + const message = {}; + if (poll.messageSecret) { + message.messageContextInfo = { messageSecret: poll.messageSecret }; + } + message[selectableOptionsCount === 1 ? 'pollCreationMessageV3' : 'pollCreationMessage'] = { + name: String(poll.name), + options: options.map(optionName => ({ optionName })), + selectableOptionsCount, + }; + return message; +} diff --git a/tests/gateway/test_whatsapp_native_delivery.py b/tests/gateway/test_whatsapp_native_delivery.py new file mode 100644 index 00000000000..1f05054972d --- /dev/null +++ b/tests/gateway/test_whatsapp_native_delivery.py @@ -0,0 +1,121 @@ +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.config import PlatformConfig +from plugins.platforms.whatsapp.adapter import WhatsAppAdapter +from tests.gateway.test_whatsapp_formatting import _AsyncCM, _make_adapter + + +class TestWhatsAppNativeFormatting: + def test_single_asterisk_markdown_italic_uses_whatsapp_underscore(self): + adapter = _make_adapter() + + assert adapter.format_message("this is *italic* text") == "this is _italic_ text" + assert adapter.format_message("- * list bullet stays literal") == "- * list bullet stays literal" + + def test_invisible_unicode_prefixes_are_sanitized(self): + adapter = _make_adapter() + + assert adapter.format_message("\u2060\u202ftext") == " text" + + +@pytest.mark.asyncio +async def test_send_poll_posts_to_bridge_poll_endpoint(): + adapter = _make_adapter() + resp = MagicMock(status=200) + resp.json = AsyncMock(return_value={"success": True, "messageId": "poll-msg"}) + adapter._http_session.post = MagicMock(return_value=_AsyncCM(resp)) + + result = await adapter.send_poll( + "15551234567", + "Proceed?", + ["Approve", "Deny"], + ) + + assert result.success + assert result.message_id == "poll-msg" + call = adapter._http_session.post.call_args + assert call.args[0] == "http://127.0.0.1:3000/send-poll" + assert call.kwargs["json"] == { + "chatId": "15551234567@s.whatsapp.net", + "question": "Proceed?", + "options": ["Approve", "Deny"], + "selectableCount": 1, + } + + +@pytest.mark.asyncio +async def test_send_location_posts_to_bridge_location_endpoint(): + adapter = _make_adapter() + resp = MagicMock(status=200) + resp.json = AsyncMock(return_value={"success": True, "messageId": "loc-msg"}) + adapter._http_session.post = MagicMock(return_value=_AsyncCM(resp)) + + result = await adapter.send_location( + "15551234567", + 41.015, + 28.979, + name="HQ", + address="Example Street", + ) + + assert result.success + assert result.message_id == "loc-msg" + call = adapter._http_session.post.call_args + assert call.args[0] == "http://127.0.0.1:3000/send-location" + assert call.kwargs["json"] == { + "chatId": "15551234567@s.whatsapp.net", + "latitude": 41.015, + "longitude": 28.979, + "name": "HQ", + "address": "Example Street", + } + + +@pytest.mark.asyncio +async def test_send_tracks_text_chunk_message_ids_in_snake_case_raw_response(): + adapter = _make_adapter() + first = MagicMock(status=200) + first.json = AsyncMock(return_value={"success": True, "messageId": "msg-1"}) + second = MagicMock(status=200) + second.json = AsyncMock(return_value={"success": True, "messageId": "msg-2"}) + adapter._http_session.post = MagicMock(side_effect=[_AsyncCM(first), _AsyncCM(second)]) + + result = await adapter.send("15551234567", "x" * (adapter.MAX_MESSAGE_LENGTH + 100)) + + assert result.success + assert result.message_id == "msg-2" + assert result.continuation_message_ids == ("msg-1",) + assert result.raw_response["message_ids"] == ["msg-1", "msg-2"] + assert "messageIds" not in result.raw_response + + +@pytest.mark.asyncio +async def test_whatsapp_reply_context_is_structured_not_prerendered(): + adapter = WhatsAppAdapter( + PlatformConfig( + enabled=True, + extra={"session_name": "test", "dm_policy": "allowlist", "allow_from": ["*"]}, + ) + ) + + event = await adapter._build_message_event( + { + "body": "what do you see here?", + "chatId": "15551234567@s.whatsapp.net", + "chatName": "Example Chat", + "senderId": "15551234567@s.whatsapp.net", + "senderName": "Example User", + "isGroup": False, + "hasQuotedMessage": True, + "quotedText": "the gateway should not inject reply context twice", + "quotedMessageId": "quoted-123", + } + ) + + assert event is not None + assert event.text == "what do you see here?" + assert event.reply_to_message_id == "quoted-123" + assert event.reply_to_text == "the gateway should not inject reply context twice" + assert not event.text.startswith("[Replying to:")