From 34ee3bbf8ec286950b1a36e01928b5bf31fcf9d2 Mon Sep 17 00:00:00 2001 From: Michel Belleau Date: Sun, 28 Jun 2026 14:15:39 -0700 Subject: [PATCH] fix(matrix): add MSC3245 duration + MSC1767 waveform metadata to voice sends Salvaged from PR #68063 (base commit, runner-path hunks superseded by the platform-aware OPUS_VOICE_PLATFORMS fix in this branch). Element and other Matrix clients render voice bubbles more reliably when m.audio events carry duration and waveform metadata; probe both best-effort via ffprobe/ffmpeg. --- plugins/platforms/matrix/adapter.py | 109 +++++++++++++++++++++++++++- tests/gateway/test_matrix_voice.py | 19 +++-- 2 files changed, 122 insertions(+), 6 deletions(-) diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py index ec6067e7c601..766a2795b4ca 100644 --- a/plugins/platforms/matrix/adapter.py +++ b/plugins/platforms/matrix/adapter.py @@ -52,11 +52,15 @@ Environment variables: from __future__ import annotations import asyncio +import array import inspect import logging import mimetypes import os import re +import shutil +import subprocess +import sys import time from urllib.parse import urljoin, urlsplit, urlunsplit from dataclasses import dataclass, field @@ -135,6 +139,87 @@ from gateway.platforms.helpers import ThreadParticipationTracker logger = logging.getLogger(__name__) +_MATRIX_VOICE_WAVEFORM_BINS = 30 + + +def _matrix_voice_metadata_for_file(path: Path) -> Dict[str, Any]: + """Return best-effort Matrix voice metadata for an audio file. + + Matrix clients such as Element render ``m.audio`` events with + ``org.matrix.msc3245.voice`` as voice bubbles. They are more reliable when + the event also includes duration and MSC1767 waveform metadata. Metadata + extraction is deliberately best-effort: media delivery must still work on + systems without ffprobe/ffmpeg. + """ + metadata: Dict[str, Any] = {} + + ffprobe = shutil.which("ffprobe") + if ffprobe: + try: + result = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + str(path), + ], + capture_output=True, + text=True, + timeout=10, + stdin=subprocess.DEVNULL, + ) + if result.returncode == 0: + duration = float((result.stdout or "").strip() or 0) + if duration > 0: + metadata["duration"] = int(duration * 1000) + except Exception: + logger.debug("Matrix: failed to probe voice duration for %s", path, exc_info=True) + + ffmpeg = shutil.which("ffmpeg") + if ffmpeg: + try: + result = subprocess.run( + [ + ffmpeg, + "-v", + "error", + "-i", + str(path), + "-ac", + "1", + "-ar", + "8000", + "-f", + "s16le", + "-", + ], + capture_output=True, + timeout=15, + stdin=subprocess.DEVNULL, + ) + if result.returncode == 0 and result.stdout: + samples = array.array("h") + samples.frombytes(result.stdout) + if sys.byteorder != "little": + samples.byteswap() + if samples: + count = len(samples) + waveform = [] + for idx in range(_MATRIX_VOICE_WAVEFORM_BINS): + start = idx * count // _MATRIX_VOICE_WAVEFORM_BINS + end = max(start + 1, (idx + 1) * count // _MATRIX_VOICE_WAVEFORM_BINS) + peak = max(abs(value) for value in samples[start:end]) + waveform.append(min(1024, int(peak / 32767 * 1024))) + metadata["waveform"] = waveform + except Exception: + logger.debug("Matrix: failed to build voice waveform for %s", path, exc_info=True) + + return metadata + _MATRIX_BANG_COMMAND_RE = re.compile( r"^!([A-Za-z][A-Za-z0-9_-]*)(?=$|\s)(.*)$", re.DOTALL, @@ -2293,6 +2378,7 @@ class MatrixAdapter(BasePlatformAdapter): reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, is_voice: bool = False, + voice_metadata: Optional[Dict[str, Any]] = None, ) -> SendResult: """Upload bytes to Matrix and send as a media message.""" if len(data) > self._max_media_bytes: @@ -2349,6 +2435,17 @@ class MatrixAdapter(BasePlatformAdapter): # Add MSC3245 voice flag for native voice messages. if is_voice: msg_content["org.matrix.msc3245.voice"] = {} + duration = (voice_metadata or {}).get("duration") + waveform = (voice_metadata or {}).get("waveform") + if duration is not None: + msg_content["info"]["duration"] = duration + if duration is not None or waveform is not None: + audio_metadata: Dict[str, Any] = {} + if duration is not None: + audio_metadata["duration"] = duration + if waveform is not None: + audio_metadata["waveform"] = waveform + msg_content["org.matrix.msc1767.audio"] = audio_metadata self._apply_relation_metadata(msg_content, reply_to=reply_to, metadata=metadata) @@ -2397,9 +2494,19 @@ class MatrixAdapter(BasePlatformAdapter): fname = file_name or p.name ct = mimetypes.guess_type(fname)[0] or "application/octet-stream" data = p.read_bytes() + voice_metadata = _matrix_voice_metadata_for_file(p) if is_voice else None return await self._upload_and_send( - room_id, data, fname, ct, msgtype, caption, reply_to, metadata, is_voice + room_id, + data, + fname, + ct, + msgtype, + caption, + reply_to, + metadata, + is_voice, + voice_metadata, ) # ------------------------------------------------------------------ diff --git a/tests/gateway/test_matrix_voice.py b/tests/gateway/test_matrix_voice.py index b113ba275caf..48055b3f460d 100644 --- a/tests/gateway/test_matrix_voice.py +++ b/tests/gateway/test_matrix_voice.py @@ -323,17 +323,26 @@ class TestMatrixSendVoiceMSC3245: self.adapter._client.send_message_event = mock_send_message_event - await self.adapter.send_voice( - chat_id="!room:example.org", - audio_path=temp_path, - caption="Test voice", - ) + with patch( + "plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file", + return_value={"duration": 1234, "waveform": [0, 512, 1024]}, + ): + await self.adapter.send_voice( + chat_id="!room:example.org", + audio_path=temp_path, + caption="Test voice", + ) assert sent_content is not None, "No message was sent" assert "org.matrix.msc3245.voice" in sent_content, \ f"MSC3245 voice field missing from content: {sent_content.keys()}" assert sent_content["msgtype"] == "m.audio" assert sent_content["info"]["mimetype"] == "audio/ogg" + assert sent_content["info"]["duration"] == 1234 + assert sent_content["org.matrix.msc1767.audio"] == { + "duration": 1234, + "waveform": [0, 512, 1024], + } assert self.upload_call is not None, "Expected upload_media() to be called" assert isinstance(self.upload_call["data"], bytes) assert self.upload_call["mime_type"] == "audio/ogg"