fix(matrix): enforce Ogg/Opus at send_voice boundary, probe metadata off-loop

Salvaged from PR #68063 (@malaiwah). MatrixAdapter.send_voice now transcodes
any non-Ogg audio to Ogg/Opus at the adapter boundary (best-effort — the
original file is sent unchanged when ffmpeg is unavailable), so MSC3245
voice bubbles render even when a caller hands the adapter MP3/WAV audio.
_matrix_voice_metadata_for_file probing now runs via asyncio.to_thread so
ffprobe/ffmpeg subprocess timeouts can't stall the adapter event loop.

The PR's tools/tts_tool.py want_opus hunk was dropped: main's
OPUS_VOICE_PLATFORMS set (PR #73072) already includes matrix; the Matrix
opus-routing test is kept.

Refs #14841
This commit is contained in:
Michel Belleau 2026-07-28 09:30:57 -07:00 committed by Teknium
parent 34ee3bbf8e
commit 33313e88f8
3 changed files with 201 additions and 11 deletions

View file

@ -220,6 +220,53 @@ def _matrix_voice_metadata_for_file(path: Path) -> Dict[str, Any]:
return metadata
def _matrix_transcode_voice_to_ogg(path: str) -> Optional[str]:
"""Best-effort transcode of an audio file to Ogg/Opus for MSC3245 delivery.
Returns the path of a NEW temporary ``.ogg`` file (caller owns cleanup), or
``None`` when ffmpeg is unavailable or fails callers then send the
original file, matching the adapter's previous behaviour. Runs blocking
subprocess work; call via ``asyncio.to_thread`` from async code.
"""
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
return None
import tempfile
fd, ogg_path = tempfile.mkstemp(prefix="matrix_voice_", suffix=".ogg")
os.close(fd)
try:
result = subprocess.run(
[
ffmpeg,
"-v",
"error",
"-y",
"-i",
str(path),
"-acodec",
"libopus",
"-ac",
"1",
"-b:a",
"64k",
ogg_path,
],
capture_output=True,
timeout=30,
stdin=subprocess.DEVNULL,
)
if result.returncode == 0 and os.path.getsize(ogg_path) > 0:
return ogg_path
except Exception:
logger.debug("Matrix: voice transcode to Ogg/Opus failed for %s", path, exc_info=True)
try:
os.unlink(ogg_path)
except OSError:
pass
return None
_MATRIX_BANG_COMMAND_RE = re.compile(
r"^!([A-Za-z][A-Za-z0-9_-]*)(?=$|\s)(.*)$",
re.DOTALL,
@ -2117,16 +2164,47 @@ class MatrixAdapter(BasePlatformAdapter):
reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> SendResult:
"""Upload an audio file as a voice message (MSC3245 native voice)."""
return await self._send_local_file(
chat_id,
audio_path,
"m.audio",
caption,
reply_to,
metadata=metadata,
is_voice=True,
)
"""Upload an audio file as a voice message (MSC3245 native voice).
Matrix voice bubbles require Opus in an Ogg container (MSC3245), but
callers can reach this with any audio format e.g. a model-invoked
``text_to_speech`` result routed through gateway media delivery, not
just ``_send_voice_reply``. Enforce the codec at this boundary:
transcode non-Ogg input to Ogg/Opus (best-effort if ffmpeg is
unavailable the original file is sent unchanged, preserving the
previous behaviour).
"""
converted_path: Optional[str] = None
send_path = audio_path
if not str(audio_path).lower().endswith((".ogg", ".oga", ".opus")):
converted_path = await asyncio.to_thread(
_matrix_transcode_voice_to_ogg, audio_path
)
if converted_path:
send_path = converted_path
try:
return await self._send_local_file(
chat_id,
send_path,
"m.audio",
caption,
reply_to,
# keep the caller's basename (the temp transcode file has a
# generated name) so the event body stays meaningful
file_name=(
Path(audio_path).with_suffix(".ogg").name
if converted_path
else None
),
metadata=metadata,
is_voice=True,
)
finally:
if converted_path:
try:
os.unlink(converted_path)
except OSError:
pass
async def send_video(
self,
@ -2494,7 +2572,13 @@ class MatrixAdapter(BasePlatformAdapter):
fname = file_name or p.name
ct = mimetypes.guess_type(fname)[0] or "application/octet-stream"
data = p.read_bytes()
voice_metadata = _matrix_voice_metadata_for_file(p) if is_voice else None
# ffprobe/ffmpeg probing is blocking (subprocess timeouts up to 15s) —
# run it off the event loop so voice uploads never stall the adapter.
voice_metadata = (
await asyncio.to_thread(_matrix_voice_metadata_for_file, p)
if is_voice
else None
)
return await self._upload_and_send(
room_id,

View file

@ -350,3 +350,82 @@ class TestMatrixSendVoiceMSC3245:
finally:
os.unlink(temp_path)
@pytest.mark.asyncio
async def test_send_voice_transcodes_non_ogg_to_opus(self):
"""Non-Ogg audio reaching send_voice (e.g. direct text_to_speech MP3)
is transcoded to Ogg/Opus at the adapter boundary (issue #14841)."""
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
f.write(b"fake mp3 data")
temp_path = f.name
with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as f:
f.write(b"fake ogg opus data")
converted_path = f.name
try:
sent_content = None
async def mock_send_message_event(room_id, event_type, content):
nonlocal sent_content
sent_content = content
return "$sent_event"
self.adapter._client.send_message_event = mock_send_message_event
with patch(
"plugins.platforms.matrix.adapter._matrix_transcode_voice_to_ogg",
return_value=converted_path,
) as mock_transcode, patch(
"plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file",
return_value={"duration": 1234, "waveform": [0, 512, 1024]},
):
await self.adapter.send_voice(
chat_id="!room:example.org",
audio_path=temp_path,
caption="Test voice",
)
mock_transcode.assert_called_once_with(temp_path)
assert sent_content is not None, "No message was sent"
assert "org.matrix.msc3245.voice" in sent_content
assert sent_content["info"]["mimetype"] == "audio/ogg"
assert self.upload_call is not None
assert self.upload_call["data"] == b"fake ogg opus data"
assert self.upload_call["mime_type"] == "audio/ogg"
assert self.upload_call["filename"].endswith(".ogg")
# converted temp file is cleaned up by send_voice
assert not os.path.exists(converted_path)
finally:
os.unlink(temp_path)
if os.path.exists(converted_path):
os.unlink(converted_path)
@pytest.mark.asyncio
async def test_send_voice_ogg_input_skips_transcode(self):
"""Already-Ogg input must not be re-transcoded."""
with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as f:
f.write(b"fake ogg data")
temp_path = f.name
try:
async def mock_send_message_event(room_id, event_type, content):
return "$sent_event"
self.adapter._client.send_message_event = mock_send_message_event
with patch(
"plugins.platforms.matrix.adapter._matrix_transcode_voice_to_ogg",
) as mock_transcode, patch(
"plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file",
return_value={},
):
await self.adapter.send_voice(
chat_id="!room:example.org",
audio_path=temp_path,
)
mock_transcode.assert_not_called()
finally:
os.unlink(temp_path)

View file

@ -68,3 +68,30 @@ def test_edge_telegram_converts_to_opus_voice(tmp_path, monkeypatch):
assert result["voice_compatible"] is True
assert result["media_tag"] == f"[[audio_as_voice]]\nMEDIA:{opus}"
convert.assert_called_once_with(str(out))
def test_edge_matrix_converts_to_opus_voice(tmp_path, monkeypatch):
"""Matrix voice bubbles need Ogg/Opus too (MSC3245, issue #14841)."""
out = tmp_path / "speech.mp3"
opus = tmp_path / "speech.ogg"
def fake_convert(path: str) -> str:
assert path == str(out)
opus.write_bytes(b"ogg")
return str(opus)
convert = Mock(side_effect=fake_convert)
monkeypatch.setenv("HERMES_SESSION_PLATFORM", "matrix")
monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: {"provider": "edge"})
monkeypatch.setattr(tts_tool, "_import_edge_tts", lambda: object())
monkeypatch.setattr(tts_tool, "_generate_edge_tts", _write_edge_output)
monkeypatch.setattr(tts_tool, "_convert_to_opus", convert)
result = json.loads(tts_tool.text_to_speech_tool("hello", output_path=str(out)))
assert result["success"] is True
assert result["file_path"] == str(opus)
assert result["voice_compatible"] is True
assert result["media_tag"] == f"[[audio_as_voice]]\nMEDIA:{opus}"
convert.assert_called_once_with(str(out))