mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(matrix): enforce Ogg/Opus at send_voice boundary, probe metadata off-loop
Salvaged from PR #68063 (@malaiwah). MatrixAdapter.send_voice now transcodes any non-Ogg audio to Ogg/Opus at the adapter boundary (best-effort — the original file is sent unchanged when ffmpeg is unavailable), so MSC3245 voice bubbles render even when a caller hands the adapter MP3/WAV audio. _matrix_voice_metadata_for_file probing now runs via asyncio.to_thread so ffprobe/ffmpeg subprocess timeouts can't stall the adapter event loop. The PR's tools/tts_tool.py want_opus hunk was dropped: main's OPUS_VOICE_PLATFORMS set (PR #73072) already includes matrix; the Matrix opus-routing test is kept. Refs #14841
This commit is contained in:
parent
34ee3bbf8e
commit
33313e88f8
3 changed files with 201 additions and 11 deletions
|
|
@ -220,6 +220,53 @@ def _matrix_voice_metadata_for_file(path: Path) -> Dict[str, Any]:
|
|||
|
||||
return metadata
|
||||
|
||||
def _matrix_transcode_voice_to_ogg(path: str) -> Optional[str]:
|
||||
"""Best-effort transcode of an audio file to Ogg/Opus for MSC3245 delivery.
|
||||
|
||||
Returns the path of a NEW temporary ``.ogg`` file (caller owns cleanup), or
|
||||
``None`` when ffmpeg is unavailable or fails — callers then send the
|
||||
original file, matching the adapter's previous behaviour. Runs blocking
|
||||
subprocess work; call via ``asyncio.to_thread`` from async code.
|
||||
"""
|
||||
ffmpeg = shutil.which("ffmpeg")
|
||||
if not ffmpeg:
|
||||
return None
|
||||
import tempfile
|
||||
|
||||
fd, ogg_path = tempfile.mkstemp(prefix="matrix_voice_", suffix=".ogg")
|
||||
os.close(fd)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
ffmpeg,
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-i",
|
||||
str(path),
|
||||
"-acodec",
|
||||
"libopus",
|
||||
"-ac",
|
||||
"1",
|
||||
"-b:a",
|
||||
"64k",
|
||||
ogg_path,
|
||||
],
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
if result.returncode == 0 and os.path.getsize(ogg_path) > 0:
|
||||
return ogg_path
|
||||
except Exception:
|
||||
logger.debug("Matrix: voice transcode to Ogg/Opus failed for %s", path, exc_info=True)
|
||||
try:
|
||||
os.unlink(ogg_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
_MATRIX_BANG_COMMAND_RE = re.compile(
|
||||
r"^!([A-Za-z][A-Za-z0-9_-]*)(?=$|\s)(.*)$",
|
||||
re.DOTALL,
|
||||
|
|
@ -2117,16 +2164,47 @@ class MatrixAdapter(BasePlatformAdapter):
|
|||
reply_to: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> SendResult:
|
||||
"""Upload an audio file as a voice message (MSC3245 native voice)."""
|
||||
return await self._send_local_file(
|
||||
chat_id,
|
||||
audio_path,
|
||||
"m.audio",
|
||||
caption,
|
||||
reply_to,
|
||||
metadata=metadata,
|
||||
is_voice=True,
|
||||
)
|
||||
"""Upload an audio file as a voice message (MSC3245 native voice).
|
||||
|
||||
Matrix voice bubbles require Opus in an Ogg container (MSC3245), but
|
||||
callers can reach this with any audio format — e.g. a model-invoked
|
||||
``text_to_speech`` result routed through gateway media delivery, not
|
||||
just ``_send_voice_reply``. Enforce the codec at this boundary:
|
||||
transcode non-Ogg input to Ogg/Opus (best-effort — if ffmpeg is
|
||||
unavailable the original file is sent unchanged, preserving the
|
||||
previous behaviour).
|
||||
"""
|
||||
converted_path: Optional[str] = None
|
||||
send_path = audio_path
|
||||
if not str(audio_path).lower().endswith((".ogg", ".oga", ".opus")):
|
||||
converted_path = await asyncio.to_thread(
|
||||
_matrix_transcode_voice_to_ogg, audio_path
|
||||
)
|
||||
if converted_path:
|
||||
send_path = converted_path
|
||||
try:
|
||||
return await self._send_local_file(
|
||||
chat_id,
|
||||
send_path,
|
||||
"m.audio",
|
||||
caption,
|
||||
reply_to,
|
||||
# keep the caller's basename (the temp transcode file has a
|
||||
# generated name) so the event body stays meaningful
|
||||
file_name=(
|
||||
Path(audio_path).with_suffix(".ogg").name
|
||||
if converted_path
|
||||
else None
|
||||
),
|
||||
metadata=metadata,
|
||||
is_voice=True,
|
||||
)
|
||||
finally:
|
||||
if converted_path:
|
||||
try:
|
||||
os.unlink(converted_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
async def send_video(
|
||||
self,
|
||||
|
|
@ -2494,7 +2572,13 @@ class MatrixAdapter(BasePlatformAdapter):
|
|||
fname = file_name or p.name
|
||||
ct = mimetypes.guess_type(fname)[0] or "application/octet-stream"
|
||||
data = p.read_bytes()
|
||||
voice_metadata = _matrix_voice_metadata_for_file(p) if is_voice else None
|
||||
# ffprobe/ffmpeg probing is blocking (subprocess timeouts up to 15s) —
|
||||
# run it off the event loop so voice uploads never stall the adapter.
|
||||
voice_metadata = (
|
||||
await asyncio.to_thread(_matrix_voice_metadata_for_file, p)
|
||||
if is_voice
|
||||
else None
|
||||
)
|
||||
|
||||
return await self._upload_and_send(
|
||||
room_id,
|
||||
|
|
|
|||
|
|
@ -350,3 +350,82 @@ class TestMatrixSendVoiceMSC3245:
|
|||
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_voice_transcodes_non_ogg_to_opus(self):
|
||||
"""Non-Ogg audio reaching send_voice (e.g. direct text_to_speech MP3)
|
||||
is transcoded to Ogg/Opus at the adapter boundary (issue #14841)."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
|
||||
f.write(b"fake mp3 data")
|
||||
temp_path = f.name
|
||||
with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as f:
|
||||
f.write(b"fake ogg opus data")
|
||||
converted_path = f.name
|
||||
|
||||
try:
|
||||
sent_content = None
|
||||
|
||||
async def mock_send_message_event(room_id, event_type, content):
|
||||
nonlocal sent_content
|
||||
sent_content = content
|
||||
return "$sent_event"
|
||||
|
||||
self.adapter._client.send_message_event = mock_send_message_event
|
||||
|
||||
with patch(
|
||||
"plugins.platforms.matrix.adapter._matrix_transcode_voice_to_ogg",
|
||||
return_value=converted_path,
|
||||
) as mock_transcode, patch(
|
||||
"plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file",
|
||||
return_value={"duration": 1234, "waveform": [0, 512, 1024]},
|
||||
):
|
||||
await self.adapter.send_voice(
|
||||
chat_id="!room:example.org",
|
||||
audio_path=temp_path,
|
||||
caption="Test voice",
|
||||
)
|
||||
|
||||
mock_transcode.assert_called_once_with(temp_path)
|
||||
assert sent_content is not None, "No message was sent"
|
||||
assert "org.matrix.msc3245.voice" in sent_content
|
||||
assert sent_content["info"]["mimetype"] == "audio/ogg"
|
||||
assert self.upload_call is not None
|
||||
assert self.upload_call["data"] == b"fake ogg opus data"
|
||||
assert self.upload_call["mime_type"] == "audio/ogg"
|
||||
assert self.upload_call["filename"].endswith(".ogg")
|
||||
# converted temp file is cleaned up by send_voice
|
||||
assert not os.path.exists(converted_path)
|
||||
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
if os.path.exists(converted_path):
|
||||
os.unlink(converted_path)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_voice_ogg_input_skips_transcode(self):
|
||||
"""Already-Ogg input must not be re-transcoded."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as f:
|
||||
f.write(b"fake ogg data")
|
||||
temp_path = f.name
|
||||
|
||||
try:
|
||||
async def mock_send_message_event(room_id, event_type, content):
|
||||
return "$sent_event"
|
||||
|
||||
self.adapter._client.send_message_event = mock_send_message_event
|
||||
|
||||
with patch(
|
||||
"plugins.platforms.matrix.adapter._matrix_transcode_voice_to_ogg",
|
||||
) as mock_transcode, patch(
|
||||
"plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file",
|
||||
return_value={},
|
||||
):
|
||||
await self.adapter.send_voice(
|
||||
chat_id="!room:example.org",
|
||||
audio_path=temp_path,
|
||||
)
|
||||
|
||||
mock_transcode.assert_not_called()
|
||||
|
||||
finally:
|
||||
os.unlink(temp_path)
|
||||
|
|
|
|||
|
|
@ -68,3 +68,30 @@ def test_edge_telegram_converts_to_opus_voice(tmp_path, monkeypatch):
|
|||
assert result["voice_compatible"] is True
|
||||
assert result["media_tag"] == f"[[audio_as_voice]]\nMEDIA:{opus}"
|
||||
convert.assert_called_once_with(str(out))
|
||||
|
||||
|
||||
def test_edge_matrix_converts_to_opus_voice(tmp_path, monkeypatch):
|
||||
"""Matrix voice bubbles need Ogg/Opus too (MSC3245, issue #14841)."""
|
||||
out = tmp_path / "speech.mp3"
|
||||
opus = tmp_path / "speech.ogg"
|
||||
|
||||
def fake_convert(path: str) -> str:
|
||||
assert path == str(out)
|
||||
opus.write_bytes(b"ogg")
|
||||
return str(opus)
|
||||
|
||||
convert = Mock(side_effect=fake_convert)
|
||||
|
||||
monkeypatch.setenv("HERMES_SESSION_PLATFORM", "matrix")
|
||||
monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: {"provider": "edge"})
|
||||
monkeypatch.setattr(tts_tool, "_import_edge_tts", lambda: object())
|
||||
monkeypatch.setattr(tts_tool, "_generate_edge_tts", _write_edge_output)
|
||||
monkeypatch.setattr(tts_tool, "_convert_to_opus", convert)
|
||||
|
||||
result = json.loads(tts_tool.text_to_speech_tool("hello", output_path=str(out)))
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["file_path"] == str(opus)
|
||||
assert result["voice_compatible"] is True
|
||||
assert result["media_tag"] == f"[[audio_as_voice]]\nMEDIA:{opus}"
|
||||
convert.assert_called_once_with(str(out))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue