diff --git a/gateway/run.py b/gateway/run.py index 3cc5ad5b1bb..9dce5a27cd9 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -11198,7 +11198,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: _pending_clarify = None if _pending_clarify is not None and _clarify_mod is not None: - _raw_clarify_reply = (event.text or "").strip() + _clarify_has_audio = bool(self._pending_event_audio_paths(event)) + _raw_clarify_reply = await self._prepare_clarify_reply_text(event) + if _clarify_has_audio and not _raw_clarify_reply: + logger.info( + "Gateway retained pending clarify after voice transcription " + "produced no usable text (session=%s, id=%s)", + _quick_key, + _pending_clarify.clarify_id, + ) + return "" # Skip slash commands — the user clearly wanted to issue a # command, not answer the clarify. Leave the clarify pending # so the user can retry; if it times out, the agent unblocks @@ -13027,6 +13036,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_key=session_key, ) + async def _prepare_clarify_reply_text(self, event) -> str: + """Return raw text or successful voice transcripts for a clarify reply.""" + if not self._pending_event_audio_paths(event): + return (event.text or "").strip() + + _, successful_transcripts = await self._transcribe_pending_audio_event_once( + event, "", + ) + return "\n\n".join( + transcript.strip() + for transcript in successful_transcripts + if transcript.strip() + ) + def _consume_pending_native_image_paths(self, session_key: str) -> List[str]: pending_native = getattr(self, "_pending_native_image_paths_by_session", None) if not pending_native: diff --git a/tests/gateway/test_unknown_command.py b/tests/gateway/test_unknown_command.py index c32f5da1bb1..8ebb5f8e52c 100644 --- a/tests/gateway/test_unknown_command.py +++ b/tests/gateway/test_unknown_command.py @@ -12,7 +12,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from gateway.config import GatewayConfig, Platform, PlatformConfig -from gateway.platforms.base import MessageEvent +from gateway.platforms.base import MessageEvent, MessageType from gateway.session import SessionEntry, SessionSource, build_session_key @@ -30,6 +30,18 @@ def _make_event(text: str) -> MessageEvent: return MessageEvent(text=text, source=_make_source(), message_id="m1") +def _make_voice_event(text: str = "voice_message_1.ogg") -> MessageEvent: + source = _make_source() + return MessageEvent( + text=text, + message_type=MessageType.VOICE, + source=source, + message_id="m1", + media_urls=["/tmp/voice_message_1.ogg"], + media_types=["audio/ogg"], + ) + + def _make_runner(): from gateway.run import GatewayRunner @@ -200,6 +212,67 @@ async def test_underscored_alias_for_hyphenated_builtin_not_flagged(monkeypatch) assert "Unknown command" not in result +@pytest.mark.asyncio +@pytest.mark.parametrize("event_text", ["voice_message_1.ogg", ""]) +async def test_pending_clarify_voice_reply_uses_transcript_and_choice_coercion( + monkeypatch, + event_text, +): + """Filename-bearing and captionless voice replies resolve from raw STT text.""" + import gateway.run as gateway_run + from tools import clarify_gateway + + runner = _make_runner() + session_key = build_session_key(_make_source()) + clarify_id = f"clarify-voice-{event_text or 'empty'}" + clarify_gateway.register( + clarify_id, + session_key, + "Pick one", + ["first choice", "second choice"], + ) + runner.hooks.emit_collect = AsyncMock(return_value=[]) + runner._transcribe_pending_audio_event_once = AsyncMock( + return_value=('"2"', ["2"]), + ) + + monkeypatch.setattr( + gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "***"} + ) + + result = await runner._handle_message(_make_voice_event(event_text)) + + assert result == "" + runner._transcribe_pending_audio_event_once.assert_awaited_once() + assert clarify_gateway.wait_for_response(clarify_id, timeout=0.1) == "second choice" + + +@pytest.mark.asyncio +async def test_failed_clarify_voice_transcription_does_not_resolve_marker(monkeypatch): + """An STT status marker is not a user answer; the clarify stays pending.""" + import gateway.run as gateway_run + from tools import clarify_gateway + + runner = _make_runner() + event = _make_voice_event("") + session_key = build_session_key(event.source) + clarify_id = "clarify-voice-stt-failure" + clarify_gateway.register(clarify_id, session_key, "Say anything", None) + runner._transcribe_pending_audio_event_once = AsyncMock( + return_value=("[voice message could not be transcribed]", []), + ) + monkeypatch.setattr( + gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "***"} + ) + + try: + assert await runner._handle_message(event) == "" + runner._transcribe_pending_audio_event_once.assert_awaited_once() + assert clarify_gateway.has_pending(session_key) is True + finally: + clarify_gateway.clear_session(session_key) + + # ------------------------------------------------------------------ # command: decision hook — deny / handled / rewrite # ------------------------------------------------------------------