diff --git a/tests/tools/test_tts_instructions.py b/tests/tools/test_tts_instructions.py new file mode 100644 index 00000000000..cdad64e238c --- /dev/null +++ b/tests/tools/test_tts_instructions.py @@ -0,0 +1,124 @@ +"""Tests for the OpenAI TTS `instructions` field passthrough. + +Covers #14196: forwarding the OpenAI-spec `instructions` parameter through the +`text_to_speech` tool so the agent can control tone/emotion/pacing on +gpt-4o-mini-tts and OpenAI-compatible voice-design servers. +""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + for key in ("OPENAI_API_KEY", "HERMES_SESSION_PLATFORM"): + monkeypatch.delenv(key, raising=False) + + +# --------------------------------------------------------------------------- +# Backend-level passthrough (_generate_openai_tts) +# --------------------------------------------------------------------------- + +class TestOpenaiBackendInstructions: + def _run(self, tmp_path, monkeypatch, *, tts_config=None, instructions=None): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + mock_client = MagicMock() + mock_client.audio.speech.create.return_value = MagicMock() + mock_cls = MagicMock(return_value=mock_client) + + with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \ + patch("tools.tts_tool._resolve_openai_audio_client_config", + return_value=("test-key", None)): + from tools.tts_tool import _generate_openai_tts + kwargs = {} + if instructions is not None: + kwargs["instructions"] = instructions + _generate_openai_tts( + "Hello", str(tmp_path / "out.mp3"), tts_config or {}, **kwargs + ) + return mock_client.audio.speech.create + + def test_instructions_forwarded_when_provided(self, tmp_path, monkeypatch): + """Tool arg `instructions` is passed to audio.speech.create as-is.""" + create = self._run(tmp_path, monkeypatch, instructions="Speak cheerfully.") + assert create.call_args[1]["instructions"] == "Speak cheerfully." + + def test_instructions_absent_by_default(self, tmp_path, monkeypatch): + """No instructions arg -> key not present in create kwargs. + + Preserves behavior on `tts-1`/`tts-1-hd` and strict servers that + reject unknown kwargs. + """ + create = self._run(tmp_path, monkeypatch) + assert "instructions" not in create.call_args[1] + + def test_empty_string_instructions_omitted(self, tmp_path, monkeypatch): + """Empty string is treated as absent (not forwarded).""" + create = self._run(tmp_path, monkeypatch, instructions="") + assert "instructions" not in create.call_args[1] + + +# --------------------------------------------------------------------------- +# Tool-level plumbing (text_to_speech_tool -> _generate_openai_tts) +# --------------------------------------------------------------------------- + +class TestToolLevelInstructions: + def _invoke_tool(self, tmp_path, monkeypatch, *, instructions=None): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + + mock_client = MagicMock() + + def fake_stream(path): + # Mimic OpenAI SDK's stream_to_file by writing a tiny payload. + with open(path, "wb") as f: + f.write(b"ID3\x03\x00\x00\x00\x00\x00\x00") + + response = MagicMock() + response.stream_to_file.side_effect = fake_stream + mock_client.audio.speech.create.return_value = response + + mock_cls = MagicMock(return_value=mock_client) + + with patch("tools.tts_tool._import_openai_client", return_value=mock_cls), \ + patch("tools.tts_tool._resolve_openai_audio_client_config", + return_value=("test-key", None)), \ + patch("tools.tts_tool._load_tts_config", + return_value={"provider": "openai"}): + from tools.tts_tool import text_to_speech_tool + kwargs = {"output_path": str(tmp_path / "out.mp3")} + if instructions is not None: + kwargs["instructions"] = instructions + result = text_to_speech_tool("Hello world", **kwargs) + return mock_client.audio.speech.create, json.loads(result) + + def test_tool_threads_instructions_to_openai_create( + self, tmp_path, monkeypatch + ): + create, result = self._invoke_tool( + tmp_path, monkeypatch, instructions="Whisper conspiratorially." + ) + assert result.get("success") is True + assert create.call_args[1]["instructions"] == "Whisper conspiratorially." + + def test_tool_omits_instructions_when_not_supplied( + self, tmp_path, monkeypatch + ): + create, result = self._invoke_tool(tmp_path, monkeypatch) + assert result.get("success") is True + assert "instructions" not in create.call_args[1] + + +# --------------------------------------------------------------------------- +# Schema +# --------------------------------------------------------------------------- + +class TestSchema: + def test_schema_exposes_instructions_parameter(self): + from tools.tts_tool import TTS_SCHEMA + props = TTS_SCHEMA["parameters"]["properties"] + assert "instructions" in props + assert props["instructions"]["type"] == "string" + # Must stay optional — current behavior must be preserved. + assert "instructions" not in TTS_SCHEMA["parameters"].get("required", []) diff --git a/tests/tools/test_tts_max_text_length.py b/tests/tools/test_tts_max_text_length.py index 49ae5ca2f4b..fbadf61aa84 100644 --- a/tests/tools/test_tts_max_text_length.py +++ b/tests/tools/test_tts_max_text_length.py @@ -127,7 +127,7 @@ class TestTextToSpeechToolTruncation: text = "A" * 5000 captured_text = {} - def fake_openai(t, out, cfg): + def fake_openai(t, out, cfg, **_kw): captured_text["text"] = t with open(out, "wb") as f: f.write(b"\x00") @@ -175,7 +175,7 @@ class TestTextToSpeechToolTruncation: text = "C" * 500 captured_text = {} - def fake_openai(t, out, cfg): + def fake_openai(t, out, cfg, **_kw): captured_text["text"] = t with open(out, "wb") as f: f.write(b"\x00") diff --git a/tools/tts_tool.py b/tools/tts_tool.py index ffb00a08c58..9ad3d2f1d29 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -1272,6 +1272,7 @@ def _generate_openai_tts( model: Optional[str] = None, voice: Optional[str] = None, speed: Optional[float] = None, + instructions: Optional[str] = None, ) -> str: """Generate audio via the OpenAI ``audio.speech.create`` SDK shape. @@ -1292,6 +1293,11 @@ def _generate_openai_tts( voice: Voice id. When None, reads ``tts.openai.voice``. speed: Playback speed. When None, reads ``tts.openai.speed`` / ``tts.speed``. + instructions: Optional voice-design guidance (tone, emotion, pacing, + accent, whispering). Forwarded to `audio.speech.create` when + truthy; omitted otherwise so ``tts-1``/``tts-1-hd`` and strict + OpenAI-compatible servers that reject unknown kwargs are + unaffected. Returns: Path to the saved audio file. @@ -1355,6 +1361,8 @@ def _generate_openai_tts( } if speed != 1.0: create_kwargs["speed"] = max(0.25, min(4.0, speed)) + if instructions: + create_kwargs["instructions"] = instructions response = client.audio.speech.create(**create_kwargs) response.stream_to_file(output_path) @@ -2568,6 +2576,7 @@ def text_to_speech_tool( text: str, output_path: Optional[str] = None, speed: Optional[float] = None, + instructions: Optional[str] = None, ) -> str: """ Convert text to speech audio. @@ -2583,6 +2592,10 @@ def text_to_speech_tool( text: The text to convert to speech. output_path: Optional custom save path. Defaults to ~/voice-memos/.mp3 speed: Optional playback speed multiplier (0.25-4.0). Overrides config.yaml. + instructions: Optional voice-design guidance (tone, emotion, pacing, + accent, whispering). Forwarded to the OpenAI backend + (gpt-4o-mini-tts and OpenAI-compatible servers). Silently + ignored by backends that don't support it. Returns: str: JSON result with success, file_path, and optionally MEDIA tag. @@ -2724,7 +2737,7 @@ def text_to_speech_tool( "error": "OpenAI provider selected but 'openai' package not installed." }, ensure_ascii=False) logger.info("Generating speech with OpenAI TTS...") - _generate_openai_tts(text, file_str, tts_config) + _generate_openai_tts(text, file_str, tts_config, instructions=instructions) elif provider == "deepinfra": try: @@ -3333,6 +3346,15 @@ TTS_SCHEMA = { "speed": { "type": "number", "description": "Playback speed multiplier. 1.0 = normal, 0.5 = very slow (language learning), 2.0 = fast. Range: 0.25-4.0. Overrides the speed configured in config.yaml." + }, + "instructions": { + "type": "string", + "description": ( + "Optional voice-design guidance: tone, emotion, pacing, accent, " + "whispering, impressions (e.g. 'Speak in a cheerful, excited whisper'). " + "Forwarded to the OpenAI backend (gpt-4o-mini-tts and OpenAI-compatible " + "voice-design servers). Silently ignored by backends that don't support it." + ) } }, "required": ["text"] @@ -3346,7 +3368,8 @@ registry.register( handler=lambda args, **kw: text_to_speech_tool( text=args.get("text", ""), output_path=args.get("output_path"), - speed=args.get("speed")), + speed=args.get("speed"), + instructions=args.get("instructions")), check_fn=check_tts_requirements, emoji="🔊", ) diff --git a/website/docs/user-guide/features/voice-mode.md b/website/docs/user-guide/features/voice-mode.md index a14e63dfd9b..86d671f3062 100644 --- a/website/docs/user-guide/features/voice-mode.md +++ b/website/docs/user-guide/features/voice-mode.md @@ -436,6 +436,11 @@ tts: model: "gpt-4o-mini-tts" voice: "alloy" # alloy, echo, fable, onyx, nova, shimmer base_url: "https://api.openai.com/v1" # optional: override for self-hosted or OpenAI-compatible endpoints + # The `text_to_speech` tool accepts an optional per-call `instructions` + # argument (tone, emotion, pacing, accent, whispering) that is forwarded + # to `gpt-4o-mini-tts` and to OpenAI-compatible voice-design servers + # (e.g. Qwen3-TTS-VoiceDesign via oMLX). See OpenAI's voice-design guide: + # https://platform.openai.com/docs/guides/text-to-speech neutts: ref_audio: '' ref_text: '' @@ -493,6 +498,12 @@ Provider priority (automatic fallback): **local** > **groq** > **openai** NeuTTS uses the `tts.neutts` config block above. +For `openai`, the `text_to_speech` tool accepts an optional `instructions` +argument that unlocks `gpt-4o-mini-tts`'s voice-design capability (tone, +emotion, pacing, accent, whispering). The same field also routes to +OpenAI-compatible voice-design servers mounted via `tts.openai.base_url` +(e.g. Qwen3-TTS-VoiceDesign via oMLX). + --- ## Troubleshooting