From 805c1c340cddb0382ce7775f1403e43ee3d7d391 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:37:21 -0700 Subject: [PATCH] feat(stt): support OpenAI gpt-transcribe transcription model Adds gpt-transcribe (OpenAI's new file-transcription model, $0.0045/min) to the OpenAI STT provider: - OPENAI_MODELS set: gpt-transcribe is recognized so provider auto-correction keeps it on OpenAI and rejects it on Groq - Language hint wiring: gpt-transcribe replaces the singular 'language' field with a 'languages' list; the API rejects the legacy field, so the hint is sent via extra_body {languages: [..]} - Config comment (DEFAULT_CONFIG), cli-config.yaml.example, desktop settings enum, and docs (en + zh-Hans) updated - Tests: model pass-through, languages-list hint shape, legacy singular hint preserved for gpt-4o-transcribe, Groq auto-correction gpt-live-transcribe (realtime WebSocket, $0.017/min) is NOT wired here: the file-based STT pipeline has no realtime session path; it belongs in a future realtime/voice-mode integration. --- apps/desktop/src/app/settings/constants.ts | 2 +- cli-config.yaml.example | 2 +- hermes_cli/config.py | 2 +- tests/tools/test_transcription_tools.py | 64 +++++++++++++++++++ tools/transcription_tools.py | 10 ++- website/docs/user-guide/configuration.md | 2 +- website/docs/user-guide/features/tts.md | 4 +- .../docs/user-guide/features/voice-mode.md | 1 + .../current/user-guide/configuration.md | 2 +- .../current/user-guide/features/tts.md | 4 +- .../current/user-guide/features/voice-mode.md | 1 + 11 files changed, 83 insertions(+), 11 deletions(-) diff --git a/apps/desktop/src/app/settings/constants.ts b/apps/desktop/src/app/settings/constants.ts index 3e12e08e70d..b0e03175bc0 100644 --- a/apps/desktop/src/app/settings/constants.ts +++ b/apps/desktop/src/app/settings/constants.ts @@ -349,7 +349,7 @@ export const ENUM_OPTIONS: Record = { 'kittentts', 'piper' ], - 'stt.openai.model': ['whisper-1', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe'], + 'stt.openai.model': ['whisper-1', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe', 'gpt-transcribe'], 'stt.mistral.model': ['voxtral-mini-latest', 'voxtral-mini-2602'], 'tts.openai.model': ['gpt-4o-mini-tts', 'tts-1', 'tts-1-hd'], 'tts.elevenlabs.model_id': ['eleven_multilingual_v2', 'eleven_turbo_v2_5', 'eleven_flash_v2_5'], diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 7d5a76f96ec..0c17e2c4615 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1143,7 +1143,7 @@ stt: # model: "whisper-large-v3-turbo" # language: "" # blank = stt.language > HERMES_LOCAL_STT_LANGUAGE > auto-detect openai: - model: "whisper-1" # whisper-1 | gpt-4o-mini-transcribe | gpt-4o-transcribe + model: "whisper-1" # whisper-1 | gpt-4o-mini-transcribe | gpt-4o-transcribe | gpt-transcribe language: "" # auto-detect; set to "en", "es", "fr", etc. to force # mistral: # model: "voxtral-mini-latest" # voxtral-mini-latest | voxtral-mini-2602 diff --git a/hermes_cli/config.py b/hermes_cli/config.py index b6676337849..cb32c10ca82 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2338,7 +2338,7 @@ DEFAULT_CONFIG = { "language": "", # auto-detect by default; set to "en", "es", "fr", etc. to force }, "openai": { - "model": "whisper-1", # whisper-1, gpt-4o-mini-transcribe, gpt-4o-transcribe + "model": "whisper-1", # whisper-1, gpt-4o-mini-transcribe, gpt-4o-transcribe, gpt-transcribe "language": "", # auto-detect by default; set to "en", "es", "fr", etc. to force }, "mistral": { diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index e5886bbae91..e1f5ecee2b4 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -1068,6 +1068,70 @@ class TestModelAutoCorrection: call_kwargs = mock_client.audio.transcriptions.create.call_args assert call_kwargs.kwargs["model"] == "gpt-4o-mini-transcribe" + def test_gpt_transcribe_model_not_overridden(self, monkeypatch, sample_wav): + monkeypatch.setenv("VOICE_TOOLS_OPENAI_KEY", "sk-test") + + mock_client = MagicMock() + mock_client.audio.transcriptions.create.return_value = "test" + + with patch("tools.transcription_tools._HAS_OPENAI", True), \ + patch("openai.OpenAI", return_value=mock_client): + from tools.transcription_tools import _transcribe_openai + _transcribe_openai(sample_wav, "gpt-transcribe") + + call_kwargs = mock_client.audio.transcriptions.create.call_args + assert call_kwargs.kwargs["model"] == "gpt-transcribe" + assert call_kwargs.kwargs["response_format"] == "json" + + def test_gpt_transcribe_language_hint_uses_languages_list(self, monkeypatch, sample_wav): + """gpt-transcribe rejects the singular ``language`` field; the hint + must be sent as a ``languages`` list via extra_body instead.""" + monkeypatch.setenv("VOICE_TOOLS_OPENAI_KEY", "sk-test") + + mock_client = MagicMock() + mock_client.audio.transcriptions.create.return_value = "test" + + with patch("tools.transcription_tools._HAS_OPENAI", True), \ + patch("openai.OpenAI", return_value=mock_client), \ + patch("tools.transcription_tools._resolve_stt_language", return_value="fr"): + from tools.transcription_tools import _transcribe_openai + _transcribe_openai(sample_wav, "gpt-transcribe") + + call_kwargs = mock_client.audio.transcriptions.create.call_args + assert "language" not in call_kwargs.kwargs + assert call_kwargs.kwargs["extra_body"] == {"languages": ["fr"]} + + def test_legacy_openai_model_language_hint_uses_singular_field(self, monkeypatch, sample_wav): + monkeypatch.setenv("VOICE_TOOLS_OPENAI_KEY", "sk-test") + + mock_client = MagicMock() + mock_client.audio.transcriptions.create.return_value = "test" + + with patch("tools.transcription_tools._HAS_OPENAI", True), \ + patch("openai.OpenAI", return_value=mock_client), \ + patch("tools.transcription_tools._resolve_stt_language", return_value="fr"): + from tools.transcription_tools import _transcribe_openai + _transcribe_openai(sample_wav, "gpt-4o-transcribe") + + call_kwargs = mock_client.audio.transcriptions.create.call_args + assert call_kwargs.kwargs["language"] == "fr" + assert "extra_body" not in call_kwargs.kwargs + + def test_gpt_transcribe_rejected_on_groq(self, monkeypatch, sample_wav): + """gpt-transcribe is OpenAI-only and must be auto-corrected on Groq.""" + monkeypatch.setenv("GROQ_API_KEY", "gsk-test") + + mock_client = MagicMock() + mock_client.audio.transcriptions.create.return_value = "test" + + with patch("tools.transcription_tools._HAS_OPENAI", True), \ + patch("openai.OpenAI", return_value=mock_client): + from tools.transcription_tools import _transcribe_groq, DEFAULT_GROQ_STT_MODEL + _transcribe_groq(sample_wav, "gpt-transcribe") + + call_kwargs = mock_client.audio.transcriptions.create.call_args + assert call_kwargs.kwargs["model"] == DEFAULT_GROQ_STT_MODEL + def test_unknown_model_passes_through_groq(self, monkeypatch, sample_wav): """A model not in either known set should not be overridden.""" monkeypatch.setenv("GROQ_API_KEY", "gsk-test") diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 54acdf72f7b..9e091abdd51 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -128,7 +128,7 @@ LOCAL_NATIVE_AUDIO_FORMATS = {".wav", ".aiff", ".aif"} MAX_FILE_SIZE = 25 * 1024 * 1024 # 25 MB # Known model sets for auto-correction -OPENAI_MODELS = {"whisper-1", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"} +OPENAI_MODELS = {"whisper-1", "gpt-4o-mini-transcribe", "gpt-4o-transcribe", "gpt-transcribe"} GROQ_MODELS = {"whisper-large-v3", "whisper-large-v3-turbo", "distil-whisper-large-v3-en"} # Singleton for the local model — loaded once, reused across calls @@ -1824,7 +1824,13 @@ def _transcribe_openai( "response_format": "text" if model_name == "whisper-1" else "json", } if language: - create_kwargs["language"] = language + if model_name == "gpt-transcribe": + # gpt-transcribe replaces the singular ``language`` + # field with a ``languages`` list; the API rejects + # requests that send the legacy field. + create_kwargs["extra_body"] = {"languages": [language]} + else: + create_kwargs["language"] = language logger.debug("Using language hint '%s' for OpenAI STT", language) return client.audio.transcriptions.create(**create_kwargs) diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index da0cf2adb9c..9ad7e55f877 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1721,7 +1721,7 @@ stt: groq: language: "" # per-provider override of stt.language openai: - model: "whisper-1" # whisper-1 | gpt-4o-mini-transcribe | gpt-4o-transcribe + model: "whisper-1" # whisper-1 | gpt-4o-mini-transcribe | gpt-4o-transcribe | gpt-transcribe language: "" # per-provider override of stt.language # model: "whisper-1" # Legacy fallback key still respected ``` diff --git a/website/docs/user-guide/features/tts.md b/website/docs/user-guide/features/tts.md index 9c20ca79968..d413f6d3a0c 100644 --- a/website/docs/user-guide/features/tts.md +++ b/website/docs/user-guide/features/tts.md @@ -465,7 +465,7 @@ stt: groq: language: "" # optional ISO-639-1 hint; blank = use HERMES_LOCAL_STT_LANGUAGE if set, else auto-detect openai: - model: "whisper-1" # whisper-1, gpt-4o-mini-transcribe, gpt-4o-transcribe + model: "whisper-1" # whisper-1, gpt-4o-mini-transcribe, gpt-4o-transcribe, gpt-transcribe mistral: model: "voxtral-mini-latest" # voxtral-mini-latest, voxtral-mini-2602 xai: @@ -487,7 +487,7 @@ stt: **Groq API** — Requires `GROQ_API_KEY`. Good cloud fallback when you want a free hosted STT option. Set `stt.groq.language` (or the global `HERMES_LOCAL_STT_LANGUAGE` env var) to skip Whisper's auto-detect and reduce latency on known-language audio. -**OpenAI API** — Accepts `VOICE_TOOLS_OPENAI_KEY` first and falls back to `OPENAI_API_KEY`. Supports `whisper-1`, `gpt-4o-mini-transcribe`, and `gpt-4o-transcribe`. +**OpenAI API** — Accepts `VOICE_TOOLS_OPENAI_KEY` first and falls back to `OPENAI_API_KEY`. Supports `whisper-1`, `gpt-4o-mini-transcribe`, `gpt-4o-transcribe`, and `gpt-transcribe`. **Mistral API (Voxtral Transcribe)** — Requires `MISTRAL_API_KEY`. Uses Mistral's [Voxtral Transcribe](https://docs.mistral.ai/capabilities/audio/speech_to_text/) models. Supports 13 languages, speaker diarization, and word-level timestamps. Install with `cd ~/.hermes/hermes-agent && uv pip install -e ".[mistral]"`. diff --git a/website/docs/user-guide/features/voice-mode.md b/website/docs/user-guide/features/voice-mode.md index fa6a0803cc4..fda828848bf 100644 --- a/website/docs/user-guide/features/voice-mode.md +++ b/website/docs/user-guide/features/voice-mode.md @@ -484,6 +484,7 @@ DISCORD_ALLOWED_USERS=... | **Groq** | `whisper-large-v3` | Fast (~1s) | Better | Free tier | Yes | | **OpenAI** | `whisper-1` | Fast (~1s) | Good | Paid | Yes | | **OpenAI** | `gpt-4o-transcribe` | Medium (~2s) | Best | Paid | Yes | +| **OpenAI** | `gpt-transcribe` | Fast | Best | Paid ($0.0045/min) | Yes | | **Mistral** | `voxtral-mini-latest` | Fast | Good | Paid | Yes | | **xAI** | `grok-stt` | Fast | Good | Paid | Yes | diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md index a092445e65c..9779e6eed73 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md @@ -1272,7 +1272,7 @@ stt: local: model: "base" # tiny、base、small、medium、large-v3 openai: - model: "whisper-1" # whisper-1 | gpt-4o-mini-transcribe | gpt-4o-transcribe + model: "whisper-1" # whisper-1 | gpt-4o-mini-transcribe | gpt-4o-transcribe | gpt-transcribe # model: "whisper-1" # 旧版回退键仍受支持 ``` diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/tts.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/tts.md index d13b4c1a9c5..2d640f9b338 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/tts.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/tts.md @@ -399,7 +399,7 @@ stt: local: model: "base" # tiny, base, small, medium, large-v3 openai: - model: "whisper-1" # whisper-1, gpt-4o-mini-transcribe, gpt-4o-transcribe + model: "whisper-1" # whisper-1, gpt-4o-mini-transcribe, gpt-4o-transcribe, gpt-transcribe mistral: model: "voxtral-mini-latest" # voxtral-mini-latest, voxtral-mini-2602 xai: @@ -420,7 +420,7 @@ stt: **Groq API** — 需要 `GROQ_API_KEY`。当你需要免费托管 STT 选项时，是良好的云端备选方案。 -**OpenAI API** — 优先使用 `VOICE_TOOLS_OPENAI_KEY`，回退至 `OPENAI_API_KEY`。支持 `whisper-1`、`gpt-4o-mini-transcribe` 和 `gpt-4o-transcribe`。 +**OpenAI API** — 优先使用 `VOICE_TOOLS_OPENAI_KEY`，回退至 `OPENAI_API_KEY`。支持 `whisper-1`、`gpt-4o-mini-transcribe`、`gpt-4o-transcribe` 和 `gpt-transcribe`。 **Mistral API（Voxtral Transcribe）** — 需要 `MISTRAL_API_KEY`。使用 Mistral 的 [Voxtral Transcribe](https://docs.mistral.ai/capabilities/audio/speech_to_text/) 模型。支持 13 种语言、说话人分离和词级时间戳。通过 `cd ~/.hermes/hermes-agent && uv pip install -e ".[mistral]"` 安装。 diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/voice-mode.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/voice-mode.md index 1138d8aa920..65930308802 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/voice-mode.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/voice-mode.md @@ -458,6 +458,7 @@ DISCORD_ALLOWED_USERS=... | **Groq** | `whisper-large-v3` | 快（约 1 秒） | 较好 | 免费额度 | 是 | | **OpenAI** | `whisper-1` | 快（约 1 秒） | 良好 | 付费 | 是 | | **OpenAI** | `gpt-4o-transcribe` | 中等（约 2 秒） | 最佳 | 付费 | 是 | +| **OpenAI** | `gpt-transcribe` | 快 | 最佳 | 付费（$0.0045/分钟） | 是 | 提供商优先级（自动回退）：**本地** > **groq** > **openai**