From bf8004e3a8ae4d8b97917066db47ce7f9a9cd2ca Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:51:32 -0700 Subject: [PATCH] fix(stt): kill faster-whisper silence hallucinations at the source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local faster-whisper called model.transcribe with bare {'beam_size': 5}: no VAD, cross-window conditioning on, no confidence filtering. Pure silence produced hallucinated tokens (E2E: 5s anullsrc WAV -> 'You', no_speech_prob=0.705) and noisy clips could produce runs of junk, often in other languages. Three-layer class fix, one shared owner for every local-whisper call site (build_local_transcribe_kwargs): 1. Silero VAD filter (bundled with faster-whisper) on by default — silence never reaches the model. stt.local.vad: false restores the raw behavior for music/ambient transcription. stt.local.vad_min_silence_ms tunes chunk splitting (default 500). 2. condition_on_previous_text=False — one hallucinated token can no longer seed a self-reinforcing run; negligible cost for voice-note-length audio. 3. Segment confidence gate (_join_confident_segments): drop a segment only when no_speech_prob > 0.6 AND avg_logprob < -1.0 (openai-whisper's own heuristic shape; both must hit so quiet-but-real speech survives). Config: stt.local.no_speech_prob_threshold / logprob_threshold. The WHISPER_HALLUCINATIONS blocklist in voice_mode.py stays as last-resort defense but should now almost never fire. E2E (real faster-whisper 'base', CPU int8): silence.wav before 'You' -> after '' noise.wav before '' -> after '' speech.wav before/after 'Hello World, this is a test of the transcription system.' (unchanged) Docs (EN + zh-Hans), DEFAULT_CONFIG, cli-config.yaml.example updated; 19 unit tests (kwargs contract, off-switch, confidence gate incl. quiet-speech survival, _transcribe_local wiring), sabotage-verified. --- cli-config.yaml.example | 6 + hermes_cli/config.py | 6 + .../tools/test_stt_silence_hallucinations.py | 159 ++++++++++++++++++ tools/transcription_tools.py | 127 ++++++++++++-- website/docs/user-guide/configuration.md | 6 +- .../current/user-guide/configuration.md | 2 +- 6 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 tests/tools/test_stt_silence_hallucinations.py diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 0c17e2c4615..2f43c509118 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1138,6 +1138,12 @@ stt: model: "base" # tiny | base | small | medium | large-v3 | turbo # language: "" # auto-detect; set to "en", "es", "fr", etc. to force # initial_prompt: "" # Optional faster-whisper prompt, e.g. bias Chinese output to simplified Chinese + # --- Anti-hallucination hardening (whisper decodes junk from silence without these) --- + # vad: true # Silero VAD filter (default on) — silence never reaches whisper. + # # Set false to restore raw behavior (e.g. transcribing music/ambient audio). + # vad_min_silence_ms: 500 # min silence (ms) that splits speech chunks when vad is on + # no_speech_prob_threshold: 0.6 # drop a segment only if no_speech_prob > this... + # logprob_threshold: -1.0 # ...AND avg_logprob < this (both must hit — quiet real speech survives) language: "en" # GLOBAL language hint for every STT provider (per-provider language wins). Set "" for auto-detect. # groq: # model: "whisper-large-v3-turbo" diff --git a/hermes_cli/config.py b/hermes_cli/config.py index cb32c10ca82..b51b4cea4d5 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2332,6 +2332,12 @@ DEFAULT_CONFIG = { "model": "base", # tiny, base, small, medium, large-v3 "language": "", # auto-detect by default; set to "en", "es", "fr", etc. to force "initial_prompt": "", + # Anti-hallucination hardening (faster-whisper decodes junk tokens + # from silence/noise without these): + "vad": True, # Silero VAD filter — silence never reaches whisper. false = old raw behavior (music/ambient). + "vad_min_silence_ms": 500, # min silence (ms) that splits speech chunks when vad is on + "no_speech_prob_threshold": 0.6, # drop a segment only if no_speech_prob is ABOVE this... + "logprob_threshold": -1.0, # ...AND its avg_logprob is BELOW this (both must hit) }, "groq": { "model": "whisper-large-v3-turbo", # whisper-large-v3, whisper-large-v3-turbo, distil-whisper-large-v3-en diff --git a/tests/tools/test_stt_silence_hallucinations.py b/tests/tools/test_stt_silence_hallucinations.py new file mode 100644 index 00000000000..291439549f2 --- /dev/null +++ b/tests/tools/test_stt_silence_hallucinations.py @@ -0,0 +1,159 @@ +"""Tests for the local faster-whisper silence-hallucination hardening. + +One shared kwargs owner (`build_local_transcribe_kwargs`) must apply the +three-layer fix at every local whisper call site: + +1. Silero VAD filter on by default (``stt.local.vad: false`` restores raw). +2. ``condition_on_previous_text=False`` always. +3. Segment confidence gate: drop segments only when the model BOTH thinks + the window is non-speech AND decoded it with low confidence — quiet but + real speech must survive. +""" + +from types import SimpleNamespace + +from tools.transcription_tools import ( + _LOGPROB_THRESHOLD_DEFAULT, + _NO_SPEECH_PROB_THRESHOLD_DEFAULT, + _is_hallucinated_segment, + _join_confident_segments, + build_local_transcribe_kwargs, +) + + +def _seg(text, no_speech_prob=0.0, avg_logprob=-0.2): + return SimpleNamespace(text=text, no_speech_prob=no_speech_prob, avg_logprob=avg_logprob) + + +class TestBuildLocalTranscribeKwargs: + def test_vad_on_by_default(self): + kwargs = build_local_transcribe_kwargs({}) + assert kwargs["vad_filter"] is True + assert kwargs["vad_parameters"] == {"min_silence_duration_ms": 500} + + def test_conditioning_always_off(self): + assert build_local_transcribe_kwargs({})["condition_on_previous_text"] is False + assert ( + build_local_transcribe_kwargs({"local": {"vad": False}})[ + "condition_on_previous_text" + ] + is False + ) + + def test_vad_off_switch_restores_raw_behavior(self): + kwargs = build_local_transcribe_kwargs({"local": {"vad": False}}) + assert kwargs["vad_filter"] is False + assert "vad_parameters" not in kwargs + + def test_null_local_section_is_safe(self): + # YAML `local: null` breaks .get("local", {}) chains — must not here. + kwargs = build_local_transcribe_kwargs({"local": None}) + assert kwargs["vad_filter"] is True + + def test_vad_min_silence_configurable(self): + kwargs = build_local_transcribe_kwargs({"local": {"vad_min_silence_ms": 750}}) + assert kwargs["vad_parameters"] == {"min_silence_duration_ms": 750} + + def test_vad_min_silence_garbage_falls_back(self): + kwargs = build_local_transcribe_kwargs({"local": {"vad_min_silence_ms": "nope"}}) + assert kwargs["vad_parameters"] == {"min_silence_duration_ms": 500} + + def test_beam_size_kept(self): + assert build_local_transcribe_kwargs({})["beam_size"] == 5 + + def test_language_and_prompt_resolved(self, monkeypatch): + monkeypatch.delenv("HERMES_LOCAL_STT_LANGUAGE", raising=False) + cfg = {"language": "en", "local": {"initial_prompt": "Hermes glossary"}} + kwargs = build_local_transcribe_kwargs(cfg) + assert kwargs["language"] == "en" + assert kwargs["initial_prompt"] == "Hermes glossary" + + +class TestConfidenceGate: + def test_high_no_speech_and_low_logprob_dropped(self): + seg = _seg(" You", no_speech_prob=0.9, avg_logprob=-1.5) + assert _is_hallucinated_segment( + seg, _NO_SPEECH_PROB_THRESHOLD_DEFAULT, _LOGPROB_THRESHOLD_DEFAULT + ) + + def test_quiet_but_confident_speech_survives(self): + # High no_speech_prob alone must NOT drop a segment the model decoded + # confidently (quiet-but-real speech). + seg = _seg(" hello there", no_speech_prob=0.8, avg_logprob=-0.3) + assert not _is_hallucinated_segment( + seg, _NO_SPEECH_PROB_THRESHOLD_DEFAULT, _LOGPROB_THRESHOLD_DEFAULT + ) + + def test_low_confidence_speech_survives(self): + # Low avg_logprob alone (mumbled real speech) must survive too. + seg = _seg(" mumble", no_speech_prob=0.1, avg_logprob=-1.8) + assert not _is_hallucinated_segment( + seg, _NO_SPEECH_PROB_THRESHOLD_DEFAULT, _LOGPROB_THRESHOLD_DEFAULT + ) + + def test_missing_attrs_never_dropped(self): + seg = SimpleNamespace(text=" plugin segment") + assert not _is_hallucinated_segment( + seg, _NO_SPEECH_PROB_THRESHOLD_DEFAULT, _LOGPROB_THRESHOLD_DEFAULT + ) + + def test_join_drops_only_hallucinated(self): + segments = [ + _seg(" Hello world."), + _seg(" Thank you.", no_speech_prob=0.95, avg_logprob=-2.0), + _seg(" This is a test."), + ] + assert _join_confident_segments(segments, {}) == "Hello world. This is a test." + + def test_thresholds_configurable(self): + seg = _seg(" borderline", no_speech_prob=0.5, avg_logprob=-0.8) + cfg = {"no_speech_prob_threshold": 0.4, "logprob_threshold": -0.5} + assert _join_confident_segments([seg], cfg) == "" + + def test_garbage_thresholds_fall_back_to_defaults(self): + seg = _seg(" ok", no_speech_prob=0.1, avg_logprob=-0.1) + cfg = {"no_speech_prob_threshold": "high", "logprob_threshold": None} + assert _join_confident_segments([seg], cfg) == "ok" + + +class TestTranscribeLocalWiring: + """_transcribe_local must pass the shared hardened kwargs to the model.""" + + def _run(self, monkeypatch, stt_config, segments=None): + import tools.transcription_tools as tt + + captured = {} + + class FakeModel: + def transcribe(self, path, **kwargs): + captured.update(kwargs) + info = SimpleNamespace(language="en", duration=1.0) + return iter(segments or [_seg(" hi")]), info + + monkeypatch.setattr(tt, "_HAS_FASTER_WHISPER", True) + monkeypatch.setattr(tt, "_local_model", FakeModel()) + monkeypatch.setattr(tt, "_local_model_name", "base") + monkeypatch.setattr(tt, "_load_stt_config", lambda: stt_config) + monkeypatch.delenv("HERMES_LOCAL_STT_LANGUAGE", raising=False) + result = tt._transcribe_local("/tmp/fake.wav", "base") + return captured, result + + def test_hardened_kwargs_reach_model(self, monkeypatch): + captured, result = self._run(monkeypatch, {}) + assert result["success"] is True + assert captured["vad_filter"] is True + assert captured["vad_parameters"] == {"min_silence_duration_ms": 500} + assert captured["condition_on_previous_text"] is False + + def test_config_off_switch_reaches_model(self, monkeypatch): + captured, _ = self._run(monkeypatch, {"local": {"vad": False}}) + assert captured["vad_filter"] is False + assert "vad_parameters" not in captured + + def test_hallucinated_segments_filtered_from_transcript(self, monkeypatch): + segments = [ + _seg(" real speech"), + _seg(" Дякую за перегляд!", no_speech_prob=0.97, avg_logprob=-1.6), + ] + _, result = self._run(monkeypatch, {}, segments=segments) + assert result["transcript"] == "real speech" diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 9e091abdd51..cb1de1b0d59 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -1494,6 +1494,117 @@ def _load_local_whisper_model(model_name: str, device: str = "auto", compute_typ return WhisperModel(model_name, device="cpu", compute_type="int8") +# Silence-hallucination hardening defaults for local faster-whisper. +# Whisper decodes SOMETHING even from pure silence/noise — often short junk +# tokens ("You", "Thank you.", other-language phrases). Three layers kill the +# class at the source (all tunable under ``stt.local``): +# 1. vad_filter (Silero VAD, bundled with faster-whisper): silence never +# reaches the model. ``stt.local.vad: false`` restores raw behavior +# (e.g. transcribing music/ambient audio). +# 2. condition_on_previous_text=False: one hallucinated token can't seed a +# run of them; negligible quality cost for voice-note-length audio. +# 3. Segment confidence gate (see _is_hallucinated_segment): drops segments +# the model itself flags as probably-not-speech AND low-confidence. +_VAD_MIN_SILENCE_MS_DEFAULT = 500 +_NO_SPEECH_PROB_THRESHOLD_DEFAULT = 0.6 +_LOGPROB_THRESHOLD_DEFAULT = -1.0 + + +def build_local_transcribe_kwargs(stt_config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Build the kwargs for EVERY local faster-whisper ``model.transcribe`` call. + + Single owner for the anti-hallucination hardening — any new local-whisper + call site must go through this helper instead of hand-rolling kwargs. + """ + stt_config = stt_config if isinstance(stt_config, dict) else _load_stt_config() + local_cfg = stt_config.get("local") or {} + + kwargs: Dict[str, Any] = { + "beam_size": 5, + # Don't feed the previous window's text back as a prompt: a single + # hallucinated token otherwise seeds a self-reinforcing run of them. + "condition_on_previous_text": False, + } + + vad_enabled = local_cfg.get("vad", True) + if vad_enabled is None: + vad_enabled = True + if bool(vad_enabled): + kwargs["vad_filter"] = True + try: + min_silence_ms = int( + local_cfg.get("vad_min_silence_ms", _VAD_MIN_SILENCE_MS_DEFAULT) + ) + except (TypeError, ValueError): + min_silence_ms = _VAD_MIN_SILENCE_MS_DEFAULT + kwargs["vad_parameters"] = {"min_silence_duration_ms": min_silence_ms} + else: + kwargs["vad_filter"] = False + + forced_lang = _resolve_stt_language("local", stt_config) + if forced_lang: + kwargs["language"] = forced_lang + + initial_prompt = local_cfg.get("initial_prompt") + if isinstance(initial_prompt, str) and initial_prompt.strip(): + kwargs["initial_prompt"] = initial_prompt + + return kwargs + + +def _confidence_thresholds(local_cfg: Dict[str, Any]) -> tuple[float, float]: + """Resolve (no_speech_prob, avg_logprob) gate thresholds from config.""" + try: + no_speech = float( + local_cfg.get("no_speech_prob_threshold", _NO_SPEECH_PROB_THRESHOLD_DEFAULT) + ) + except (TypeError, ValueError): + no_speech = _NO_SPEECH_PROB_THRESHOLD_DEFAULT + try: + logprob = float(local_cfg.get("logprob_threshold", _LOGPROB_THRESHOLD_DEFAULT)) + except (TypeError, ValueError): + logprob = _LOGPROB_THRESHOLD_DEFAULT + return no_speech, logprob + + +def _is_hallucinated_segment(segment: Any, no_speech_threshold: float, logprob_threshold: float) -> bool: + """True when a segment is very likely a silence hallucination. + + Conservative AND gate (matches openai-whisper's own heuristic): the model + must BOTH think the window is non-speech (high no_speech_prob) AND have + decoded it with low confidence (low avg_logprob). Quiet-but-real speech + fails one of the two conditions and survives. + """ + no_speech_prob = getattr(segment, "no_speech_prob", None) + avg_logprob = getattr(segment, "avg_logprob", None) + if no_speech_prob is None or avg_logprob is None: + return False + try: + no_speech_prob = float(no_speech_prob) + avg_logprob = float(avg_logprob) + except (TypeError, ValueError): + # Unknown segment shape (plugin/test doubles) — never drop. + return False + return no_speech_prob > no_speech_threshold and avg_logprob < logprob_threshold + + +def _join_confident_segments(segments: Any, local_cfg: Dict[str, Any]) -> str: + """Join segment texts, dropping probable silence hallucinations.""" + no_speech_threshold, logprob_threshold = _confidence_thresholds(local_cfg) + kept: list[str] = [] + for segment in segments: + if _is_hallucinated_segment(segment, no_speech_threshold, logprob_threshold): + logger.debug( + "Dropping probable hallucinated segment %r (no_speech_prob=%.3f, avg_logprob=%.3f)", + getattr(segment, "text", ""), + getattr(segment, "no_speech_prob", float("nan")), + getattr(segment, "avg_logprob", float("nan")), + ) + continue + kept.append(segment.text.strip()) + return " ".join(kept).strip() + + def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: """Transcribe using faster-whisper (local, free).""" global _local_model, _local_model_name @@ -1523,20 +1634,16 @@ def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: ) _local_model_name = model_name - # Language: stt.local.language > stt.language > env var > auto-detect. + # Shared hardened kwargs: VAD filter (default on), no cross-window + # conditioning, language/initial_prompt resolution — one owner for + # every local faster-whisper call site. stt_config = _load_stt_config() local_config = stt_config.get("local") or {} - _forced_lang = _resolve_stt_language("local", stt_config) - transcribe_kwargs = {"beam_size": 5} - if _forced_lang: - transcribe_kwargs["language"] = _forced_lang - initial_prompt = local_config.get("initial_prompt") - if isinstance(initial_prompt, str) and initial_prompt.strip(): - transcribe_kwargs["initial_prompt"] = initial_prompt + transcribe_kwargs = build_local_transcribe_kwargs(stt_config) try: segments, info = _local_model.transcribe(file_path, **transcribe_kwargs) - transcript = " ".join(segment.text.strip() for segment in segments) + transcript = _join_confident_segments(segments, local_config) except Exception as exc: # CUDA runtime libs sometimes only fail at dlopen-on-first-use, # AFTER the model loaded successfully. Evict the broken cached @@ -1556,7 +1663,7 @@ def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: _local_model = WhisperModel(model_name, device="cpu", compute_type="int8") _local_model_name = model_name segments, info = _local_model.transcribe(file_path, **transcribe_kwargs) - transcript = " ".join(segment.text.strip() for segment in segments) + transcript = _join_confident_segments(segments, local_config) logger.info( "Transcribed %s via local whisper (%s, lang=%s, %.1fs audio)", diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 9ad7e55f877..09558a4e9c9 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1718,6 +1718,10 @@ stt: model: "base" # tiny, base, small, medium, large-v3 language: "" # per-provider override of stt.language initial_prompt: "" # optional whisper prompt to bias vocabulary/script (e.g. Simplified Chinese) + vad: true # Silero VAD filter (default on) — silence never reaches whisper; false = raw behavior (music/ambient) + vad_min_silence_ms: 500 # min silence (ms) that splits speech chunks when vad is on + no_speech_prob_threshold: 0.6 # drop a segment only when no_speech_prob > this... + logprob_threshold: -1.0 # ...AND avg_logprob < this (both must hit — quiet real speech survives) groq: language: "" # per-provider override of stt.language openai: @@ -1732,7 +1736,7 @@ Set `stt.echo_transcripts: false` when the gateway should transcribe voice notes Provider behavior: -- `local` uses `faster-whisper` running on your machine. Install it separately with `pip install faster-whisper`. +- `local` uses `faster-whisper` running on your machine. Install it separately with `pip install faster-whisper`. Silence-hallucination hardening is on by default: a Silero VAD filter keeps silence/noise from ever reaching Whisper, cross-window conditioning is disabled, and segments the model itself flags as probably-not-speech *and* low-confidence are dropped. Set `stt.local.vad: false` to transcribe non-speech audio (music, ambient) with the raw behavior. - `groq` uses Groq's Whisper-compatible endpoint and reads `GROQ_API_KEY`. Pass `stt.groq.language` (or the global `HERMES_LOCAL_STT_LANGUAGE` env var) to skip auto-detection and reduce latency. - `openai` uses the OpenAI speech API and reads `VOICE_TOOLS_OPENAI_KEY`. diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md index 9779e6eed73..2fc64c0c76a 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md @@ -1278,7 +1278,7 @@ stt: Provider 行为： -- `local` 使用在您机器上运行的 `faster-whisper`。使用 `pip install faster-whisper` 单独安装。 +- `local` 使用在您机器上运行的 `faster-whisper`。使用 `pip install faster-whisper` 单独安装。静音幻觉防护默认开启:Silero VAD 过滤器让静音/噪声不会进入 Whisper,跨窗口条件预测被禁用,并且模型自己标记为"很可能不是语音"且低置信度的片段会被丢弃。设置 `stt.local.vad: false` 可用原始行为转录非语音音频(音乐、环境声)。 - `groq` 使用 Groq 的 Whisper 兼容端点，读取 `GROQ_API_KEY`。 - `openai` 使用 OpenAI 语音 API，读取 `VOICE_TOOLS_OPENAI_KEY`。