diff --git a/mlx_audio/stt/models/whisper/whisper.py b/mlx_audio/stt/models/whisper/whisper.py
index 24c2b7686..34dbcb3d1 100644
--- a/mlx_audio/stt/models/whisper/whisper.py
+++ b/mlx_audio/stt/models/whisper/whisper.py
@@ -987,6 +987,8 @@ def decode_with_fallback(segment: mx.array) -> DecodingResult:
                 if (
                     no_speech_threshold is not None
                     and decode_result.no_speech_prob > no_speech_threshold
+                    and logprob_threshold is not None
+                    and decode_result.avg_logprob < logprob_threshold
                 ):
                     needs_fallback = False  # silence
                 if not needs_fallback: