diff --git a/CHANGELOG.md b/CHANGELOG.md index 9615e4c8..a4b0f605 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Per-PR attribution and contributor credits are published automatically on the co ## [Unreleased] ### Fixed +- A sentence that fails to generate or encode now fails the request instead of being left out of an otherwise successful response. Non-streaming requests get an error; a stream is cut off rather than finished, and its download link file is removed instead of being kept with the sentence missing - CUDA out-of-memory during generation is retried once and then reported, instead of retrying the same request without limit; a retry no longer repeats audio already sent or drops word timestamps - `/dev/generate_from_phonemes` checks its voice like `/v1/audio/speech` does, returning 400 for an unknown voice. It used to pass the voice straight to the mixer, where a file path as the voice could write a mixed voice pack into the voices directory - `/dev/phonemize` returns the phonemes for all of the text. Input the pipeline splits into several segments, such as separate paragraphs, used to come back with only the first segment diff --git a/api/src/services/temp_manager.py b/api/src/services/temp_manager.py index 09c1ac19..587163e2 100644 --- a/api/src/services/temp_manager.py +++ b/api/src/services/temp_manager.py @@ -4,6 +4,7 @@ import os import tempfile import time +from contextlib import suppress import aiofiles import aiofiles.os @@ -139,6 +140,10 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except Exception as e: logger.error(f"Error closing temp file: {e}") self._write_error = True + # a failed request must not leave a truncated file at its download link + if exc_type is not None and self.temp_file: + with suppress(FileNotFoundError): + os.remove(self.temp_path) async def write(self, chunk: bytes) -> None: """Write a chunk of audio data diff --git a/api/src/services/tts_service.py b/api/src/services/tts_service.py index cdb09733..ee331e8d 100644 --- a/api/src/services/tts_service.py +++ b/api/src/services/tts_service.py @@ -66,111 +66,102 @@ async def _process_chunk( ) -> AsyncGenerator[AudioChunk, None]: """Process tokens into audio.""" async with self._chunk_semaphore: - try: - # Handle stream finalization - if is_last: - # Skip format conversion for raw audio mode - if not output_format: - yield AudioChunk(np.array([], dtype=np.int16), output=b"") - return - chunk_data = await AudioService.convert_audio( - AudioChunk( - np.array([], dtype=np.float32) - ), # Dummy data for type checking - output_format, - writer, - speed, - "", - normalizer=normalizer, - is_last_chunk=True, - ) - yield chunk_data + # Handle stream finalization + if is_last: + # Skip format conversion for raw audio mode + if not output_format: + yield AudioChunk(np.array([], dtype=np.int16), output=b"") return + chunk_data = await AudioService.convert_audio( + AudioChunk( + np.array([], dtype=np.float32) + ), # Dummy data for type checking + output_format, + writer, + speed, + "", + normalizer=normalizer, + is_last_chunk=True, + ) + yield chunk_data + return - # Skip empty chunks - if not tokens and not chunk_text: - return - - await self.model_manager.ensure_backend() - backend = self.model_manager.get_backend() - - # Generate audio using pre-warmed model - if isinstance(backend, KokoroV1): - chunk_index = 0 - # For Kokoro V1, pass text and voice info with lang_code - async for chunk_data in self.model_manager.generate( - chunk_text, - (voice_name, voice_path), - speed=speed, - lang_code=lang_code, - return_timestamps=return_timestamps, - ): - chunk_data.audio *= volume_multiplier - # For streaming, convert to bytes - if output_format: - try: - chunk_data = await AudioService.convert_audio( - chunk_data, - output_format, - writer, - speed, - chunk_text, - is_last_chunk=is_last, - normalizer=normalizer, - ) - yield chunk_data - except Exception as e: - logger.error(f"Failed to convert audio: {str(e)}") - else: - chunk_data = AudioService.trim_audio( - chunk_data, chunk_text, speed, is_last, normalizer - ) - yield chunk_data - chunk_index += 1 - else: - # For legacy backends, load voice tensor - voice_tensor = await self._voice_manager.load_voice( - voice_name, device=backend.device - ) - chunk_data = await self.model_manager.generate( - tokens, - voice_tensor, - speed=speed, - return_timestamps=return_timestamps, - ) - - if chunk_data.audio is None: - logger.error("Model generated None for audio chunk") - return + # Skip empty chunks + if not tokens and not chunk_text: + return - if len(chunk_data.audio) == 0: - logger.error("Model generated empty audio chunk") - return + await self.model_manager.ensure_backend() + backend = self.model_manager.get_backend() + # Generate audio using pre-warmed model + if isinstance(backend, KokoroV1): + chunk_index = 0 + # For Kokoro V1, pass text and voice info with lang_code + async for chunk_data in self.model_manager.generate( + chunk_text, + (voice_name, voice_path), + speed=speed, + lang_code=lang_code, + return_timestamps=return_timestamps, + ): chunk_data.audio *= volume_multiplier - # For streaming, convert to bytes if output_format: - try: - chunk_data = await AudioService.convert_audio( - chunk_data, - output_format, - writer, - speed, - chunk_text, - normalizer=normalizer, - is_last_chunk=is_last, - ) - yield chunk_data - except Exception as e: - logger.error(f"Failed to convert audio: {str(e)}") + chunk_data = await AudioService.convert_audio( + chunk_data, + output_format, + writer, + speed, + chunk_text, + is_last_chunk=is_last, + normalizer=normalizer, + ) + yield chunk_data else: - trimmed = AudioService.trim_audio( + chunk_data = AudioService.trim_audio( chunk_data, chunk_text, speed, is_last, normalizer ) - yield trimmed - except Exception as e: - logger.error(f"Failed to process tokens: {str(e)}") + yield chunk_data + chunk_index += 1 + else: + # For legacy backends, load voice tensor + voice_tensor = await self._voice_manager.load_voice( + voice_name, device=backend.device + ) + chunk_data = await self.model_manager.generate( + tokens, + voice_tensor, + speed=speed, + return_timestamps=return_timestamps, + ) + + if chunk_data.audio is None: + logger.error("Model generated None for audio chunk") + return + + if len(chunk_data.audio) == 0: + logger.error("Model generated empty audio chunk") + return + + chunk_data.audio *= volume_multiplier + + # For streaming, convert to bytes + if output_format: + chunk_data = await AudioService.convert_audio( + chunk_data, + output_format, + writer, + speed, + chunk_text, + normalizer=normalizer, + is_last_chunk=is_last, + ) + yield chunk_data + else: + trimmed = AudioService.trim_audio( + chunk_data, chunk_text, speed, is_last, normalizer + ) + yield trimmed async def _load_voice_from_path(self, path: str, weight: float): # Check if the path is None and raise a ValueError if it is not @@ -403,91 +394,76 @@ async def generate_audio_stream( tokens or chunk_text.strip() ): # Process if there are tokens OR non-whitespace text # --- Handle Text Chunk --- - try: - chunk_start_offset = current_offset - # Process audio for chunk - async for chunk_data in self._process_chunk( - chunk_text, # Pass text for Kokoro V1 - tokens, # Pass tokens for legacy backends - voice_name, # Pass voice name - voice_path, # Pass voice path - clamp_rate(speed * segment_rate), - writer, - output_format, - is_first=(chunk_index == 0), - volume_multiplier=volume_multiplier, - is_last=False, # We'll update the last chunk later - normalizer=stream_normalizer, - lang_code=pipeline_lang_code, # Pass lang_code - return_timestamps=return_timestamps, - ): - if chunk_data.word_timestamps is not None: - for timestamp in chunk_data.word_timestamps: - timestamp.start_time += current_offset - timestamp.end_time += current_offset - if allow_voice_tags: - timestamp.voice = voice_name - - # Update offset based on the actual duration of the generated audio chunk - chunk_duration = 0 - if ( - chunk_data.audio is not None - and len(chunk_data.audio) > 0 - ): - chunk_duration = len(chunk_data.audio) / 24000 - current_offset += chunk_duration - - # Yield the processed chunk (either formatted or raw) - if chunk_data.output is not None: - yield chunk_data - elif ( - chunk_data.audio is not None - and len(chunk_data.audio) > 0 - ): - yield chunk_data - else: - logger.warning( - f"No audio generated for chunk: '{chunk_text[:100]}...'" - ) - - chunk_index += 1 # Increment chunk index after processing text - if timings is not None and current_offset > chunk_start_offset: - entry = { - "text": chunk_text, - "start": round(chunk_start_offset, 3), - "end": round(current_offset, 3), - } - if allow_voice_tags: - entry["voice"] = voice_name - timings.append(entry) - except Exception as e: - logger.error( - f"Failed to process audio for chunk: '{chunk_text[:100]}...'. Error: {str(e)}" - ) - continue - - # Only finalize if we successfully processed at least one chunk - if chunk_index > 0: - try: - # Empty tokens list to finalize audio + chunk_start_offset = current_offset + # Process audio for chunk async for chunk_data in self._process_chunk( - "", # Empty text - [], # Empty tokens - voice_name, - voice_path, - speed, + chunk_text, # Pass text for Kokoro V1 + tokens, # Pass tokens for legacy backends + voice_name, # Pass voice name + voice_path, # Pass voice path + clamp_rate(speed * segment_rate), writer, output_format, - is_first=False, - is_last=True, # Signal this is the last chunk + is_first=(chunk_index == 0), volume_multiplier=volume_multiplier, + is_last=False, # We'll update the last chunk later normalizer=stream_normalizer, lang_code=pipeline_lang_code, # Pass lang_code + return_timestamps=return_timestamps, ): + if chunk_data.word_timestamps is not None: + for timestamp in chunk_data.word_timestamps: + timestamp.start_time += current_offset + timestamp.end_time += current_offset + if allow_voice_tags: + timestamp.voice = voice_name + + # Update offset based on the actual duration of the generated audio chunk + chunk_duration = 0 + if chunk_data.audio is not None and len(chunk_data.audio) > 0: + chunk_duration = len(chunk_data.audio) / 24000 + current_offset += chunk_duration + + # Yield the processed chunk (either formatted or raw) if chunk_data.output is not None: yield chunk_data - except Exception as e: - logger.error(f"Failed to finalize audio stream: {str(e)}") + elif chunk_data.audio is not None and len(chunk_data.audio) > 0: + yield chunk_data + else: + logger.warning( + f"No audio generated for chunk: '{chunk_text[:100]}...'" + ) + + chunk_index += 1 # Increment chunk index after processing text + if timings is not None and current_offset > chunk_start_offset: + entry = { + "text": chunk_text, + "start": round(chunk_start_offset, 3), + "end": round(current_offset, 3), + } + if allow_voice_tags: + entry["voice"] = voice_name + timings.append(entry) + + # Only finalize if we successfully processed at least one chunk + if chunk_index > 0: + # Empty tokens list to finalize audio + async for chunk_data in self._process_chunk( + "", # Empty text + [], # Empty tokens + voice_name, + voice_path, + speed, + writer, + output_format, + is_first=False, + is_last=True, # Signal this is the last chunk + volume_multiplier=volume_multiplier, + normalizer=stream_normalizer, + lang_code=pipeline_lang_code, # Pass lang_code + ): + if chunk_data.output is not None: + yield chunk_data except Exception as e: logger.error(f"Error in phoneme audio generation: {str(e)}") diff --git a/api/tests/test_temp_manager.py b/api/tests/test_temp_manager.py index b63365e5..646cba33 100644 --- a/api/tests/test_temp_manager.py +++ b/api/tests/test_temp_manager.py @@ -4,7 +4,7 @@ import pytest from api.src.core.config import settings -from api.src.services.temp_manager import cleanup_temp_files +from api.src.services.temp_manager import TempFileWriter, cleanup_temp_files @pytest.mark.asyncio @@ -35,6 +35,17 @@ async def test_cleanup_removes_timing_sidecar_with_its_audio(tmp_path, monkeypat assert fresh_sidecar.exists() +async def test_failed_write_removes_partial_download(tmp_path, monkeypatch): + monkeypatch.setattr(settings, "temp_file_dir", str(tmp_path)) + writer = await TempFileWriter("mp3").__aenter__() + await writer.write(b"partial") + + error = RuntimeError("generation failed") + await writer.__aexit__(type(error), error, None) + + assert not os.path.exists(writer.temp_path) + + @pytest.mark.asyncio async def test_cleanup_ages_files_against_the_clock(tmp_path, monkeypatch): """Age is measured from now, not from the directory's last change.""" diff --git a/api/tests/test_tts_service.py b/api/tests/test_tts_service.py index f6270955..b02608c0 100644 --- a/api/tests/test_tts_service.py +++ b/api/tests/test_tts_service.py @@ -589,3 +589,25 @@ async def test_generate_audio_with_nothing_speakable_raises(): MagicMock(), allow_voice_tags=True, ) + + +@pytest.mark.asyncio +async def test_failed_chunk_raises_instead_of_dropping_speech(): + """A chunk that fails to generate must fail the request, not vanish from a 200.""" + model_manager = AsyncMock() + model_manager.get_backend = MagicMock() + model_manager.generate.side_effect = RuntimeError("model failed") + with ( + patch("api.src.services.tts_service.get_model_manager") as mock_get_model, + patch("api.src.services.tts_service.get_voice_manager") as mock_get_voice, + ): + mock_get_model.return_value = model_manager + mock_get_voice.return_value = AsyncMock() + service = await TTSService.create() + service.get_voices_path = AsyncMock(return_value=("af_heart", "/p/af_heart.pt")) + + with pytest.raises(RuntimeError, match="model failed"): + async for _ in service.generate_audio_stream( + "Hello there.", "af_heart", MagicMock(), output_format=None + ): + pass