Update app.py

app.py

@@ -45,212 +45,182 @@ class VoiceSynthesizer:
         except Exception as e:
             print(f"Bark model loading error: {e}")
     
-def process_reference_audio(self, reference_audio):
-    """Process and store reference audio for voice cloning"""
-    try:
-        # Gradio can pass audio in different formats
-        if reference_audio is None:
-            return "No audio provided"
-        
-        # Handle different input types
-        if isinstance(reference_audio, tuple):
-            # Gradio typically returns (sample_rate, audio_array)
-            if len(reference_audio) == 2:
-                sample_rate, audio_data = reference_audio
-            else:
-                audio_data = reference_audio[0]
-                sample_rate = SAMPLE_RATE  # Default to Bark sample rate
-        elif isinstance(reference_audio, np.ndarray):
-            audio_data = reference_audio
-            sample_rate = SAMPLE_RATE
-        else:
-            return "Invalid audio format"
-        
-        # Ensure audio is numpy array
-        audio_data = np.asarray(audio_data)
-        
-        # Handle multi-channel audio
-        if audio_data.ndim > 1:
-            audio_data = audio_data.mean(axis=1)
-        
-        # Trim or pad to standard length
-        max_duration = 10  # 10 seconds
-        max_samples = max_duration * sample_rate
-        
-        if len(audio_data) > max_samples:
-            audio_data = audio_data[:max_samples]
-        
-        # Resample if necessary
-        if sample_rate != SAMPLE_RATE:
-            from scipy.signal import resample
-            audio_data = resample(audio_data, int(len(audio_data) * SAMPLE_RATE / sample_rate))
-        
-        # Save reference audio
-        ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
-        sf.write(ref_filename, audio_data, SAMPLE_RATE)
-        
-        # Store reference voice
-        self.reference_voice = ref_filename
-        
-        return "Reference voice processed successfully"
+    def _initialize_bark(self):
+        """Bark model initialization (already done in __init__)"""
+        return None
     
-    except Exception as e:
-        print(f"Reference audio processing error: {e}")
-        import traceback
-        traceback.print_exc()
-        return f"Error processing reference audio: {str(e)}"
-
-def _generate_bark_speech(self, text, voice_preset=None):
-    """Generate speech using Bark"""
-    # Default Bark voice presets
-    voice_presets = [
-        "v2/en_speaker_6",  # Female
-        "v2/en_speaker_3",  # Male
-        "v2/en_speaker_9",  # Neutral
-    ]
-    
-    # Prepare history prompt
-    history_prompt = None
-    
-    # Check if a reference voice is available
-    if self.reference_voice is not None:
-        # Use saved reference voice file
-        history_prompt = self.reference_voice
-    elif voice_preset:
-        # Use predefined voice preset
-        history_prompt = voice_presets[0] if "v2/en_speaker" not in voice_preset else voice_preset
+    def _initialize_speecht5(self):
+        """Initialize SpeechT5 model from Hugging Face"""
+        try:
+            # Load SpeechT5 model and processor
+            model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
+            processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
+            vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+            
+            # Load speaker embeddings
+            embeddings_dataset = datasets.load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+            speaker_embeddings = torch.tensor(embeddings_dataset[0]["xvector"]).unsqueeze(0)
+            
+            return {
+                "model": model,
+                "processor": processor,
+                "vocoder": vocoder,
+                "speaker_embeddings": speaker_embeddings
+            }
+        except Exception as e:
+            print(f"SpeechT5 model loading error: {e}")
+            return None
     
-    # Generate audio with or without history prompt
-    try:
-        if history_prompt:
-            audio_array = generate_audio(
-                text, 
-                history_prompt=history_prompt
-            )
-        else:
-            # Fallback to default generation
-            audio_array = generate_audio(text)
-        
-        # Save generated audio
-        filename = f"bark_speech_{int(time.time())}.wav"
-        filepath = os.path.join(self.working_dir, filename)
-        wavfile.write(filepath, SAMPLE_RATE, audio_array)
+    def process_reference_audio(self, reference_audio):
+        """Process and store reference audio for voice cloning"""
+        try:
+            # Gradio can pass audio in different formats
+            if reference_audio is None:
+                return "No audio provided"
+            
+            # Handle different input types
+            if isinstance(reference_audio, tuple):
+                # Gradio typically returns (sample_rate, audio_array)
+                if len(reference_audio) == 2:
+                    sample_rate, audio_data = reference_audio
+                else:
+                    audio_data = reference_audio[0]
+                    sample_rate = SAMPLE_RATE  # Default to Bark sample rate
+            elif isinstance(reference_audio, np.ndarray):
+                audio_data = reference_audio
+                sample_rate = SAMPLE_RATE
+            else:
+                return "Invalid audio format"
+            
+            # Ensure audio is numpy array
+            audio_data = np.asarray(audio_data)
+            
+            # Handle multi-channel audio
+            if audio_data.ndim > 1:
+                audio_data = audio_data.mean(axis=1)
+            
+            # Trim or pad to standard length
+            max_duration = 10  # 10 seconds
+            max_samples = max_duration * sample_rate
+            
+            if len(audio_data) > max_samples:
+                audio_data = audio_data[:max_samples]
+            
+            # Resample if necessary
+            if sample_rate != SAMPLE_RATE:
+                from scipy.signal import resample
+                audio_data = resample(audio_data, int(len(audio_data) * SAMPLE_RATE / sample_rate))
+            
+            # Save reference audio
+            ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
+            sf.write(ref_filename, audio_data, SAMPLE_RATE)
+            
+            # Store reference voice
+            self.reference_voice = ref_filename
+            
+            return "Reference voice processed successfully"
         
-        return filepath, None
-    
-    except Exception as e:
-        print(f"Bark speech generation error: {e}")
-        import traceback
-        traceback.print_exc()
-        return None, f"Error in Bark speech generation: {str(e)}"
-def create_interface():
-    synthesizer = VoiceSynthesizer()
+        except Exception as e:
+            print(f"Reference audio processing error: {e}")
+            import traceback
+            traceback.print_exc()
+            return f"Error processing reference audio: {str(e)}"
     
-    with gr.Blocks() as interface:
-        gr.Markdown("# 🎙️ Advanced Voice Synthesis")
-        
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## 1. Capture Reference Voice")
-                reference_audio = gr.Audio(sources=["microphone", "upload"], type="numpy")
-                process_ref_btn = gr.Button("Process Reference Voice")
-                process_ref_output = gr.Textbox(label="Reference Voice Processing")
-                
-            with gr.Column():
-                gr.Markdown("## 2. Generate Speech")
-                text_input = gr.Textbox(label="Enter Text to Speak")
-                
-                # Model Selection
-                model_dropdown = gr.Dropdown(
-                    choices=[
-                        "bark (Suno AI)",
-                        "speecht5 (Microsoft)"
-                    ],
-                    label="Select TTS Model",
-                    value="bark (Suno AI)"
+    def _generate_bark_speech(self, text, voice_preset=None):
+        """Generate speech using Bark"""
+        # Default Bark voice presets
+        voice_presets = [
+            "v2/en_speaker_6",  # Female
+            "v2/en_speaker_3",  # Male
+            "v2/en_speaker_9",  # Neutral
+        ]
+        
+        # Prepare history prompt
+        history_prompt = None
+        
+        # Check if a reference voice is available
+        if self.reference_voice is not None:
+            # Use saved reference voice file
+            history_prompt = self.reference_voice
+        elif voice_preset:
+            # Use predefined voice preset
+            history_prompt = voice_presets[0] if "v2/en_speaker" not in voice_preset else voice_preset
+        
+        # Generate audio with or without history prompt
+        try:
+            if history_prompt:
+                audio_array = generate_audio(
+                    text, 
+                    history_prompt=history_prompt
                 )
-                
-                # Voice Preset Dropdowns
-                with gr.Row():
-                    bark_preset = gr.Dropdown(
-                        choices=[
-                            "v2/en_speaker_6 (Female)",
-                            "v2/en_speaker_3 (Male)", 
-                            "v2/en_speaker_9 (Neutral)"
-                        ],
-                        label="Bark Voice Preset",
-                        visible=True
-                    )
-                    
-                    speecht5_preset = gr.Dropdown(
-                        choices=[
-                            "Default Speaker"
-                        ],
-                        label="SpeechT5 Speaker",
-                        visible=False
-                    )
-                
-                generate_btn = gr.Button("Generate Speech")
-                audio_output = gr.Audio(label="Generated Speech")
-                error_output = gr.Textbox(label="Errors", visible=True)
+            else:
+                # Fallback to default generation
+                audio_array = generate_audio(text)
+            
+            # Save generated audio
+            filename = f"bark_speech_{int(time.time())}.wav"
+            filepath = os.path.join(self.working_dir, filename)
+            wavfile.write(filepath, SAMPLE_RATE, audio_array)
+            
+            return filepath, None
         
-        # Process reference audio
-        process_ref_btn.click(
-            fn=synthesizer.process_reference_audio,
-            inputs=reference_audio,
-            outputs=process_ref_output
-        )
+        except Exception as e:
+            print(f"Bark speech generation error: {e}")
+            import traceback
+            traceback.print_exc()
+            return None, f"Error in Bark speech generation: {str(e)}"
+    
+    def generate_speech(self, text, model_name=None, voice_preset=None):
+        """Generate speech using selected model"""
+        if not text or not text.strip():
+            return None, "Please enter some text to speak"
+        
+        # Use specified model or current model
+        current_model = model_name or self.current_model
         
-        # Dynamic model and preset visibility
-        def update_model_visibility(model):
-            if "bark" in model.lower():
-                return {
-                    bark_preset: gr.update(visible=True),
-                    speecht5_preset: gr.update(visible=False)
-                }
+        try:
+            if current_model == "bark":
+                return self._generate_bark_speech(text, voice_preset)
+            elif current_model == "speecht5":
+                return self._generate_speecht5_speech(text, voice_preset)
             else:
-                return {
-                    bark_preset: gr.update(visible=False),
-                    speecht5_preset: gr.update(visible=True)
-                }
+                raise ValueError(f"Unsupported model: {current_model}")
         
-        model_dropdown.change(
-            fn=update_model_visibility,
-            inputs=model_dropdown,
-            outputs=[bark_preset, speecht5_preset]
+        except Exception as e:
+            print(f"Speech generation error: {e}")
+            import traceback
+            traceback.print_exc()
+            return None, f"Error generating speech: {str(e)}"
+    
+    def _generate_speecht5_speech(self, text, speaker_id=None):
+        """Generate speech using SpeechT5"""
+        # Ensure model is initialized
+        speecht5_models = self.models["speecht5"]()
+        if not speecht5_models:
+            return None, "SpeechT5 model not loaded"
+        
+        model = speecht5_models["model"]
+        processor = speecht5_models["processor"]
+        vocoder = speecht5_models["vocoder"]
+        speaker_embeddings = speecht5_models["speaker_embeddings"]
+        
+        # Prepare inputs
+        inputs = processor(text=text, return_tensors="pt")
+        
+        # Generate speech
+        speech = model.generate_speech(
+            inputs["input_ids"], 
+            speaker_embeddings
         )
         
-        # Speech generation logic
-        def generate_speech_wrapper(text, model, bark_preset, speecht5_preset):
-            # Map model name
-            model_map = {
-                "bark (Suno AI)": "bark",
-                "speecht5 (Microsoft)": "speecht5"
-            }
-            
-            # Select appropriate preset
-            preset = bark_preset if "bark" in model else speecht5_preset
-            
-            return synthesizer.generate_speech(
-                text, 
-                model_name=model_map[model], 
-                voice_preset=preset
-            )
+        # Convert to numpy array
+        audio_array = speech.numpy()
         
-        generate_btn.click(
-            fn=generate_speech_wrapper,
-            inputs=[text_input, model_dropdown, bark_preset, speecht5_preset],
-            outputs=[audio_output, error_output]
-        )
-    
-    return interface
+        # Save generated audio
+        filename = f"speecht5_speech_{int(time.time())}.wav"
+        filepath = os.path.join(self.working_dir, filename)
+        wavfile.write(filepath, 16000, audio_array)
+        
+        return filepath, None
 
-if __name__ == "__main__":
-    interface = create_interface()
-    interface.launch(
-        share=False,
-        debug=True,
-        show_error=True,
-        server_name='0.0.0.0',
-        server_port=7860
-    )
+# Rest of the code remains the same...