Update app.py

app.py

@@ -2,114 +2,235 @@ import gradio as gr
 import numpy as np
 import os
 import time
+import torch
 from scipy.io import wavfile
 
-# Explicitly import Bark components
+# Bark imports
 from bark import generate_audio, SAMPLE_RATE
 from bark.generation import preload_models
 
-class VoiceCloningApp:
+# Hugging Face Transformers
+from transformers import AutoModelForTextToSpeech, AutoProcessor, AutoTokenizer
+from transformers import SpeechT5HifiGan, SpeechT5ForTextToSpeech, SpeechT5Processor
+
+class VoiceSynthesizer:
     def __init__(self):
         # Create working directory
         self.base_dir = os.path.dirname(os.path.abspath(__file__))
         self.working_dir = os.path.join(self.base_dir, "working_files")
         os.makedirs(self.working_dir, exist_ok=True)
         
-        # Explicit model loading with error handling
+        # Initialize models dictionary
+        self.models = {
+            "bark": self._initialize_bark,
+            "speecht5": self._initialize_speecht5
+        }
+        
+        # Default model
+        self.current_model = "bark"
+        
+        # Initialize Bark models
         try:
             print("Attempting to load Bark models...")
             preload_models()
             print("Bark models loaded successfully.")
         except Exception as e:
-            print(f"Error loading Bark models: {e}")
-            import traceback
-            traceback.print_exc()
-            raise RuntimeError(f"Could not load Bark models. Error: {e}")
-
-    def process_reference_audio(self, audio_data):
-        """Simple audio processing"""
-        if audio_data is None:
-            return "Please provide an audio input"
-        
+            print(f"Bark model loading error: {e}")
+    
+    def _initialize_bark(self):
+        """Bark model initialization (already done in __init__)"""
+        return None
+    
+    def _initialize_speecht5(self):
+        """Initialize SpeechT5 model from Hugging Face"""
         try:
-            # Unpack audio data
-            sample_rate, audio_array = audio_data
+            # Load SpeechT5 model and processor
+            model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
+            processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
+            vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
             
-            # Normalize audio
-            audio_array = audio_array / np.max(np.abs(audio_array))
+            # Load speaker embeddings
+            embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+            speaker_embeddings = torch.tensor(embeddings_dataset[0]["xvector"]).unsqueeze(0)
             
-            # Save reference audio
-            filename = f"reference_{int(time.time())}.wav"
-            filepath = os.path.join(self.working_dir, filename)
-            wavfile.write(filepath, sample_rate, audio_array)
-            
-            return "✅ Audio captured successfully!"
-        
+            return {
+                "model": model,
+                "processor": processor,
+                "vocoder": vocoder,
+                "speaker_embeddings": speaker_embeddings
+            }
         except Exception as e:
-            return f"Error processing audio: {str(e)}"
-
-    def generate_speech(self, text):
-        """Generate speech using Bark"""
+            print(f"SpeechT5 model loading error: {e}")
+            return None
+    
+    def set_model(self, model_name):
+        """Set the current model for speech synthesis"""
+        if model_name not in self.models:
+            raise ValueError(f"Model {model_name} not supported")
+        self.current_model = model_name
+    
+    def generate_speech(self, text, model_name=None, voice_preset=None):
+        """Generate speech using selected model"""
         if not text or not text.strip():
             return None, "Please enter some text to speak"
         
+        # Use specified model or current model
+        current_model = model_name or self.current_model
+        
         try:
-            # Generate audio with explicit error handling
-            print(f"Generating speech for text: {text}")
-            
-            # Simplified audio generation
-            audio_array = generate_audio(
-                text,
-                history_prompt=None,
-            )
-            
-            # Save generated audio
-            filename = f"generated_speech_{int(time.time())}.wav"
-            filepath = os.path.join(self.working_dir, filename)
-            wavfile.write(filepath, SAMPLE_RATE, audio_array)
-            
-            return filepath, None
+            if current_model == "bark":
+                return self._generate_bark_speech(text, voice_preset)
+            elif current_model == "speecht5":
+                return self._generate_speecht5_speech(text, voice_preset)
+            else:
+                raise ValueError(f"Unsupported model: {current_model}")
         
         except Exception as e:
             print(f"Speech generation error: {e}")
             import traceback
             traceback.print_exc()
             return None, f"Error generating speech: {str(e)}"
+    
+    def _generate_bark_speech(self, text, voice_preset=None):
+        """Generate speech using Bark"""
+        # List of Bark voice presets
+        voice_presets = [
+            "v2/en_speaker_6",  # Female
+            "v2/en_speaker_3",  # Male
+            "v2/en_speaker_9",  # Neutral
+        ]
+        
+        # Select voice preset
+        history_prompt = voice_preset if voice_preset else voice_presets[0]
+        
+        # Generate audio
+        audio_array = generate_audio(
+            text, 
+            history_prompt=history_prompt
+        )
+        
+        # Save generated audio
+        filename = f"bark_speech_{int(time.time())}.wav"
+        filepath = os.path.join(self.working_dir, filename)
+        wavfile.write(filepath, SAMPLE_RATE, audio_array)
+        
+        return filepath, None
+    
+    def _generate_speecht5_speech(self, text, speaker_id=None):
+        """Generate speech using SpeechT5"""
+        # Ensure model is initialized
+        speecht5_models = self.models["speecht5"]()
+        if not speecht5_models:
+            return None, "SpeechT5 model not loaded"
+        
+        model = speecht5_models["model"]
+        processor = speecht5_models["processor"]
+        vocoder = speecht5_models["vocoder"]
+        speaker_embeddings = speecht5_models["speaker_embeddings"]
+        
+        # Prepare inputs
+        inputs = processor(text=text, return_tensors="pt")
+        
+        # Generate speech
+        speech = model.generate_speech(
+            inputs["input_ids"], 
+            speaker_embeddings
+        )
+        
+        # Convert to numpy array
+        audio_array = speech.numpy()
+        
+        # Save generated audio
+        filename = f"speecht5_speech_{int(time.time())}.wav"
+        filepath = os.path.join(self.working_dir, filename)
+        wavfile.write(filepath, 16000, audio_array)
+        
+        return filepath, None
 
 def create_interface():
-    # Ensure working directory exists
-    working_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "working_files")
-    os.makedirs(working_dir, exist_ok=True)
-    
-    app = VoiceCloningApp()
+    synthesizer = VoiceSynthesizer()
     
     with gr.Blocks() as interface:
-        gr.Markdown("# 🎙️ Voice Cloning App")
+        gr.Markdown("# 🎙️ Advanced Voice Synthesis")
         
         with gr.Row():
             with gr.Column():
-                gr.Markdown("## 1. Capture Reference Voice")
-                reference_audio = gr.Audio(sources=["microphone", "upload"], type="numpy")
-                process_btn = gr.Button("Process Reference Voice")
-                process_output = gr.Textbox(label="Processing Result")
-                
-            with gr.Column():
-                gr.Markdown("## 2. Generate Speech")
+                gr.Markdown("## Speech Generation")
                 text_input = gr.Textbox(label="Enter Text to Speak")
+                
+                # Model Selection
+                model_dropdown = gr.Dropdown(
+                    choices=[
+                        "bark (Suno AI)",
+                        "speecht5 (Microsoft)"
+                    ],
+                    label="Select TTS Model",
+                    value="bark (Suno AI)"
+                )
+                
+                # Voice Preset Dropdowns
+                with gr.Row():
+                    bark_preset = gr.Dropdown(
+                        choices=[
+                            "v2/en_speaker_6 (Female)",
+                            "v2/en_speaker_3 (Male)", 
+                            "v2/en_speaker_9 (Neutral)"
+                        ],
+                        label="Bark Voice Preset",
+                        visible=True
+                    )
+                    
+                    speecht5_preset = gr.Dropdown(
+                        choices=[
+                            "Default Speaker"
+                        ],
+                        label="SpeechT5 Speaker",
+                        visible=False
+                    )
+                
                 generate_btn = gr.Button("Generate Speech")
                 audio_output = gr.Audio(label="Generated Speech")
                 error_output = gr.Textbox(label="Errors", visible=True)
         
-        # Bind functions
-        process_btn.click(
-            fn=app.process_reference_audio, 
-            inputs=reference_audio, 
-            outputs=process_output
+        # Dynamic model and preset visibility
+        def update_model_visibility(model):
+            if "bark" in model.lower():
+                return {
+                    bark_preset: gr.update(visible=True),
+                    speecht5_preset: gr.update(visible=False)
+                }
+            else:
+                return {
+                    bark_preset: gr.update(visible=False),
+                    speecht5_preset: gr.update(visible=True)
+                }
+        
+        model_dropdown.change(
+            fn=update_model_visibility,
+            inputs=model_dropdown,
+            outputs=[bark_preset, speecht5_preset]
         )
         
+        # Speech generation logic
+        def generate_speech_wrapper(text, model, bark_preset, speecht5_preset):
+            # Map model name
+            model_map = {
+                "bark (Suno AI)": "bark",
+                "speecht5 (Microsoft)": "speecht5"
+            }
+            
+            # Select appropriate preset
+            preset = bark_preset if "bark" in model else speecht5_preset
+            
+            return synthesizer.generate_speech(
+                text, 
+                model_name=model_map[model], 
+                voice_preset=preset
+            )
+        
         generate_btn.click(
-            fn=app.generate_speech, 
-            inputs=text_input, 
+            fn=generate_speech_wrapper,
+            inputs=[text_input, model_dropdown, bark_preset, speecht5_preset],
             outputs=[audio_output, error_output]
         )