Update app.py

app.py

@@ -4,11 +4,12 @@ import os
 import time
 import torch
 from scipy.io import wavfile
+import soundfile as sf
 import datasets
 
 # Bark imports
 from bark import generate_audio, SAMPLE_RATE
-from bark.generation import preload_models
+from bark.generation import preload_models, generate_text_semantic
 
 # Hugging Face Transformers
 from transformers import (
@@ -24,6 +25,9 @@ class VoiceSynthesizer:
         self.working_dir = os.path.join(self.base_dir, "working_files")
         os.makedirs(self.working_dir, exist_ok=True)
         
+        # Store reference voice
+        self.reference_voice = None
+        
         # Initialize models dictionary
         self.models = {
             "bark": self._initialize_bark,
@@ -41,6 +45,38 @@ class VoiceSynthesizer:
         except Exception as e:
             print(f"Bark model loading error: {e}")
     
+    def process_reference_audio(self, reference_audio):
+        """Process and store reference audio for voice cloning"""
+        try:
+            # Ensure audio is in the right format
+            if reference_audio is None:
+                return "No audio provided"
+            
+            # Convert to numpy array if needed
+            if isinstance(reference_audio, tuple):
+                reference_audio = reference_audio[0]
+            
+            # Ensure the audio is mono and normalized
+            if reference_audio.ndim > 1:
+                reference_audio = reference_audio.mean(axis=1)
+            
+            # Resample or trim if necessary
+            if len(reference_audio) > SAMPLE_RATE * 10:  # Limit to 10 seconds
+                reference_audio = reference_audio[:SAMPLE_RATE * 10]
+            
+            # Save reference audio
+            ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
+            sf.write(ref_filename, reference_audio, SAMPLE_RATE)
+            
+            # Store reference voice
+            self.reference_voice = reference_audio
+            
+            return "Reference voice processed successfully"
+        
+        except Exception as e:
+            print(f"Reference audio processing error: {e}")
+            return f"Error processing reference audio: {str(e)}"
+    
     def _initialize_bark(self):
         """Bark model initialization (already done in __init__)"""
         return None
@@ -67,12 +103,6 @@ class VoiceSynthesizer:
             print(f"SpeechT5 model loading error: {e}")
             return None
     
-    def set_model(self, model_name):
-        """Set the current model for speech synthesis"""
-        if model_name not in self.models:
-            raise ValueError(f"Model {model_name} not supported")
-        self.current_model = model_name
-    
     def generate_speech(self, text, model_name=None, voice_preset=None):
         """Generate speech using selected model"""
         if not text or not text.strip():
@@ -97,21 +127,34 @@ class VoiceSynthesizer:
     
     def _generate_bark_speech(self, text, voice_preset=None):
         """Generate speech using Bark"""
-        # List of Bark voice presets
+        # Default Bark voice presets
         voice_presets = [
             "v2/en_speaker_6",  # Female
             "v2/en_speaker_3",  # Male
             "v2/en_speaker_9",  # Neutral
         ]
         
-        # Select voice preset
-        history_prompt = voice_preset if voice_preset else voice_presets[0]
+        # Prepare history prompt
+        history_prompt = None
         
-        # Generate audio
-        audio_array = generate_audio(
-            text, 
-            history_prompt=history_prompt
-        )
+        # Check if a reference voice is available
+        if self.reference_voice is not None:
+            # Save reference voice for Bark
+            ref_filename = os.path.join(self.working_dir, "reference_voice.wav")
+            history_prompt = ref_filename
+        elif voice_preset:
+            # Use predefined voice preset
+            history_prompt = voice_presets[0] if "v2/en_speaker" not in voice_preset else voice_preset
+        
+        # Generate audio with or without history prompt
+        if history_prompt:
+            audio_array = generate_audio(
+                text, 
+                history_prompt=history_prompt
+            )
+        else:
+            # Fallback to default generation
+            audio_array = generate_audio(text)
         
         # Save generated audio
         filename = f"bark_speech_{int(time.time())}.wav"
@@ -159,7 +202,13 @@ def create_interface():
         
         with gr.Row():
             with gr.Column():
-                gr.Markdown("## Speech Generation")
+                gr.Markdown("## 1. Capture Reference Voice")
+                reference_audio = gr.Audio(sources=["microphone", "upload"], type="numpy")
+                process_ref_btn = gr.Button("Process Reference Voice")
+                process_ref_output = gr.Textbox(label="Reference Voice Processing")
+                
+            with gr.Column():
+                gr.Markdown("## 2. Generate Speech")
                 text_input = gr.Textbox(label="Enter Text to Speak")
                 
                 # Model Selection
@@ -196,6 +245,13 @@ def create_interface():
                 audio_output = gr.Audio(label="Generated Speech")
                 error_output = gr.Textbox(label="Errors", visible=True)
         
+        # Process reference audio
+        process_ref_btn.click(
+            fn=synthesizer.process_reference_audio,
+            inputs=reference_audio,
+            outputs=process_ref_output
+        )
+        
         # Dynamic model and preset visibility
         def update_model_visibility(model):
             if "bark" in model.lower():