add audio cloning functionality (test)

app.py

@@ -1,118 +1,128 @@
 import gradio as gr
-import random
-import re
-import difflib
-import torch
-from functools import lru_cache
-from transformers import pipeline
-
-# ------------------- Sentence Bank (customize freely) -------------------
-SENTENCE_BANK = [
-    "The quick brown fox jumps over the lazy dog.",
-    "I promise to speak clearly and at a steady pace.",
-    "Open source makes AI more transparent and inclusive.",
-    "Hugging Face Spaces make demos easy to share.",
-    "Today the weather in Berlin is pleasantly cool.",
-    "Privacy and transparency should go hand in hand.",
-    "Please generate a new sentence for me to read.",
-    "Machine learning can amplify or reduce inequality.",
-    "Responsible AI requires participation from everyone.",
-    "This microphone test checks my pronunciation accuracy.",
-]
-
-# ------------------- Utilities -------------------
-def normalize_text(t: str) -> str:
-    # English-only normalization: lowercase, keep letters/digits/' and -
-    t = t.lower()
-    t = re.sub(r"[^a-z0-9'\-]+", " ", t)
-    t = re.sub(r"\s+", " ", t).strip()
-    return t
-
-def similarity_and_diff(ref: str, hyp: str):
-    """Return similarity ratio (0..1) and HTML diff highlighting changes."""
-    ref_tokens = ref.split()
-    hyp_tokens = hyp.split()
-    sm = difflib.SequenceMatcher(a=ref_tokens, b=hyp_tokens)
-    ratio = sm.ratio()
 
+import src.generate as generate
+import src.process as process
+import src.tts as tts
+
+
+# ------------------- UI printing functions -------------------
+def clear_all():
+    # target, user_transcript, score_html, diff_html, result_html,
+    # tts_text, clone_status, tts_audio
+    return "", "", "", "", "", "", "", None
+
+
+def make_result_html(pass_threshold, passed, ratio):
+    """Returns summary and score label."""
+    summary = (
+        f"✅ Correct (≥ {int(pass_threshold * 100)}%)"
+        if passed else
+        f"❌ Not a match (need ≥ {int(pass_threshold * 100)}%)"
+    )
+    score = f"Similarity: {ratio * 100:.1f}%"
+    return summary, score
+
+
+def make_alignment_html(ref_tokens, hyp_tokens, alignments):
+    """Returns HTML showing alignment between target and recognized user audio."""
     out = []
-    for op, i1, i2, j1, j2 in sm.get_opcodes():
+    no_match_html = ' <span style="background:#ffe0e0;text-decoration:line-through;">'
+    match_html = ' <span style="background:#e0ffe0;">'
+    for span in alignments:
+        op, i1, i2, j1, j2 = span
+        ref_string = " ".join(ref_tokens[i1:i2])
+        hyp_string = " ".join(hyp_tokens[j1:j2])
         if op == "equal":
-            out.append(" " + " ".join(ref_tokens[i1:i2]))
+            out.append(" " + ref_string)
         elif op == "delete":
-            out.append(
-                ' <span style="background:#ffe0e0;text-decoration:line-through;">'
-                + " ".join(ref_tokens[i1:i2]) + "</span>"
-            )
+            out.append(no_match_html + ref_string + "</span>")
         elif op == "insert":
-            out.append(
-                ' <span style="background:#e0ffe0;">'
-                + " ".join(hyp_tokens[j1:j2]) + "</span>"
-            )
+            out.append(match_html + hyp_string + "</span>")
         elif op == "replace":
-            out.append(
-                ' <span style="background:#ffe0e0;text-decoration:line-through;">'
-                + " ".join(ref_tokens[i1:i2]) + "</span>"
-            )
-            out.append(
-                ' <span style="background:#e0ffe0;">'
-                + " ".join(hyp_tokens[j1:j2]) + "</span>"
-            )
+            out.append(no_match_html + ref_string + "</span>")
+            out.append(match_html + hyp_string + "</span>")
     html = '<div style="line-height:1.6;font-size:1rem;">' + "".join(out).strip() + "</div>"
-    return ratio, html
-
-@lru_cache(maxsize=2)
-def get_asr(model_id: str, device_preference: str):
-    """Cache an ASR pipeline. device_preference: 'auto'|'cpu'|'cuda'."""
-    if device_preference == "cuda" and torch.cuda.is_available():
-        device = 0
-    elif device_preference == "auto":
-        device = 0 if torch.cuda.is_available() else -1
-    else:
-        device = -1
-    return pipeline(
-        "automatic-speech-recognition",
-        model=model_id,           # use English-only Whisper models (.en)
-        device=device,
-        chunk_length_s=30,
-        return_timestamps=False,
-    )
+    return html
 
-def gen_sentence():
-    return random.choice(SENTENCE_BANK)
 
-def clear_all():
-    # target, hyp_out, score_out, diff_out, summary_out
-    return "", "", "", "", ""
+def make_html(sentence_match):
+    """Build diff + results HTML."""
+    diff_html = make_alignment_html(sentence_match.target_tokens,
+                                    sentence_match.user_tokens,
+                                    sentence_match.alignments)
+    result_html, score_html = make_result_html(sentence_match.pass_threshold,
+                                               sentence_match.passed,
+                                               sentence_match.ratio)
+    return score_html, result_html, diff_html
+
 
 # ------------------- Core Check (English-only) -------------------
-def check_pronunciation(audio_path, target_sentence, model_id, device_pref, pass_threshold):
+def get_user_transcript(audio_path: gr.Audio, target_sentence: str, model_id: str, device_pref: str) -> (str, str):
+    """ASR for the input audio and basic validation."""
     if not target_sentence:
-        return "", "", "", "Please generate a sentence first."
-
-    asr = get_asr(model_id, device_pref)
-
-    try:
-        # IMPORTANT: For English-only Whisper (.en), do NOT pass language/task args.
-        result = asr(audio_path)
-        hyp_raw = result["text"].strip()
-    except Exception as e:
-        return "", "", "", f"Transcription failed: {e}"
-
-    ref_norm = normalize_text(target_sentence)
-    hyp_norm = normalize_text(hyp_raw)
-
-    ratio, diff_html = similarity_and_diff(ref_norm, hyp_norm)
-    passed = ratio >= pass_threshold
+        return "Please generate a sentence first.", ""
+    if audio_path is None:
+        return "Please start, record, then stop the audio recording before trying to transcribe.", ""
+
+    user_transcript = process.run_asr(audio_path, model_id, device_pref)
+    if isinstance(user_transcript, Exception):
+        return f"Transcription failed: {user_transcript}", ""
+    return "", user_transcript
+
+
+def transcribe_check(audio_path, target_sentence, model_id, device_pref, pass_threshold):
+    """Transcribe user audio, compute match, and render results."""
+    error_msg, user_transcript = get_user_transcript(audio_path, target_sentence, model_id, device_pref)
+    if error_msg:
+        score_html = ""
+        diff_html = ""
+        result_html = error_msg
+    else:
+        sentence_match = process.SentenceMatcher(target_sentence, user_transcript, pass_threshold)
+        score_html, result_html, diff_html = make_html(sentence_match)
+    return user_transcript, score_html, result_html, diff_html
+
+
+# ------------------- Voice cloning gate -------------------
+def clone_if_pass(
+    audio_path,              # ref voice (the same recorded clip)
+    target_sentence,         # sentence user was supposed to say
+    user_transcript,         # what ASR heard
+    tts_text,                # what we want to synthesize (in cloned voice)
+    pass_threshold,          # must meet or exceed this
+    tts_model_id,            # e.g., "coqui/XTTS-v2"
+    tts_language,            # e.g., "en"
+):
+    """
+    If user correctly read the target (>= threshold), clone their voice from the
+    recorded audio and speak 'tts_text'. Otherwise, refuse.
+    """
+    # Basic validations
+    if audio_path is None:
+        return None, "Record audio first (reference voice is required)."
+    if not target_sentence:
+        return None, "Generate a target sentence first."
+    if not user_transcript:
+        return None, "Transcribe first to verify the sentence."
+    if not tts_text:
+        return None, "Enter the sentence to synthesize."
+
+    # Recompute pass/fail to avoid relying on UI state
+    sm = process.SentenceMatcher(target_sentence, user_transcript, pass_threshold)
+    if not sm.passed:
+        return None, (
+            f"❌ Cloning blocked: your reading did not reach the threshold "
+            f"({sm.ratio*100:.1f}% < {int(pass_threshold*100)}%)."
+        )
 
-    summary = (
-        f"✅ Correct (≥ {int(pass_threshold*100)}%)"
-        if passed else
-        f"❌ Not a match (need ≥ {int(pass_threshold*100)}%)"
-    )
-    score = f"Similarity: {ratio*100:.1f}%"
+    # Run zero-shot cloning
+    out = tts.run_tts_clone(audio_path, tts_text, model_id=tts_model_id, language=tts_language)
+    if isinstance(out, Exception):
+        return None, f"Voice cloning failed: {out}"
+    sr, wav = out
+    # Gradio Audio can take a tuple (sr, np.array)
+    return (sr, wav), f"✅ Cloned and synthesized with {tts_model_id} ({tts_language})."
 
-    return hyp_raw, score, diff_html, summary
 
 # ------------------- UI -------------------
 with gr.Blocks(title="Say the Sentence (English)") as demo:
@@ -122,25 +132,28 @@ with gr.Blocks(title="Say the Sentence (English)") as demo:
         1) Generate a sentence.  
         2) Record yourself reading it.  
         3) Transcribe & check your accuracy.  
+        4) If matched, clone your voice to speak any sentence you enter.
         """
     )
 
     with gr.Row():
-        target = gr.Textbox(label="Target sentence", interactive=False, placeholder="Click 'Generate sentence'")
+        target = gr.Textbox(label="Target sentence", interactive=False,
+                            placeholder="Click 'Generate sentence'")
 
     with gr.Row():
         btn_gen = gr.Button("🎲 Generate sentence", variant="primary")
         btn_clear = gr.Button("🧹 Clear")
 
     with gr.Row():
-        audio = gr.Audio(sources=["microphone"], type="filepath", label="Record your voice")
+        audio = gr.Audio(sources=["microphone"], type="filepath",
+                         label="Record your voice")
 
     with gr.Accordion("Advanced settings", open=False):
         model_id = gr.Dropdown(
             choices=[
-                "openai/whisper-tiny.en",        # fastest (CPU-friendly)
-                "openai/whisper-base.en",        # better accuracy, a bit slower
-                "distil-whisper/distil-small.en" # optional distil English model
+                "openai/whisper-tiny.en",
+                "openai/whisper-base.en",
+                "distil-whisper/distil-small.en",
             ],
             value="openai/whisper-tiny.en",
             label="ASR model (English only)",
@@ -150,26 +163,68 @@ with gr.Blocks(title="Say the Sentence (English)") as demo:
             value="auto",
             label="Device preference"
         )
-        pass_threshold = gr.Slider(0.50, 1.00, value=0.85, step=0.01, label="Match threshold")
+        pass_threshold = gr.Slider(0.50, 1.00, value=0.85, step=0.01,
+                                   label="Match threshold")
 
     with gr.Row():
         btn_check = gr.Button("✅ Transcribe & Check", variant="primary")
+    with gr.Row():
+        user_transcript = gr.Textbox(label="Transcription", interactive=False)
+    with gr.Row():
+        score_html = gr.Label(label="Score")
+        result_html = gr.Label(label="Result")
+    diff_html = gr.HTML(
+        label="Word-level diff (red = expected but missing / green = extra or replacement)")
+
+    gr.Markdown("## 🔁 Voice cloning (gated)")
+    with gr.Row():
+        tts_text = gr.Textbox(
+            label="Text to synthesize (voice clone)",
+            placeholder="Type the sentence you want the cloned voice to say",
+        )
+    with gr.Row():
+        tts_model_id = gr.Dropdown(
+            choices=[
+                "coqui/XTTS-v2",
+                # add others if you like, e.g. "myshell-ai/MeloTTS"
+            ],
+            value="coqui/XTTS-v2",
+            label="TTS (voice cloning) model",
+        )
+        tts_language = gr.Dropdown(
+            choices=["en", "de", "fr", "es", "it", "pt", "pl", "tr", "ru", "nl", "cs", "ar", "zh"],
+            value="en",
+            label="Language",
+        )
 
     with gr.Row():
-        hyp_out = gr.Textbox(label="Transcription", interactive=False)
+        btn_clone = gr.Button("🔁 Clone voice (if passed)", variant="secondary")
     with gr.Row():
-        score_out = gr.Label(label="Score")
-        summary_out = gr.Label(label="Result")
-    diff_out = gr.HTML(label="Word-level diff (red = expected but missing / green = extra or replacement)")
+        tts_audio = gr.Audio(label="Cloned speech output", interactive=False)
+        clone_status = gr.Label(label="Cloning status")
+
+    # -------- Events --------
+    # Use pre-specified sentence bank by default
+    btn_gen.click(fn=generate.gen_sentence_set, outputs=target)
+    # Or use LLM generation:
+    # btn_gen.click(fn=generate.gen_sentence_llm, outputs=target)
+
+    btn_clear.click(
+        fn=clear_all,
+        outputs=[target, user_transcript, score_html, result_html, diff_html, tts_text, clone_status, tts_audio]
+    )
 
-    # Events
-    btn_gen.click(fn=gen_sentence, outputs=target)
-    btn_clear.click(fn=clear_all, outputs=[target, hyp_out, score_out, diff_out, summary_out])
     btn_check.click(
-        fn=check_pronunciation,
+        fn=transcribe_check,
         inputs=[audio, target, model_id, device_pref, pass_threshold],
-        outputs=[hyp_out, score_out, diff_out, summary_out]
+        outputs=[user_transcript, score_html, result_html, diff_html]
+    )
+
+    btn_clone.click(
+        fn=clone_if_pass,
+        inputs=[audio, target, user_transcript, tts_text, pass_threshold, tts_model_id, tts_language],
+        outputs=[tts_audio, clone_status],
     )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()

requirements.txt

@@ -3,3 +3,5 @@ transformers>=4.44.0
 torch>=2.2.0
 accelerate>=0.33.0
 sentencepiece>=0.2.0
+numpy
+

src/generate.py

@@ -0,0 +1,46 @@
+import random
+
+from transformers import pipeline, AutoTokenizer
+
+import src.process as process
+
+# You can choose to use either:
+# (1) a list of pre-specified sentences, in SENTENCE_BANK
+# (2) an LLM-generated sentence.
+# SENTENCE_BANK is used in the `gen_sentence_set` function.
+# LLM generation is used in the `gen_sentence_llm` function.
+
+# ------------------- Sentence Bank (customize freely) -------------------
+SENTENCE_BANK = [
+    "The quick brown fox jumps over the lazy dog.",
+    "I promise to speak clearly and at a steady pace.",
+    "Open source makes AI more transparent and inclusive.",
+    "Hugging Face Spaces make demos easy to share.",
+    "Today the weather in Berlin is pleasantly cool.",
+    "Privacy and transparency should go hand in hand.",
+    "Please generate a new sentence for me to read.",
+    "Machine learning can amplify or reduce inequality.",
+    "Responsible AI requires participation from everyone.",
+    "This microphone test checks my pronunciation accuracy.",
+]
+
+
+def gen_sentence_llm():
+    """Generates a sentence using an LLM.
+    Returns:
+        Normalized text string to display in the UI.
+    """
+    prompt = ""
+    tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
+    generator = pipeline('text-generation', model='gpt2')
+    result = generator(prompt, stop_strings=[".", ], num_return_sequences=1,
+                       tokenizer=tokenizer, pad_token_id=tokenizer.eos_token_id)
+    display_text = process.normalize_text(result[0]["generated_text"],
+                                          lower=False)
+    return display_text
+
+
+def gen_sentence_set():
+    """Returns a sentence for the user to say using a prespecified set of options."""
+    return random.choice(SENTENCE_BANK)
+

src/process.py

@@ -0,0 +1,88 @@
+import difflib
+import re
+from functools import lru_cache
+
+import gradio.components.audio as gr_audio
+import torch
+from transformers import pipeline
+
+
+# ------------------- Utilities -------------------
+def normalize_text(t: str, lower: bool = True) -> str:
+    """For normalizing LLM-generated and human-generated strings.
+    For LLMs, this removes extraneous quote marks and spaces."""
+    # English-only normalization: lowercase, keep letters/digits/' and -
+    if lower:
+        t = t.lower()
+    # TODO: Previously was re.sub(r"[^a-z0-9'\-]+", " ", t); discuss normalizing for LLMs too.
+    t = re.sub(r"[^a-zA-Z0-9'\-.,]+", " ", t)
+    t = re.sub(r"\s+", " ", t).strip()
+    return t
+
+
+@lru_cache(maxsize=2)
+def get_asr_pipeline(model_id: str, device_preference: str) -> pipeline:
+    """Cache an ASR pipeline.
+    Parameters:
+        model_id: String of desired ASR model.
+        device_preference: String of desired device for ASR processing, "cuda", "cpu", or "auto".
+    Returns:
+        transformers.pipeline ASR component.
+    """
+    if device_preference == "cuda" and torch.cuda.is_available():
+        device = 0
+    elif device_preference == "auto":
+        device = 0 if torch.cuda.is_available() else -1
+    else:
+        device = -1
+    return pipeline(
+        "automatic-speech-recognition",
+        model=model_id,           # use English-only Whisper models (.en)
+        device=device,
+        chunk_length_s=30,
+        return_timestamps=False,
+    )
+
+def run_asr(audio_path: gr_audio, model_id: str, device_pref: str) -> str | Exception:
+    """Returns the recognized user utterance from the input audio stream.
+    Parameters:
+        audio_path: gradio.Audio component.
+        model_id: String of desired ASR model.
+        device_preference: String of desired device for ASR processing, "cuda", "cpu", or "auto".
+    Returns:
+        hyp_raw: Recognized user utterance.
+    """
+    asr = get_asr_pipeline(model_id, device_pref)
+    try:
+        # IMPORTANT: For English-only Whisper (.en), do NOT pass language/task args.
+        result = asr(audio_path)
+        hyp_raw = result["text"].strip()
+    except Exception as e:
+        return e
+    return hyp_raw
+
+def similarity_and_diff(ref_tokens: list, hyp_tokens: list) -> (float, list[str, int, int, int]):
+    """
+    Returns:
+        ratio: Similarity ratio (0..1).
+        opcodes: List of differences between target and recognized user utterance.
+    """
+    sm = difflib.SequenceMatcher(a=ref_tokens, b=hyp_tokens)
+    ratio = sm.ratio()
+    opcodes = sm.get_opcodes()
+    return ratio, opcodes
+
+class SentenceMatcher:
+    """Class for keeping track of (target sentence, user utterance) match features."""
+    def __init__(self, target_sentence, user_transcript, pass_threshold):
+        self.target_sentence: str = target_sentence
+        self.user_transcript: str = user_transcript
+        self.pass_threshold: float = pass_threshold
+        self.target_tokens: list = normalize_text(target_sentence).split()
+        self.user_tokens: list = normalize_text(user_transcript).split()
+        self.ratio: float
+        self.alignments: list
+        self.ratio, self.alignments = similarity_and_diff(self.target_tokens,
+                                                          self.user_tokens)
+        self.passed: bool = self.ratio >= self.pass_threshold
+

src/prompts.py

@@ -0,0 +1,47 @@
+# src/utils/prompts.py
+
+def get_consent_generation_prompt(audio_model_name: str, short_prompt: bool = False) -> str:
+    """
+    Returns a text prompt instructing the model to generate a natural-sounding
+    consent sentence for voice cloning with the specified model.
+
+    Args:
+        audio_model_name (str): Name of the audio model to mention in the prompt.
+        short_prompt (bool): If True, returns a concise one-line prompt suitable
+            for direct model input. If False (default), returns the full detailed prompt.
+
+    Returns:
+        str: The prompt text.
+    """
+
+    if short_prompt:
+        return (
+            f"Generate one natural, spoken-style English sentence (10–20 words) in which a person "
+            f"clearly gives informed consent to use their voice for generating synthetic audio "
+            f"with the model {audio_model_name}. The sentence should sound conversational, include "
+            f"a clear consent phrase like 'I give my consent' or 'I agree', mention {audio_model_name} "
+            f"by name, and be phonetically varied but neutral in tone. Output only the final sentence."
+        )
+
+    return f"""
+        Generate a short, natural-sounding English sentence (10–20 words) that a person could say aloud 
+        to clearly state their informed consent to use their voice for generating synthetic audio with 
+        an AI model called {audio_model_name}.
+
+        The sentence should:
+        - Sound natural and conversational, not like legal text.
+        - Explicitly include a consent phrase, such as “I give my consent,” “I agree,” or “I allow.”
+        - Mention the model name ({audio_model_name}) clearly in the sentence.
+        - Include a neutral descriptive clause before or after the consent phrase to add phonetic variety 
+        (e.g., “The weather today is bright and calm” or “This recording is made clearly and freely.”)
+        - Have a neutral or polite tone (no emotional extremes).
+        - Be comfortable to read aloud and phonetically rich, covering diverse vowels and consonants naturally.
+        - Be self-contained, so the full sentence can serve as an independent audio clip.
+
+        Examples of structure to follow:
+        - “The weather is clear and warm today. I give my consent to use my voice for generating audio with the model {audio_model_name}.”
+        - “I give my consent to use my voice for generating audio with the model {audio_model_name}. This statement is made freely and clearly.”
+        - “Good afternoon. I agree to the use of my recorded voice for audio generation with the model {audio_model_name}.”
+
+        The output should be a single, natural sentence ready to be spoken aloud for recording purposes.
+        """

src/tts.py

@@ -0,0 +1,43 @@
+# src/tts.py
+from __future__ import annotations
+from typing import Tuple, Union
+
+import numpy as np
+from transformers import pipeline
+
+# We use the text-to-speech pipeline with XTTS v2 (zero-shot cloning)
+# Example forward params: {"speaker_wav": "/path/to/ref.wav", "language": "en"}
+
+def get_tts_pipeline(model_id: str):
+    """
+    Create a TTS pipeline for the given model.
+    XTTS v2 works well for zero-shot cloning and is available on the Hub.
+    """
+    # NOTE: Add device selection similar to ASR if needed
+    return pipeline("text-to-speech", model=model_id)
+
+def run_tts_clone(
+    ref_audio_path: str,
+    text_to_speak: str,
+    model_id: str = "coqui/XTTS-v2",
+    language: str = "en",
+) -> Union[Tuple[int, np.ndarray], Exception]:
+    """
+    Synthesize 'text_to_speak' in the cloned voice from 'ref_audio_path'.
+
+    Returns:
+        (sampling_rate, waveform) on success, or Exception on failure.
+    """
+    try:
+        tts = get_tts_pipeline(model_id)
+        result = tts(
+            text_to_speak,
+            forward_params={"speaker_wav": ref_audio_path, "language": language},
+        )
+        # transformers TTS returns dict like: {"audio": {"array": np.ndarray, "sampling_rate": 24000}}
+        audio = result["audio"]
+        sr = int(audio["sampling_rate"])
+        wav = audio["array"].astype(np.float32)
+        return sr, wav
+    except Exception as e:
+        return e