Upload 3 files

app.py

@@ -0,0 +1,134 @@
+import streamlit as st 
+from audiorecorder import audiorecorder
+
+import torch
+from transformers import pipeline
+import torch 
+import torchaudio
+
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain import HuggingFaceHub, LLMChain, PromptTemplate
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import ConversationalRetrievalChain
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.vectorstores import FAISS
+import tempfile 
+from streamlit_chat import message
+import streamlit as st
+from elevenlabs import set_api_key
+from elevenlabs import clone, generate, play
+from pydub import AudioSegment
+
+import os 
+import re
+import sys
+import pandas as pd
+
+from helper import parse_transcription,hindi_to_english,translate_english_to_hindi,hindi_tts
+
+def extract_text_from_html(html):
+    cleanr = re.compile('<.*?>')
+    cleantext = re.sub(cleanr, '', html)
+def conversational_chat(query):    
+    result = llm_chain({"question": query, 
+    "chat_history": st.session_state['history']})
+    st.session_state['history'].append((query, result["answer"]))
+
+    return result["answer"]
+
+def save_uploaded_file_as_mp3(uploaded_file, output_file_path):
+    audio = AudioSegment.from_file(uploaded_file)
+    audio.export(output_file_path, format="mp3")
+
+def ui():
+    user_api_key = st.sidebar.text_input(
+        label="#### Your OpenAI API key 👇",
+        placeholder="Paste your openAI API key, sk-",
+        type="password")
+    
+    eleven_labs_api_key = st.sidebar.text_input(
+        label="#### Your OpenAI API key 👇",
+        placeholder="Paste your openAI API key, sk-",
+        type="password")
+    if user_api_key is not None and user_api_key.strip() != "":
+        os.environ["OPENAI_API_KEY"] =user_api_key
+        template = """
+            Behave like a  Telecomm customer servce call agent and don't include any website address, compnay name or any other parameter in your output
+            {history}
+            Me:{human_input}
+            Jack:
+            """
+        
+        prompt = PromptTemplate(
+        input_variables=["history", "human_input"],
+        template=template
+    )
+        
+    llm_chain = LLMChain(
+        llm = ChatOpenAI(temperature=0.0,model_name='gpt-3.5-turbo'),
+        prompt=prompt,
+        verbose=True,
+        memory=ConversationBufferWindowMemory(k=2)
+    )
+
+    if 'history' not in st.session_state:
+        st.session_state['history'] = []
+
+    if 'generated' not in st.session_state:
+        st.session_state['generated'] = ["Hello ! Ask me anything about " + " 🤗"]
+
+    if 'past' not in st.session_state:
+        st.session_state['past'] = ["Hey ! 👋"]
+
+    if user_api_key is not None and user_api_key.strip() != "":
+        eleven_labs_api_key = st.sidebar.text_input(
+            label="#### Your Eleven Labs API key 👇",
+            placeholder="Paste your Eleven Labs API key",
+            type="password")
+        
+        set_api_key(user_api_key)
+        
+        #container for the chat history
+        response_container = st.container()
+        #container for the user's text input
+        container = st.container()
+
+        with container:
+            audio_file = audiorecorder("Click to record", "Recording...")
+            audio_file=audio_file.tobytes()
+            submit_button = st.form_submit_button(label='Send')
+            if submit_button and audio_file:
+                output_file_path = "./output_audio.mp3"
+                save_uploaded_file_as_mp3(audio_file,output_file_path )
+                hindi_input_audio,sample_rate=torchaudio.load(output_file_path)
+                #applying the audio recognition 
+                hindi_transcription=parse_transcription('./output_audio.mp3')
+                st.success(f"Audio file saved as {output_file_path}")
+                #convert hindi to english
+                english_input=hindi_to_english(hindi_transcription)
+                #feeding the input to the LLM
+                english_output = conversational_chat(english_input)
+                #converting english to hindi 
+                hin_output=translate_english_to_hindi(english_output)
+                #getting the hindi_tts 
+                hindi_output_audio=hindi_tts(hin_output)
+
+                st.session_state['past'].append(hindi_input_audio)
+                st.session_state['generated'].append(hindi_output_audio)
+            
+            if st.session_state['generated']:
+                with response_container:
+                    for i in range(len(st.session_state['generated'])):
+                        st.audio(st.session_state["past"][i])
+                        st.audio(st.session_state["generated"][i])
+
+if __name__ == '__main__':
+    ui()
+
+            
+            
+
+
+
+

helper.py

@@ -0,0 +1,77 @@
+import torch 
+import torchaudio
+import torch
+from transformers import pipeline
+import soundfile as sf
+import torch
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+import argparse
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+import librosa
+from huggingface_hub.hf_api import HfFolder
+from transformers import MarianMTModel, MarianTokenizer
+from elevenlabs import set_api_key
+from elevenlabs import clone, generate, play
+HfFolder.save_token('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')
+access_token = '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'
+
+
+tokenizer_en_hn = AutoTokenizer.from_pretrained("vasudevgupta/mbart-bhasha-hin-eng")
+model_translate_en_hm = AutoModelForSeq2SeqLM.from_pretrained("vasudevgupta/mbart-bhasha-hin-eng")
+
+processor = Wav2Vec2Processor.from_pretrained("Harveenchadha/vakyansh-wav2vec2-hindi-him-4200")
+model_speech = Wav2Vec2ForCTC.from_pretrained("Harveenchadha/vakyansh-wav2vec2-hindi-him-4200")
+
+
+
+def parse_transcription(wav_file):
+    # load audio
+    audio_input, sample_rate = librosa.load(wav_file, sr=16000)
+
+    # pad input values and return pt tensor
+    input_values = processor(audio_input, sampling_rate=sample_rate, return_tensors="pt").input_values
+
+    # INFERENCE
+    # retrieve logits & take argmax
+    logits = model_speech(input_values).logits
+    predicted_ids = torch.argmax(logits, dim=-1)
+
+    # transcribe
+    transcription = processor.decode(predicted_ids[0], skip_special_tokens=True)
+    return transcription
+
+def hindi_to_english(text): 
+  inputs = tokenizer_en_hn.encode(
+    text, return_tensors="pt",padding=True,max_length=512,truncation=True)
+
+  outputs = model_translate_en_hm.generate(
+    inputs, max_length=128, num_beams=4, early_stopping=True
+    )
+  translated = tokenizer_en_hn.decode(outputs[0]).replace('<pad>',"").replace('<s>', "").strip().lower()
+  return translated
+
+def translate_english_to_hindi(input_text):
+    # Load the pre-trained English to Hindi translation model and tokenizer
+    model_name = "Helsinki-NLP/opus-mt-en-hi"
+    tokenizer = MarianTokenizer.from_pretrained(model_name)
+    model = MarianMTModel.from_pretrained(model_name)
+
+    # Tokenize the input text and generate translation
+    inputs = tokenizer(input_text, return_tensors="pt", padding=True)
+    translated_ids = model.generate(inputs.input_ids)
+
+    # Decode the translated output
+    translated_text = tokenizer.decode(translated_ids[0], skip_special_tokens=True)
+
+    return translated_text
+
+def hindi_tts(text): 
+   audio = generate(
+  text="Hi! My name is Bella, nice to meet you!",
+  voice="Customer Service",
+  model="eleven_monolingual_v1"
+    )
+   
+   return audio
+   
+

requirements.txt

@@ -0,0 +1,14 @@
+streamlit 
+streamlit-audiorecorder
+langchain 
+streamlit 
+openai 
+tiktoken 
+faiss-cpu 
+streamlit_chat
+huggingface_hub
+librosa
+sentencepiece
+elevenlabs 
+pydub
+torchaudio