Update files to support multi-turn

README.md

@@ -4,7 +4,7 @@ emoji: 💬
 colorFrom: yellow
 colorTo: purple
 sdk: gradio
-sdk_version: 5.0.1
+sdk_version: 5.21.0
 app_file: app.py
 pinned: true
 license: apache-2.0

app.py

@@ -11,14 +11,6 @@ from vllm import LLM, SamplingParams
 import vllm
 import re
 
-from huggingface_hub import login
-TOKEN = os.environ.get("TOKEN", None)
-login(token=TOKEN)
-
-print("transformers version:", transformers.__version__)
-print("vllm version:", vllm.__version__)
-print("gradio version:", gr.__version__)
-
 
 def load_model_processor(model_path):
     processor = AutoProcessor.from_pretrained(model_path)
@@ -32,24 +24,15 @@ def load_model_processor(model_path):
 model_path1 = "SeaLLMs/SeaLLMs-Audio-7B"
 model1, processor1 = load_model_processor(model_path1)
 
-def response_to_audio(audio_url, text, model=None, processor=None, temperature = 0,repetition_penalty=1.1, top_p = 0.9,max_new_tokens = 2048):
-    if text == None:
-        conversation = [
-            {"role": "user", "content": [
-                {"type": "audio", "audio_url": audio_url},
-            ]},]
-    elif audio_url == None:
-        conversation = [
-            {"role": "user", "content": [
-                {"type": "text", "text": text},
-           ]},]
-    else:
-        conversation = [
-            {"role": "user", "content": [
-                {"type": "audio", "audio_url": audio_url},
-                {"type": "text", "text": text},
-           ]},]
-
+def response_to_audio_conv(conversation, model=None, processor=None, temperature = 0.7,repetition_penalty=1.1, top_p = 0.5,max_new_tokens = 2048):
+    turn = conversation[-1]
+    if turn["role"] == "user":
+        for content in turn['content']:
+            if content["type"] == "text":
+                if contains_chinese(content["text"]):
+                    return "Caution! This demo does not support Chinese!"
+    
+    
     text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
     audios = []
     for message in conversation:
@@ -76,45 +59,76 @@ def response_to_audio(audio_url, text, model=None, processor=None, temperature =
 
     output = model.generate([input], sampling_params=sampling_params)[0]
     response = output.outputs[0].text
+    if contains_chinese(response):
+        return "ERROR! Try a different instruction/prompt!"
     return response
 
-def clear_inputs():
-    return None, "", ""
+def print_like_dislike(x: gr.LikeData):
+    print(x.index, x.value, x.liked)
 
 def contains_chinese(text):
     # Regular expression for Chinese characters
     chinese_char_pattern = re.compile(r'[\u4e00-\u9fff]')
     return bool(chinese_char_pattern.search(text))
 
-def compare_responses(audio_url, text):
-    if contains_chinese(text): 
-        return "Caution! This demo does not support Chinese!"
+def add_message(history, message):
+    paths = []
+    for turn in history: 
+        if turn['role'] == "user" and type(turn['content']) != str:
+            paths.append(turn['content'][0])
+    for x in message["files"]:
+        if x not in paths:
+            history.append({"role": "user", "content": {"path": x}})
+    if message["text"] is not None:
+        history.append({"role": "user", "content": message["text"]})
+    return history, gr.MultimodalTextbox(value=None, interactive=False)
+
+def format_user_messgae(message):
+    if type(message['content']) == str:
+        return {"role": "user", "content": [{"type": "text", "text": message['content']}]}
+    else:
+        return {"role": "user", "content": [{"type": "audio", "audio_url": message['content'][0]}]}
+
+def history_to_conversation(history):
+    conversation = []
+    audio_paths = []
+    for turn in history:
+        if turn['role'] == "user":
+            if not turn['content']: 
+                continue
+            turn = format_user_messgae(turn)
+            if turn['content'][0]['type'] == 'audio':
+                if turn['content'][0]['audio_url'] in audio_paths:
+                    continue
+                else: 
+                    audio_paths.append(turn['content'][0]['audio_url'])
+
+            if len(conversation) > 0 and conversation[-1]["role"] == "user":
+                conversation[-1]['content'].append(turn['content'][0])
+            else:
+                conversation.append(turn)
+        else:
+            conversation.append(turn)
     
-    response1 = response_to_audio(audio_url, text, model1, processor1)
-    if contains_chinese(response1): 
-        return "ERROR! Try another example!"
-
-    return response1
+    print(json.dumps(conversation, indent=4, ensure_ascii=False))
+    return conversation
+
+def bot(history: list, temperature = 0.7,repetition_penalty=1.1, top_p = 0.5,
+                    max_new_tokens = 2048):
+    conversation = history_to_conversation(history)
+    response = response_to_audio_conv(conversation, model=model1, processor=processor1, temperature = temperature,repetition_penalty=repetition_penalty, top_p = top_p, max_new_tokens = max_new_tokens)
+    # response = "Nice to meet you!"
+    print("Bot:",response)
+
+    history.append({"role": "assistant", "content": ""})
+    for character in response:
+        history[-1]["content"] += character
+        time.sleep(0.01)
+        yield history
 
 with gr.Blocks() as demo:
-    # gr.Markdown(f"Evaluate {model_path1}")
     gr.HTML("""<p align="center"><img src="https://DAMO-NLP-SG.github.io/SeaLLMs-Audio/static/images/seallm-audio-logo.png" style="height: 80px"/><p>""")
-    # gr.Image("images/seal_logo.png", elem_id="seal_logo", show_label=False,height=80,show_fullscreen_button=False)
     gr.HTML("""<h1 align="center" id="space-title">SeaLLMs-Audio-Demo</h1>""")
-#     gr.Markdown(
-#         """\
-# <center><font size=4>This WebUI is based on SeaLLMs-Audio-7B, developed by Alibaba DAMO Academy.<br>
-#     You can interact with the chatbot in <b>English, Chinese, Indonesian, Thai, or Vietnamese</b>.<br>
-#     For the input, you can input <b>audio and/or text</center>.""")
-
-#     # Links with proper formatting
-#     gr.Markdown(
-#         """<center><font size=4>
-#         <a href="https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B-Chat">[Website]</a> &nbsp; 
-#         <a href="https://huggingface.co/SeaLLMs/SeaLLMs-Audio-7B">[Model🤗]</a> &nbsp; 
-#         <a href="https://github.com/DAMO-NLP-SG/SeaLLMs-Audio">[Github]</a>
-#         </center>""",
-#     )
 
     gr.HTML(
         """<div style="text-align: center; font-size: 16px;">
@@ -141,42 +155,27 @@ with gr.Blocks() as demo:
     #     with gr.Column():
     #         repetition_penalty = gr.Slider(minimum=0, maximum=2, value=1.1, step=0.1, label="Repetition Penalty")
     
-    with gr.Row():
-        with gr.Column():
-            # mic_input = gr.Microphone(label="Record Audio", type="filepath", elem_id="mic_input")
-            mic_input = gr.Audio(sources = ['upload', 'microphone'], label="Record Audio", type="filepath", elem_id="mic_input")
-        with gr.Column():
-            additional_input = gr.Textbox(label="Text Input")
-    
-    # Button to trigger the function
-    with gr.Row():
-        btn_submit = gr.Button("Submit")
-        btn_clear = gr.Button("Clear")
-
-    with gr.Row():
-        output_text1 = gr.Textbox(label=model_path1.split('/')[-1], interactive=False, elem_id="output_text1")
-
-    btn_submit.click(
-        fn=compare_responses,
-        inputs=[mic_input, additional_input],
-        outputs=[output_text1],
+    chatbot = gr.Chatbot(elem_id="chatbot", bubble_full_width=False, type="messages")
+
+    chat_input = gr.MultimodalTextbox(
+        interactive=True,
+        file_count="single",
+        file_types=['.wav'],
+        placeholder="Enter message (optional) ...",
+        show_label=False,
+        sources=["microphone", "upload"],
     )
 
-    btn_clear.click(
-        fn=clear_inputs,
-        inputs=None,
-        outputs=[mic_input, additional_input, output_text1],
-        queue=False,
+    chat_msg = chat_input.submit(
+        add_message, [chatbot, chat_input], [chatbot, chat_input]
     )
+    bot_msg = chat_msg.then(bot, chatbot, chatbot, api_name="bot_response")
+    # bot_msg = chat_msg.then(bot, [chatbot, temperature, repetition_penalty, top_p], chatbot, api_name="bot_response")
+    bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])
 
+    # chatbot.like(print_like_dislike, None, None, like_user_message=True)
 
-# demo.launch(
-#     share=False,
-#     inbrowser=True,
-#     server_port=7950,
-#     server_name="0.0.0.0",
-#     max_threads=40
-# )
+    clear_button = gr.ClearButton([chatbot, chat_input])
 
 demo.launch(share=True)
 demo.queue(default_concurrency_limit=40).launch(share=True)