Upload 2 files

app.py

@@ -0,0 +1,87 @@
+from threading import Thread
+
+import gradio as gr
+import spaces
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+
+BANNER_HTML = """
+<p align="center">
+    <a href="https://github.com/ymcui/Chinese-LLaMA-Alpaca-3">
+        <img src="https://ymcui.com/images/chinese-llama-alpaca-3-banner.png" width="600"/>
+    </a>
+</p>
+<h3>
+    <center>Check our 
+        <a href='https://github.com/ymcui/Chinese-LLaMA-Alpaca-3' target='_blank'>Chinese-LLaMA-Alpaca-3 GitHub Project</a> 
+        for more information.
+    </center>
+</h3>
+<p>
+    <center><em>The demo is mainly for academic purposes and users are not expected to use this demo for illegal activities.</em></center>
+</p>
+"""
+
+DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant. 你是一个乐于助人的助手。"
+
+# Load different instruct models based on the selected version
+def load_model(version):
+    global tokenizer, model
+    if version == "v1":
+        model_name = "hfl/llama-3-chinese-8b-instruct"
+    elif version == "v2":
+        model_name = "hfl/llama-3-chinese-8b-instruct-v2"
+    elif version == "v3":
+        model_name = "hfl/llama-3-chinese-8b-instruct-v3"
+    
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
+    return f"Model {model_name} loaded."
+
+@spaces.GPU(duration=50)
+def stream_chat(message: str, history: list, system_prompt: str, model_version: str, temperature: float, max_new_tokens: int):
+    conversation = [{"role": "system", "content": system_prompt or DEFAULT_SYSTEM_PROMPT}]
+    for prompt, answer in history:
+        conversation.extend([{"role": "user", "content": prompt}, {"role": "assistant", "content": answer}])
+
+    conversation.append({"role": "user", "content": message})
+
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
+    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+
+    generate_kwargs = {
+        "input_ids": input_ids,
+        "streamer": streamer,
+        "max_new_tokens": max_new_tokens,
+        "temperature": temperature,
+        "do_sample": temperature != 0,
+    }
+
+    generation_thread = Thread(target=model.generate, kwargs=generate_kwargs)
+    generation_thread.start()
+
+    output = ""
+    for new_token in streamer:
+        output += new_token
+        yield output
+
+chatbot = gr.Chatbot(height=500)
+
+with gr.Blocks() as demo:
+    gr.HTML(BANNER_HTML)
+    gr.ChatInterface(
+        fn=stream_chat,
+        chatbot=chatbot,
+        fill_height=True,
+        additional_inputs_accordion=gr.Accordion(label="Parameters / 参数设置", open=False, render=False),
+        additional_inputs=[
+            gr.Text(value=DEFAULT_SYSTEM_PROMPT, label="System Prompt / 系统提示词", render=False),
+            gr.Radio(choices=["v1", "v2", "v3"], label="Model Version / 模型版本", value="v3", interactive=False, render=False),
+            gr.Slider(minimum=0, maximum=1, step=0.1, value=0.5, label="Temperature / 温度系数", render=False),
+            gr.Slider(minimum=128, maximum=2048, step=1, value=256, label="Max new tokens / 最大生成长度", render=False),
+        ],
+        cache_examples=False,
+    )
+
+if __name__ == "__main__":
+    load_model("v3")  # Load the default model
+    demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+huggingface_hub==0.22.2
+transformers
+accelerate