Update chat_demo.py

chat_demo.py

@@ -7,8 +7,8 @@ import tempfile
 import subprocess
 import threading
 
-BASE_URL = "http://localhost:8080/v1"
-MODEL_NAME = "bn"
+BASE_URL = "http://localhost:5100/v1"
+MODEL_NAME = "placeholder-model-id"
 
 def read_output(process):
     """Reads the output from the subprocess and prints it to the console."""
@@ -33,7 +33,7 @@ def start_server(command):
 
     return process
 
-server_process = start_server(["./ik_llama.cpp/build/bin/llama-server", "-m" ,"./ik_llama.cpp/build/model-out.gguf", "--chat-template", "vicuna"])
+server_process = start_server(["./llama.cpp/build/bin/llama-server", "-m" ,"./llama.cpp/build/ERNIE-4.5-0.3B-PT-UD-Q8_K_XL.gguf", "-c", "32000", "--jinja", "--no-mmap", "--port", "5100"])
 
 
 cli = OpenAI(api_key="sk-nokey", base_url=BASE_URL)
@@ -52,7 +52,7 @@ def openai_call(message, history, system_prompt, max_new_tokens):
         model=MODEL_NAME,
         messages=history,
         max_tokens=max_new_tokens,
-        stop=["<|im_end|>", "</s>"],
+        #stop=["<|im_end|>", "</s>"],
         stream=True
     )
     reply = ""
@@ -97,11 +97,11 @@ with gr.Blocks() as demo:
         type="messages",
         additional_inputs=[
             gr.Textbox("You are a helpful AI assistant.", label="System Prompt"),
-            gr.Slider(30, 2048, label="Max new tokens"),
+            gr.Slider(30, 8192, label="Max new tokens"),
         ],
         additional_outputs=[conv_state],
-        title="Chat with bitnet using ik_llama",
-        description="Warning: Do not input sensitive info - assume everything is public! Also note this is experimental and ik_llama server doesn't seems to support arbitrary chat template, we're using vicuna as approximate match - so there might be intelligence degradation."
+        title="Edge level LLM Chat demo",
+        description="In this demo, you can chat with sub-1B param range LLM - they are small enough to run with reasonable speed on most end user device. **Warning:** Do not input sensitive info - assume everything is public!"
     )
     download_file = gr.File()
     download_btn = gr.Button("Export Conversation for Download") \