Update app.py

app.py

@@ -27,32 +27,50 @@ def respond(
     top_p,
 ):
     from llama_cpp import Llama
-    from llama_cpp_agent import LlamaCppAgent
-    from llama_cpp_agent import MessagesFormatterType
-    from llama_cpp_agent.providers import LlamaCppPythonProvider
+    llm = Llama(
+      model_path="models/mistral-7b-instruct-v0.2.Q6_K.gguf",
+      chat_format="mistral"
+    )
+    stream = llm.create_chat_completion(
+        messages = [
+            {"role": "system", "content": f"{system_message}"},
+            {
+                "role": "user",
+                "content": f"{message}"
+            }
+        ],
+        stream=True,
+    )
+    for output in stream:
+        yield json.dumps(output, indent=2)
+    # from llama_cpp import Llama
+    # from llama_cpp_agent import LlamaCppAgent
+    # from llama_cpp_agent import MessagesFormatterType
+    # from llama_cpp_agent.providers import LlamaCppPythonProvider
     
-    llama_model = Llama(r"models/mistral-7b-instruct-v0.2.Q6_K.gguf", n_batch=1024, n_threads=0, n_gpu_layers=33, n_ctx=8192, verbose=False)
+    # llama_model = Llama(r"models/mistral-7b-instruct-v0.2.Q6_K.gguf", n_batch=1024, n_threads=0, n_gpu_layers=33, n_ctx=8192, verbose=False)
 
-    provider = LlamaCppPythonProvider(llama_model)
+    # provider = LlamaCppPythonProvider(llama_model)
 
-    agent = LlamaCppAgent(
-      provider,
-      system_prompt=f"{system_message}",
-      predefined_messages_formatter_type=MessagesFormatterType.MISTRAL,
-      debug_output=True
-    )
+    # agent = LlamaCppAgent(
+    #   provider,
+    #   system_prompt=f"{system_message}",
+    #   predefined_messages_formatter_type=MessagesFormatterType.MISTRAL,
+    #   debug_output=True
+    # )
 
-    settings = provider.get_provider_default_settings()
-    settings.stream = True
-    settings.max_tokens = max_tokens
-    settings.temperature = temperature
-    settings.top_p = top_p
-    partial_message = ""
-    for new_token in agent.get_chat_response(message, llm_sampling_settings=settings, returns_streaming_generator=True):
-        partial_message += new_token
-        if '<|im_end|>' in partial_message:
-            break
-        yield partial_message
+    # settings = provider.get_provider_default_settings()
+    # settings.stream = True
+    # settings.max_tokens = max_tokens
+    # settings.temperature = temperature
+    # settings.top_p = top_p
+    # partial_message = ""
+    # for new_token in agent.get_chat_response(message, llm_sampling_settings=settings, returns_streaming_generator=True):
+    #     partial_message += new_token
+    #     if '<|im_end|>' in partial_message:
+    #         break
+    #     yield partial_message
+    
     # stop_tokens = ["</s>", "[INST]", "[INST] ", "<s>", "[/INST]", "[/INST] "]
     # chat_template = '<s>[INST] ' + system_message
     # # for human, assistant in history: