llama-cpp-agent

Paused

pabloce commited on May 20, 2024

Commit

102611c

verified ·

1 Parent(s): 13e498a

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -29,7 +29,8 @@ def respond(
 ):
     from llama_cpp import Llama
     llm = Llama(
-      model_path="models/mistral-7b-instruct-v0.2.Q6_K.gguf"
     )
     stream = llm.create_chat_completion(
         messages = [

 ):
     from llama_cpp import Llama
     llm = Llama(
+        model_path="models/mistral-7b-instruct-v0.2.Q6_K.gguf",
+        n_gpu_layers=33,
     )
     stream = llm.create_chat_completion(
         messages = [