Update app.py
Browse files
app.py
CHANGED
|
@@ -20,7 +20,7 @@ def respond(
|
|
| 20 |
temperature,
|
| 21 |
top_p,
|
| 22 |
):
|
| 23 |
-
llama_model = Llama(r"models/mistral-7b-instruct-v0.2.Q6_K.gguf", n_batch=1024, n_threads=0,
|
| 24 |
|
| 25 |
provider = LlamaCppPythonProvider(llama_model)
|
| 26 |
|
|
|
|
| 20 |
temperature,
|
| 21 |
top_p,
|
| 22 |
):
|
| 23 |
+
llama_model = Llama(r"models/mistral-7b-instruct-v0.2.Q6_K.gguf", n_batch=1024, n_threads=0, n_gpu_layers=64, n_ctx=8192, verbose=False)
|
| 24 |
|
| 25 |
provider = LlamaCppPythonProvider(llama_model)
|
| 26 |
|