Update app.py

app.py

@@ -36,6 +36,8 @@ def respond(
     max_tokens,
     temperature,
     top_p,
+    top_k,
+    repeat_penalty,
     model,
 ):
     from llama_cpp import Llama
@@ -47,10 +49,11 @@ def respond(
 
     llm = Llama(
         model_path=f"models/{model}",
+        flash_attn=True,
         n_gpu_layers=81,
         n_ctx=8192,
     )
-    provider = LlamaCppPythonProvider(llm)
+    provider = LlamaCppPythonProvider(llm, temperature, top_k, top_p, max_tokens, repeat_penalty)
 
     agent = LlamaCppAgent(
         provider,
@@ -89,14 +92,28 @@ demo = gr.ChatInterface(
     respond,
     additional_inputs=[
         gr.Textbox(value="You are a helpful assistant.", label="System message"),
-        gr.Slider(minimum=1, maximum=8192, value=8192, step=1, label="Max new tokens"),
+        gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
             minimum=0.1,
             maximum=1.0,
             value=0.95,
             step=0.05,
-            label="Top-p (nucleus sampling)",
+            label="Top-p",
+        ),
+        gr.Slider(
+            minimum=0,
+            maximum=100,
+            value=40,
+            step=1,
+            label="Top-k",
+        ),
+        gr.Slider(
+            minimum=0.0,
+            maximum=2.0,
+            value=1.1,
+            step=0.1,
+            label="Repetition penalty",
         ),
         gr.Dropdown(['Meta-Llama-3-70B-Instruct-Q3_K_M.gguf', 'Llama-3-8B-Synthia-v3.5-f16.gguf'], value="Meta-Llama-3-70B-Instruct-Q3_K_M.gguf", label="Model"),
     ],