Update main.py

main.py

@@ -5,6 +5,7 @@ import prompt_style
 import os
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
+import time
 
 
 model_id = "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3-GGUF"
@@ -40,14 +41,17 @@ def format_prompt(item: Item):
 def generate(item: Item):
     formatted_prompt = format_prompt(item)
     output = model.create_chat_completion(messages=formatted_prompt, seed=item.seed,  
-                                          temperature=item.temperature, ax_tokens=item.max_new_tokens)
+                                          temperature=item.temperature, max_tokens=item.max_new_tokens)
 
     out = output['choices'][0]['message']['content']
     return out
 
 @app.post("/generate/")
 async def generate_text(item: Item):
+    t1 = time.time()
     ans = generate(item)
+    print(ans)
+    print(f"time: {str(time.time() - t1)}")
     return {"response": ans}