Updated settings

app.py

@@ -31,9 +31,8 @@ model = AutoModelForCausalLM.from_pretrained(
     model_name,
     torch_dtype=torch.bfloat16,
     device_map="auto",
-    # load_in_8bit=True,
-    # try to make it fit on a T4?
-    load_in_4bit=True
+    load_in_8bit=True,
+    # load_in_4bit=True
 )
 streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
 
@@ -50,6 +49,7 @@ def chat(message, history):
         max_new_tokens=200,
         do_sample=True,
         temperature=0.7,
+        repetition_penalty=1.15,
         top_p=0.95,
         eos_token_id=tokenizer.eos_token_id,
     )