comment out device='cuda' & input generate_kwargs

app.py

@@ -5,7 +5,7 @@ import gradio as gr
 import spaces
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
-device = "cuda" # for GPU usage or "cpu" for CPU usage
+# device = "cuda" # for GPU usage or "cpu" for CPU usage
 
 tokenizer = AutoTokenizer.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct")
 model = AutoModelForCausalLM.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct", torch_dtype=torch.float16, device_map="auto")
@@ -33,13 +33,17 @@ Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE":
 @spaces.GPU()
 def model_call(question, document, answer):
     NEW_FORMAT = PROMPT.format(question=question, document=document, answer=answer)
-    inputs = tokenizer(NEW_FORMAT, return_tensors="pt").to(device)
-    model.generate(
-        inputs.input_ids,
-        attention_mask=inputs.attention_mask,
+    inputs = tokenizer(NEW_FORMAT, return_tensors="pt")
+    input_ids = inputs.input_ids.to(model.device)
+    attention_mask = inputs.attention_mask
+    generate_kwargs = dict(
+        input_ids=input_ids,
+        do_sample=True,
+        attention_mask=attention_mask,
         pad_token_id=tokenizer.eos_token_id,
     )
-    generated_text = tokenizer.decode(inputs.input_ids[0])
+    outputs = model.generate(**generate_kwargs)
+    generated_text = tokenizer.decode(outputs[0])
     print(generated_text)
     return generated_text