specified device

utils/models.py

@@ -193,7 +193,7 @@ def run_inference(model_name, context, question, result_queue):
         # else:
         #     # Decode the generated tokens, excluding the input tokens
         #     result = tokenizer.decode(outputs[0][input_length:], skip_special_tokens=True)
-        llm = LLM(model_name, dtype=torch.bfloat16, hf_token=True, enforce_eager=True)
+        llm = LLM(model_name, dtype=torch.bfloat16, hf_token=True, enforce_eager=True, device="cpu")
         params = SamplingParams(
             max_tokens=512,
             )