use 4-bit

model.py

@@ -10,10 +10,11 @@ model_id = 'baichuan-inc/Baichuan2-13B-Chat'
 if torch.cuda.is_available():
     model = AutoModelForCausalLM.from_pretrained(
         model_id,
-        device_map='auto',
-        torch_dtype=torch.bfloat16,
+        # device_map='auto',
+        torch_dtype=torch.float16,
         trust_remote_code=True
     )
+    model = model.quantize(4).cuda()
     model.generation_config = GenerationConfig.from_pretrained(model_id)
 else:
     model = None