set to fp16

app.py

@@ -125,10 +125,10 @@ def get_response(params):
 
             if type(images) is list:
                 images = [
-                    image.to(model.device, dtype=torch.float32) for image in images
+                    image.to(model.device, dtype=torch.float16) for image in images
                 ]
             else:
-                images = images.to(model.device, dtype=torch.float32)
+                images = images.to(model.device, dtype=torch.float16)
 
             replace_token = DEFAULT_IMAGE_TOKEN
             if getattr(model.config, "mm_use_im_start_end", False):
@@ -491,7 +491,7 @@ if __name__ == "__main__":
         device="cpu",
         load_4bit=args.load_4bit,
         load_8bit=args.load_8bit,
-        torch_dtype=torch.float32,
+        torch_dtype=torch.float16,
     )
 
     demo = build_demo()