Update app.py

app.py

@@ -1,7 +1,8 @@
 import gradio as gr
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import bitsandbytes
+from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
+
+quant_config = HqqConfig(nbits=8, group_size=64)
 
 MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
@@ -12,8 +13,12 @@ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
 
 model =\
         AutoModelForCausalLM\
-                .from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16)\
-                .to(DEVICE)
+                .from_pretrained(
+                        MODEL_ID, 
+                        torch_dtype=torch.float16, 
+                        # device_map="cuda", 
+                        quantization_config=quant_config
+        ).to(DEVICE)
 
 #########