Update app.py

Try Unsloth...

app.py

@@ -1,32 +1,40 @@
 
 import gc
 
-import gradio as gr
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
+# import gradio as gr
+# import torch
+# from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
 
-# quant_config = HqqConfig(nbits=8, group_size=64)
+# # quant_config = HqqConfig(nbits=8, group_size=64)
 
-MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+# MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
+# DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
-print("Loading tokenizer & model…")
-tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-# model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
+# print("Loading tokenizer & model…")
+# tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+# # model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
 
-model =\
-        AutoModelForCausalLM\
-                .from_pretrained(
-                        MODEL_ID, 
-                        torch_dtype=torch.float16, 
-                        # device_map="cuda", 
-                        # quantization_config=quant_config
-        ).to(DEVICE)
+# model =\
+#         AutoModelForCausalLM\
+#                 .from_pretrained(
+#                         MODEL_ID, 
+#                         torch_dtype=torch.float16, 
+#                         # device_map="cuda", 
+#                         # quantization_config=quant_config
+#         ).to(DEVICE)
 
 #gc.collect()
 
 #########
 
+model, tokenizer = FastLanguageModel.from_pretrained(
+    "unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
+    max_seq_length=128_000,
+    load_in_4bit=True
+)
+
+#########
+
 # import gc
 
 # import gradio as gr