Update app.py

Revert ONNX quantization attempt...

app.py

@@ -1,45 +1,45 @@
 
-# import gc
+import gc
 
-# import gradio as gr
-# import torch
-# from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
+import gradio as gr
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
 
-# quant_config = HqqConfig(nbits=8, group_size=64)
+quant_config = HqqConfig(nbits=8, group_size=64)
 
-# MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
-# DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
-# print("Loading tokenizer & model…")
-# tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
-# # model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
+print("Loading tokenizer & model…")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+# model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16).to(DEVICE)
 
-# model =\
-#         AutoModelForCausalLM\
-#                 .from_pretrained(
-#                         MODEL_ID, 
-#                         torch_dtype=torch.float16, 
-#                         # device_map="cuda", 
-#                         quantization_config=quant_config
-#         ).to(DEVICE)
+model =\
+        AutoModelForCausalLM\
+                .from_pretrained(
+                        MODEL_ID, 
+                        torch_dtype=torch.float16, 
+                        # device_map="cuda", 
+                        quantization_config=quant_config
+        ).to(DEVICE)
 
 #gc.collect()
 
 #########
 
-import gc
+# import gc
 
-import gradio as gr
-from transformers import AutoTokenizer
-from optimum.onnxruntime import ORTModelForCausalLM, ORTQuantizer
-from optimum.onnxruntime.configuration import AutoQuantizationConfig
+# import gradio as gr
+# from transformers import AutoTokenizer
+# from optimum.onnxruntime import ORTModelForCausalLM, ORTQuantizer
+# from optimum.onnxruntime.configuration import AutoQuantizationConfig
 
-MODEL_NAME = "HuggingFaceTB/SmolLM3-3B"
+# MODEL_NAME = "HuggingFaceTB/SmolLM3-3B"
 
 
 
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-model = ORTModelForCausalLM.from_pretrained(MODEL_NAME, export=True)
+# tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+# model = ORTModelForCausalLM.from_pretrained(MODEL_NAME, export=True)
 
 # print("Creating quant config")
 # qconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=True)