Update app.py

app.py

@@ -41,30 +41,30 @@ MODEL_NAME = "HuggingFaceTB/SmolLM3-3B"
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 model = ORTModelForCausalLM.from_pretrained(MODEL_NAME, export=True)
 
-print("Creating quant config")
-qconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=True)
-print("Creating quant config successful")
-
-print("Creating quantizer")
-quantizer = ORTQuantizer.from_pretrained(model)
-print("Creating quantizer successful")
-# Step 4: Perform quantization saving output in a new directory
-quantized_model_dir = "./quantized_model"
-print("Starting quantization...")
-quantizer.quantize(save_dir=quantized_model_dir, quantization_config=qconfig)
-print("Quantization was successful. Garbage collecting...")
-
-del(quantizer)
-del(qconfig)
-del(model)
+# print("Creating quant config")
+# qconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=True)
+# print("Creating quant config successful")
+
+# print("Creating quantizer")
+# quantizer = ORTQuantizer.from_pretrained(model)
+# print("Creating quantizer successful")
+# # Step 4: Perform quantization saving output in a new directory
+# quantized_model_dir = "./quantized_model"
+# print("Starting quantization...")
+# quantizer.quantize(save_dir=quantized_model_dir, quantization_config=qconfig)
+# print("Quantization was successful. Garbage collecting...")
+
+# del(quantizer)
+# del(qconfig)
+# del(model)
 
 # Run garbage collection again to release memory from quantizer objects
 gc.collect()
 
-# Step 5: Load the quantized ONNX model for inference
-print("Loading quantized ONNX model for inference...")
-model = ORTModelForCausalLM.from_pretrained(quantized_model_dir)
-print("Loading model was succcessful. Garbage collecting.")
+# # Step 5: Load the quantized ONNX model for inference
+# print("Loading quantized ONNX model for inference...")
+# model = ORTModelForCausalLM.from_pretrained(quantized_model_dir)
+# print("Loading model was succcessful. Garbage collecting.")
 
 # Garbage collection again after final loading
 gc.collect()