Update app.py

Added export to inference mode.

app.py

@@ -29,9 +29,9 @@ import gradio as gr
 
 import torch
 from transformers import TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
-from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig
+from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig, Float4DynamicActivationFloat4WeightConfig
 # quant_config = Float8WeightOnlyConfig()
-quant_config = Float8DynamicActivationFloat8WeightConfig()
+quant_config = Float4DynamicActivationFloat4WeightConfig() # Float8DynamicActivationFloat8WeightConfig()
 quantization_config = TorchAoConfig(quant_type=quant_config)
 
 MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
@@ -45,6 +45,10 @@ model = AutoModelForCausalLM.from_pretrained(
 
 gc.collect()
 
+pt2e.move_exported_model_to_eval(model)
+
+gc.collect()
+
 #########
 
 # from unsloth import FastLanguageModel