Update app.py

Added garbage collection after quantization and generation.

app.py

@@ -1,3 +1,6 @@
+
+import gc
+
 import gradio as gr
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, HqqConfig
@@ -20,6 +23,8 @@ model =\
                         quantization_config=quant_config
         ).to(DEVICE)
 
+gc.collect()
+
 #########
 
 # print("Loading tokenizer & model…")
@@ -73,6 +78,7 @@ def chat_fn(history, enable_thinking, temperature, top_p, top_k, repetition_pena
         # xml_tools=TOOLS
     )
     inputs = tokenizer(text, return_tensors="pt")
+    gc.collect()
     with torch.inference_mode():
         streamer = model.generate(
             **inputs,
@@ -85,6 +91,7 @@ def chat_fn(history, enable_thinking, temperature, top_p, top_k, repetition_pena
             pad_token_id=tokenizer.eos_token_id,
             streamer=None          # we'll yield manually
         )
+    gc.collect()
     output_ids = streamer[0][len(inputs.input_ids[0]):]
     response = tokenizer.decode(output_ids, skip_special_tokens=True)
     if isinstance(response, str):