Loading in 8bits

app.py

@@ -17,16 +17,16 @@ import gradio as gr
 #
 model_name = "google/madlad400-10b-mt"
 
-quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_compute_dtype=torch.float16,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4"
-)
 #quantization_config = BitsAndBytesConfig(
-#    load_in_8bit=True,
-#    llm_int8_threshold=200.0 # https://discuss.huggingface.co/t/correct-usage-of-bitsandbytesconfig/33809/5
+#    load_in_4bit=True,
+#    bnb_4bit_compute_dtype=torch.float16,
+#    bnb_4bit_use_double_quant=True,
+#    bnb_4bit_quant_type="nf4"
 #)
+quantization_config = BitsAndBytesConfig(
+    load_in_8bit=True,
+    llm_int8_threshold=200.0 # https://discuss.huggingface.co/t/correct-usage-of-bitsandbytesconfig/33809/5
+)
 
 tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
 model = AutoModelForSeq2SeqLM.from_pretrained(