Upload 3 files

README.md

@@ -5,7 +5,6 @@ colorFrom: yellow
 colorTo: yellow
 sdk: gradio
 sdk_version: 5.6.0
-python_version: 3.11
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -16,7 +16,7 @@ HF_TOKEN = os.environ.get("HF_TOKEN")
 
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
 # Set `torch_dtype=torch.float16` to load model in float16, otherwise it will be loaded as float32 and might cause OOM Error.
-model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.float16).eval().to(device)
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.float16, device_map="auto").eval()
 terminators = [tokenizer.eos_token_id, 
                tokenizer.convert_tokens_to_ids("<|eot_id|>"), 
                tokenizer.convert_tokens_to_ids("<|reserved_special_token_26|>")]
@@ -59,4 +59,4 @@ interface = gr.Interface(
 )
 
 # Launch the Gradio app
-interface.launch(ssr_mode=False)
+interface.queue().launch(ssr_mode=False)

requirements.txt

@@ -4,3 +4,4 @@ torch
 numpy<2
 einops
 sentencepiece
+accelerate