Update app.py

app.py

@@ -18,8 +18,7 @@ def tokenize(input_text):
     codeqwen_tokens = len(codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     rwkv4_tokens = len(rwkv4_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     rwkv5_tokens = len(rwkv5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    deepseekllm_tokens = len(deepseekllm_tokenizer(input_text, add_special_tokens=True)["input_ids"])
-    deepseekv2_tokens = len(deepseekv2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    deepseek_tokens = len(deepseek_tokenizer(input_text, add_special_tokens=True)["input_ids"])
 
     results = {
         "LLaMa-1/LLaMa-2": llama_tokens,
@@ -37,8 +36,7 @@ def tokenize(input_text):
         "CodeQwen": codeqwen_tokens,
         "RWKV-v4": rwkv4_tokens,
         "RWKV-v5/RWKV-v6": rwkv5_tokens,
-        "DeepSeek-LLM": deepseekllm_tokens,
-        "DeepSeek-V2": deepseekv2_tokens
+        "DeepSeek": deepseek_tokens
     }
 
     # Sort the results in descending order based on token length
@@ -63,8 +61,7 @@ if __name__ == "__main__":
     codeqwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/CodeQwen1.5-7B")
     rwkv4_tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-4-14b-pile", trust_remote_code=True)
     rwkv5_tokenizer = AutoTokenizer.from_pretrained("RWKV/v5-EagleX-v2-7B-HF", trust_remote_code=True)
-    deepseekllm_tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-llm-7b-base", trust_remote_code=True)
-    deepseekv2_tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2", trust_remote_code=True)
+    deepseek_tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2", trust_remote_code=True)
 
     iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=17), outputs="text")
     iface.launch()