Update app.py

app.py

@@ -1,17 +1,29 @@
 from transformers import AutoTokenizer
 import gradio as gr
 
-
 def tokenize(input_text):
-    llama_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    mistral_tokens = mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    llama_tokens = len(llama_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    mistral_tokens = len(mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt_neox_tokens = len(gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    falcon_tokens = len(falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    phi2_tokens = len(phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+
+    token_lengths = {
+        "LLaMa": llama_tokens,
+        "Mistral": mistral_tokens,
+        "GPT-2/GPT-J": gpt2_tokens,
+        "GPT-NeoX": gpt_neox_tokens,
+        "Falcon": falcon_tokens,
+        "Phi-2": phi2_tokens,
+        "T5": t5_tokens
+    }
+
+    sorted_tokens = sorted(token_lengths.items(), key=lambda x: x[1], reverse=True)
+    result = "\n".join([f"{name}: {length}" for name, length in sorted_tokens])
 
-    return f"LLaMa: {len(llama_tokens)}\nMistral: {len(mistral_tokens)}\nGPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}"
+    return result
 
 
 if __name__ == "__main__":