Update app.py

app.py

@@ -1,10 +1,11 @@
-from sentencepiece import SentencePieceProcessor
+from transformers import AutoTokenizer
 import gradio as gr
 
-sp = SentencePieceProcessor(model_file="tokenizer.model")
+
+tokenizer = AutoTokenizer.from_pretrained("gpt2")
 
 def tokenize(input_text):
-    tokens = sp.EncodeAsIds(input_text)
+    tokens = tokenizer(input_text)["input_ids"]
     return f"Number of tokens: {len(tokens)}"
 
 iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="text")