Update app.py

app.py

@@ -357,7 +357,7 @@ def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
     
     # Train BPE tokenizer
 #    tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
-    trainer = BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+    trainer = models.BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
     tokenizer.train_from_iterator(optimized_texts, trainer)
     
     return tokenizer, optimized_texts