Update app.py

app.py

@@ -20,7 +20,7 @@ import jellyfish
 from gensim.models import Word2Vec
 from gensim.models.fasttext import FastText
 from collections import Counter
-from tokenizers import Tokenizer
+from tokenizers import Tokenizer, models
 from tokenizers.models import WordLevel
 from tokenizers.trainers import WordLevelTrainer
 from tokenizers.pre_tokenizers import Whitespace
@@ -344,6 +344,8 @@ def visualize_results(results_df, stats_df):
     return fig
 
 def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
+    tokenizer = Tokenizer(models.BPE(unk_token="[UNK]"))
+
     # Count word frequencies
     word_freq = Counter(word for text in texts for word in text.split())
     
@@ -354,7 +356,7 @@ def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
     ]
     
     # Train BPE tokenizer
-    tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
+#    tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
     trainer = BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
     tokenizer.train_from_iterator(optimized_texts, trainer)