Update app.py

app.py

@@ -343,6 +343,23 @@ def visualize_results(results_df, stats_df):
     plt.tight_layout()
     return fig
 
+def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
+    # Count word frequencies
+    word_freq = Counter(word for text in texts for word in text.split())
+    
+    # Remove rare words
+    optimized_texts = [
+        ' '.join(word for word in text.split() if word_freq[word] >= min_frequency)
+        for text in texts
+    ]
+    
+    # Train BPE tokenizer
+    tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
+    trainer = BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+    tokenizer.train_from_iterator(optimized_texts, trainer)
+    
+    return tokenizer, optimized_texts
+
 # Main Comparison Function
 def compare_embeddings(file, query, model_types, model_names, split_strategy, chunk_size, overlap_size, custom_separators, vector_store_type, search_type, top_k, lang='german', use_custom_embedding=False, optimize_vocab=False, phonetic_weight=0.3, custom_tokenizer_file=None):
     all_results = []