Update app.py

app.py

@@ -22,29 +22,27 @@ def merge_vocab(pair, v_in):
     bigram = ' '.join(pair)
     replacement = ''.join(pair)
     for word in v_in:
-        w_out = word.replace(bigram, replacement)
+        # Use regex to ensure whole-word replacement
+        w_out = re.sub(r'(?<!\S)' + re.escape(bigram) + r'(?!\S)', replacement, word)
         v_out[w_out] = v_in[word]
     return v_out
 
 def apply_bpe(text, bpe_codes):
     word_list = text.split()
     for pair, _ in bpe_codes:
-        if ' ' in pair:
-            p = re.compile(r'(?<!\S)' + re.escape(' '.join(pair)) + r'(?!\S)')
-            word_list = [p.sub(''.join(pair), word) for word in word_list]
+        p = re.compile(r'(?<!\S)' + re.escape(' '.join(pair)) + r'(?!\S)')
+        word_list = [p.sub(''.join(pair), word) for word in word_list]
     return ' '.join(word_list)
 
 def bpe_process(input_text, target_vocab_size):
     preprocessed_text = preprocess_text(input_text)
     
-    # Initialize vocabulary with character-level tokens and common subwords
+    # Initialize vocabulary
     vocab = Counter(preprocessed_text.split())
-    vocab.update(Counter([preprocessed_text[i:i+2] for i in range(len(preprocessed_text)-1)]))
-    vocab.update(Counter([preprocessed_text[i:i+3] for i in range(len(preprocessed_text)-2)]))
     
     # Perform BPE merges
     bpe_codes = []
-    while len(vocab) < target_vocab_size:
+    while len(vocab) < target_vocab_size and len(vocab) > 1:
         pairs = get_stats(vocab)
         if not pairs:
             break
@@ -56,8 +54,8 @@ def bpe_process(input_text, target_vocab_size):
     encoded_text = apply_bpe(preprocessed_text, bpe_codes)
     
     # Calculate compression ratio
-    original_size = len(preprocessed_text)
-    compressed_size = len(encoded_text)
+    original_size = len(preprocessed_text.split())
+    compressed_size = len(encoded_text.split())
     compression_ratio = original_size / compressed_size if compressed_size != 0 else 0
     
     # Check if criteria are met