Update app.py

app.py

@@ -3,8 +3,11 @@ import re
 from collections import Counter
 
 def preprocess_text(text):
+    # Remove punctuation and special characters, keep Hindi characters and spaces
     text = re.sub(r'[^\u0900-\u097F\s]', '', text)
-    return ' '.join(text.split())
+    # Remove extra whitespace
+    text = ' '.join(text.split())
+    return text
 
 def get_stats(vocab):
     pairs = Counter()
@@ -29,12 +32,12 @@ def apply_bpe(text, bpe_codes):
         if ' ' in pair:
             p = re.compile(r'(?<!\S)' + re.escape(' '.join(pair)) + r'(?!\S)')
             word_list = [p.sub(''.join(pair), word) for word in word_list]
-    return word_list
+    return ' '.join(word_list)
 
 def bpe_process(input_text, target_vocab_size):
     preprocessed_text = preprocess_text(input_text)
     
-    # Initialize vocabulary
+    # Initialize vocabulary with character-level tokens and common subwords
     vocab = Counter(preprocessed_text.split())
     vocab.update(Counter([preprocessed_text[i:i+2] for i in range(len(preprocessed_text)-1)]))
     vocab.update(Counter([preprocessed_text[i:i+3] for i in range(len(preprocessed_text)-2)]))
@@ -53,9 +56,9 @@ def bpe_process(input_text, target_vocab_size):
     encoded_text = apply_bpe(preprocessed_text, bpe_codes)
     
     # Calculate compression ratio
-    original_size = len(preprocessed_text.split())
+    original_size = len(preprocessed_text)
     compressed_size = len(encoded_text)
-    compression_ratio = original_size / compressed_size
+    compression_ratio = original_size / compressed_size if compressed_size != 0 else 0
     
     # Check if criteria are met
     criteria_met = {
@@ -64,7 +67,7 @@ def bpe_process(input_text, target_vocab_size):
     }
     
     return (
-        " ".join(encoded_text),
+        encoded_text,
         len(vocab),
         compression_ratio,
         criteria_met
@@ -88,4 +91,4 @@ iface = gr.Interface(
 )
 
 # Launch the Gradio app
-iface.launch(share=True)
+iface.launch(share=True)