Updated app with code for deduplication

app.py

@@ -6,6 +6,7 @@ from reach import Reach
 from difflib import ndiff
 import sys
 import tqdm
+from tqdm.utils import format_interval, format_num, format_sizeof
 
 # Load the model at startup
 model = StaticModel.from_pretrained("minishlab/M2V_base_output")
@@ -26,13 +27,13 @@ def deduplicate(embedding_matrix: np.ndarray, threshold: float, batch_size: int
     """
     Deduplicate embeddings and return the deduplicated indices and a mapping of removed indices to their corresponding original indices.
     """
-    # Build the index
+    # Building the index
     reach = Reach(vectors=embedding_matrix, items=[str(i) for i in range(len(embedding_matrix))])
 
     deduplicated_indices = set(range(len(embedding_matrix)))
     duplicate_to_original_mapping = {}
 
-    # Find nearest neighbors
+    # Finding nearest neighbors
     results = reach.nearest_neighbor_threshold(
         embedding_matrix,
         threshold=threshold,
@@ -40,7 +41,7 @@ def deduplicate(embedding_matrix: np.ndarray, threshold: float, batch_size: int
         show_progressbar=True  # Allow internal progress bar
     )
 
-    # Process duplicates
+    # Processing duplicates
     for i, similar_items in enumerate(results):
         if i not in deduplicated_indices:
             continue
@@ -58,13 +59,13 @@ def deduplicate_across_datasets(embedding_matrix_1: np.ndarray, embedding_matrix
     """
     Deduplicate embeddings across two datasets and return the indices of duplicates between them.
     """
-    # Build the index from Dataset 1
+    # Building the index from Dataset 1
     reach = Reach(vectors=embedding_matrix_1, items=[str(i) for i in range(len(embedding_matrix_1))])
 
     duplicate_indices_in_test = []
     duplicate_to_original_mapping = {}
 
-    # Find nearest neighbors between datasets
+    # Finding nearest neighbors between datasets
     results = reach.nearest_neighbor_threshold(
         embedding_matrix_2,
         threshold=threshold,
@@ -72,7 +73,7 @@ def deduplicate_across_datasets(embedding_matrix_1: np.ndarray, embedding_matrix
         show_progressbar=True  # Allow internal progress bar
     )
 
-    # Process duplicates
+    # Processing duplicates
     for i, similar_items in enumerate(results):
         similar_indices = [int(item[0]) for item in similar_items if item[1] >= threshold]
 
@@ -103,9 +104,9 @@ def perform_deduplication(
             super().__init__(*args, **kwargs)
 
     # Copy module-level attributes from original tqdm module
-    TqdmWrapper.format_interval = staticmethod(tqdm.format_interval)
-    TqdmWrapper.format_num = staticmethod(tqdm.format_num)
-    TqdmWrapper.format_sizeof = staticmethod(tqdm.format_sizeof)
+    TqdmWrapper.format_interval = staticmethod(format_interval)
+    TqdmWrapper.format_num = staticmethod(format_num)
+    TqdmWrapper.format_sizeof = staticmethod(format_sizeof)
 
     # Monkey-patch tqdm.tqdm with our wrapper
     original_tqdm_tqdm = tqdm.tqdm
@@ -313,12 +314,12 @@ with gr.Blocks() as demo:
     compute_button.click(
         fn=perform_deduplication,
         inputs=[
-            deduplication_type, 
-            dataset1_name, 
-            dataset1_split, 
+            deduplication_type,
+            dataset1_name,
+            dataset1_split,
             dataset1_text_column,
-            dataset2_name, 
-            dataset2_split, 
+            dataset2_name,
+            dataset2_split,
             dataset2_text_column,
             threshold
         ],
@@ -328,6 +329,7 @@ with gr.Blocks() as demo:
 demo.launch()
 
 
+
 # import gradio as gr
 # from datasets import load_dataset
 # import numpy as np