Updates

app.py

@@ -14,19 +14,19 @@ default_dataset_split = "train"
 default_text_column = "sentence"
 default_threshold = 0.9
 
-def batch_iterable(iterable, batch_size):
-    """Yield successive batches from an iterable."""
-    for i in range(0, len(iterable), batch_size):
-        yield iterable[i:i + batch_size]
-
-def compute_embeddings(texts, batch_size, progress, desc):
-    """Compute embeddings for a list of texts with progress tracking."""
-    embeddings = []
-    total_batches = (len(texts) + batch_size - 1) // batch_size
-    for i, batch_texts in enumerate(batch_iterable(texts, batch_size)):
-        embeddings.append(model.encode(batch_texts, show_progressbar=False))
-        progress((i + 1) / total_batches, desc=desc)
-    return np.concatenate(embeddings, axis=0)
+# def batch_iterable(iterable, batch_size):
+#     """Yield successive batches from an iterable."""
+#     for i in range(0, len(iterable), batch_size):
+#         yield iterable[i:i + batch_size]
+
+# def compute_embeddings(texts, batch_size, progress, desc):
+#     """Compute embeddings for a list of texts with progress tracking."""
+#     embeddings = []
+#     total_batches = (len(texts) + batch_size - 1) // batch_size
+#     for i, batch_texts in enumerate(batch_iterable(texts, batch_size)):
+#         embeddings.append(model.encode(batch_texts, show_progressbar=False))
+#         progress((i + 1) / total_batches, desc=desc)
+#     return np.concatenate(embeddings, axis=0)
 
 def deduplicate_embeddings(
     embeddings_a: np.ndarray,
@@ -90,8 +90,8 @@ def perform_deduplication(
         yield "Loading Dataset 1...", ""
         texts1 = load_dataset_texts(dataset1_name, dataset1_split, dataset1_text_column)
         yield "Computing embeddings for Dataset 1...", ""
-        embeddings1 = compute_embeddings(texts1, batch_size=64, progress=progress, desc="Dataset 1 embeddings")
-
+        #embeddings1 = compute_embeddings(texts1, batch_size=64, progress=progress, desc="Dataset 1 embeddings")
+        embeddings1 = model.encode(texts1, show_progressbar=True)
         if deduplication_type == "Single dataset":
             # Deduplicate within Dataset 1
             yield "Deduplicating within Dataset 1...", ""
@@ -128,8 +128,8 @@ def perform_deduplication(
             yield "Loading Dataset 2...", ""
             texts2 = load_dataset_texts(dataset2_name, dataset2_split, dataset2_text_column)
             yield "Computing embeddings for Dataset 2...", ""
-            embeddings2 = compute_embeddings(texts2, batch_size=64, progress=progress, desc="Dataset 2 embeddings")
-
+            #embeddings2 = compute_embeddings(texts2, batch_size=64, progress=progress, desc="Dataset 2 embeddings")
+            embeddings2 = model.encode(texts2, show_progressbar=True)
             # Deduplicate Dataset 2 against Dataset 1
             yield "Deduplicating Dataset 2 against Dataset 1...", ""
             duplicate_indices, duplicate_mapping = deduplicate_embeddings(