Updates

app.py

@@ -69,17 +69,11 @@ def perform_deduplication(
             batch_size = 64
             total_batches = (len(texts) + batch_size - 1) // batch_size
 
-            def compute_embeddings():
-                for batch_texts in progress.tqdm(batch_iterable(texts, batch_size), desc="Computing embeddings for Dataset 1", total=total_batches):
-                    batch_embeddings = model.encode(batch_texts, show_progressbar=False)
-                    embeddings.append(batch_embeddings)
-                return np.concatenate(embeddings, axis=0)
-
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                future = executor.submit(compute_embeddings)
-                while not future.done():
-                    pass  # Wait for embeddings to be computed
-                embedding_matrix = future.result()
+            for batch_texts in progress.tqdm(batch_iterable(texts, batch_size), desc="Computing embeddings for Dataset 1", total=total_batches):
+                batch_embeddings = model.encode(batch_texts, show_progressbar=False)
+                embeddings.append(batch_embeddings)
+
+            embedding_matrix = np.concatenate(embeddings, axis=0)
 
             # Deduplicate
             status = "Deduplicating embeddings..."
@@ -125,6 +119,7 @@ def perform_deduplication(
         yield f"An error occurred: {e}", ""
         raise e
 
+
 def deduplicate(embedding_matrix: np.ndarray, threshold: float, batch_size: int = 1024, progress=None) -> tuple[np.ndarray, dict[int, int]]:
     """
     Deduplicate embeddings and return the deduplicated indices and a mapping of removed indices to their corresponding original indices.