improvements

app.py

@@ -4,6 +4,7 @@ import pandas as pd
 import numpy as np
 import torch
 from transformers import AlbertTokenizer, AlbertModel
+from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 from io import BytesIO
 
@@ -12,6 +13,14 @@ model_size='base'
 tokenizer = AlbertTokenizer.from_pretrained('albert-' + model_size + '-v2')
 model = AlbertModel.from_pretrained('albert-' + model_size + '-v2')
 
+model_sbert = SentenceTransformer('sentence-transformers/paraphrase-albert-base-v2')
+# for regular burt 0.98
+similarity_threshold = 0.9
+
+def get_sbert_embedding(input_text):
+    embedding = model_sbert.encode(input_text)
+    return embedding.tolist()
+
 def get_embedding(input_text):
     encoded_input = tokenizer(input_text, return_tensors='pt')
     input_ids = encoded_input.input_ids
@@ -31,7 +40,7 @@ def get_embedding(input_text):
         #sentence_embedding = output.last_hidden_state[0][0]
         return sentence_embedding.tolist()
 
-st. set_page_config(layout="wide")
+st.set_page_config(layout="wide")
 st.title('Upload the Address Dataset')
 
 st.markdown('Upload an Excel file to view the data in a table.')
@@ -92,7 +101,7 @@ if uploaded_file is not None:
             end = num_items
             stop_iter = True
 
-        data_caqh.iloc[start:end, embedding_col_index]  = data_caqh.iloc[start:end, full_addr_col_index].apply(get_embedding)
+        data_caqh.iloc[start:end, embedding_col_index]  = data_caqh.iloc[start:end, full_addr_col_index].apply(get_sbert_embedding)
 
         progress_bar.progress(value=progress, text=f"CAQH embeddings: {(i + 1) * step_size} processed out of {num_items}")
 
@@ -123,7 +132,8 @@ if uploaded_file is not None:
             end = num_items
             stop_iter = True
 
-        data_ndb.iloc[start:end, embedding_col_index]  = data_ndb.iloc[start:end, full_addr_col_index].apply(get_embedding)
+        # or get_embedding
+        data_ndb.iloc[start:end, embedding_col_index]  = data_ndb.iloc[start:end, full_addr_col_index].apply(get_sbert_embedding)
 
         progress_bar.progress(value=progress, text=f"NDB embeddings: {(i + 1) * step_size} processed out of {num_items}")
 
@@ -142,10 +152,11 @@ if uploaded_file is not None:
             if sim > max_similarity:
                 max_similarity = sim
                 matched_row = ndb_row
-        if max_similarity >= 0.98:
+        if max_similarity >= similarity_threshold:
             data_caqh.at[i, 'matched-addr'] = matched_row['full-addr']
             data_caqh.at[i, 'similarity-score'] = max_similarity
         else:
+            print(f"max similarity was {max_similarity}")
             data_caqh.at[i, 'matched-addr'] = 'No Matches'
 
         progress = i / num_items