small updates

app.py

@@ -2,8 +2,8 @@ import math
 import streamlit as st
 import pandas as pd
 import numpy as np
-import torch
-from transformers import AlbertTokenizer, AlbertModel
+#import torch
+#from transformers import AlbertTokenizer, AlbertModel
 from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 from io import BytesIO
@@ -14,7 +14,7 @@ from io import BytesIO
 #model = AlbertModel.from_pretrained('albert-' + model_size + '-v2')
 
 # For baseline 'sentence-transformers/paraphrase-albert-base-v2'
-model_name = 'output/training_OnlineConstrativeLoss-2023-03-11_23-47-34'
+model_name = 'output/training_OnlineConstrativeLoss-2023-03-14_00-40-03'
 
 similarity_threshold = 0.9
 
@@ -60,12 +60,16 @@ if uploaded_file is not None:
     data_ndb = pd.read_excel(uploaded_file, sheet_name='NDB', dtype=str)
 
     # Data cleaning CAQH
-    data_caqh['postalcode'] = data_caqh['postalcode'].astype(str).apply(lambda x: x[:5] + '-' + x[5:] if len(x) > 5 and not '-' in x else x)
+    # if you need to format with 00000-0000
+    # lambda x: x[:5] + '-' + x[5:] if len(x) > 5 and not '-' in x else x
+    data_caqh['postalcode'] = data_caqh['postalcode'].astype(str).apply(lambda x: x[:5])
+
     data_caqh['full-addr'] = data_caqh['address1'].astype(str) + ', ' \
                              + np.where(data_caqh['address2'].isnull(),  '' , data_caqh['address2'].astype(str)+ ', ')  \
                              + data_caqh['city'].astype(str) + ', '\
                              + data_caqh['state'].astype(str) + ' ' \
                              + data_caqh['postalcode'].astype(str)
+    data_caqh['full-addr'] = data_caqh['full-addr'].str.upper()
 
     st.write(f"CAQH before duplicate removal {len(data_caqh)}")
     data_caqh.drop_duplicates(subset='full-addr',inplace=True)
@@ -73,15 +77,16 @@ if uploaded_file is not None:
     st.write(f"CAQH after duplicate removal {len(data_caqh)}")
 
     # Data cleaning NDB
-    data_ndb['zip_pls_4_cd'] = data_ndb['zip_pls_4_cd'].astype(str).apply(lambda x: x if (x[-1] != '0' and x[-1] != '1') else '')
+    #data_ndb['zip_pls_4_cd'] = data_ndb['zip_pls_4_cd'].astype(str).apply(lambda x: x if (x[-1] != '0' and x[-1] != '1') else '')
 
-    data_ndb['zip_cd_zip_pls_4_cd'] = data_ndb['zip_cd'].astype(str) +\
-                                      np.where( data_ndb['zip_pls_4_cd'] == '', '', '-' \
-                                      + data_ndb['zip_pls_4_cd'].astype(str))
+    #data_ndb['zip_cd_zip_pls_4_cd'] = data_ndb['zip_cd'].astype(str) +\
+    #                                  np.where( data_ndb['zip_pls_4_cd'] == '', '', '-' \
+    #                                  + data_ndb['zip_pls_4_cd'].astype(str))
 
     data_ndb['full-addr'] = data_ndb['adr_ln_1_txt'].astype(str).str.strip() + ', ' \
-                            + data_ndb['cty_nm'].astype(str).str.strip() + ',  ' \
-                            + data_ndb['st_cd'].astype(str) + ' ' + data_ndb['zip_cd_zip_pls_4_cd']
+                            + data_ndb['cty_nm'].astype(str).str.strip() + ', ' \
+                            + data_ndb['st_cd'].astype(str) + ' ' + data_ndb['zip_cd'].astype(str)
+    data_ndb['full-addr'] = data_ndb['full-addr'].str.upper()
 
     # Calculate similarity For CAQH
     num_items = len(data_caqh)

data_set_training.csv

@@ -65,12 +65,15 @@ ADDRESS1|ADDRESS2|ARE_SAME
 145 34 23TH ST, JACKSONVILLE, FL 32258|145-50 23TH ST, JACKSONVILLE, FL 32258|0
 145-12 23TH ST, JACKSONVILLE, FL 32258|145 29 23TH ST, JACKSONVILLE, FL 32258|0
 15 49 RT 9, HALFMOON, NY 12065|15-49 RT 9, HALFMOON, NY 12065|1
+15 49 RT 9, HALFMOON, NY 12065|15-49 ROUTE 9, HALFMOON, NY 12065|1
 15 49 RT 9, HALFMOON, NY 12065|15-59 RT 9, HALFMOON, NY 12065|0
+15 49 RT 9, HALFMOON, NY 12065|15-59 ROUTE 9, HALFMOON, NY 12065|0
 15 49 RT 9, HALFMOON, NY 12065|1549 RT 9, HALFMOON, NY 12065|1
 152 13 GOLD STAR HWY, GROTON, CT 63403|152-18 GOLD STAR HWY, GROTON, CT 63403|0
 152 43 GOLD STAR HWY, GROTON, CT 63403|152-43 GOLD STAR HWY, GROTON, CT 63403|0
 152 43 GOLD STAR HWY, GROTON, CT 63403|152-44 GOLD STAR HWY, GROTON, CT 63403|0
 154-9 RT 9, HALFMOON, NY 12065|1549 RT 9, HALFMOON, NY 12065|1
+154-9 RT 9, HALFMOON, NY 12065|1549 ROUTE 9, HALFMOON, NY 12065|1
 160-10 N MIDLAND AVE, NYACK, NY 10960|160 10 NORTH MIDLAND AVENUE, NYACK, NY 10960|1
 160-10 N MIDLAND AVE, NYACK, NY 10960|160 20 NORTH MIDLAND AVE, NYACK, NY 10960|0
 160-10 N MIDLAND AVE, NYACK, NY 10960|160-10 N MIDLAND AVENUE, NYACK, NY 10960|1

quick_evaluate.py

@@ -33,7 +33,7 @@ a10= "440 TECHNOLOGY CENTER DR., BOSTON, MA 10034"
 a11="87-22 ROUTE 13, CORTLANDVILLE, NY 13045"
 a12="87 22 ROUTE 13, CORTLANDVILLE, NY 13045"
 a13="87-55 ROUTE 13, CORTLANDVILLE, NY 13045"
-a14="257 37 US RT 11, EVANS MILLS, NY 13637"
+a14="257 37 US ROUTE 11, EVANS MILLS, NY 13637"
 a15="257-37 US ROUTE 11, EVANS MILLS, NY 13637"
 
 a16="15645 S MAIN ST SUITE D, PENNINGTON, NJ 08534"