wip

app.py

@@ -15,6 +15,7 @@ from io import BytesIO
 
 # For baseline 'sentence-transformers/paraphrase-albert-base-v2'
 model_name = 'output/training_OnlineConstrativeLoss-2023-03-14_01-24-44'
+model_name = 'output/training_OnlineConstrativeLoss-2023-03-17_16-10-39'
 
 similarity_threshold = 0.9
 

data_set_training.csv

@@ -239,4 +239,33 @@ VALLEY HEALTHCARE SYSTEM 1600 FORT BENNING RD, COLUMBUS, GA 31903|1600 FORT BENN
 165 10 VILLAGE DR W, UPPER MARLBORO, MD 20772|165 12 VILLAGE DR W, UPPER MARLBORO, MD 20772|0
 345 12 OLD WASHINGTON RD, WALDORF, MD 20602|345-12 OLD WASHINGTON RD, WALDORF, MD 20602|1
 144 12 ONYX CT, FREDERICKSBURG, VA 22407|144-11 ONYX CT, FREDERICKSBURG, VA 22407|0
-144 12 ONYX CT, FREDERICKSBURG, VA 22407|144-12 ONYX CT, FREDERICKSBURG, VA 22407|1
+144 12 ONYX CT, FREDERICKSBURG, VA 22407|144-12 ONYX CT, FREDERICKSBURG, VA 22407|1
+14453 UNION ST, Mc Coll, SC 29570|144-53 UNION ST, Mc Coll, SC 29570|1
+14453 UNION ST, Mc Coll, SC 29570|144 53 UNION ST, Mc Coll, SC 29570|0
+14453 UNION ST, Mc Coll, SC 29570|14 453 UNION STREET, Mc Coll, SC 29570|1
+14453 UNION ST APT 343, Mc Coll, SC 29570|144 53 UNION ST APT 343, Mc Coll, SC 29570|1
+14453 UNION ST, Mc Coll, SC 29570|144-53A UNION STREET, Mc Coll, SC 29570|0
+14453 UNION ST, Mc Coll, SC 29570|14443 UNION ST, Mc Coll, SC 29570|0
+14453 UNION ST, Mc Coll, SC 29570|144-53 UNION ST APT 343, Mc Coll, SC 29570|0
+20334 PARK AVE, PARK CITY, UT 84060|20234 PARK AVE, PARK CITY, UT 84060|0
+20334 PARK AVE, PARK CITY, UT 84060|20-334 PARK AVE, PARK CITY, UT 84060|0
+20334 PARK AVE, PARK CITY, UT 84060|202-34 PARK AVENUE, PARK CITY, UT 84060|1
+20334 PARK AVE, PARK CITY SUITE 2, UT 84060|202 34 PARK AVENUE STE 2, PARK CITY, UT 84060|1
+203 MAPLE AVE FL 2, ENGLEWOOD, NJ 07631|203 MAPLE AVE, ENGLEWOOD, NJ 07631|1
+203 MAPLE AVE FL 2, ENGLEWOOD, NJ 07631|203 MAPLE AVENUE, ENGLEWOOD, NJ 07631|1
+203 MAPLE AVE FL 2 STE 3, ENGLEWOOD, NJ 07631|203 MAPLE AVE, ENGLEWOOD, NJ 07631|0
+203 MAPLE AVE, ENGLEWOOD, NJ 07631|205 MAPLE AVE, ENGLEWOOD, NJ 07631|0
+2032 MAPLE AVE, ENGLEWOOD, NJ 07631|2031 MAPLE AVE, ENGLEWOOD, NJ 07631|0
+1427 MARVIN GRIFFIN RD, AUGUSTA, GA 30906|1417 MARVIN GRIFFIN RD, AUGUSTA, GA 30906|0
+32 GRAND ST, NEDERLAND, TX 77627|33 GRAND ST, NEDERLAND, TX 77627|0
+32 GRAND ST, NEDERLAND, TX 77627|32 GRAND ST #4, NEDERLAND, TX 77627|0
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|80 HOSPITAL DRIVE SUITE 6, BARBOURVILLE, KY 40906|1
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|80 HOSPITAL DR. STE. 6, BARBOURVILLE KY, 40906|1
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|SUITE #6, 80 HOSPITAL DRIVE, BARBOURVILLE, KY 40906|1
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|STE #6 - 80 HOSPITAL DR., BARBOURVILLE, KY 40906|1
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|#6-80 HOSPITAL DRIVE, BARBOURVILLE, KY 40906|1
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|80-2 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|0
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|80 HOSPITAL DR SUITE 6A, BARBOURVILLE, KY 40906|0
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|81 HOSPITAL DRIVE STE 6, BARBOURVILLE, KY 40906|0
+80 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|82 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|0
+80 22 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|8022 HOSPITAL DR SUITE 6, BARBOURVILLE, KY 40906|1

dev_set_training.csv

@@ -26,4 +26,12 @@ ADDRESS1|ADDRESS2|ARE_SAME
 8724 ROUTE 13, CORTLANDVILLE, NY 13045|87-24 ROUTE 13, CORTLANDVILLE, NY 13045|1
 HEART HEALTH, 90 N COLUMBUS AVE, LOUISVILLE, MS 39339|90 N COLUMBUS AVE, LOUISVILLE, MS 39339|1
 115 34 SHOREWAY DR, QUEENSTOWN, MD 21658|115-43 SHOREWAY DR, QUEENSTOWN, MD 21658|0
-112 24 SHOREWAY DR, QUEENSTOWN, MD 21658|112-24 SHOREWAY DR, QUEENSTOWN, MD 21658|1
+112 24 SHOREWAY DR, QUEENSTOWN, MD 21658|112-24 SHOREWAY DR, QUEENSTOWN, MD 21658|1
+3619 S 22ND DR, YUMA, AZ 85364|3636 S 22ND DR, YUMA, AZ 85364|0
+7325 FRANKLIN BLVD, SACRAMENTO, CA 95823|73235 FRANKLIN BLVD, SACRAMENTO, CA 95823|0
+3660 MAIN ST, TUCSON, AZ 85721|3701 MAIN ST, TUCSON, AZ 85721|0
+3910 MAGNET RD, MALVERN, AR 72104|3910 MAGNET RD, STE 206 MALVERN, AR 72104|0
+15702 OBERLIN RD, RALEIGH, NC 27605|15702 OBERLIN RD FL 1, RALEIGH, NC 27605|1
+14425 ROOSOVELT AVE APT 322, LA JOLLA, CA 92092|14325 ROOSOVELT AVE, LA JOLLA, CA 92092|0
+14425 ROOSOVELT AVE APT 322, LA JOLLA, CA 92092|144-25 ROOSOVELT AVE APT 322, LA JOLLA, CA 92092|1
+14425 ROOSOVELT AVE, LA JOLLA, CA 92092|144-25A ROOSOVELT AVENUE, LA JOLLA, CA 92092|0

eval.py

@@ -13,6 +13,12 @@ logger = logging.getLogger(__name__)
 
 model_name = 'sentence-transformers/paraphrase-albert-base-v2'
 
+#model_name='output/training_OnlineConstrativeLoss-2023-03-11_23-47-34'
+#model_name= 'output/training_OnlineConstrativeLoss-2023-03-14_01-24-44'
+
+#86% so far
+model_name = 'output/training_OnlineConstrativeLoss-2023-03-17_16-10-39'
+
 
 model_sbert = SentenceTransformer(model_name)
 

quick_evaluate.py

@@ -1,4 +1,4 @@
-#from transformers import AlbertTokenizer, AlbertModel
+from transformers import AlbertTokenizer, AlbertModel
 from sklearn.metrics.pairwise import cosine_similarity
 from sentence_transformers import SentenceTransformer
 
@@ -6,15 +6,16 @@ from sentence_transformers import SentenceTransformer
 
 # base
 # large
-#tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
 #model = AlbertModel.from_pretrained("albert-base-v2")
 #'sentence-transformers/paraphrase-albert-base-v2'
 model_name = 'output/training_OnlineConstrativeLoss-2023-03-10_11-17-15'
 model_name = 'output/training_OnlineConstrativeLoss-2023-03-11_00-24-35'
 model_name = 'output/training_OnlineConstrativeLoss-2023-03-11_01-00-19'
-model_name='output/training_OnlineConstrativeLoss-2023-03-14_01-24-44'
+model_name='output/training_OnlineConstrativeLoss-2023-03-17_16-10-39'
 model_sbert = SentenceTransformer(model_name)
 
+
 def get_sbert_embedding(input_text):
     embedding = model_sbert.encode(input_text)
     return embedding.tolist()
@@ -40,6 +41,17 @@ a16="15645 S MAIN ST SUITE D, PENNINGTON, NJ 08534"
 a17="156-45 S MAIN ST SUITE D, PENNINGTON, NJ 08534"
 a18="156-46 S MAIN ST SUITE D, PENNINGTON, NJ 08534"
 
+a19 = "THE PAVILION AT QUEENS FOR REHABILITAION AND NURSING 36-17 PARSONS BOULEVARD, FLUSHING, NY 11354"
+a20 = "136-17 39TH AVENUE, 4TH FLOOR, SUITE CF-E, FLUSHING, NY 11354"
+a21="WISDOM MEDICAL P.C., 136-20 38 TH AVE  6E, FLUSHING, NY 11354"
+
+encoded_input = tokenizer(a21, return_tensors='pt')
+input_ids = encoded_input.input_ids
+input_num_tokens = input_ids.shape[1]
+print(input_num_tokens)
+list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
+#
+print( "Tokens : " + ' '.join(list_of_tokens))
 #def get_embedding(input_text):
 #    encoded_input = tokenizer(input_text, return_tensors='pt')
 #    input_ids = encoded_input.input_ids