Update ✨Entity Linking Application✨.py

✨Entity Linking Application✨.py

@@ -1,4 +1,3 @@
-import pandas as pd
 import json
 import numpy as np
 import re
@@ -15,20 +14,18 @@ from openai import OpenAI
 import sys
 import time
 from bs4 import BeautifulSoup
-from fake_useragent import UserAgent
 import requests
-    
-ua = UserAgent()
+import nest_asyncio
+import httpx
+
+
+nest_asyncio.apply()
 
-headers = {
-    "User-Agent": f"{ua.random}"
-    }
 
 folder_path = '/home/user/app/qids_folder'
 
 if not os.path.exists(folder_path):
     os.mkdir(folder_path)
-    print(f"folder created at {folder_path}")
 else:
     pass
 
@@ -54,14 +51,11 @@ async def combination_method(name, session):
         x = itertools_combinations(new_name, 2)
         for i in x:
             new_word = (i[0] + " " + i[1])
-            url = f"https://www.google.com/search?q={new_word} site:en.wikipedia.org inurl:/wiki/ -inurl:? -inurl:Category: -inurl:File: -inurl:Special: -inurl:Help:&num=5"
-            html = requests.get(url, headers=headers)
-            soup = BeautifulSoup(html.text, "html.parser")
-            elements_with_href = soup.find_all(href=True)
-            href_links = [element['href'] for element in elements_with_href]
-            for link in href_links:
-                if link.startswith('https://en.wikipedia.org/wiki/'):
-                    data.add(link.split("/")[-1])
+            url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={new_word}&srlimit=20&srprop=&srenablerewrites=True&format=json"
+            json_data = await fetch_json(url, session)
+            suggestion = json_data.get('query', {}).get('search', {})
+            for pageid in suggestion:
+                data.add(pageid.get('title', {}))
     return data
 
 async def single_method(name, session):
@@ -69,30 +63,24 @@ async def single_method(name, session):
         data = set()
         new_name = name.replace("-", " ").replace("/", " ").split()
         for i in new_name:
-            url = f"https://www.google.com/search?q={i} site:en.wikipedia.org inurl:/wiki/ -inurl:? -inurl:Category: -inurl:File: -inurl:Special: -inurl:Help:&num=5"
-            html = requests.get(url, headers=headers)
-            soup = BeautifulSoup(html.text, "html.parser")
-            elements_with_href = soup.find_all(href=True)
-            href_links = [element['href'] for element in elements_with_href]
-            for link in href_links:
-                if link.startswith('https://en.wikipedia.org/wiki/'):
-                    data.add(link.split("/")[-1])
+            url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={i}&srlimit=20&srprop=&srenablerewrites=True&format=json"
+            json_data = await fetch_json(url, session)
+            suggestion = json_data.get('query', {}).get('search', {})
+            for pageid in suggestion:
+                data.add(pageid.get('title', {}))
     return data
 
-async def mains(name, single, combi):
+async def mains(name, deep_search):
     data = set()
     disam_data = set()
     qids = set()
     
     async with aiohttp.ClientSession() as session:
-        url = f"https://www.google.com/search?q={name} site:en.wikipedia.org inurl:/wiki/ -inurl:? -inurl:Category: -inurl:File: -inurl:Special: -inurl:Help:"
-        html = requests.get(url, headers=headers)
-        soup = BeautifulSoup(html.text, "html.parser")
-        elements_with_href = soup.find_all(href=True)
-        href_links = [element['href'] for element in elements_with_href]
-        for link in href_links:
-            if link.startswith('https://en.wikipedia.org/wiki/'):
-                data.add(link.split("/")[-1])
+        url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={name}&srlimit=20&srprop=&srenablerewrites=True&format=json"
+        json_data = await fetch_json(url, session)
+        suggestion = json_data.get('query', {}).get('search', {})
+        for pageid in suggestion:
+            data.add(pageid.get('title', {}))
 
         wikipedia_url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={name}&srlimit=1&srprop=&srenablerewrites=True&srinfo=suggestion&format=json"
         json_data = await fetch_json(wikipedia_url, session)
@@ -126,14 +114,14 @@ async def mains(name, single, combi):
                     disam_data.add(ids)
 
         # Makes combinations of the name
-        if combi == "Yes":
+        if deep_search == "Yes":
             if len(name.replace("-", " ").split()) >= 3: 
                 combination_names = await combination_method(name, session)
                 for i in combination_names:
                     disam_data.add(i)
 
         # Checks every word alone
-        if single == "Yes":
+        if deep_search == "Yes":
             if len(name.replace("-", " ").replace("/", " ").split()) >= 2:
                 singles = await single_method(name, session)
                 for i in singles:
@@ -270,18 +258,6 @@ async def main(name):
         with open(f"/home/user/app/info_extraction/{name}.json", "w", encoding="utf-8") as flast:
             json.dump(final_list, flast)
 
-#def check_sentence(sentence):
-#    two_consecutive_uppercase = r"[A-Z]{2}"
-#    uppercase_followed_by_fullstop = r"[A-Z]\."
-
-#    if re.search(two_consecutive_uppercase, sentence):
-#        return True
-
-#    if re.search(uppercase_followed_by_fullstop, sentence):
-#        return True
-
-#    return False 
-
 def main_cli():
     st.title("✨ Entity Linking Application ✨")
     st.caption("This web application is part of my master’s dissertation.")
@@ -306,8 +282,7 @@ def main_cli():
         
     input_sentence_user = st.text_input("Enter a sentence:", "", disabled=st.session_state.running)
     input_mention_user = st.text_input("Enter a textural reference (mention) that is inside the sentence:", "", disabled=st.session_state.running)
-    single = st.selectbox("Search each word individually? (Useful for difficult mentions)", ['Yes', 'No'], index=1, disabled=st.session_state.running)
-    combi = st.selectbox("Make combinations of each word? (Useful for difficult mentions)", ['Yes', 'No'], index=1, disabled=st.session_state.running)
+    deep_search = st.selectbox("Perform deep search? (Useful for difficult mentions)", ['Yes', 'No'], index=1, disabled=st.session_state.running)
     disambi = st.selectbox("Run acronym disambiguation? (Enable it if the mention include an acronym or if it is nested)", ['Yes', 'No'], index=0, disabled=st.session_state.running)
     
     if st.button("Run Entity Linking", key="run_button", disabled=st.session_state.running):
@@ -426,18 +401,18 @@ def main_cli():
                         list_with_contexts.append(context)
                 st.write("✅ Applied Data Normilzation module (1/5)")
                 # Candidate Retrieval & Information Gathering
-                async def big_main(mention, single, combi):
+                async def big_main(mention, deep_search):
                     mention = mention.split(",")
                     with st.spinner("Applying Candidate Retrieval module... (2/5)"):
                         for i in mention:
-                            await mains(i, single, combi)
+                            await mains(i, deep_search)
                     st.write("✅ Applied Candidate Retrieval module (2/5)")
                     with st.spinner("Applying Information Gathering module... (3/5)"):
                         for i in mention:
                             await main(i)
                     st.write("✅ Applied Information Gathering module (3/5)")
                         
-                asyncio.run(big_main(name, single, combi))
+                asyncio.run(big_main(name, deep_search))
 
                 number = 0
                 for i,j,o in zip(list_with_full_names,list_with_contexts,list_with_names_to_show):