Update ✨Entity Linking Application✨.py

✨Entity Linking Application✨.py

@@ -15,6 +15,12 @@ from openai import OpenAI
 import sys
 from googlesearch import search
 
+
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+}
+
+
 folder_path = '/home/user/app/qids_folder'
 
 if not os.path.exists(folder_path):
@@ -55,24 +61,24 @@ async def fetch_json(url, session):
         return await response.json()
 
 async def combination_method(name, session):
-    async with aiohttp.ClientSession() as session:
+    async with aiohttp.ClientSession(headers=headers) as session:
         data = set()
         new_name = name.split()
         x = itertools_combinations(new_name, 2)
         for i in x:
             new_word = (i[0] + " " + i[1])
-            url = f"{new_word} site:en.wikipedia.org"
+            url = f"{new_word} site:en.wikipedia.org inurl:/wiki/ -inurl:?"
             s = search(url, num_results = 12, lang="en")
             for i in s:
                 data.add(i.split("/")[-1])
     return data
 
 async def single_method(name, session):
-    async with aiohttp.ClientSession() as session:
+    async with aiohttp.ClientSession(headers=headers) as session:
         data = set()
         new_name = name.replace("-", " ").replace("/", " ").split()
         for i in new_name:
-            url = f"{i} site:en.wikipedia.org"
+            url = f"{i} site:en.wikipedia.org inurl:/wiki/ -inurl:?"
             s = search(url, num_results = 12, lang="en")
             for i in s:
                 data.add(i.split("/")[-1])
@@ -83,8 +89,8 @@ async def mains(name, single, combi):
     disam_data = set()
     qids = set()
     
-    async with aiohttp.ClientSession() as session:
-        url = f"{name} site:en.wikipedia.org"
+    async with aiohttp.ClientSession(headers=headers) as session:
+        url = f"{name} site:en.wikipedia.org inurl:/wiki/ -inurl:?"
         s = search(url, num_results = 30, lang="en")
         for i in s:
             data.add(i.split("/")[-1])
@@ -172,7 +178,7 @@ def cleaner(text):
     return text
 
 async def retriever(qid):
-    async with aiohttp.ClientSession() as session: 
+    async with aiohttp.ClientSession(headers=headers) as session: 
         list_with_sent = []
 
         query_label = f"""SELECT ?subjectLabel