Update ✨Entity Linking Application✨.py

✨Entity Linking Application✨.py

@@ -13,6 +13,7 @@ import streamlit as st
 import time
 from openai import OpenAI
 import sys
+from googlesearch import search
 
 folder_path = '/home/user/app/qids_folder'
 
@@ -49,21 +50,6 @@ else:
     st.warning("Please enter an API token to proceed.")
 
 
-async def fetch_url(session, url):
-    pageids_list = []
-    async with session.get(url) as response:
-        x = await response.text()
-        objective_list = x.split('"objectiveResults\\":')[-1].split(',\\"wikipediaResults\\"')[0].replace('\\\\\\"', "").replace("\\", "")
-        wikipedia_list = x.split(',\\"wikipediaResults\\":')[-1].split(',\\"data-sentry-element\\"')[0].replace('\\\\\\"', "").replace("\\", "")
-        data_1 = json.loads(objective_list)
-        data_2 = json.loads(wikipedia_list)
-        for i in data_1:
-            pageids_list.append(i.get("page_id"))
-        for i in data_2:
-            pageids_list.append(i.get("pageid"))
-        return pageids_list
-        
-        
 async def fetch_json(url, session):
     async with session.get(url) as response:
         return await response.json()
@@ -71,26 +57,25 @@ async def fetch_json(url, session):
 async def combination_method(name, session):
     async with aiohttp.ClientSession() as session:
         data = set()
-        new_name = name.replace("+", " ").split()
+        new_name = name.split()
         x = itertools_combinations(new_name, 2)
         for i in x:
-            new_word = (i[0] + " " + i[1]).replace(" ", "+")
-            url = f"https://www.objective.inc/demos/wikipedia?query={new_word}"
-            page_source = await fetch_url(session, url)
-            for i in page_source:
-                data.add(i)
+            new_word = (i[0] + " " + i[1])
+            url = f"{new_word} site:en.wikipedia.org"
+            search = search(url, num_results = 12, lang="en")
+            for i in search:
+                data.add(i.split("/")[-1])
     return data
 
 async def single_method(name, session):
     async with aiohttp.ClientSession() as session:
         data = set()
-        new_name = name.replace("+", " ").replace("-", " ").replace("/", " ").split()
+        new_name = name.replace("-", " ").replace("/", " ").split()
         for i in new_name:
-            new_word = i.replace(" ", "+")
-            url = f"https://www.objective.inc/demos/wikipedia?query={new_word}"
-            page_source = await fetch_url(session, url)
-            for i in page_source:
-                data.add(i)
+            url = f"{i} site:en.wikipedia.org"
+            search = search(url, num_results = 12, lang="en")
+            for i in search:
+                data.add(i.split("/")[-1])
     return data
 
 async def mains(name, single, combi):
@@ -99,10 +84,10 @@ async def mains(name, single, combi):
     qids = set()
     
     async with aiohttp.ClientSession() as session:
-        url = f"https://www.objective.inc/demos/wikipedia?query={name}"
-        page_source = await fetch_url(session, url)
-        for i in page_source:
-            data.add(i)
+        url = f"{name} site:en.wikipedia.org"
+        search= search(url, num_results = 30, lang="en")
+        for i in search:
+            data.add(i.split("/")[-1])
 
         wikipedia_url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={name}&srlimit=1&srprop=&srenablerewrites=True&srinfo=suggestion&format=json"
         json_data = await fetch_json(wikipedia_url, session)
@@ -113,7 +98,7 @@ async def mains(name, single, combi):
             json_suggestion = await fetch_json(suggested_url, session)
             results = json_suggestion.get('query', {}).get('search')
             for i in results:
-                data.add(int(i.get('pageid')))
+                data.add(int(i.get('title')))
 
         # Handle disambiguation links
         if data != {0}:
@@ -133,28 +118,28 @@ async def mains(name, single, combi):
                         if ":" not in i:
                             wikipedia_disamb = f"https://en.wikipedia.org/w/api.php?action=query&format=json&titles={i}&indexpageids"
                             json_id = await fetch_json(wikipedia_disamb, session)
-                            real_title = json_id.get('query').get('pageids')
+                            real_title = json_id.get('query').get('title')
                             disam_data.add(int(real_title[0]))
                 else:
                     disam_data.add(ids)
 
         # Makes combinations of the name
         if combi == "Yes":
-            if len(name.replace("+", " ").replace("-", " ").split()) >= 3: 
+            if len(name.replace("-", " ").split()) >= 3: 
                 combination_names = await combination_method(name, session)
                 for i in combination_names:
                     disam_data.add(i)
 
         # Checks every word alone
         if single == "Yes":
-            if len(name.replace("+", " ").replace("-", " ").replace("/", " ").split()) >= 2:
+            if len(name.replace("-", " ").replace("/", " ").split()) >= 2:
                 singles = await single_method(name, session)
                 for i in singles:
                     disam_data.add(i)
 
         for ids in disam_data:
             try:
-                wikibase_url = f"https://en.wikipedia.org/w/api.php?action=query&pageids={ids}&prop=pageprops&format=json"
+                wikibase_url = f"https://en.wikipedia.org/w/api.php?action=query&titles={ids}&prop=pageprops&format=json"
                 json_qid = await fetch_json(wikibase_url, session)
                 wikidata_qid = json_qid.get('query', {}).get('pages', {}).get(str(ids), {}).get('pageprops', {}).get('wikibase_item', {})
                 if wikidata_qid: