Upload 2 files

wikipedia_utils.py

@@ -0,0 +1,60 @@
+import wikipedia
+import spacy
+
+def get_wiki_query(query):
+    try:
+        ### spacy code
+
+        # Load the English model
+        nlp = spacy.load("en_core_web_sm")
+
+        # Parse the sentence
+        doc = nlp(query)
+
+
+        # Entity path (people, evenrs, books)
+        entities_components = [entity_substring.text for entity_substring in doc.ents]
+        if len(entities_components) > 0:
+            subject_of_the_query= ""
+            for substrings in entities_components:
+                subject_of_the_query = subject_of_the_query + substrings
+
+            if subject_of_the_query == "":
+                print("Entity query not parsed.")
+            return subject_of_the_query
+
+
+
+        else:
+            first_noun = next((t for t in doc if t.pos_ in {"NOUN", "PROPN"}), None).text
+            print("Returning first noun from the query.")
+            return first_noun
+
+
+
+
+    except Exception as e:
+        print("Failed parsing a query subject from query" ,query)
+        print(e)
+
+
+def fetch_wikipedia_page(wiki_query):
+    try:
+        matched_articles = wikipedia.search(wiki_query)
+        if len(matched_articles) > 0:
+            used_article = matched_articles[0]
+            page_content = wikipedia.page(used_article,auto_suggest=False)
+            return page_content.content
+        else:
+            return ""
+    except Exception as e:
+        print("Could not fetch the wikipedia article using ", wiki_query)
+        print(e)
+
+test_queries = ["How many albums did Amy Winehouse publish?", "Who is Evora Cesaria?","Is cat an animal?"]
+wiki_queries = []
+wiki_pages= []
+for tq in test_queries:
+    wiki_queries.append(get_wiki_query(tq))
+for wq in wiki_queries:
+    wiki_pages.append(fetch_wikipedia_page(wq))

youtube_utils.py

@@ -0,0 +1,24 @@
+from youtube_transcript_api import YouTubeTranscriptApi
+import re
+
+def get_youtube_video_id(query):
+    try:
+        match = re.search(r'(?:youtu\.be/|youtube\.com/(?:watch\?v=|embed/|v/|shorts/))([\w-]{11})', query)
+        if match:
+            video_id = match.group(1)
+            print(video_id)
+            return video_id
+    except:
+        print("Did not find youtube video id from query ", query)
+
+def fetch_transcript_english(video_id):
+    try:
+        ytt_api = YouTubeTranscriptApi()
+        transcript = ytt_api.fetch(video_id,languages=['en'])
+        return transcript
+    except:
+        print("Error ")
+
+def post_process_transcript(transcript_snippets):
+    full_transcript = " ".join([transcript_snippet.text for transcript_snippet in transcript_snippets])
+    return full_transcript