Request urls instead of reading from file, clean code and improve readability

app.py

@@ -4,84 +4,57 @@ from langchain import hub
 from langchain_chroma import Chroma
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnablePassthrough
-from langchain_mistralai import MistralAIEmbeddings
 from langchain_community.embeddings import HuggingFaceInstructEmbeddings
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_mistralai import ChatMistralAI
-from langchain_community.document_loaders import PyPDFLoader
 import requests
-from pathlib import Path
 from langchain_community.document_loaders import WebBaseLoader
 import bs4
 from langchain_core.rate_limiters import InMemoryRateLimiter
 from urllib.parse import urljoin
 
+# Define a limiter to avoid rate limit issues with MistralAI
 rate_limiter = InMemoryRateLimiter(
     requests_per_second=0.1,  # <-- MistralAI free. We can only make a request once every second
     check_every_n_seconds=0.01,  # Wake up every 100 ms to check whether allowed to make a request,
     max_bucket_size=10,  # Controls the maximum burst size.
 )
 
-# # Get data from url
-# url = 'https://camels.readthedocs.io/_/downloads/en/latest/pdf/'
-# r = requests.get(url, stream=True)
-# document_path = Path('data.pdf')
-
-# document_path.write_bytes(r.content)
-# # document_path = "camels-readthedocs-io-en-latest.pdf"
-# loader = PyPDFLoader(document_path)
-# docs = loader.load()
-
-# # Load, chunk and index the contents of the blog.
-# url = "https://lilianweng.github.io/posts/2023-06-23-agent/"
-# loader = WebBaseLoader(
-#     web_paths=(url,),
-#     bs_kwargs=dict(
-#         parse_only=bs4.SoupStrainer(
-#             class_=("post-content", "post-title", "post-header")
-#         )
-#     ),
-# )
-# loader = WebBaseLoader(url)
-# docs = loader.load()
-
-# def get_subpages(base_url):
-#     visited_urls = []
-#     urls_to_visit = [base_url]
-
-#     while urls_to_visit:
-#         url = urls_to_visit.pop(0)
-#         if url in visited_urls:
-#             continue
-        
-#         visited_urls.append(url)
-#         response = requests.get(url)
-#         soup = bs4.BeautifulSoup(response.content, "html.parser")
+# Function to get all the subpages from a base url
+def get_subpages(base_url):
+    visited_urls = []
+    urls_to_visit = [base_url]
 
-#         for link in soup.find_all("a", href=True):
-#             full_url = urljoin(base_url, link['href'])
-#             if base_url in full_url and not full_url.endswith(".html") and full_url not in visited_urls:
-#                 urls_to_visit.append(full_url)
-#     visited_urls = visited_urls[1:]
+    while urls_to_visit:
+        url = urls_to_visit.pop(0)
+        if url in visited_urls:
+            continue
+        
+        visited_urls.append(url)
+        response = requests.get(url)
+        soup = bs4.BeautifulSoup(response.content, "html.parser")
 
-#     return visited_urls
+        for link in soup.find_all("a", href=True):
+            full_url = urljoin(base_url, link['href'])
+            if base_url in full_url and full_url.endswith(".html") and full_url not in visited_urls:
+                urls_to_visit.append(full_url)
+    visited_urls = visited_urls[1:]
 
-# base_url = "https://camels.readthedocs.io/en/latest/"
-# # base_url = "https://carla.readthedocs.io/en/latest/"
-# # urls = get_subpages(base_url)
+    return visited_urls
 
-urlsfile = open("urls.txt")
-urls = urlsfile.readlines()
-urls = [url.replace("\n","") for url in urls]
-urlsfile.close()
+# Get urls
+base_url = "https://camels.readthedocs.io/en/latest/"
+urls = get_subpages(base_url)
 
 # Load, chunk and index the contents of the blog.
 loader = WebBaseLoader(urls)
 docs = loader.load()
 
+# Join content pages for processing
 def format_docs(docs):
     return "\n\n".join(doc.page_content for doc in docs)
 
+# Create a RAG chain
 def RAG(llm, docs, embeddings):
 
     # Split text
@@ -114,7 +87,6 @@ llm = ChatMistralAI(model="mistral-large-latest", rate_limiter=rate_limiter)
 embed_model = "sentence-transformers/multi-qa-distilbert-cos-v1"
 # embed_model = "nvidia/NV-Embed-v2"
 embeddings = HuggingFaceInstructEmbeddings(model_name=embed_model)
-# embeddings = MistralAIEmbeddings()
 
 # RAG chain
 rag_chain = RAG(llm, docs, embeddings)
@@ -131,12 +103,13 @@ def handle_prompt(message, history):
 
 greetingsmessage = "Hi, I'm the CAMELS DocBot, I'm here to assist you with any question related to the CAMELS simulations documentation"
 example_questions = [
-                    "How can i read a halo file?",
+                    "How can I read a halo file?",
                     "Which simulation suites are included in CAMELS?",
                     "Which are the largest volumes in CAMELS simulations?",
-                    "How can I get the power spectrum of a simulation?"
+                    "Write a complete snippet of code getting the power spectrum of a simulation"
                      ]
 
+# Define Gradio interface
 demo = gr.ChatInterface(handle_prompt, type="messages", title="CAMELS DocBot", examples=example_questions, theme=gr.themes.Soft(), description=greetingsmessage)#, chatbot=chatbot)
 
 demo.launch()

urls.txt

@@ -1,42 +0,0 @@
-https://camels.readthedocs.io/en/latest/
-https://camels.readthedocs.io/en/latest/news.html
-https://camels.readthedocs.io/en/latest/goals.html
-https://camels.readthedocs.io/en/latest/publications.html
-https://camels.readthedocs.io/en/latest/data_access.html
-https://camels.readthedocs.io/en/latest/citation.html
-https://camels.readthedocs.io/en/latest/description.html
-https://camels.readthedocs.io/en/latest/suites_sets.html
-https://camels.readthedocs.io/en/latest/codes.html
-https://camels.readthedocs.io/en/latest/parameters.html
-https://camels.readthedocs.io/en/latest/organization.html
-https://camels.readthedocs.io/en/latest/snapshots.html
-https://camels.readthedocs.io/en/latest/subfind.html
-https://camels.readthedocs.io/en/latest/SubLink.html
-https://camels.readthedocs.io/en/latest/rockstar.html
-https://camels.readthedocs.io/en/latest/ahf.html
-https://camels.readthedocs.io/en/latest/caesar.html
-https://camels.readthedocs.io/en/latest/Pk.html
-https://camels.readthedocs.io/en/latest/Bk.html
-https://camels.readthedocs.io/en/latest/pdf.html
-https://camels.readthedocs.io/en/latest/VIDE.html
-https://camels.readthedocs.io/en/latest/Lya.html
-https://camels.readthedocs.io/en/latest/Xrays.html
-https://camels.readthedocs.io/en/latest/Profiles.html
-https://camels.readthedocs.io/en/latest/CMD.html
-https://camels.readthedocs.io/en/latest/SAM.html
-https://camels.readthedocs.io/en/latest/zoomGZ.html
-https://camels.readthedocs.io/en/latest/tutorials.html
-https://camels.readthedocs.io/en/latest/images.html
-https://camels.readthedocs.io/en/latest/camels_library.html
-https://camels.readthedocs.io/en/latest/pylians3.html
-https://camels.readthedocs.io/en/latest/team.html
-https://camels.readthedocs.io/en/latest/contact.html
-https://camels.readthedocs.io/en/latest/logo.html
-https://camels.readthedocs.io/en/latest/examples/Reading_Manipulating_Snapshots.html
-https://camels.readthedocs.io/en/latest/examples/Pk.html
-https://camels.readthedocs.io/en/latest/examples/Images.html
-https://camels.readthedocs.io/en/latest/examples/particles_subhalos.html
-https://camels.readthedocs.io/en/latest/index.html
-https://camels.readthedocs.io/en/latest/Images.html
-https://camels.readthedocs.io/en/latest/particles_subhalos.html
-https://camels.readthedocs.io/en/latest/Reading_Manipulating_Snapshots.html