added timeout

code/modules/config/constants.py

@@ -3,6 +3,8 @@ import os
 
 load_dotenv()
 
+TIMEOUT = 60
+
 # API Keys - Loaded from the .env file
 
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

code/modules/dataloader/data_loader.py

@@ -22,6 +22,7 @@ from modules.dataloader.pdf_readers.base import PDFReader
 from modules.dataloader.pdf_readers.llama import LlamaParser
 from modules.dataloader.pdf_readers.gpt import GPTParser
 from modules.dataloader.helpers import get_metadata
+from modules.config.constants import TIMEOUT
 
 logger = logging.getLogger(__name__)
 BASE_DIR = os.getcwd()
@@ -32,7 +33,7 @@ class HTMLReader:
         pass
 
     def read_url(self, url):
-        response = requests.get(url)
+        response = requests.get(url, timeout=TIMEOUT)
         if response.status_code == 200:
             return response.text
         else:
@@ -52,7 +53,7 @@ class HTMLReader:
             absolute_url = urljoin(base_url, href)
             link["href"] = absolute_url
 
-            resp = requests.head(absolute_url)
+            resp = requests.head(absolute_url, timeout=TIMEOUT)
             if resp.status_code != 200:
                 logger.warning(
                     f"Link {absolute_url} is broken. Status code: {resp.status_code}"
@@ -127,7 +128,7 @@ class FileReader:
         return [Document(page_content=self.web_reader.read_html(url))]
 
     def read_tex_from_url(self, tex_url):
-        response = requests.get(tex_url)
+        response = requests.get(tex_url, timeout=TIMEOUT)
         if response.status_code == 200:
             return [Document(page_content=response.text)]
         else:

code/modules/dataloader/helpers.py

@@ -2,6 +2,7 @@ import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse
 import tempfile
+from modules.config.constants import TIMEOUT
 
 
 def get_urls_from_file(file_path: str):
@@ -27,11 +28,11 @@ def get_metadata(lectures_url, schedule_url):
     lecture_metadata = {}
 
     # Get the main lectures page content
-    r_lectures = requests.get(lectures_url)
+    r_lectures = requests.get(lectures_url, timeout=TIMEOUT)
     soup_lectures = BeautifulSoup(r_lectures.text, "html.parser")
 
     # Get the main schedule page content
-    r_schedule = requests.get(schedule_url)
+    r_schedule = requests.get(schedule_url, timeout=TIMEOUT)
     soup_schedule = BeautifulSoup(r_schedule.text, "html.parser")
 
     # Find all lecture blocks
@@ -119,7 +120,7 @@ def download_pdf_from_url(pdf_url):
     Returns:
         str: The local file path of the downloaded PDF file.
     """
-    response = requests.get(pdf_url)
+    response = requests.get(pdf_url, timeout=TIMEOUT)
     if response.status_code == 200:
         with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
             temp_file.write(response.content)

code/modules/dataloader/pdf_readers/gpt.py

@@ -6,6 +6,7 @@ from io import BytesIO
 from openai import OpenAI
 from pdf2image import convert_from_path
 from langchain.schema import Document
+from modules.config.constants import TIMEOUT
 
 
 class GPTParser:
@@ -59,6 +60,7 @@ class GPTParser:
                 "https://api.openai.com/v1/chat/completions",
                 headers=headers,
                 json=payload,
+                timeout=TIMEOUT,
             )
 
             resp = response.json()

code/modules/dataloader/pdf_readers/llama.py

@@ -2,7 +2,7 @@ import os
 import requests
 from llama_parse import LlamaParse
 from langchain.schema import Document
-from modules.config.constants import OPENAI_API_KEY, LLAMA_CLOUD_API_KEY
+from modules.config.constants import OPENAI_API_KEY, LLAMA_CLOUD_API_KEY, TIMEOUT
 from modules.dataloader.helpers import download_pdf_from_url
 
 
@@ -52,7 +52,11 @@ class LlamaParser:
         files = [
             (
                 "file",
-                ("file", requests.get(pdf_url).content, "application/octet-stream"),
+                (
+                    "file",
+                    requests.get(pdf_url, timeout=TIMEOUT).content,
+                    "application/octet-stream",
+                ),
             )
         ]
 

code/modules/dataloader/webpage_crawler.py

@@ -4,6 +4,7 @@ import asyncio
 import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urldefrag
+from modules.config.constants import TIMEOUT
 
 
 class WebpageCrawler:
@@ -19,7 +20,7 @@ class WebpageCrawler:
 
     def url_exists(self, url: str) -> bool:
         try:
-            response = requests.head(url)
+            response = requests.head(url, timeout=TIMEOUT)
             return response.status_code == 200
         except requests.ConnectionError:
             return False
@@ -89,7 +90,7 @@ class WebpageCrawler:
 
     def is_webpage(self, url: str) -> bool:
         try:
-            response = requests.head(url, allow_redirects=True)
+            response = requests.head(url, allow_redirects=True, timeout=TIMEOUT)
             content_type = response.headers.get("Content-Type", "").lower()
             return "text/html" in content_type
         except requests.RequestException: