Multi processing for reading urls

README.md

@@ -5,4 +5,4 @@ app_file: app.py
 licese: mit
 ---
 
-install  nltk.download("punkt")s
+install  nltk.download("punkt")

__init__.py

@@ -1 +1 @@
-from google import GoogleSearch, Document, SemanticSearch
+import search

app.py

@@ -1,6 +1,5 @@
-import sys
+from search import SemanticSearch, GoogleSearch, Document
 import streamlit as st
-from google import SemanticSearch, GoogleSearch, Document
 from model import RAGModel, load_configs
 
 
@@ -38,7 +37,7 @@ if prompt := st.chat_input("Search Here insetad of Google"):
     st.session_state.messages.append({"role": "user", "content": prompt})
 
     search(prompt)
-    s = SemanticSearch(
+    s, u = SemanticSearch(
         prompt,
         st.session_state.doc,
         configs["model"]["embeding_model"],
@@ -51,3 +50,4 @@ if prompt := st.chat_input("Search Here insetad of Google"):
         st.markdown(response)
 
     st.session_state.messages.append({"role": "assistant", "content": response})
+    

model.py

@@ -1,4 +1,4 @@
-from google import SemanticSearch, GoogleSearch, Document
+from search import SemanticSearch, GoogleSearch, Document
 from transformers import AutoTokenizer, AutoModelForCausalLM
 from transformers import BitsAndBytesConfig
 from transformers.utils import is_flash_attn_2_available
@@ -71,8 +71,8 @@ if __name__ == "__main__":
     # g = GoogleSearch(query)
     # data = g.all_page_data
     # d = Document(data, 512)
-    # s = SemanticSearch( "all-mpnet-base-v2", "mps")
+    # s, u = SemanticSearch( "all-mpnet-base-v2", "mps")
     # topk = s.semantic_search(query=query, k=32)
-    r = RAGModel(configs)
-    output = r.answer_query(query=query, topk_items=[""])
-    print(output)
+    # r = RAGModel(configs)
+    # output = r.answer_query(query=query, topk_items=[""])
+    # print(output)

google.py → search.py

@@ -5,6 +5,7 @@ import nltk
 import torch
 from typing import Union
 from sentence_transformers import SentenceTransformer, util
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 
 class GoogleSearch:
@@ -33,10 +34,12 @@ class GoogleSearch:
             for link in sublist
             if len(link) > 0
         ]
+        print(links)
         return links
 
     def read_url_page(self, url: str) -> str:
 
+        print(url)
         response = requests.get(url, headers=self.headers)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, "html.parser")
@@ -55,11 +58,25 @@ class GoogleSearch:
     def all_pages(self) -> list[tuple[str, str]]:
 
         data: list[tuple[str, str]] = []
-        for url in self.links:
-            try:
-                data.append((url, self.read_url_page(url)))
-            except requests.exceptions.HTTPError as e:
-                print(e)
+        with ThreadPoolExecutor(max_workers=4) as executor:
+
+            future_to_url = {
+                executor.submit(self.read_url_page, url): url for url in self.links
+            }
+            for future in as_completed(future_to_url):
+                url = future_to_url[future]
+                try:
+                    output = future.result()
+                    data.append((url, output))
+
+                except requests.exceptions.HTTPError as e:
+                    print(e)
+
+        # for url in self.links:
+        #     try:
+        #         data.append((url, self.read_url_page(url)))
+        #     except requests.exceptions.HTTPError as e:
+        #         print(e)
 
         return data
 
@@ -111,7 +128,7 @@ class SemanticSearch:
     def __init__(
         self, doc_chunks: tuple[list, list], model_path: str, device: str
     ) -> None:
-        query = query
+
         self.doc_chunks, self.urls = doc_chunks
         self.st = SentenceTransformer(
             model_path,
@@ -125,7 +142,7 @@ class SemanticSearch:
         scores = util.dot_score(a=query_embeding, b=doc_embeding)[0]
 
         top_k = torch.topk(scores, k=k)[1].cpu().tolist()
-        return [(self.doc_chunks[i], scores[i]) for i in top_k]
+        return [self.doc_chunks[i] for i in top_k], self.urls
 
     def get_embeding(self, text: Union[list[str], str]):
         en = self.st.encode(text)
@@ -137,10 +154,12 @@ if __name__ == "__main__":
     query = "what is LLM"
     g = GoogleSearch(query)
     data = g.all_page_data
-    d = Document(data, 333)
-
-    s = SemanticSearch("all-mpnet-base-v2", "mps")
-    print(len(s.semantic_search(query, k=64)))
+    # d = Document(data, 333)
+    # doc_chunks = d.doc()
+    # s = SemanticSearch(doc_chunks, "all-mpnet-base-v2", "mps")
+    # topk, u = s.semantic_search(query, k=64)
+    # print(len(topk))
+    # print(topk, u)
 
     # g = GoogleSearch("what is LLM")
     # d = Document(g.all_page_data)