Update app.py

app.py

@@ -8,8 +8,10 @@ from langchain.llms import HuggingFaceHub
 from langchain.embeddings import HuggingFaceHubEmbeddings
 from langchain.vectorstores import Chroma
 from langchain.chains import RetrievalQA
+
 from trafilatura import fetch_url, extract
 from trafilatura.spider import focused_crawler
+from trafilatura.settings import use_config
 
 
 
@@ -20,11 +22,14 @@ def url_changes(url, pages_to_visit, urls_to_scrape, repo_id):
     to_visit, links = focused_crawler(url, max_seen_urls=pages_to_visit, max_known_urls=urls_to_scrape)
     print(f"{len(links)} to be crawled")
 
+    config = use_config()
+    config.set("DEFAULT", "EXTRACTION_TIMEOUT", "0")
+
     results_df = pd.DataFrame()
     for url in links:
         downloaded = fetch_url(url)
         if downloaded:
-          result = extract(downloaded, output_format='json')
+          result = extract(downloaded, output_format='json', config=config)
           result = json.loads(result)
 
           results_df = pd.concat([results_df, pd.DataFrame.from_records([result])])