Latest changes

pages/fetch_resources_store_collection.py

@@ -0,0 +1,59 @@
+import streamlit as st
+from langchain_community.document_loaders import AsyncHtmlLoader
+from langchain.schema import Document
+import json
+from typing import Iterable
+
+# Placeholder for async fetch function (adjust based on actual async handling in your environment)
+async def fetch_documents(urls):
+    loader = AsyncHtmlLoader(urls)
+    docs = await loader.load()
+    return docs
+
+def save_docs_to_jsonl(array: Iterable[Document], file_path: str) -> None:
+    with open(file_path, 'w') as jsonl_file:
+        for doc in array:
+            jsonl_file.write(json.dumps(doc.to_dict()) + '\n')  # Assuming Document objects have a to_dict method
+
+def load_docs_from_jsonl(file_path) -> Iterable[Document]:
+    array = []
+    with open(file_path, 'r') as jsonl_file:
+        for line in jsonl_file:
+            data = json.loads(line)
+            obj = Document(**data)
+            array.append(obj)
+    return array
+
+def fetch_clean_organize_page():
+    st.title("Fetch, Clean, and Organize Documents")
+    
+    # Initialize 'selected_urls' at the start of your app if it doesn't exist
+    if 'selected_urls' not in st.session_state:
+        st.session_state['selected_urls'] = []  # Default to an empty list
+    
+
+
+    urls = st.session_state['selected_urls']
+    if st.button("Fetch Documents"):
+        # Async fetching operation placeholder
+        # Adjust based on your async handling strategy
+        docs = fetch_documents(urls)  # This needs proper async handling
+        st.session_state['docs'] = docs  # Assuming docs are fetched and stored correctly
+
+    if 'docs' in st.session_state:
+        st.write(f"Fetched {len(st.session_state['docs'])} documents.")
+
+        if st.button("Save Documents as JSON"):
+            save_docs_to_jsonl(st.session_state['docs'], "documents.jsonl")
+            st.success("Documents saved as JSON.")
+
+            # Provide download link (streamlit >= 0.88.0)
+            with open("documents.jsonl", "rb") as file:
+                btn = st.download_button(
+                        label="Download JSON",
+                        data=file,
+                        file_name="documents.jsonl",
+                        mime="application/octet-stream")
+
+# Assuming this function is called in your app
+fetch_clean_organize_page()

pages/file_web_source_collection.py

@@ -19,12 +19,7 @@ def find_linked_urls(url):
     return set()
 
 def convert_to_absolute_urls(base_url, links):
-    absolute_urls = []
-    for link in links:
-        if not link.startswith('http'):
-            link = urljoin(base_url, link)
-        absolute_urls.append(link)
-    return set(absolute_urls)
+    return {urljoin(base_url, link) if not link.startswith('http') else link for link in links}
 
 def categorize_links(base_url, links):
     internal_links, external_links = set(), set()
@@ -38,20 +33,26 @@ def categorize_links(base_url, links):
 def main():
     st.title("Data Source Configuration")
     
+    
+    
+    if 'scanned_urls' not in st.session_state:
+        st.session_state['scanned_urls'] = {}
+    
     st.subheader("Scan Websites for URLs")
     url_input = st.text_area("Enter URLs to scan, separated by new lines:")
     url_list = [url.strip() for url in url_input.strip().split('\n') if url.strip()]  # Splitting and cleaning input
 
-    if st.button("Scan URLs"):
-        all_links = {}
-        for url in url_list:
-            unique_urls = find_linked_urls(url)
-            absolute_urls = convert_to_absolute_urls(url, unique_urls)
-            internal_links, external_links = categorize_links(url, absolute_urls)
-            all_links[url] = {"internal": internal_links, "external": external_links}
-
+    scan_button_clicked = st.button("Scan URLs")
+    if scan_button_clicked or st.session_state['scanned_urls']:
+        if scan_button_clicked:
+            for url in url_list:
+                unique_urls = find_linked_urls(url)
+                absolute_urls = convert_to_absolute_urls(url, unique_urls)
+                internal_links, external_links = categorize_links(url, absolute_urls)
+                st.session_state['scanned_urls'][url] = {"internal": internal_links, "external": external_links}
+        
         selected_urls = []
-        for base_url, links in all_links.items():
+        for base_url, links in st.session_state['scanned_urls'].items():
             st.write(f"Base URL: {base_url}")
             include_all_internal = st.checkbox(f"Include all internal links from {base_url}", key=f"all_{base_url}")
 
@@ -61,21 +62,26 @@ def main():
                 selected_internal = [link for link in links["internal"] if st.checkbox(link, key=link)]
                 selected_urls.extend(selected_internal)
             
-            # Displaying external links for informational purposes
             if links["external"]:
                 st.write("External links:")
                 for link in links["external"]:
                     st.write(link)
 
-        # Convert selected URLs to a DataFrame and display
         if selected_urls:
             df_selected_urls = pd.DataFrame(selected_urls, columns=['Selected URLs'])
             st.write(df_selected_urls)
 
-            # Saving the DataFrame as CSV
-            if st.button("Save Selected URLs to CSV"):
-                df_selected_urls.to_csv('selected_urls.csv', index=False)
-                st.success("Saved selected URLs to selected_urls.csv")
+            st.session_state['selected_urls'] = df_selected_urls
+
+            # Convert DataFrame to CSV for download
+            csv = df_selected_urls.to_csv(index=False).encode('utf-8')
+
+            st.download_button(
+                label="Download selected URLs as CSV",
+                data=csv,
+                file_name='selected_urls.csv',
+                mime='text/csv',
+            )
 
 if __name__ == "__main__":
     main()

selected_urls.csv

@@ -0,0 +1,3 @@
+Selected URLs
+https://tlt.rutgers.edu/blog
+https://tlt.rutgers.edu/coursera