Update app.py

app.py

@@ -4,12 +4,15 @@ import os
 import urllib
 import base64
 from bs4 import BeautifulSoup
+import hashlib
+import json
 
 EXCLUDED_FILES = ['app.py', 'requirements.txt', 'pre-requirements.txt', 'packages.txt', 'README.md','.gitattributes', "backup.py","Dockerfile"]
 
-# Create a history.txt file if it doesn't exist yet
-with open("history.txt", "a+") as f:
-    f.close()
+# Create a history.json file if it doesn't exist yet
+if not os.path.exists("history.json"):
+    with open("history.json", "w") as f:
+        json.dump({}, f)
 
 def download_file(url, local_filename):
     if url.startswith('http://') or url.startswith('https://'):
@@ -23,17 +26,17 @@ def download_file(url, local_filename):
         except requests.exceptions.HTTPError as err:
             print(f"HTTP error occurred: {err}")
 
-def download_html_and_files(url):
+def download_html_and_files(url, subdir):
     html_content = requests.get(url).text
     soup = BeautifulSoup(html_content, 'html.parser')
     base_url = urllib.parse.urlunparse(urllib.parse.urlparse(url)._replace(path='', params='', query='', fragment=''))
     for link in soup.find_all('a'):
         file_url = urllib.parse.urljoin(base_url, link.get('href'))
-        local_filename = urllib.parse.urlparse(file_url).path.split('/')[-1]
+        local_filename = os.path.join(subdir, urllib.parse.urlparse(file_url).path.split('/')[-1])
         if local_filename:
             link['href'] = local_filename
             download_file(file_url, local_filename)
-    with open("index.html", "w") as file:
+    with open(os.path.join(subdir, "index.html"), "w") as file:
         file.write(str(soup))
 
 def list_files(directory_path='.'):
@@ -44,33 +47,45 @@ def get_download_link(file):
     with open(file, "rb") as f:
         bytes = f.read()
         b64 = base64.b64encode(bytes).decode()
-        href = f'<a href="data:file/octet-stream;base64,{b64}" download=\'{file}\'>Click to download {file}</a>'
+        href = f'<a href="data:file/octet-stream;base64,{b64}" download=\'{file.split("/")[-1]}\'>Click to download {file.split("/")[-1]}</a>'
     return href
 
-def show_download_links():
-    st.sidebar.write('Here are the files you can download:')
-    for file in list_files():
-        st.sidebar.markdown(get_download_link(file), unsafe_allow_html=True)
+def show_download_links(subdir):
+    st.write(f'Files for {subdir}:')
+    for file in list_files(subdir):
+        st.markdown(get_download_link(file), unsafe_allow_html=True)
 
 def main():
     st.sidebar.title('Web Datasets Bulk Downloader')
     url = st.sidebar.text_input('Please enter a Web URL to bulk download text and files')
 
-    # Save the history of URL entered as a text file
+    # Load history
+    with open("history.json", "r") as f:
+        history = json.load(f)
+
+    # Save the history of URL entered as a json file
     if url:
-        with open("history.txt", "a") as f:
-            f.write(url + "\n")
+        subdir = hashlib.md5(url.encode()).hexdigest()
+        if not os.path.exists(subdir):
+            os.makedirs(subdir)
+        if url not in history:
+            history[url] = subdir
+            with open("history.json", "w") as f:
+                json.dump(history, f)
 
     if st.sidebar.button('📥 Get All the Content'):
-        download_html_and_files(url)
-        show_download_links()
+        download_html_and_files(url, history[url])
+        show_download_links(history[url])
+
     if st.sidebar.button('📂 Show Download Links'):
-        show_download_links()
+        for subdir in history.values():
+            show_download_links(subdir)
 
     # Display history as markdown
-    with open("history.txt", "r") as f:
-        history = f.read()
-    st.markdown(f"### History\n\n{history}")
+    with st.expander("URL History and Downloaded Files"):
+        for url, subdir in history.items():
+            st.markdown(f"#### {url}")
+            show_download_links(subdir)
 
 if __name__ == "__main__":
-    main()
+    main()