Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -59,6 +59,21 @@ def list_files(directory_path='.'):
|
|
| 59 |
files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]
|
| 60 |
return [f for f in files if f not in EXCLUDED_FILES]
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
def show_file_operations(file_path, sequence_number):
|
| 63 |
st.write(f"File: {os.path.basename(file_path)}")
|
| 64 |
unique_key = hashlib.md5(file_path.encode()).hexdigest()
|
|
@@ -117,6 +132,8 @@ def get_download_link(file):
|
|
| 117 |
|
| 118 |
def main():
|
| 119 |
st.sidebar.title('Web Datasets Bulk Downloader')
|
|
|
|
|
|
|
| 120 |
url_input_method = st.sidebar.radio("Choose URL Input Method", ["Enter URL", "Select from List"])
|
| 121 |
url = ""
|
| 122 |
if url_input_method == "Enter URL":
|
|
@@ -125,9 +142,15 @@ def main():
|
|
| 125 |
selected_site = st.sidebar.selectbox("Select a Website", list(URLS.keys()))
|
| 126 |
url = URLS[selected_site]
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
with open("history.json", "r") as f:
|
| 129 |
history = json.load(f)
|
| 130 |
|
|
|
|
| 131 |
if url:
|
| 132 |
subdir = hashlib.md5(url.encode()).hexdigest()
|
| 133 |
if not os.path.exists(subdir):
|
|
@@ -137,18 +160,22 @@ def main():
|
|
| 137 |
with open("history.json", "w") as f:
|
| 138 |
json.dump(history, f)
|
| 139 |
|
|
|
|
| 140 |
if st.sidebar.button('📥 Get All the Content'):
|
| 141 |
download_html_and_files(url, history[url])
|
| 142 |
show_download_links(history[url])
|
| 143 |
|
|
|
|
| 144 |
if st.sidebar.button('📂 Show Download Links'):
|
| 145 |
for subdir in history.values():
|
| 146 |
show_download_links(subdir)
|
| 147 |
|
|
|
|
| 148 |
with st.expander("URL History and Downloaded Files"):
|
| 149 |
for url, subdir in history.items():
|
| 150 |
st.markdown(f"#### {url}")
|
| 151 |
show_download_links(subdir)
|
| 152 |
|
|
|
|
| 153 |
if __name__ == "__main__":
|
| 154 |
main()
|
|
|
|
| 59 |
files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]
|
| 60 |
return [f for f in files if f not in EXCLUDED_FILES]
|
| 61 |
|
| 62 |
+
def file_editor(file_path):
|
| 63 |
+
st.write(f"Editing File: {os.path.basename(file_path)}")
|
| 64 |
+
file_content = ""
|
| 65 |
+
|
| 66 |
+
with open(file_path, "r") as f:
|
| 67 |
+
file_content = f.read()
|
| 68 |
+
|
| 69 |
+
file_content = st.text_area("Edit the file content:", value=file_content, height=250)
|
| 70 |
+
|
| 71 |
+
if st.button("💾 Save"):
|
| 72 |
+
with open(file_path, "w") as f:
|
| 73 |
+
f.write(file_content)
|
| 74 |
+
st.success(f"File '{os.path.basename(file_path)}' saved!")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
def show_file_operations(file_path, sequence_number):
|
| 78 |
st.write(f"File: {os.path.basename(file_path)}")
|
| 79 |
unique_key = hashlib.md5(file_path.encode()).hexdigest()
|
|
|
|
| 132 |
|
| 133 |
def main():
|
| 134 |
st.sidebar.title('Web Datasets Bulk Downloader')
|
| 135 |
+
|
| 136 |
+
# Selecting URL input method
|
| 137 |
url_input_method = st.sidebar.radio("Choose URL Input Method", ["Enter URL", "Select from List"])
|
| 138 |
url = ""
|
| 139 |
if url_input_method == "Enter URL":
|
|
|
|
| 142 |
selected_site = st.sidebar.selectbox("Select a Website", list(URLS.keys()))
|
| 143 |
url = URLS[selected_site]
|
| 144 |
|
| 145 |
+
# Reading or creating history.json
|
| 146 |
+
if not os.path.exists("history.json"):
|
| 147 |
+
with open("history.json", "w") as f:
|
| 148 |
+
json.dump({}, f)
|
| 149 |
+
|
| 150 |
with open("history.json", "r") as f:
|
| 151 |
history = json.load(f)
|
| 152 |
|
| 153 |
+
# Handling URL submission
|
| 154 |
if url:
|
| 155 |
subdir = hashlib.md5(url.encode()).hexdigest()
|
| 156 |
if not os.path.exists(subdir):
|
|
|
|
| 160 |
with open("history.json", "w") as f:
|
| 161 |
json.dump(history, f)
|
| 162 |
|
| 163 |
+
# Button for downloading content
|
| 164 |
if st.sidebar.button('📥 Get All the Content'):
|
| 165 |
download_html_and_files(url, history[url])
|
| 166 |
show_download_links(history[url])
|
| 167 |
|
| 168 |
+
# Button for showing download links
|
| 169 |
if st.sidebar.button('📂 Show Download Links'):
|
| 170 |
for subdir in history.values():
|
| 171 |
show_download_links(subdir)
|
| 172 |
|
| 173 |
+
# Expander for showing URL history and download links
|
| 174 |
with st.expander("URL History and Downloaded Files"):
|
| 175 |
for url, subdir in history.items():
|
| 176 |
st.markdown(f"#### {url}")
|
| 177 |
show_download_links(subdir)
|
| 178 |
|
| 179 |
+
|
| 180 |
if __name__ == "__main__":
|
| 181 |
main()
|