Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,195 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+venv/bin/python filter=lfs diff=lfs merge=lfs -text
+venv/bin/python3 filter=lfs diff=lfs merge=lfs -text
+venv/bin/python3.9 filter=lfs diff=lfs merge=lfs -text
+venv/bin/ruff filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/PIL/__pycache__/Image.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/PIL/_imaging.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/PIL/_imagingcms.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/PIL/_imagingft.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/PIL/_imagingmath.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/__pycache__/typing_extensions.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/_cffi_backend.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/charset_normalizer/md__mypyc.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/contourpy/_contourpy.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/__pycache__/agl.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/cu2qu/cu2qu.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/feaLib/lexer.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/misc/bezierTools.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/otlLib/__pycache__/builder.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/pens/momentsPen.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/qu2cu/qu2cu.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/subset/__pycache__/__init__.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/fontTools/varLib/iup.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/component-test/src/lib/images/Duck.glb filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/component-test/src/lib/images/base.webp filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/component-test/src/lib/images/cantina.wav filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/component-test/src/lib/images/tower.jpg filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/component-test/src/lib/images/world.mp4 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/core/public/static/img/Duck.glb filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/spa/public/static/img/Duck.glb filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/spa/test/files/contract.pdf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/spa/test/files/file_test.ogg filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/_frontend_code/spa/test/files/world.mp4 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/frpc_linux_amd64_v0.2 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio/templates/frontend/static/img/Duck.glb filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/gradio_client/__pycache__/media_data.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/huggingface_hub/__pycache__/hf_api.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/huggingface_hub/inference/__pycache__/_client.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/idna/__pycache__/uts46data.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/kiwisolver/_cext.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/_elementpath.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/builder.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/etree.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/html/_difflib.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/html/diff.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/objectify.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/lxml/sax.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/__pycache__/backend_bases.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/__pycache__/figure.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/__pycache__/patches.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/__pycache__/pyplot.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/__pycache__/widgets.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/_c_internal_utils.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/_image.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/_path.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/_qhull.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/_tri.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/_ttconv.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/axes/__pycache__/_axes.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/axes/__pycache__/_base.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/backends/_backend_agg.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/backends/_tkagg.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/ft2font.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans-Bold.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans-BoldOblique.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans-Oblique.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSansMono-Bold.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSansMono-BoldOblique.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSansMono-Oblique.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSansMono.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSerif-Bold.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSerif-BoldItalic.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSerif-Italic.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSerif.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/STIXGeneral.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/STIXGeneralBol.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/STIXGeneralBolIta.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/mpl-data/fonts/ttf/STIXGeneralItalic.ttf filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/matplotlib/tests/__pycache__/test_axes.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/mpl_toolkits/mplot3d/__pycache__/axes3d.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/__pycache__/_add_newdocs.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/__pycache__/fromnumeric.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/_multiarray_tests.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/_multiarray_umath.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/_simd.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/tests/__pycache__/test_multiarray.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/tests/__pycache__/test_numeric.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/tests/__pycache__/test_regression.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/tests/__pycache__/test_ufunc.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/_core/tests/__pycache__/test_umath.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/fft/_pocketfft_umath.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/lib/__pycache__/_function_base_impl.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/lib/tests/__pycache__/test_function_base.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/linalg/__pycache__/_linalg.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/linalg/_umath_linalg.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/ma/__pycache__/core.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/ma/tests/__pycache__/test_core.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/_bounded_integers.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/_common.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/_generator.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/_mt19937.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/_pcg64.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/_philox.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/bit_generator.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy/random/mtrand.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy.libs/libgfortran-040039e1-0352e75f.so.5.0.0 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy.libs/libquadmath-96973f99-934c22de.so.0.0.0 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/numpy.libs/libscipy_openblas64_-99b71e71.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/orjson/orjson.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/algos.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/arrays.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/groupby.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/hashing.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/hashtable.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/index.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/internals.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/interval.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/join.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/lib.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/missing.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/ops.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/parsers.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/reshape.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/sas.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/sparse.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/testing.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslib.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/conversion.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/dtypes.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/fields.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/nattype.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/np_datetime.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/offsets.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/parsing.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/period.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/strptime.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/timedeltas.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/timestamps.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/timezones.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/tzconversion.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/tslibs/vectorized.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/window/aggregations.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/window/indexers.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/_libs/writers.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/__pycache__/frame.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/__pycache__/generic.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/__pycache__/series.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/groupby/__pycache__/groupby.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/indexes/__pycache__/base.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/indexes/__pycache__/multi.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/core/strings/__pycache__/accessor.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/io/__pycache__/pytables.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/io/__pycache__/stata.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/io/formats/__pycache__/style.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/tests/frame/__pycache__/test_constructors.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/tests/indexing/__pycache__/test_loc.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/tests/io/__pycache__/test_sql.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pandas/tests/tools/__pycache__/test_to_datetime.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pdfminer/__pycache__/glyphlist.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libbrotlicommon-3ecfe81c.so.1 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libfreetype-be14bf51.so.6.20.1 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libharfbuzz-89381d8f.so.0.60850.0 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libjpeg-77ae51ab.so.62.4.0 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/liblcms2-e69eef39.so.2.0.16 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/liblzma-13fa198c.so.5.4.5 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libopenjp2-05423b53.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libpng16-58efbb84.so.16.43.0 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libtiff-0a86184d.so.6.0.2 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libwebp-2fd3cdca.so.7.1.9 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pillow.libs/libxcb-b8a56d01.so.1.1.0 filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/distlib/t64-arm.exe filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/distlib/t64.exe filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/distlib/w64-arm.exe filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/idna/__pycache__/uts46data.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/pkg_resources/__pycache__/__init__.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/pyparsing/__pycache__/core.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pip/_vendor/rich/__pycache__/_emoji_codes.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pkg_resources/__pycache__/__init__.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pkg_resources/_vendor/__pycache__/pyparsing.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pycparser/__pycache__/yacctab.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pydantic_core/__pycache__/core_schema.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pydantic_core/_pydantic_core.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pygments/lexers/__pycache__/lisp.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pyparsing/__pycache__/core.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/pypdfium2_raw/libpdfium.so filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/rich/__pycache__/_emoji_codes.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/setuptools/_vendor/__pycache__/pyparsing.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/setuptools/_vendor/more_itertools/__pycache__/more.cpython-39.pyc filter=lfs diff=lfs merge=lfs -text
+venv/lib/python3.9/site-packages/yaml/_yaml.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,6 @@
 ---
-title: Web Scraping To Markdown For RAG
-emoji: 👀
-colorFrom: purple
-colorTo: indigo
+title: Web_Scraping_to_Markdown_for_RAG
+app_file: file_cleaning_ui.py
 sdk: gradio
-sdk_version: 5.49.0
-app_file: app.py
-pinned: false
+sdk_version: 4.44.1
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

file_cleaning.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+import argparse
+import re
+import pdfplumber
+import os
+
+def clean_text_for_rag(text: str) -> str:
+    """
+    Nettoie le texte pour RAG : normalise les caractères spéciaux,
+    puis garde uniquement lettres (accents), chiffres, espaces et ponctuation simple.
+    """
+    # Normalisation des caractères spéciaux
+    text = re.sub(
+        r"[’‘“”«»–—\u00A0\u202F…œŒæÆ©®™§°±×÷]",
+        lambda m: {
+            "’": "'", "‘": "'", "“": '"', "”": '"', "«": '"', "»": '"',
+            "–": "-", "—": "-", "…": "...", "œ": "oe", "Œ": "OE", "æ": "ae", "Æ": "AE",
+            "©": "(c)", "®": "(R)", "™": "TM", "§": "§", "°": "°", "±": "+/-", "×": "x", "÷": "/"
+        }.get(m.group(0), m.group(0)),
+        text
+    )
+    
+    # Nettoyage strict : garder lettres, chiffres, espaces, ponctuation simple
+    text = re.sub(r'[^a-zA-ZÀ-ÿæ-œ0-9\s\.\,\:\;\!\?\-\_\'\"\\(\)\–\…]', '', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    
+    return text
+
+def extract_and_clean_pdf(pdf_path, output_path):
+    """
+    Extrait le texte du PDF et le nettoie pour RAG.
+    """
+    print(f"[+] Extraction du texte depuis : {pdf_path}")
+    
+    with pdfplumber.open(pdf_path) as pdf:
+        all_text = []
+        for page in pdf.pages:
+            text = page.extract_text()
+            if text:
+                all_text.append(text)
+    
+    # Concaténer tout le texte
+    full_text = ' '.join(all_text)
+    # Nettoyer pour RAG
+    cleaned_text = clean_text_for_rag(full_text)
+    
+    # Sauvegarder
+    with open(output_path, 'w', encoding='utf-8') as f:
+        f.write(cleaned_text)
+    
+    print(f"[+] Texte nettoyé sauvegardé dans : {output_path}")
+
+def extract_text(file_path, output_path):
+    """
+    Extrait le texte des autres formats de fichier et le nettoie.
+    """
+    print(f"[+] Extraction du texte depuis : {file_path}")
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+        
+        # Nettoyer chaque ligne
+        cleaned_lines = [clean_text_for_rag(line.strip()) for line in lines if line.strip()]
+        
+        # Joindre avec sauts de ligne pour un bon format .md
+        cleaned_text = '\n'.join(cleaned_lines)
+        
+        # Sauvegarder dans le fichier .md
+        with open(output_path, 'w', encoding='utf-8') as f:
+            f.write(cleaned_text)
+        
+        print(f"[+] Texte nettoyé sauvegardé dans : {output_path}")
+    except Exception as e:
+        print(f"Erreur lors de l'ouverture du fichier {file_path} : {e}")
+        raise
+
+def main():
+    parser = argparse.ArgumentParser(description='Extraire et nettoyer un fichier pour le RAG. Sortie toujours en .md.')
+    parser.add_argument('input_file', type=str, help='Chemin vers le fichier à traiter (PDF, TXT, MD, etc.).')
+    parser.add_argument('output_md', type=str, help='Chemin du fichier Markdown (.md) de sortie.')
+    
+    args = parser.parse_args()
+    
+    input_path = args.input_file
+    output_path = args.output_md  # On garde le nom original
+
+    # Vérifier que le fichier existe
+    if not os.path.exists(input_path):
+        print(f"Erreur : Le fichier {input_path} n'existe pas.")
+        return
+
+    # Détecter l'extension du fichier d'entrée
+    _, ext = os.path.splitext(input_path.lower())
+
+    # Vérifier et corriger l'extension de sortie si nécessaire
+    if not output_path.lower().endswith('.md'):
+        print(f"[!] Avertissement : Le fichier de sortie n'a pas l'extension .md. Il sera renommé en .md.")
+        output_path = os.path.splitext(output_path)[0] + '.md'
+
+    # Appeler la bonne fonction selon le format
+    if ext == '.pdf':
+        extract_and_clean_pdf(input_path, output_path)
+    else:
+        extract_text(input_path, output_path)
+
+if __name__ == '__main__':
+    main()

file_cleaning_ui.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+
+import os
+import re
+import tempfile
+from pathlib import Path
+
+import pdfplumber
+import gradio as gr
+
+def clean_text_for_rag(text: str) -> str:
+    """Normalise et nettoie le texte pour un usage RAG."""
+    # Normalisation des caractères typographiques
+    text = re.sub(
+        r"[’‘“”«»–—\u00A0\u202F…œŒæÆ©®™§°±×÷]",
+        lambda m: {
+            "’": "'", "‘": "'", "“": '"', "”": '"',
+            "«": '"', "»": '"', "–": "-", "—": "-",
+            "…": "...", "œ": "oe", "Œ": "OE",
+            "æ": "ae", "Æ": "AE", "©": "(c)", "®": "(R)",
+            "™": "TM", "§": "§", "°": "°", "±": "+/-",
+            "×": "x", "÷": "/"
+        }.get(m.group(0), m.group(0)),
+        text,
+    )
+    # Conserver uniquement les caractères suivants
+    text = re.sub(r'[^a-zA-ZÀ-ÿæ-œ0-9\s\.\,\:\;\!\?\-\_\'\"\\\(\)]', '', text)
+    # Réduire les espaces multiples
+    return re.sub(r'\s+', ' ', text).strip()
+
+
+def extract_and_clean_pdf(pdf_path: str) -> str:
+    """Ouvre le PDF, récupère le texte et le nettoie."""
+    print(f"[+] Extraction du PDF : {pdf_path}")
+    all_pages = []
+    with pdfplumber.open(pdf_path) as pdf:
+        for page in pdf.pages:
+            txt = page.extract_text()
+            if txt:
+                all_pages.append(txt)
+    return clean_text_for_rag(" ".join(all_pages))
+
+
+def extract_and_clean_txt(txt_path: str) -> str:
+    """Lit un fichier texte (txt, md, …) et le nettoie."""
+    print(f"[+] Lecture du fichier texte : {txt_path}")
+    with open(txt_path, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    cleaned = [
+        clean_text_for_rag(line.strip())
+        for line in lines
+        if line.strip()
+    ]
+    return "\n".join(cleaned)
+
+def process_file(input_file: gr.File, output_name: str) -> str:
+    """
+    - Detecte le type (PDF ou texte)
+    - Effectue l'extraction + nettoyage
+    - Crée un fichier temporaire **avec le nom choisi** (output_name)
+    - Retourne le chemin du fichier temporaire (Gradio le propose en téléchargement)
+    """
+    input_path = input_file.name
+    _, ext = os.path.splitext(input_path.lower())
+
+    if ext == ".pdf":
+        cleaned_text = extract_and_clean_pdf(input_path)
+    else:
+        cleaned_text = extract_and_clean_txt(input_path)
+
+    output_name = output_name.strip()
+    if not output_name.lower().endswith(".md"):
+        output_name = f"{output_name}.md"
+
+    temp_dir = tempfile.mkdtemp()          
+    out_path = os.path.join(temp_dir, output_name)
+
+    with open(out_path, "w", encoding="utf-8") as f:
+        f.write(cleaned_text)
+
+    return out_path
+
+with gr.Blocks(title="Nettoyage de texte pour RAG") as demo:
+    gr.Markdown("# 📄 Nettoyage de texte pour RAG (PDF, TXT, MD)")
+    gr.Markdown(
+        "Déposez un fichier PDF, TXT ou MD. Le texte sera extrait, nettoyé "
+        "et vous pourrez le télécharger sous **le nom que vous choisissez**."
+    )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            input_file = gr.File(
+                label="Déposez votre fichier ici",
+                file_types=["pdf", "txt", "md", "file"],
+            )
+            output_name = gr.Textbox(
+                value="output.md",
+                label="Nom du fichier de sortie (en .md)",
+                placeholder="exemple.md",
+                interactive=True,
+            )
+            submit_btn = gr.Button("Traiter le fichier", variant="primary")
+        with gr.Column(scale=1):
+            output_file = gr.File(
+                label="Fichier nettoyé (.md)",
+                file_types=["md"],
+            )
+
+    submit_btn.click(
+        fn=process_file,
+        inputs=[input_file, output_name],
+        outputs=output_file,
+    )
+
+    gr.Markdown(
+        """
+        ---
+        **Nettoyage effectué :**
+        - Suppression des symboles non imprimables / caractères parasites  
+        - Conservation : lettres (avec accents), chiffres, espaces, ponctuation simple  
+        - Normalisation des espaces  
+        - Sortie toujours au format **`.md`**
+        """
+    )
+
+if __name__ == "__main__":
+    demo.launch(share=True)

requirements.txt

@@ -0,0 +1 @@
+pdfplumber

venv/bin/Activate.ps1

@@ -0,0 +1,241 @@
+<#
+.Synopsis
+Activate a Python virtual environment for the current PowerShell session.
+
+.Description
+Pushes the python executable for a virtual environment to the front of the
+$Env:PATH environment variable and sets the prompt to signify that you are
+in a Python virtual environment. Makes use of the command line switches as
+well as the `pyvenv.cfg` file values present in the virtual environment.
+
+.Parameter VenvDir
+Path to the directory that contains the virtual environment to activate. The
+default value for this is the parent of the directory that the Activate.ps1
+script is located within.
+
+.Parameter Prompt
+The prompt prefix to display when this virtual environment is activated. By
+default, this prompt is the name of the virtual environment folder (VenvDir)
+surrounded by parentheses and followed by a single space (ie. '(.venv) ').
+
+.Example
+Activate.ps1
+Activates the Python virtual environment that contains the Activate.ps1 script.
+
+.Example
+Activate.ps1 -Verbose
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and shows extra information about the activation as it executes.
+
+.Example
+Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
+Activates the Python virtual environment located in the specified location.
+
+.Example
+Activate.ps1 -Prompt "MyPython"
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and prefixes the current prompt with the specified string (surrounded in
+parentheses) while the virtual environment is active.
+
+.Notes
+On Windows, it may be required to enable this Activate.ps1 script by setting the
+execution policy for the user. You can do this by issuing the following PowerShell
+command:
+
+PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+
+For more information on Execution Policies: 
+https://go.microsoft.com/fwlink/?LinkID=135170
+
+#>
+Param(
+    [Parameter(Mandatory = $false)]
+    [String]
+    $VenvDir,
+    [Parameter(Mandatory = $false)]
+    [String]
+    $Prompt
+)
+
+<# Function declarations --------------------------------------------------- #>
+
+<#
+.Synopsis
+Remove all shell session elements added by the Activate script, including the
+addition of the virtual environment's Python executable from the beginning of
+the PATH variable.
+
+.Parameter NonDestructive
+If present, do not remove this function from the global namespace for the
+session.
+
+#>
+function global:deactivate ([switch]$NonDestructive) {
+    # Revert to original values
+
+    # The prior prompt:
+    if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
+        Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
+        Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
+    }
+
+    # The prior PYTHONHOME:
+    if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
+        Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
+        Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
+    }
+
+    # The prior PATH:
+    if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
+        Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
+        Remove-Item -Path Env:_OLD_VIRTUAL_PATH
+    }
+
+    # Just remove the VIRTUAL_ENV altogether:
+    if (Test-Path -Path Env:VIRTUAL_ENV) {
+        Remove-Item -Path env:VIRTUAL_ENV
+    }
+
+    # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
+    if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
+        Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
+    }
+
+    # Leave deactivate function in the global namespace if requested:
+    if (-not $NonDestructive) {
+        Remove-Item -Path function:deactivate
+    }
+}
+
+<#
+.Description
+Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
+given folder, and returns them in a map.
+
+For each line in the pyvenv.cfg file, if that line can be parsed into exactly
+two strings separated by `=` (with any amount of whitespace surrounding the =)
+then it is considered a `key = value` line. The left hand string is the key,
+the right hand is the value.
+
+If the value starts with a `'` or a `"` then the first and last character is
+stripped from the value before being captured.
+
+.Parameter ConfigDir
+Path to the directory that contains the `pyvenv.cfg` file.
+#>
+function Get-PyVenvConfig(
+    [String]
+    $ConfigDir
+) {
+    Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
+
+    # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
+    $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
+
+    # An empty map will be returned if no config file is found.
+    $pyvenvConfig = @{ }
+
+    if ($pyvenvConfigPath) {
+
+        Write-Verbose "File exists, parse `key = value` lines"
+        $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
+
+        $pyvenvConfigContent | ForEach-Object {
+            $keyval = $PSItem -split "\s*=\s*", 2
+            if ($keyval[0] -and $keyval[1]) {
+                $val = $keyval[1]
+
+                # Remove extraneous quotations around a string value.
+                if ("'""".Contains($val.Substring(0, 1))) {
+                    $val = $val.Substring(1, $val.Length - 2)
+                }
+
+                $pyvenvConfig[$keyval[0]] = $val
+                Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
+            }
+        }
+    }
+    return $pyvenvConfig
+}
+
+
+<# Begin Activate script --------------------------------------------------- #>
+
+# Determine the containing directory of this script
+$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
+$VenvExecDir = Get-Item -Path $VenvExecPath
+
+Write-Verbose "Activation script is located in path: '$VenvExecPath'"
+Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
+Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
+
+# Set values required in priority: CmdLine, ConfigFile, Default
+# First, get the location of the virtual environment, it might not be
+# VenvExecDir if specified on the command line.
+if ($VenvDir) {
+    Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
+}
+else {
+    Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
+    $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
+    Write-Verbose "VenvDir=$VenvDir"
+}
+
+# Next, read the `pyvenv.cfg` file to determine any required value such
+# as `prompt`.
+$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
+
+# Next, set the prompt from the command line, or the config file, or
+# just use the name of the virtual environment folder.
+if ($Prompt) {
+    Write-Verbose "Prompt specified as argument, using '$Prompt'"
+}
+else {
+    Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
+    if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
+        Write-Verbose "  Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
+        $Prompt = $pyvenvCfg['prompt'];
+    }
+    else {
+        Write-Verbose "  Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
+        Write-Verbose "  Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
+        $Prompt = Split-Path -Path $venvDir -Leaf
+    }
+}
+
+Write-Verbose "Prompt = '$Prompt'"
+Write-Verbose "VenvDir='$VenvDir'"
+
+# Deactivate any currently active virtual environment, but leave the
+# deactivate function in place.
+deactivate -nondestructive
+
+# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
+# that there is an activated venv.
+$env:VIRTUAL_ENV = $VenvDir
+
+if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
+
+    Write-Verbose "Setting prompt to '$Prompt'"
+
+    # Set the prompt to include the env name
+    # Make sure _OLD_VIRTUAL_PROMPT is global
+    function global:_OLD_VIRTUAL_PROMPT { "" }
+    Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
+    New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
+
+    function global:prompt {
+        Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
+        _OLD_VIRTUAL_PROMPT
+    }
+}
+
+# Clear PYTHONHOME
+if (Test-Path -Path Env:PYTHONHOME) {
+    Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
+    Remove-Item -Path Env:PYTHONHOME
+}
+
+# Add the venv to the PATH
+Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
+$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"

venv/bin/activate

@@ -0,0 +1,66 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+deactivate () {
+    # reset old environment variables
+    if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
+        PATH="${_OLD_VIRTUAL_PATH:-}"
+        export PATH
+        unset _OLD_VIRTUAL_PATH
+    fi
+    if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
+        PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
+        export PYTHONHOME
+        unset _OLD_VIRTUAL_PYTHONHOME
+    fi
+
+    # This should detect bash and zsh, which have a hash command that must
+    # be called to get it to forget past commands.  Without forgetting
+    # past commands the $PATH changes we made may not be respected
+    if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+        hash -r 2> /dev/null
+    fi
+
+    if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
+        PS1="${_OLD_VIRTUAL_PS1:-}"
+        export PS1
+        unset _OLD_VIRTUAL_PS1
+    fi
+
+    unset VIRTUAL_ENV
+    if [ ! "${1:-}" = "nondestructive" ] ; then
+    # Self destruct!
+        unset -f deactivate
+    fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV="/home/l.sottani/rag/preprocessing/venv"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
+# could use `if (set -u; : $PYTHONHOME) ;` in bash
+if [ -n "${PYTHONHOME:-}" ] ; then
+    _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
+    unset PYTHONHOME
+fi
+
+if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
+    _OLD_VIRTUAL_PS1="${PS1:-}"
+    PS1="(venv) ${PS1:-}"
+    export PS1
+fi
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands.  Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+    hash -r 2> /dev/null
+fi

venv/bin/activate.csh

@@ -0,0 +1,25 @@
+# This file must be used with "source bin/activate.csh" *from csh*.
+# You cannot run it directly.
+# Created by Davide Di Blasi <davidedb@gmail.com>.
+# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
+
+alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate'
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+setenv VIRTUAL_ENV "/home/l.sottani/rag/preprocessing/venv"
+
+set _OLD_VIRTUAL_PATH="$PATH"
+setenv PATH "$VIRTUAL_ENV/bin:$PATH"
+
+
+set _OLD_VIRTUAL_PROMPT="$prompt"
+
+if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
+    set prompt = "(venv) $prompt"
+endif
+
+alias pydoc python -m pydoc
+
+rehash

venv/bin/activate.fish

@@ -0,0 +1,64 @@
+# This file must be used with "source <venv>/bin/activate.fish" *from fish*
+# (https://fishshell.com/); you cannot run it directly.
+
+function deactivate  -d "Exit virtual environment and return to normal shell environment"
+    # reset old environment variables
+    if test -n "$_OLD_VIRTUAL_PATH"
+        set -gx PATH $_OLD_VIRTUAL_PATH
+        set -e _OLD_VIRTUAL_PATH
+    end
+    if test -n "$_OLD_VIRTUAL_PYTHONHOME"
+        set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
+        set -e _OLD_VIRTUAL_PYTHONHOME
+    end
+
+    if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
+        functions -e fish_prompt
+        set -e _OLD_FISH_PROMPT_OVERRIDE
+        functions -c _old_fish_prompt fish_prompt
+        functions -e _old_fish_prompt
+    end
+
+    set -e VIRTUAL_ENV
+    if test "$argv[1]" != "nondestructive"
+        # Self-destruct!
+        functions -e deactivate
+    end
+end
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+set -gx VIRTUAL_ENV "/home/l.sottani/rag/preprocessing/venv"
+
+set -gx _OLD_VIRTUAL_PATH $PATH
+set -gx PATH "$VIRTUAL_ENV/bin" $PATH
+
+# Unset PYTHONHOME if set.
+if set -q PYTHONHOME
+    set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
+    set -e PYTHONHOME
+end
+
+if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
+    # fish uses a function instead of an env var to generate the prompt.
+
+    # Save the current fish_prompt function as the function _old_fish_prompt.
+    functions -c fish_prompt _old_fish_prompt
+
+    # With the original prompt function renamed, we can override with our own.
+    function fish_prompt
+        # Save the return status of the last command.
+        set -l old_status $status
+
+        # Output the venv prompt; color taken from the blue of the Python logo.
+        printf "%s%s%s" (set_color 4B8BBE) "(venv) " (set_color normal)
+
+        # Restore the return status of the previous command.
+        echo "exit $old_status" | .
+        # Output the original/"old" prompt.
+        _old_fish_prompt
+    end
+
+    set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
+end

venv/bin/chardetect

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from chardet.cli.chardetect import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/dumppdf.py

@@ -0,0 +1,480 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+"""Extract pdf structure in XML format"""
+
+import logging
+import os.path
+import re
+import sys
+from argparse import ArgumentParser
+from typing import Any, Container, Dict, Iterable, List, Optional, TextIO, Union, cast
+
+import pdfminer
+from pdfminer.pdfdocument import PDFDocument, PDFNoOutlines, PDFXRefFallback
+from pdfminer.pdfexceptions import (
+    PDFIOError,
+    PDFObjectNotFound,
+    PDFTypeError,
+    PDFValueError,
+)
+from pdfminer.pdfpage import PDFPage
+from pdfminer.pdfparser import PDFParser
+from pdfminer.pdftypes import PDFObjRef, PDFStream, resolve1, stream_value
+from pdfminer.psparser import LIT, PSKeyword, PSLiteral
+from pdfminer.utils import isnumber
+
+logging.basicConfig()
+logger = logging.getLogger(__name__)
+
+ESC_PAT = re.compile(r'[\000-\037&<>()"\042\047\134\177-\377]')
+
+
+def escape(s: Union[str, bytes]) -> str:
+    if isinstance(s, bytes):
+        us = str(s, "latin-1")
+    else:
+        us = s
+    return ESC_PAT.sub(lambda m: "&#%d;" % ord(m.group(0)), us)
+
+
+def dumpxml(out: TextIO, obj: object, codec: Optional[str] = None) -> None:
+    if obj is None:
+        out.write("<null />")
+        return
+
+    if isinstance(obj, dict):
+        out.write('<dict size="%d">\n' % len(obj))
+        for k, v in obj.items():
+            out.write("<key>%s</key>\n" % k)
+            out.write("<value>")
+            dumpxml(out, v)
+            out.write("</value>\n")
+        out.write("</dict>")
+        return
+
+    if isinstance(obj, list):
+        out.write('<list size="%d">\n' % len(obj))
+        for v in obj:
+            dumpxml(out, v)
+            out.write("\n")
+        out.write("</list>")
+        return
+
+    if isinstance(obj, (str, bytes)):
+        out.write('<string size="%d">%s</string>' % (len(obj), escape(obj)))
+        return
+
+    if isinstance(obj, PDFStream):
+        if codec == "raw":
+            # Bug: writing bytes to text I/O. This will raise TypeError.
+            out.write(obj.get_rawdata())  # type: ignore [arg-type]
+        elif codec == "binary":
+            # Bug: writing bytes to text I/O. This will raise TypeError.
+            out.write(obj.get_data())  # type: ignore [arg-type]
+        else:
+            out.write("<stream>\n<props>\n")
+            dumpxml(out, obj.attrs)
+            out.write("\n</props>\n")
+            if codec == "text":
+                data = obj.get_data()
+                out.write('<data size="%d">%s</data>\n' % (len(data), escape(data)))
+            out.write("</stream>")
+        return
+
+    if isinstance(obj, PDFObjRef):
+        out.write('<ref id="%d" />' % obj.objid)
+        return
+
+    if isinstance(obj, PSKeyword):
+        # Likely bug: obj.name is bytes, not str
+        out.write("<keyword>%s</keyword>" % obj.name)  # type: ignore [str-bytes-safe]
+        return
+
+    if isinstance(obj, PSLiteral):
+        # Likely bug: obj.name may be bytes, not str
+        out.write("<literal>%s</literal>" % obj.name)  # type: ignore [str-bytes-safe]
+        return
+
+    if isnumber(obj):
+        out.write("<number>%s</number>" % obj)
+        return
+
+    raise PDFTypeError(obj)
+
+
+def dumptrailers(
+    out: TextIO,
+    doc: PDFDocument,
+    show_fallback_xref: bool = False,
+) -> None:
+    for xref in doc.xrefs:
+        if not isinstance(xref, PDFXRefFallback) or show_fallback_xref:
+            out.write("<trailer>\n")
+            dumpxml(out, xref.get_trailer())
+            out.write("\n</trailer>\n\n")
+    no_xrefs = all(isinstance(xref, PDFXRefFallback) for xref in doc.xrefs)
+    if no_xrefs and not show_fallback_xref:
+        msg = (
+            "This PDF does not have an xref. Use --show-fallback-xref if "
+            "you want to display the content of a fallback xref that "
+            "contains all objects."
+        )
+        logger.warning(msg)
+
+
+def dumpallobjs(
+    out: TextIO,
+    doc: PDFDocument,
+    codec: Optional[str] = None,
+    show_fallback_xref: bool = False,
+) -> None:
+    visited = set()
+    out.write("<pdf>")
+    for xref in doc.xrefs:
+        for objid in xref.get_objids():
+            if objid in visited:
+                continue
+            visited.add(objid)
+            try:
+                obj = doc.getobj(objid)
+                if obj is None:
+                    continue
+                out.write('<object id="%d">\n' % objid)
+                dumpxml(out, obj, codec=codec)
+                out.write("\n</object>\n\n")
+            except PDFObjectNotFound as e:
+                print("not found: %r" % e)
+    dumptrailers(out, doc, show_fallback_xref)
+    out.write("</pdf>")
+
+
+def dumpoutline(
+    outfp: TextIO,
+    fname: str,
+    objids: Any,
+    pagenos: Container[int],
+    password: str = "",
+    dumpall: bool = False,
+    codec: Optional[str] = None,
+    extractdir: Optional[str] = None,
+) -> None:
+    fp = open(fname, "rb")
+    parser = PDFParser(fp)
+    doc = PDFDocument(parser, password)
+    pages = {
+        page.pageid: pageno
+        for (pageno, page) in enumerate(PDFPage.create_pages(doc), 1)
+    }
+
+    def resolve_dest(dest: object) -> Any:
+        if isinstance(dest, (str, bytes)):
+            dest = resolve1(doc.get_dest(dest))
+        elif isinstance(dest, PSLiteral):
+            dest = resolve1(doc.get_dest(dest.name))
+        if isinstance(dest, dict):
+            dest = dest["D"]
+        if isinstance(dest, PDFObjRef):
+            dest = dest.resolve()
+        return dest
+
+    try:
+        outlines = doc.get_outlines()
+        outfp.write("<outlines>\n")
+        for level, title, dest, a, se in outlines:
+            pageno = None
+            if dest:
+                dest = resolve_dest(dest)
+                pageno = pages[dest[0].objid]
+            elif a:
+                action = a
+                if isinstance(action, dict):
+                    subtype = action.get("S")
+                    if subtype and repr(subtype) == "/'GoTo'" and action.get("D"):
+                        dest = resolve_dest(action["D"])
+                        pageno = pages[dest[0].objid]
+            s = escape(title)
+            outfp.write(f'<outline level="{level!r}" title="{s}">\n')
+            if dest is not None:
+                outfp.write("<dest>")
+                dumpxml(outfp, dest)
+                outfp.write("</dest>\n")
+            if pageno is not None:
+                outfp.write("<pageno>%r</pageno>\n" % pageno)
+            outfp.write("</outline>\n")
+        outfp.write("</outlines>\n")
+    except PDFNoOutlines:
+        pass
+    parser.close()
+    fp.close()
+
+
+LITERAL_FILESPEC = LIT("Filespec")
+LITERAL_EMBEDDEDFILE = LIT("EmbeddedFile")
+
+
+def extractembedded(fname: str, password: str, extractdir: str) -> None:
+    def extract1(objid: int, obj: Dict[str, Any]) -> None:
+        filename = os.path.basename(obj.get("UF") or cast(bytes, obj.get("F")).decode())
+        fileref = obj["EF"].get("UF") or obj["EF"].get("F")
+        fileobj = doc.getobj(fileref.objid)
+        if not isinstance(fileobj, PDFStream):
+            error_msg = (
+                "unable to process PDF: reference for %r is not a "
+                "PDFStream" % filename
+            )
+            raise PDFValueError(error_msg)
+        if fileobj.get("Type") is not LITERAL_EMBEDDEDFILE:
+            raise PDFValueError(
+                "unable to process PDF: reference for %r "
+                "is not an EmbeddedFile" % (filename),
+            )
+        path = os.path.join(extractdir, "%.6d-%s" % (objid, filename))
+        if os.path.exists(path):
+            raise PDFIOError("file exists: %r" % path)
+        print("extracting: %r" % path)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        out = open(path, "wb")
+        out.write(fileobj.get_data())
+        out.close()
+
+    with open(fname, "rb") as fp:
+        parser = PDFParser(fp)
+        doc = PDFDocument(parser, password)
+        extracted_objids = set()
+        for xref in doc.xrefs:
+            for objid in xref.get_objids():
+                obj = doc.getobj(objid)
+                if (
+                    objid not in extracted_objids
+                    and isinstance(obj, dict)
+                    and obj.get("Type") is LITERAL_FILESPEC
+                ):
+                    extracted_objids.add(objid)
+                    extract1(objid, obj)
+
+
+def dumppdf(
+    outfp: TextIO,
+    fname: str,
+    objids: Iterable[int],
+    pagenos: Container[int],
+    password: str = "",
+    dumpall: bool = False,
+    codec: Optional[str] = None,
+    extractdir: Optional[str] = None,
+    show_fallback_xref: bool = False,
+) -> None:
+    fp = open(fname, "rb")
+    parser = PDFParser(fp)
+    doc = PDFDocument(parser, password)
+    if objids:
+        for objid in objids:
+            obj = doc.getobj(objid)
+            dumpxml(outfp, obj, codec=codec)
+    if pagenos:
+        for pageno, page in enumerate(PDFPage.create_pages(doc)):
+            if pageno in pagenos:
+                if codec:
+                    for obj in page.contents:
+                        obj = stream_value(obj)
+                        dumpxml(outfp, obj, codec=codec)
+                else:
+                    dumpxml(outfp, page.attrs)
+    if dumpall:
+        dumpallobjs(outfp, doc, codec, show_fallback_xref)
+    if (not objids) and (not pagenos) and (not dumpall):
+        dumptrailers(outfp, doc, show_fallback_xref)
+    fp.close()
+    if codec not in ("raw", "binary"):
+        outfp.write("\n")
+
+
+def create_parser() -> ArgumentParser:
+    parser = ArgumentParser(description=__doc__, add_help=True)
+    parser.add_argument(
+        "files",
+        type=str,
+        default=None,
+        nargs="+",
+        help="One or more paths to PDF files.",
+    )
+
+    parser.add_argument(
+        "--version",
+        "-v",
+        action="version",
+        version=f"pdfminer.six v{pdfminer.__version__}",
+    )
+    parser.add_argument(
+        "--debug",
+        "-d",
+        default=False,
+        action="store_true",
+        help="Use debug logging level.",
+    )
+    procedure_parser = parser.add_mutually_exclusive_group()
+    procedure_parser.add_argument(
+        "--extract-toc",
+        "-T",
+        default=False,
+        action="store_true",
+        help="Extract structure of outline",
+    )
+    procedure_parser.add_argument(
+        "--extract-embedded",
+        "-E",
+        type=str,
+        help="Extract embedded files",
+    )
+
+    parse_params = parser.add_argument_group(
+        "Parser",
+        description="Used during PDF parsing",
+    )
+    parse_params.add_argument(
+        "--page-numbers",
+        type=int,
+        default=None,
+        nargs="+",
+        help="A space-seperated list of page numbers to parse.",
+    )
+    parse_params.add_argument(
+        "--pagenos",
+        "-p",
+        type=str,
+        help="A comma-separated list of page numbers to parse. Included for "
+        "legacy applications, use --page-numbers for more idiomatic "
+        "argument entry.",
+    )
+    parse_params.add_argument(
+        "--objects",
+        "-i",
+        type=str,
+        help="Comma separated list of object numbers to extract",
+    )
+    parse_params.add_argument(
+        "--all",
+        "-a",
+        default=False,
+        action="store_true",
+        help="If the structure of all objects should be extracted",
+    )
+    parse_params.add_argument(
+        "--show-fallback-xref",
+        action="store_true",
+        help="Additionally show the fallback xref. Use this if the PDF "
+        "has zero or only invalid xref's. This setting is ignored if "
+        "--extract-toc or --extract-embedded is used.",
+    )
+    parse_params.add_argument(
+        "--password",
+        "-P",
+        type=str,
+        default="",
+        help="The password to use for decrypting PDF file.",
+    )
+
+    output_params = parser.add_argument_group(
+        "Output",
+        description="Used during output generation.",
+    )
+    output_params.add_argument(
+        "--outfile",
+        "-o",
+        type=str,
+        default="-",
+        help='Path to file where output is written. Or "-" (default) to '
+        "write to stdout.",
+    )
+    codec_parser = output_params.add_mutually_exclusive_group()
+    codec_parser.add_argument(
+        "--raw-stream",
+        "-r",
+        default=False,
+        action="store_true",
+        help="Write stream objects without encoding",
+    )
+    codec_parser.add_argument(
+        "--binary-stream",
+        "-b",
+        default=False,
+        action="store_true",
+        help="Write stream objects with binary encoding",
+    )
+    codec_parser.add_argument(
+        "--text-stream",
+        "-t",
+        default=False,
+        action="store_true",
+        help="Write stream objects as plain text",
+    )
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> None:
+    parser = create_parser()
+    args = parser.parse_args(args=argv)
+
+    if args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    if args.outfile == "-":
+        outfp = sys.stdout
+    else:
+        outfp = open(args.outfile, "w")
+
+    if args.objects:
+        objids = [int(x) for x in args.objects.split(",")]
+    else:
+        objids = []
+
+    if args.page_numbers:
+        pagenos = {x - 1 for x in args.page_numbers}
+    elif args.pagenos:
+        pagenos = {int(x) - 1 for x in args.pagenos.split(",")}
+    else:
+        pagenos = set()
+
+    password = args.password
+
+    if args.raw_stream:
+        codec: Optional[str] = "raw"
+    elif args.binary_stream:
+        codec = "binary"
+    elif args.text_stream:
+        codec = "text"
+    else:
+        codec = None
+
+    for fname in args.files:
+        if args.extract_toc:
+            dumpoutline(
+                outfp,
+                fname,
+                objids,
+                pagenos,
+                password=password,
+                dumpall=args.all,
+                codec=codec,
+                extractdir=None,
+            )
+        elif args.extract_embedded:
+            extractembedded(fname, password=password, extractdir=args.extract_embedded)
+        else:
+            dumppdf(
+                outfp,
+                fname,
+                objids,
+                pagenos,
+                password=password,
+                dumpall=args.all,
+                codec=codec,
+                extractdir=None,
+                show_fallback_xref=args.show_fallback_xref,
+            )
+
+    outfp.close()
+
+
+if __name__ == "__main__":
+    main()

venv/bin/f2py

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from numpy.f2py.f2py2e import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/fastapi

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fastapi.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/fonttools

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.__main__ import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/gradio

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from gradio.cli import cli
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(cli())

venv/bin/hf

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from huggingface_hub.cli.hf import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/httpx

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from httpx import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/huggingface-cli

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from huggingface_hub.commands.huggingface_cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/markdown-it

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from markdown_it.cli.parse import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/markdownify

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from markdownify.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/normalizer

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from charset_normalizer.cli import cli_detect
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(cli_detect())

venv/bin/numpy-config

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from numpy._configtool import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pdf2txt.py

@@ -0,0 +1,323 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+"""A command line tool for extracting text and images from PDF and
+output it to plain text, html, xml or tags.
+"""
+
+import argparse
+import logging
+import sys
+from typing import Any, Container, Iterable, List, Optional
+
+import pdfminer.high_level
+from pdfminer.layout import LAParams
+from pdfminer.pdfexceptions import PDFValueError
+from pdfminer.utils import AnyIO
+
+logging.basicConfig()
+
+OUTPUT_TYPES = ((".htm", "html"), (".html", "html"), (".xml", "xml"), (".tag", "tag"))
+
+
+def float_or_disabled(x: str) -> Optional[float]:
+    if x.lower().strip() == "disabled":
+        return None
+    try:
+        return float(x)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"invalid float value: {x}")
+
+
+def extract_text(
+    files: Iterable[str] = [],
+    outfile: str = "-",
+    laparams: Optional[LAParams] = None,
+    output_type: str = "text",
+    codec: str = "utf-8",
+    strip_control: bool = False,
+    maxpages: int = 0,
+    page_numbers: Optional[Container[int]] = None,
+    password: str = "",
+    scale: float = 1.0,
+    rotation: int = 0,
+    layoutmode: str = "normal",
+    output_dir: Optional[str] = None,
+    debug: bool = False,
+    disable_caching: bool = False,
+    **kwargs: Any,
+) -> AnyIO:
+    if not files:
+        raise PDFValueError("Must provide files to work upon!")
+
+    if output_type == "text" and outfile != "-":
+        for override, alttype in OUTPUT_TYPES:
+            if outfile.endswith(override):
+                output_type = alttype
+
+    if outfile == "-":
+        outfp: AnyIO = sys.stdout
+        if sys.stdout.encoding is not None:
+            codec = "utf-8"
+    else:
+        outfp = open(outfile, "wb")
+
+    for fname in files:
+        with open(fname, "rb") as fp:
+            pdfminer.high_level.extract_text_to_fp(fp, **locals())
+    return outfp
+
+
+def create_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__, add_help=True)
+    parser.add_argument(
+        "files",
+        type=str,
+        default=None,
+        nargs="+",
+        help="One or more paths to PDF files.",
+    )
+
+    parser.add_argument(
+        "--version",
+        "-v",
+        action="version",
+        version=f"pdfminer.six v{pdfminer.__version__}",
+    )
+    parser.add_argument(
+        "--debug",
+        "-d",
+        default=False,
+        action="store_true",
+        help="Use debug logging level.",
+    )
+    parser.add_argument(
+        "--disable-caching",
+        "-C",
+        default=False,
+        action="store_true",
+        help="If caching or resources, such as fonts, should be disabled.",
+    )
+
+    parse_params = parser.add_argument_group(
+        "Parser",
+        description="Used during PDF parsing",
+    )
+    parse_params.add_argument(
+        "--page-numbers",
+        type=int,
+        default=None,
+        nargs="+",
+        help="A space-seperated list of page numbers to parse.",
+    )
+    parse_params.add_argument(
+        "--pagenos",
+        "-p",
+        type=str,
+        help="A comma-separated list of page numbers to parse. "
+        "Included for legacy applications, use --page-numbers "
+        "for more idiomatic argument entry.",
+    )
+    parse_params.add_argument(
+        "--maxpages",
+        "-m",
+        type=int,
+        default=0,
+        help="The maximum number of pages to parse.",
+    )
+    parse_params.add_argument(
+        "--password",
+        "-P",
+        type=str,
+        default="",
+        help="The password to use for decrypting PDF file.",
+    )
+    parse_params.add_argument(
+        "--rotation",
+        "-R",
+        default=0,
+        type=int,
+        help="The number of degrees to rotate the PDF "
+        "before other types of processing.",
+    )
+
+    la_params = LAParams()  # will be used for defaults
+    la_param_group = parser.add_argument_group(
+        "Layout analysis",
+        description="Used during layout analysis.",
+    )
+    la_param_group.add_argument(
+        "--no-laparams",
+        "-n",
+        default=False,
+        action="store_true",
+        help="If layout analysis parameters should be ignored.",
+    )
+    la_param_group.add_argument(
+        "--detect-vertical",
+        "-V",
+        default=la_params.detect_vertical,
+        action="store_true",
+        help="If vertical text should be considered during layout analysis",
+    )
+    la_param_group.add_argument(
+        "--line-overlap",
+        type=float,
+        default=la_params.line_overlap,
+        help="If two characters have more overlap than this they "
+        "are considered to be on the same line. The overlap is specified "
+        "relative to the minimum height of both characters.",
+    )
+    la_param_group.add_argument(
+        "--char-margin",
+        "-M",
+        type=float,
+        default=la_params.char_margin,
+        help="If two characters are closer together than this margin they "
+        "are considered to be part of the same line. The margin is "
+        "specified relative to the width of the character.",
+    )
+    la_param_group.add_argument(
+        "--word-margin",
+        "-W",
+        type=float,
+        default=la_params.word_margin,
+        help="If two characters on the same line are further apart than this "
+        "margin then they are considered to be two separate words, and "
+        "an intermediate space will be added for readability. The margin "
+        "is specified relative to the width of the character.",
+    )
+    la_param_group.add_argument(
+        "--line-margin",
+        "-L",
+        type=float,
+        default=la_params.line_margin,
+        help="If two lines are close together they are considered to "
+        "be part of the same paragraph. The margin is specified "
+        "relative to the height of a line.",
+    )
+    la_param_group.add_argument(
+        "--boxes-flow",
+        "-F",
+        type=float_or_disabled,
+        default=la_params.boxes_flow,
+        help="Specifies how much a horizontal and vertical position of a "
+        "text matters when determining the order of lines. The value "
+        "should be within the range of -1.0 (only horizontal position "
+        "matters) to +1.0 (only vertical position matters). You can also "
+        "pass `disabled` to disable advanced layout analysis, and "
+        "instead return text based on the position of the bottom left "
+        "corner of the text box.",
+    )
+    la_param_group.add_argument(
+        "--all-texts",
+        "-A",
+        default=la_params.all_texts,
+        action="store_true",
+        help="If layout analysis should be performed on text in figures.",
+    )
+
+    output_params = parser.add_argument_group(
+        "Output",
+        description="Used during output generation.",
+    )
+    output_params.add_argument(
+        "--outfile",
+        "-o",
+        type=str,
+        default="-",
+        help="Path to file where output is written. "
+        'Or "-" (default) to write to stdout.',
+    )
+    output_params.add_argument(
+        "--output_type",
+        "-t",
+        type=str,
+        default="text",
+        help="Type of output to generate {text,html,xml,tag}.",
+    )
+    output_params.add_argument(
+        "--codec",
+        "-c",
+        type=str,
+        default="utf-8",
+        help="Text encoding to use in output file.",
+    )
+    output_params.add_argument(
+        "--output-dir",
+        "-O",
+        default=None,
+        help="The output directory to put extracted images in. If not given, "
+        "images are not extracted.",
+    )
+    output_params.add_argument(
+        "--layoutmode",
+        "-Y",
+        default="normal",
+        type=str,
+        help="Type of layout to use when generating html "
+        "{normal,exact,loose}. If normal,each line is"
+        " positioned separately in the html. If exact"
+        ", each character is positioned separately in"
+        " the html. If loose, same result as normal "
+        "but with an additional newline after each "
+        "text line. Only used when output_type is html.",
+    )
+    output_params.add_argument(
+        "--scale",
+        "-s",
+        type=float,
+        default=1.0,
+        help="The amount of zoom to use when generating html file. "
+        "Only used when output_type is html.",
+    )
+    output_params.add_argument(
+        "--strip-control",
+        "-S",
+        default=False,
+        action="store_true",
+        help="Remove control statement from text. "
+        "Only used when output_type is xml.",
+    )
+
+    return parser
+
+
+def parse_args(args: Optional[List[str]]) -> argparse.Namespace:
+    parsed_args = create_parser().parse_args(args=args)
+
+    # Propagate parsed layout parameters to LAParams object
+    if parsed_args.no_laparams:
+        parsed_args.laparams = None
+    else:
+        parsed_args.laparams = LAParams(
+            line_overlap=parsed_args.line_overlap,
+            char_margin=parsed_args.char_margin,
+            line_margin=parsed_args.line_margin,
+            word_margin=parsed_args.word_margin,
+            boxes_flow=parsed_args.boxes_flow,
+            detect_vertical=parsed_args.detect_vertical,
+            all_texts=parsed_args.all_texts,
+        )
+
+    if parsed_args.page_numbers:
+        parsed_args.page_numbers = {x - 1 for x in parsed_args.page_numbers}
+
+    if parsed_args.pagenos:
+        parsed_args.page_numbers = {int(x) - 1 for x in parsed_args.pagenos.split(",")}
+
+    if parsed_args.output_type == "text" and parsed_args.outfile != "-":
+        for override, alttype in OUTPUT_TYPES:
+            if parsed_args.outfile.endswith(override):
+                parsed_args.output_type = alttype
+
+    return parsed_args
+
+
+def main(args: Optional[List[str]] = None) -> int:
+    parsed_args = parse_args(args)
+    outfp = extract_text(**vars(parsed_args))
+    outfp.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

venv/bin/pdfplumber

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pdfplumber.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pip

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pip._internal.cli.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pip3

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pip._internal.cli.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pip3.9

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pip._internal.cli.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pyftmerge

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.merge import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pyftsubset

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.subset import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pygmentize

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pygments.cmdline import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/pypdfium2

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pypdfium2.__main__ import cli_main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(cli_main())

venv/bin/python

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c54ab0ae7c3b248e10f847477b01639de71d9b1d67aee809f1f4da196caca8cc
+size 15880080

venv/bin/python3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c54ab0ae7c3b248e10f847477b01639de71d9b1d67aee809f1f4da196caca8cc
+size 15880080

venv/bin/python3.9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c54ab0ae7c3b248e10f847477b01639de71d9b1d67aee809f1f4da196caca8cc
+size 15880080

venv/bin/ruff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5321df702f31419551f21b921c00fa3a5c8347ac399d0a779bdca4eca3daad9c
+size 37014240

venv/bin/tiny-agents

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from huggingface_hub.inference._mcp.cli import app
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(app())

venv/bin/tqdm

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from tqdm.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/ttx

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from fontTools.ttx import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/typer

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from typer.cli import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/upload_theme

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from gradio.themes.upload_theme import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/bin/uvicorn

@@ -0,0 +1,8 @@
+#!/home/l.sottani/rag/preprocessing/venv/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from uvicorn.main import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

venv/lib/python3.9/site-packages/MarkupSafe-2.1.5.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

venv/lib/python3.9/site-packages/MarkupSafe-2.1.5.dist-info/LICENSE.rst

@@ -0,0 +1,28 @@
+Copyright 2010 Pallets
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1.  Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+3.  Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

venv/lib/python3.9/site-packages/MarkupSafe-2.1.5.dist-info/METADATA

@@ -0,0 +1,93 @@
+Metadata-Version: 2.1
+Name: MarkupSafe
+Version: 2.1.5
+Summary: Safely add untrusted strings to HTML/XML markup.
+Home-page: https://palletsprojects.com/p/markupsafe/
+Maintainer: Pallets
+Maintainer-email: contact@palletsprojects.com
+License: BSD-3-Clause
+Project-URL: Donate, https://palletsprojects.com/donate
+Project-URL: Documentation, https://markupsafe.palletsprojects.com/
+Project-URL: Changes, https://markupsafe.palletsprojects.com/changes/
+Project-URL: Source Code, https://github.com/pallets/markupsafe/
+Project-URL: Issue Tracker, https://github.com/pallets/markupsafe/issues/
+Project-URL: Chat, https://discord.gg/pallets
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst
+License-File: LICENSE.rst
+
+MarkupSafe
+==========
+
+MarkupSafe implements a text object that escapes characters so it is
+safe to use in HTML and XML. Characters that have special meanings are
+replaced so that they display as the actual characters. This mitigates
+injection attacks, meaning untrusted user input can safely be displayed
+on a page.
+
+
+Installing
+----------
+
+Install and update using `pip`_:
+
+.. code-block:: text
+
+    pip install -U MarkupSafe
+
+.. _pip: https://pip.pypa.io/en/stable/getting-started/
+
+
+Examples
+--------
+
+.. code-block:: pycon
+
+    >>> from markupsafe import Markup, escape
+
+    >>> # escape replaces special characters and wraps in Markup
+    >>> escape("<script>alert(document.cookie);</script>")
+    Markup('&lt;script&gt;alert(document.cookie);&lt;/script&gt;')
+
+    >>> # wrap in Markup to mark text "safe" and prevent escaping
+    >>> Markup("<strong>Hello</strong>")
+    Markup('<strong>hello</strong>')
+
+    >>> escape(Markup("<strong>Hello</strong>"))
+    Markup('<strong>hello</strong>')
+
+    >>> # Markup is a str subclass
+    >>> # methods and operators escape their arguments
+    >>> template = Markup("Hello <em>{name}</em>")
+    >>> template.format(name='"World"')
+    Markup('Hello <em>&#34;World&#34;</em>')
+
+
+Donate
+------
+
+The Pallets organization develops and supports MarkupSafe and other
+popular packages. In order to grow the community of contributors and
+users, and allow the maintainers to devote more time to the projects,
+`please donate today`_.
+
+.. _please donate today: https://palletsprojects.com/donate
+
+
+Links
+-----
+
+-   Documentation: https://markupsafe.palletsprojects.com/
+-   Changes: https://markupsafe.palletsprojects.com/changes/
+-   PyPI Releases: https://pypi.org/project/MarkupSafe/
+-   Source Code: https://github.com/pallets/markupsafe/
+-   Issue Tracker: https://github.com/pallets/markupsafe/issues/
+-   Chat: https://discord.gg/pallets